ClientRetryPolicy: Fixes behavior to Meta-data write operations in multimaster accounts

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -185,6 +185,20 @@ private async Task<ShouldRetryResult> ShouldRetryInternalAsync(
                     this.documentServiceRequest?.RequestContext?.LocationEndpointToRoute?.ToString() ?? string.Empty,
                     this.documentServiceRequest?.ResourceAddress ?? string.Empty);
 
+                if (this.globalEndpointManager.IsMetadataWriteRequestMultimaster(this.documentServiceRequest))
+                {
+                    Task<ShouldRetryResult> retryResult = this.ShouldRetryOnEndpointFailureAsync(
+                        isReadRequest: false,
+                        markBothReadAndWriteAsUnavailable: false,
+                        forceRefresh: true,
+                        retryOnPreferredLocations: false,
+                        overwriteEndpointDiscovery: true);
+
+                    this.retryContext.RetryLocationIndex = 0;
+
+                    return await retryResult;
+                }
+
                 return await this.ShouldRetryOnEndpointFailureAsync(
                     isReadRequest: false,
                     markBothReadAndWriteAsUnavailable: false,
@@ -237,9 +251,10 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
             bool isReadRequest,
             bool markBothReadAndWriteAsUnavailable,
             bool forceRefresh,
-            bool retryOnPreferredLocations)
+            bool retryOnPreferredLocations,
+            bool overwriteEndpointDiscovery = false)
         {
-            if (!this.enableEndpointDiscovery || this.failoverRetryCount > MaxRetryCount)
+            if (this.failoverRetryCount > MaxRetryCount || (!this.enableEndpointDiscovery && !overwriteEndpointDiscovery))
             {
                 DefaultTrace.TraceInformation("ClientRetryPolicy: ShouldRetryOnEndpointFailureAsync() Not retrying. Retry count = {0}, Endpoint = {1}", 
                     this.failoverRetryCount,
@@ -249,7 +264,7 @@ private async Task<ShouldRetryResult> ShouldRetryOnEndpointFailureAsync(
 
             this.failoverRetryCount++;
 
-            if (this.locationEndpoint != null)
+            if (this.locationEndpoint != null && !overwriteEndpointDiscovery)
             {
                 if (isReadRequest || markBothReadAndWriteAsUnavailable)
                 {

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -95,6 +95,11 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
 
         public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
 
+        public bool IsMetadataWriteRequestMultimaster(DocumentServiceRequest request)
+        {
+            return this.locationCache.IsMetadataWriteRequestOnMultimasterAccount(request);
+        }
+
         /// <summary>
         /// This will get the account information.
         /// It will try the global endpoint first. 
@@ -541,7 +546,6 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
                 }
             }
         }
-
         internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh = false)
         {
 #nullable disable  // Needed because AsyncCache does not have nullable enabled

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -78,7 +78,10 @@ public LocationCache(
 #endif  
 #endif
         }
-
+        /// <summary>
+        /// Gets total availible write locations from the current location information for a Cosmos account
+        /// </summary>
+        public int TotalAvailableWriteLocations => this.locationInfo.AvailableWriteLocations.Count;
         /// <summary>
         /// Gets list of read endpoints ordered by
         /// 1. Preferred location
@@ -206,6 +209,11 @@ public void OnLocationPreferenceChanged(ReadOnlyCollection<string> preferredLoca
                 preferenceList: preferredLocations);
         }
 
+        public bool IsMetadataWriteRequestOnMultimasterAccount(DocumentServiceRequest request)
+        {
+            return !request.IsReadOnlyRequest && this.TotalAvailableWriteLocations > 1 && !this.CanUseMultipleWriteLocations(request);
+        }
+
         /// <summary>
         /// Resolves request to service endpoint. 
         /// 1. If this is a write request
@@ -243,7 +251,7 @@ public Uri ResolveServiceEndpoint(DocumentServiceRequest request)
                 // first and the second writable region in DatabaseAccount (for manual failover)
                 DatabaseAccountLocationsInfo currentLocationInfo = this.locationInfo;
 
-                if (this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0)
+                if ((this.enableEndpointDiscovery && currentLocationInfo.AvailableWriteLocations.Count > 0) || this.IsMetadataWriteRequestOnMultimasterAccount(request))
                 {
                     locationIndex = Math.Min(locationIndex % 2, currentLocationInfo.AvailableWriteLocations.Count - 1);
                     string writeLocation = currentLocationInfo.AvailableWriteLocations[locationIndex];