Availability: Adds account-level read regions as effective preferred …

…regions when preferred regions is not set on client. (#4709)

# Pull Request Template

## Problem Statement

As of today, customers who do not configure
`ApplicationPreferredRegions` or `ApplicationRegion` are pinned to
either the hub region (in multi-write accounts) or the primary region
(in single-write accounts). In outage scenarios, availability is scoped
to just a single region and is not ideal (both reads/writes in
multi-write accounts and reads in single-write multi-region accounts
have their availability curbed to just the one region). Setting
`ApplicationPreferredRegions` or `ApplicationRegion` as empty is not an
opt-out of availability decision from the customer's perspective unless
of course on the client a regional endpoint has been set. This PR aims
to fix this issue when preferred regions is not set, and a global
endpoint is set on the client.

## Approach taken in this PR

The idea is to construct an _effective preferred region list_ and to
rely on account-level read and account-level write regions for it. If
there is client-perceived unavailability or account-level topology
changes - then this _effective preferred regions list_ is reordered
accordingly or reflects the account-level regions post a cached
`DatabaseAccount` refresh in the SDK.

There are also changes made to the flow which decides when
`DatabaseAccount` refresh is triggered. The decision depends on a check
whether the SDK has a different effective first preferred read / write
region from the first account-level read / write region.

## Closing issues
closes #4665

---------

Co-authored-by: Kiran Kumar Kolli <kirankk@microsoft.com>
Co-authored-by: Nalu Tripician <27316859+NaluTripician@users.noreply.github.com>
Co-authored-by: Debdatta Kunda <87335885+kundadebdatta@users.noreply.github.com>

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -918,8 +918,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
         {
             this.ValidateDirectTCPSettings();
             this.ValidateLimitToEndpointSettings();
-            this.ValidatePartitionLevelFailoverSettings();
-            this.ValidateAvailabilityStrategy();
+            this.ValidatePartitionLevelFailoverSettings();
 
             ConnectionPolicy connectionPolicy = new ConnectionPolicy()
             {
@@ -1096,15 +1095,6 @@ private void ValidatePartitionLevelFailoverSettings()
             {
                 throw new ArgumentException($"{nameof(this.ApplicationPreferredRegions)} is required when {nameof(this.EnablePartitionLevelFailover)} is enabled.");
             }
-        }
-
-        private void ValidateAvailabilityStrategy()
-        {
-            if (this.AvailabilityStrategy != null
-                && this.ApplicationPreferredRegions == null && this.ApplicationRegion == null)
-            {
-                throw new ArgumentException($"{nameof(this.ApplicationPreferredRegions)} or {nameof(this.ApplicationRegion)} must be set to use {nameof(this.AvailabilityStrategy)}");
-            }
         }
 
         private void ValidateDirectTCPSettings()

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -94,8 +94,16 @@ public GlobalEndpointManager(IDocumentClientInternal owner, ConnectionPolicy con
         public ReadOnlyCollection<Uri> AccountReadEndpoints => this.locationCache.AccountReadEndpoints;
 
         public ReadOnlyCollection<Uri> WriteEndpoints => this.locationCache.WriteEndpoints;
+
+        public int PreferredLocationCount
+        {
+            get
+            {
+                IList<string> effectivePreferredLocations = this.GetEffectivePreferredLocations();
 
-        public int PreferredLocationCount => this.connectionPolicy.PreferredLocations != null ? this.connectionPolicy.PreferredLocations.Count : 0;
+                return effectivePreferredLocations.Count;
+            }
+        }
 
         public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request)
         {
@@ -273,8 +281,7 @@ private async Task TryGetAccountPropertiesFromAllLocationsAsync()
                         return;
                     }
 
-                    await this.GetAndUpdateAccountPropertiesAsync(
-                        endpoint: serviceEndpoint);
+                    await this.GetAndUpdateAccountPropertiesAsync(endpoint: serviceEndpoint);
                 }
             }
 
@@ -495,8 +502,8 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
             if (this.cancellationTokenSource.IsCancellationRequested)
             {
                 return;
-            }
-
+            }
+
             this.locationCache.OnDatabaseAccountRead(databaseAccount);
 
             if (this.isBackgroundAccountRefreshActive)
@@ -544,7 +551,7 @@ public virtual async Task RefreshLocationAsync(bool forceRefresh = false)
         public bool CanSupportMultipleWriteLocations(DocumentServiceRequest request)
         {
             return this.locationCache.CanUseMultipleWriteLocations()
-                && this.locationCache.GetAvailableWriteLocations()?.Count > 1
+                && this.locationCache.GetAvailableAccountLevelWriteLocations()?.Count > 1
                 && (request.ResourceType == ResourceType.Document ||
                 (request.ResourceType == ResourceType.StoredProcedure && request.OperationType == OperationType.Execute));
         }
@@ -647,7 +654,10 @@ private async Task RefreshDatabaseAccountInternalAsync(bool forceRefresh)
             try
             {
                 this.LastBackgroundRefreshUtc = DateTime.UtcNow;
-                this.locationCache.OnDatabaseAccountRead(await this.GetDatabaseAccountAsync(true));
+                AccountProperties accountProperties = await this.GetDatabaseAccountAsync(true);
+
+                this.locationCache.OnDatabaseAccountRead(accountProperties);
+
             }
             catch (Exception ex)
             {
@@ -671,7 +681,7 @@ internal async Task<AccountProperties> GetDatabaseAccountAsync(bool forceRefresh
                               obsoleteValue: null,
                               singleValueInitFunc: () => GlobalEndpointManager.GetDatabaseAccountFromAnyLocationsAsync(
                                   this.defaultEndpoint,
-                                  this.connectionPolicy.PreferredLocations,
+                                  this.GetEffectivePreferredLocations(),
                                   this.connectionPolicy.AccountInitializationCustomEndpoints,
                                   this.GetDatabaseAccountAsync,
                                   this.cancellationTokenSource.Token),
@@ -689,6 +699,17 @@ private bool SkipRefresh(bool forceRefresh)
             TimeSpan timeSinceLastRefresh = DateTime.UtcNow - this.LastBackgroundRefreshUtc;
             return (this.isAccountRefreshInProgress || this.MinTimeBetweenAccountRefresh > timeSinceLastRefresh)
                 && !forceRefresh;
+        }
+
+        public IList<string> GetEffectivePreferredLocations()
+        {
+            if (this.connectionPolicy.PreferredLocations != null && this.connectionPolicy.PreferredLocations.Count > 0)
+            {
+                return this.connectionPolicy.PreferredLocations;
+            }
+
+            return this.connectionPolicy.PreferredLocations?.Count > 0 ? 
+                this.connectionPolicy.PreferredLocations : this.locationCache.EffectivePreferredLocations;
         }
     }
 }