update http client timeout for scope sync (#4822)

HttpClient has default timeout to 100seconds and when it times out it throws TaskCanceledException and call is not retried.
This change is setting timeout to 5 min for http client and uses TimeoutAfter which throws TimeoutException

edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/NestedDeviceScopeApiClient.cs

@@ -24,6 +24,10 @@ public class NestedDeviceScopeApiClient : IDeviceScopeApiClient
 
         const string NestedApiVersion = "2020-06-30-preview";
 
+        static readonly TimeSpan OperationTimeout = TimeSpan.FromMinutes(2);
+        // Timeout set to HttpClient is > OperationTimeout because HttpClient throws TaskCanceledException when it timeouts and retry policy won't retry it in that case
+        static readonly TimeSpan HttpOperationTimeout = TimeSpan.FromMinutes(5);
+
         static readonly ITransientErrorDetectionStrategy TransientErrorDetectionStrategy = new ErrorDetectionStrategy();
 
         static readonly RetryStrategy TransientRetryStrategy =
@@ -124,7 +128,7 @@ async Task<ScopeResult> GetIdentityOnBehalfOfWithRetry(Uri uri, string deviceId,
             try
             {
                 return await ExecuteWithRetry(
-                    () => this.GetIdentityOnBehalfOfInternalAsync(uri, deviceId, moduleId, onBehalfOfDevice),
+                    () => this.GetIdentityOnBehalfOfInternalAsync(uri, deviceId, moduleId, onBehalfOfDevice).TimeoutAfter(OperationTimeout),
                     Events.RetryingGetIdentities,
                     this.retryStrategy);
             }
@@ -143,7 +147,7 @@ async Task<ScopeResult> GetIdentitiesInTargetScopeWithRetry(Uri uri, Option<stri
             try
             {
                 return await ExecuteWithRetry(
-                    () => this.GetIdentitiesInTargetScopeInternalAsync(uri, continuation),
+                    () => this.GetIdentitiesInTargetScopeInternalAsync(uri, continuation).TimeoutAfter(OperationTimeout),
                     Events.RetryingGetIdentities,
                     this.retryStrategy);
             }
@@ -159,6 +163,7 @@ async Task<ScopeResult> GetIdentityOnBehalfOfInternalAsync(Uri uri, string devic
             HttpClient client = this.proxy
                 .Map(p => new HttpClient(new HttpClientHandler { Proxy = p }, disposeHandler: true))
                 .GetOrElse(() => new HttpClient());
+            client.Timeout = HttpOperationTimeout;
             using (var msg = new HttpRequestMessage(HttpMethod.Post, uri))
             {
                 // Get the auth-chain for the Edge we're acting OnBehalfOf
@@ -190,6 +195,7 @@ async Task<ScopeResult> GetIdentitiesInTargetScopeInternalAsync(Uri uri, Option<
             HttpClient client = this.proxy
                 .Map(p => new HttpClient(new HttpClientHandler { Proxy = p }, disposeHandler: true))
                 .GetOrElse(() => new HttpClient());
+            client.Timeout = HttpOperationTimeout;
             using (var msg = new HttpRequestMessage(HttpMethod.Post, uri))
             {
                 // Get the auth-chain for the target device