diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/CloudProxy.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/CloudProxy.cs index 04b7142d90d..e0a2157339a 100644 --- a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/CloudProxy.cs +++ b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/CloudProxy.cs @@ -395,6 +395,11 @@ Task HandleException(Exception ex) Events.HandleNre(ex, this); return this.CloseAsync(); } + else if (ex.IsFailOver()) + { + Events.FailOverDetected(ex, this); + return this.CloseAsync(); + } } catch (Exception e) { @@ -599,7 +604,8 @@ enum EventIds ErrorUpdatingReportedProperties, ErrorSendingFeedbackMessageAsync, ErrorGettingTwin, - HandleNre + HandleNre, + FailOverDetected } public static void Closed(CloudProxy cloudProxy) @@ -712,6 +718,11 @@ public static void HandleNre(Exception ex, CloudProxy cloudProxy) Log.LogDebug((int)EventIds.HandleNre, ex, Invariant($"Got a non-recoverable error from client for {cloudProxy.clientId}. Closing the cloud proxy since it may be in a bad state.")); } + public static void FailOverDetected(Exception ex, CloudProxy cloudProxy) + { + Log.LogInformation((int)EventIds.FailOverDetected, ex, Invariant($"Fail-over detected, closing cloud proxy for {cloudProxy.clientId}.")); + } + internal static void ExceptionInHandleException(CloudProxy cloudProxy, Exception handlingException, Exception caughtException) { Log.LogDebug((int)EventIds.ExceptionInHandleException, Invariant($"Cloud proxy {cloudProxy.id} got exception {caughtException} while handling exception {handlingException}")); diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ConnectivityAwareClient.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ConnectivityAwareClient.cs index 67d33988610..9fd12f90512 100644 --- a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ConnectivityAwareClient.cs +++ b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ConnectivityAwareClient.cs @@ -163,6 +163,14 @@ async Task InvokeFunc(Func> func, string operation, bool useForCon await this.deviceConnectivityManager.CallTimedOut(); } } + else if (ex.IsFailOver()) + { + Events.FailOverDetected(this.identity, operation, mappedException); + if (useForConnectivityCheck) + { + await this.deviceConnectivityManager.CallTimedOut(); + } + } else { Events.OperationFailed(this.identity, operation, mappedException); @@ -199,7 +207,8 @@ enum EventIds OperationTimedOut, OperationFailed, OperationSucceeded, - ChangingStatus + ChangingStatus, + FailOverDetected } public static void ReceivedDeviceSdkCallback(IIdentity identity, ConnectionStatus status, ConnectionStatusChangeReason reason) @@ -226,6 +235,11 @@ public static void ChangingStatus(AtomicBoolean isConnected, IIdentity identity) { Log.LogInformation((int)EventIds.ChangingStatus, $"Cloud connection for {identity.Id} is {isConnected.Get()}"); } + + public static void FailOverDetected(IIdentity identity, string operation, Exception ex) + { + Log.LogInformation((int)EventIds.FailOverDetected, ex, $"Operation {operation} failed for {identity.Id} because of fail-over"); + } } } } diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ExceptionMapper.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ExceptionMapper.cs index 6ed8bc2fe12..821a4d3b1be 100644 --- a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ExceptionMapper.cs +++ b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/ExceptionMapper.cs @@ -3,10 +3,13 @@ namespace Microsoft.Azure.Devices.Edge.Hub.CloudProxy { using System; using DotNetty.Transport.Channels; + using Microsoft.Azure.Devices.Client.Exceptions; using Microsoft.Azure.Devices.Edge.Util; public static class ExceptionMapper { + const string FailOverMessage = "(condition='com.microsoft:iot-hub-not-found-error')"; + public static Exception GetEdgeException(this Exception sdkException, string operation) { Preconditions.CheckNonWhiteSpace(operation, nameof(operation)); @@ -20,5 +23,15 @@ public static Exception GetEdgeException(this Exception sdkException, string ope return sdkException; } + + public static bool IsFailOver(this Exception ex) + { + var isFailOver = ex is IotHubException + && ex.InnerException != null + && !string.IsNullOrEmpty(ex.InnerException.Message) + && ex.InnerException.Message.Contains(FailOverMessage); + + return isFailOver; + } } } diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs index f89a548d0d3..39ce1a0c6f0 100644 --- a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs +++ b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test/CloudProxyTest.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Devices.Edge.Hub.CloudProxy.Test using System.Text; using System.Threading.Tasks; using Microsoft.Azure.Devices.Client; + using Microsoft.Azure.Devices.Client.Exceptions; using Microsoft.Azure.Devices.Edge.Hub.Core; using Microsoft.Azure.Devices.Edge.Hub.Core.Cloud; using Microsoft.Azure.Devices.Edge.Hub.Core.Identity; @@ -246,6 +247,32 @@ public async Task TestHandleNonRecoverableExceptions(Type exceptionType) client.VerifyAll(); } + [Fact] + public async Task TestHandleFailOverExceptions() + { + // Arrange + var symbol = new Amqp.Encoding.AmqpSymbol("com.microsoft:iot-hub-not-found-error"); + var failOverException = new IotHubException(new Amqp.AmqpException(symbol, $"(condition='{symbol}')")); + + var messageConverter = Mock.Of>(m => m.FromMessage(It.IsAny()) == new Message()); + var messageConverterProvider = Mock.Of(m => m.Get() == messageConverter); + string clientId = "d1"; + var cloudListener = Mock.Of(); + TimeSpan idleTimeout = TimeSpan.FromSeconds(60); + Action connectionStatusChangedHandler = (s, status) => { }; + var client = new Mock(MockBehavior.Strict); + client.Setup(c => c.SendEventAsync(It.IsAny())).ThrowsAsync(failOverException); + client.Setup(c => c.CloseAsync()).Returns(Task.CompletedTask); + var cloudProxy = new CloudProxy(client.Object, messageConverterProvider, clientId, connectionStatusChangedHandler, cloudListener, idleTimeout, false); + IMessage message = new EdgeMessage.Builder(new byte[0]).Build(); + + // Act + await Assert.ThrowsAsync(() => cloudProxy.SendMessageAsync(message)); + + // Assert. + client.VerifyAll(); + } + static async Task CheckMessageInEventHub(Dictionary> sentMessagesByDevice, DateTime startTime) { string eventHubConnectionString = await SecretsHelper.GetSecretFromConfigKey("eventHubConnStrKey");