Handle messaging service connection error (#568)

* Handle messaging service connection error
* Update package versions

Signed-off-by: Victor Chang <vicchang@nvidia.com>

Author: mocsharp

doc/dependency_decisions.yml

@@ -819,14 +819,15 @@
   - :who: mocsharp
     :why: MIT (https://github.com/nunit/nunit3-vs-adapter/raw/master/LICENSE)
     :versions:
-    - 4.3.0
+    - 4.3.1
     :when: 2022-10-14 23:37:28.273089349 Z
 - - :approve
   - Newtonsoft.Json
   - :who: mocsharp
     :why: MIT (https://github.com/JamesNK/Newtonsoft.Json/raw/master/LICENSE.md)
     :versions:
     - 13.0.1
+    - 13.0.2
     :when: 2022-10-14 23:37:28.643149375 Z
 - - :approve
   - Newtonsoft.Json
@@ -973,14 +974,14 @@
   - :who: mocsharp
     :why: MIT (https://github.com/SwissLife-OSS/snapshooter/raw/master/LICENSE)
     :versions:
-    - 0.7.1
+    - 0.12.0
     :when: 2022-10-14 23:37:37.042360494 Z
 - - :approve
   - Snapshooter.NUnit
   - :who: mocsharp
     :why: MIT (https://github.com/SwissLife-OSS/snapshooter/raw/master/LICENSE)
     :versions:
-    - 0.7.1
+    - 0.12.0
     :when: 2022-10-14 23:37:37.474483432 Z
 - - :approve
   - SpecFlow
@@ -2187,21 +2188,21 @@
   - :who: mocsharp
     :why: BSD 3-Clause License (https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
     :versions:
-    - 5.0.5
+    - 5.1.0
     :when: 2022-10-12 03:14:06.538744982 Z
 - - :approve
   - NLog.Extensions.Logging
   - :who: mocsharp
     :why: BSD 2-Clause Simplified License (https://github.com/NLog/NLog.Extensions.Logging/raw/master/LICENSE)
     :versions:
-    - 5.1.0
+    - 5.2.0
     :when: 2022-10-12 03:14:06.964203977 Z
 - - :approve
   - NLog.Web.AspNetCore
   - :who: mocsharp
     :why: BSD 3-Clause License (https://github.com/NLog/NLog.Web/raw/master/LICENSE)
     :versions:
-    - 5.1.5
+    - 5.2.0
     :when: 2022-10-12 03:14:07.396706995 Z
 - - :approve
   - AspNetCore.HealthChecks.MongoDb

docs/compliance/third-party-licenses.md

@@ -9804,14 +9804,14 @@ SOFTWARE.
 
 
 <details>
-<summary>NLog 5.0.5</summary>
+<summary>NLog 5.1.0</summary>
 
 ## NLog
 
-- Version: 5.0.5
+- Version: 5.1.0
 - Authors: Jarek Kowalski,Kim Christensen,Julian Verdurmen
 - Project URL: https://nlog-project.org/
-- Source: [NuGet](https://www.nuget.org/packages/NLog/5.0.5)
+- Source: [NuGet](https://www.nuget.org/packages/NLog/5.1.0)
 - License: [BSD 3-Clause License](https://github.com/NLog/NLog/raw/dev/LICENSE.txt)
 
 
@@ -9852,14 +9852,14 @@ THE POSSIBILITY OF SUCH DAMAGE.
 
 
 <details>
-<summary>NLog.Extensions.Logging 5.1.0</summary>
+<summary>NLog.Extensions.Logging 5.2.0</summary>
 
 ## NLog.Extensions.Logging
 
-- Version: 5.1.0
+- Version: 5.2.0
 - Authors: Microsoft,Julian Verdurmen
 - Project URL: https://github.com/NLog/NLog.Extensions.Logging
-- Source: [NuGet](https://www.nuget.org/packages/NLog.Extensions.Logging/5.1.0)
+- Source: [NuGet](https://www.nuget.org/packages/NLog.Extensions.Logging/5.2.0)
 - License: [BSD 2-Clause Simplified License](https://github.com/NLog/NLog.Extensions.Logging/raw/master/LICENSE)
 
 
@@ -9893,14 +9893,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 <details>
-<summary>NLog.Web.AspNetCore 5.1.4</summary>
+<summary>NLog.Web.AspNetCore 5.2.0</summary>
 
 ## NLog.Web.AspNetCore
 
-- Version: 5.1.4
+- Version: 5.2.0
 - Authors: Julian Verdurmen
 - Project URL: https://github.com/NLog/NLog.Web
-- Source: [NuGet](https://www.nuget.org/packages/NLog.Web.AspNetCore/5.1.4)
+- Source: [NuGet](https://www.nuget.org/packages/NLog.Web.AspNetCore/5.2.0)
 - License: [BSD 3-Clause License](https://github.com/NLog/NLog.Web/raw/master/LICENSE)
 
 
@@ -9978,14 +9978,14 @@ THE SOFTWARE.
 
 
 <details>
-<summary>NUnit3TestAdapter 4.3.0</summary>
+<summary>NUnit3TestAdapter 4.3.1</summary>
 
 ## NUnit3TestAdapter
 
-- Version: 4.3.0
+- Version: 4.3.1
 - Authors: Charlie Poole, Terje Sandstrom
 - Project URL: https://docs.nunit.org/articles/vs-test-adapter/Index.html
-- Source: [NuGet](https://www.nuget.org/packages/NUnit3TestAdapter/4.3.0)
+- Source: [NuGet](https://www.nuget.org/packages/NUnit3TestAdapter/4.3.1)
 - License: [MIT](https://github.com/nunit/nunit3-vs-adapter/raw/master/LICENSE)
 
 
@@ -10017,14 +10017,14 @@ THE SOFTWARE.
 
 
 <details>
-<summary>Newtonsoft.Json 13.0.1</summary>
+<summary>Newtonsoft.Json 13.0.2</summary>
 
 ## Newtonsoft.Json
 
-- Version: 13.0.1
+- Version: 13.0.2
 - Authors: James Newton-King
 - Project URL: https://www.newtonsoft.com/json
-- Source: [NuGet](https://www.nuget.org/packages/Newtonsoft.Json/13.0.1)
+- Source: [NuGet](https://www.nuget.org/packages/Newtonsoft.Json/13.0.2)
 - License: [MIT](https://github.com/JamesNK/Newtonsoft.Json/raw/master/LICENSE.md)
 
 
@@ -13087,15 +13087,15 @@ Some of the benchmark data in testdata/ is licensed differently:
 
 
 <details>
-<summary>Snapshooter 0.7.1</summary>
+<summary>Snapshooter 0.12.0</summary>
 
 ## Snapshooter
 
-- Version: 0.7.1
+- Version: 0.12.0
 - Authors: Swiss Life authors and contributors
 - Owners: Swiss Life authors and contributors
 - Project URL: https://github.com/SwissLife-OSS/Snapshooter
-- Source: [NuGet](https://www.nuget.org/packages/Snapshooter/0.7.1)
+- Source: [NuGet](https://www.nuget.org/packages/Snapshooter/0.12.0)
 - License: [MIT](https://github.com/SwissLife-OSS/snapshooter/raw/master/LICENSE)
 
 
@@ -13127,15 +13127,15 @@ SOFTWARE.
 
 
 <details>
-<summary>Snapshooter.NUnit 0.7.1</summary>
+<summary>Snapshooter.NUnit 0.12.0</summary>
 
 ## Snapshooter.NUnit
 
-- Version: 0.7.1
+- Version: 0.12.0
 - Authors: Swiss Life authors and contributors
 - Owners: Swiss Life authors and contributors
 - Project URL: https://github.com/SwissLife-OSS/Snapshooter
-- Source: [NuGet](https://www.nuget.org/packages/Snapshooter.NUnit/0.7.1)
+- Source: [NuGet](https://www.nuget.org/packages/Snapshooter.NUnit/0.12.0)
 - License: [MIT](https://github.com/SwissLife-OSS/snapshooter/raw/master/LICENSE)
 
 

src/Shared/Configuration/Monai.Deploy.WorkflowManager.Configuration.csproj

@@ -119,5 +119,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 119, Level = LogLevel.Debug, Message = "Processing task cancellation event")]
         public static partial void PrecessingTaskCancellationEvent(this ILogger logger);
+
+        [LoggerMessage(EventId = 120, Level = LogLevel.Error, Message = "Recovering connection to storage service:  {reason}.")]
+        public static partial void MessagingServiceErrorRecover(this ILogger logger, string reason);
     }
 }

src/Shared/Configuration/packages.lock.json

@@ -66,8 +66,8 @@
     <PackageReference Include="Monai.Deploy.Messaging" Version="0.1.17-rc0020" />
     <PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.10" />
     <PackageReference Include="Monai.Deploy.Messaging.RabbitMQ" Version="0.1.17-rc0020" />
-    <PackageReference Include="NLog" Version="5.0.5" />
-    <PackageReference Include="NLog.Web.AspNetCore" Version="5.1.5" />
+    <PackageReference Include="NLog" Version="5.1.0" />
+    <PackageReference Include="NLog.Web.AspNetCore" Version="5.2.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/TaskManager/API/Monai.Deploy.WorkflowManager.TaskManager.API.csproj

@@ -15,7 +15,6 @@
  */
 
 using Ardalis.GuardClauses;
-using Argo;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -86,16 +85,26 @@ public Task StartAsync(CancellationToken cancellationToken)
             _messageBrokerPublisherService = _scope.ServiceProvider.GetRequiredService<IMessageBrokerPublisherService>() ?? throw new ServiceNotFoundException(nameof(IMessageBrokerPublisherService));
             _messageBrokerSubscriberService = _scope.ServiceProvider.GetRequiredService<IMessageBrokerSubscriberService>() ?? throw new ServiceNotFoundException(nameof(IMessageBrokerSubscriberService));
 
-            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskDispatchRequest, _options.Value.Messaging.Topics.TaskDispatchRequest, TaskDispatchEventReceivedCallback);
-            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskCallbackRequest, _options.Value.Messaging.Topics.TaskCallbackRequest, TaskCallbackEventReceivedCallback);
-            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskCancellationRequest, _options.Value.Messaging.Topics.TaskCancellationRequest, TaskCancelationEventCallback);
+            _messageBrokerSubscriberService.OnConnectionError += (sender, args) =>
+            {
+                _logger.MessagingServiceErrorRecover(args.ErrorMessage);
+                SubscribeToEvents();
+            };
+            SubscribeToEvents();
 
             Status = ServiceStatus.Running;
             _logger.ServiceStarted(ServiceName);
 
             return Task.CompletedTask;
         }
 
+        private void SubscribeToEvents()
+        {
+            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskDispatchRequest, _options.Value.Messaging.Topics.TaskDispatchRequest, TaskDispatchEventReceivedCallback);
+            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskCallbackRequest, _options.Value.Messaging.Topics.TaskCallbackRequest, TaskCallbackEventReceivedCallback);
+            _messageBrokerSubscriberService.SubscribeAsync(_options.Value.Messaging.Topics.TaskCancellationRequest, _options.Value.Messaging.Topics.TaskCancellationRequest, TaskCancelationEventCallback);
+        }
+
         public Task StopAsync(CancellationToken cancellationToken)
         {
             _logger.ServiceStopping(ServiceName);

src/TaskManager/API/packages.lock.json

@@ -131,17 +131,17 @@
       },
       "NLog": {
         "type": "Direct",
-        "requested": "[5.0.5, )",
-        "resolved": "5.0.5",
-        "contentHash": "NOTWSyUEljmjMq7OqZ1X9iu4bJ+rW/o6pt79Jq8j2Q7s8DyoMMCJwe0HoCKcNjhYRJ++b+E8erH6E6WwaCTshQ=="
+        "requested": "[5.1.0, )",
+        "resolved": "5.1.0",
+        "contentHash": "oW7ekrkRG9okpDMUcEglunWj8Qf2RY8qkgl+/chJoavzg3dbT13y32t19R54FKkmq80fKzw4ZekZkCrRGanKgQ=="
       },
       "NLog.Web.AspNetCore": {
         "type": "Direct",
-        "requested": "[5.1.5, )",
-        "resolved": "5.1.5",
-        "contentHash": "a7Pe6KdwIxPxcFYy6M7wLseU++tx1bBf/ROVNlcZPLfp40DPLA0KGOk1K9kvbcwPYKKLMikdSwiOyTRZZFlaXg==",
+        "requested": "[5.2.0, )",
+        "resolved": "5.2.0",
+        "contentHash": "DqFgdydAWW+pshPdzh0ydk2jJrrVaZmBNz5+p9K8N9q/4BOPJ94S2fD8t9erd7ZMhnigaqOq/HqZH4nGGOYTbA==",
         "dependencies": {
-          "NLog.Extensions.Logging": "5.1.0"
+          "NLog.Extensions.Logging": "5.2.0"
         }
       },
       "StyleCop.Analyzers": {
@@ -707,12 +707,12 @@
       },
       "NLog.Extensions.Logging": {
         "type": "Transitive",
-        "resolved": "5.1.0",
-        "contentHash": "cQCKF2/iYjZUkn0d2o6VD1xkTUhIFHPYmZEm29KlTthLEzMht5aY80SwWlHZCKy0w19kaSq1jgLJSGrKsapUfg==",
+        "resolved": "5.2.0",
+        "contentHash": "wzVFG5p8Nwbs1Ws29T8YJg+UbJfsh61h6U4xArnDSrtVvOoccwKtoFPZWwbym3ZTiTFmHIf7Ugu1j/WnT7z3vg==",
         "dependencies": {
           "Microsoft.Extensions.Configuration.Abstractions": "6.0.0",
           "Microsoft.Extensions.Logging": "6.0.0",
-          "NLog": "5.0.5"
+          "NLog": "5.1.0"
         }
       },
       "Polly": {

src/TaskManager/Database/packages.lock.json

@@ -40,5 +40,8 @@ public static partial class Log
 
         [LoggerMessage(EventId = 900006, Level = LogLevel.Trace, Message = "{ServiceName} Worker completed in: {Elapsed_millis} milliseconds.")]
         public static partial void ServiceCompleted(this ILogger logger, string serviceName, int elapsed_millis);
+
+        [LoggerMessage(EventId = 900007, Level = LogLevel.Error, Message = "Recovering connection to storage service:  {reason}.")]
+        public static partial void MessagingServiceErrorRecover(this ILogger logger, string reason);
     }
 }

src/TaskManager/Plug-ins/AideClinicalReview/packages.lock.json

@@ -70,6 +70,11 @@ public PayloadListenerService(
             _eventPayloadListenerService = eventPayloadListenerService ?? throw new ArgumentNullException(nameof(eventPayloadListenerService));
 
             _messageSubscriber = _scope.ServiceProvider.GetRequiredService<IMessageBrokerSubscriberService>();
+            _messageSubscriber.OnConnectionError += (sender, args) =>
+            {
+                _logger.MessagingServiceErrorRecover(args.ErrorMessage);
+                SetupPolling();
+            };
         }
 
         public Task StartAsync(CancellationToken cancellationToken)

src/TaskManager/Plug-ins/Argo/packages.lock.json

@@ -1,4 +1,4 @@
-<!--
+<!--
   ~ Copyright 2022 MONAI Consortium
   ~
   ~ Licensed under the Apache License, Version 2.0 (the "License");
@@ -47,9 +47,9 @@
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
     <PackageReference Include="Monai.Deploy.Storage.MinIO" Version="0.2.10" />
     <PackageReference Include="Monai.Deploy.Messaging.RabbitMQ" Version="0.1.17-rc0020" />
-    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="NLog" Version="5.0.5" />
-    <PackageReference Include="NLog.Web.AspNetCore" Version="5.1.5" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
+    <PackageReference Include="NLog" Version="5.1.0" />
+    <PackageReference Include="NLog.Web.AspNetCore" Version="5.2.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/TaskManager/TaskManager/Logging/Log.cs

@@ -97,23 +97,23 @@
       },
       "Newtonsoft.Json": {
         "type": "Direct",
-        "requested": "[13.0.1, )",
-        "resolved": "13.0.1",
-        "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A=="
+        "requested": "[13.0.2, )",
+        "resolved": "13.0.2",
+        "contentHash": "R2pZ3B0UjeyHShm9vG+Tu0EBb2lC8b0dFzV9gVn50ofHXh9Smjk6kTn7A/FdAsC8B5cKib1OnGYOXxRBz5XQDg=="
       },
       "NLog": {
         "type": "Direct",
-        "requested": "[5.0.5, )",
-        "resolved": "5.0.5",
-        "contentHash": "NOTWSyUEljmjMq7OqZ1X9iu4bJ+rW/o6pt79Jq8j2Q7s8DyoMMCJwe0HoCKcNjhYRJ++b+E8erH6E6WwaCTshQ=="
+        "requested": "[5.1.0, )",
+        "resolved": "5.1.0",
+        "contentHash": "oW7ekrkRG9okpDMUcEglunWj8Qf2RY8qkgl+/chJoavzg3dbT13y32t19R54FKkmq80fKzw4ZekZkCrRGanKgQ=="
       },
       "NLog.Web.AspNetCore": {
         "type": "Direct",
-        "requested": "[5.1.5, )",
-        "resolved": "5.1.5",
-        "contentHash": "a7Pe6KdwIxPxcFYy6M7wLseU++tx1bBf/ROVNlcZPLfp40DPLA0KGOk1K9kvbcwPYKKLMikdSwiOyTRZZFlaXg==",
+        "requested": "[5.2.0, )",
+        "resolved": "5.2.0",
+        "contentHash": "DqFgdydAWW+pshPdzh0ydk2jJrrVaZmBNz5+p9K8N9q/4BOPJ94S2fD8t9erd7ZMhnigaqOq/HqZH4nGGOYTbA==",
         "dependencies": {
-          "NLog.Extensions.Logging": "5.1.0"
+          "NLog.Extensions.Logging": "5.2.0"
         }
       },
       "StyleCop.Analyzers": {
@@ -613,12 +613,12 @@
       },
       "NLog.Extensions.Logging": {
         "type": "Transitive",
-        "resolved": "5.1.0",
-        "contentHash": "cQCKF2/iYjZUkn0d2o6VD1xkTUhIFHPYmZEm29KlTthLEzMht5aY80SwWlHZCKy0w19kaSq1jgLJSGrKsapUfg==",
+        "resolved": "5.2.0",
+        "contentHash": "wzVFG5p8Nwbs1Ws29T8YJg+UbJfsh61h6U4xArnDSrtVvOoccwKtoFPZWwbym3ZTiTFmHIf7Ugu1j/WnT7z3vg==",
         "dependencies": {
           "Microsoft.Extensions.Configuration.Abstractions": "6.0.0",
           "Microsoft.Extensions.Logging": "6.0.0",
-          "NLog": "5.0.5"
+          "NLog": "5.1.0"
         }
       },
       "Polly": {

src/TaskManager/TaskManager/Monai.Deploy.WorkflowManager.TaskManager.csproj

@@ -32,7 +32,7 @@
     <PackageReference Include="SpecFlow.Plus.LivingDocPlugin" Version="3.9.57" />
     <PackageReference Include="SpecFlow.NUnit" Version="3.9.74" />
     <PackageReference Include="nunit" Version="3.13.3" />
-    <PackageReference Include="NUnit3TestAdapter" Version="4.3.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.3.1" />
     <PackageReference Include="FluentAssertions" Version="6.8.0" />
   </ItemGroup>