Resilient Startup

rubberduckvba.Server/GitHubSettings.cs

@@ -47,6 +47,9 @@ public record class GitHubSettings
 
 public record class HangfireSettings
 {
+    public int MaxInitializationAttempts { get; set; } = 5;
+    public int InitializationRetryDelaySeconds { get; set; } = 10;
+
     public int ServerCheckIntervalMinutes { get; set; } = 15;
     public int QueuePollIntervalSeconds { get; set; } = 30;
     public int SchedulePollIntervalSeconds { get; set; } = 30;

rubberduckvba.Server/Program.cs

@@ -7,6 +7,8 @@
 using NLog.Config;
 using NLog.Extensions.Logging;
 using NLog.Targets;
+using Polly;
+using Polly.Retry;
 using Rubberduck.SmartIndenter;
 using RubberduckServices;
 using rubberduckvba.Server.Api.Admin;
@@ -110,7 +112,25 @@ public static void Main(string[] args)
         app.MapControllers();
         app.MapFallbackToFile("/index.html");
 
-        StartHangfire(app);
+        var logger = app.Services.GetRequiredService<ILogger<Program>>();
+        logger.LogInformation("App configuration completed. Starting hangfire...");
+
+        var hangfireOptions = app.Services.GetService<IOptions<HangfireSettings>>()?.Value ?? new();
+        new ResiliencePipelineBuilder().AddRetry(new RetryStrategyOptions
+        {
+            Delay = TimeSpan.FromSeconds(10),
+            MaxRetryAttempts = hangfireOptions.MaxInitializationAttempts,
+            OnRetry = (context) =>
+            {
+                var retryCount = context.AttemptNumber;
+                var delay = context.RetryDelay;
+
+                logger.LogError(context.Outcome.Exception, $"Hangfire failed to start | Retrying storage connection in {delay.TotalSeconds} seconds. Attempt {retryCount} of {hangfireOptions.MaxInitializationAttempts}");
+                return ValueTask.CompletedTask;
+            }
+        }).Build().Execute(() => StartHangfire(app));
+
+        logger.LogInformation("Hangfire initialization completed. Starting application...");
         app.Run();
     }
 

rubberduckvba.Server/rubberduckvba.Server.csproj

@@ -30,6 +30,7 @@
     <PackageReference Include="NLog.Extensions.Logging" Version="5.4.0" />
     <PackageReference Include="NLog.WindowsEventLog" Version="5.4.0" />
     <PackageReference Include="Octokit" Version="14.0.0" />
+    <PackageReference Include="Polly" Version="8.5.2" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="7.2.0" />
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.3.1" />
   </ItemGroup>