Adds StorageInitialized health check

src/Microsoft.Health.Fhir.Api.UnitTests/Microsoft.Health.Fhir.R4.Api.UnitTests.csproj

@@ -15,6 +15,9 @@
     <ProjectReference Include="..\Microsoft.Health.Fhir.R4.Api\Microsoft.Health.Fhir.R4.Api.csproj" />
     <ProjectReference Include="..\Microsoft.Health.Fhir.Tests.Common\Microsoft.Health.Fhir.Tests.Common.csproj" />
   </ItemGroup>
+  <ItemGroup Condition="'$(TargetFramework)' != 'net6.0'">
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
+  </ItemGroup>
   <Import Project="..\Microsoft.Health.Fhir.Shared.Tests\Microsoft.Health.Fhir.Shared.Tests.projitems" Label="Shared" />
   <Import Project="..\Microsoft.Health.Fhir.Shared.Api.UnitTests\Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems" Label="Shared" />
 </Project>

src/Microsoft.Health.Fhir.Api/Features/Health/StorageInitializedHealthCheck.cs

@@ -0,0 +1,45 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using MediatR;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Health.Core;
+using Microsoft.Health.Fhir.Core.Extensions;
+using Microsoft.Health.Fhir.Core.Messages.Storage;
+
+namespace Microsoft.Health.Fhir.Api.Features.Health
+{
+    public class StorageInitializedHealthCheck : IHealthCheck, INotificationHandler<StorageInitializedNotification>
+    {
+        private const string SuccessfullyInitializedMessage = "Successfully initialized.";
+        private bool _storageReady;
+        private readonly DateTimeOffset _started = Clock.UtcNow;
+
+        public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+        {
+            if (_storageReady)
+            {
+                return Task.FromResult(HealthCheckResult.Healthy(SuccessfullyInitializedMessage));
+            }
+
+            TimeSpan waited = Clock.UtcNow - _started;
+            if (waited < TimeSpan.FromMinutes(5))
+            {
+                return Task.FromResult(new HealthCheckResult(HealthStatus.Degraded, $"Storage is initializing. Waited: {(int)waited.TotalSeconds}s."));
+            }
+
+            return Task.FromResult(new HealthCheckResult(HealthStatus.Unhealthy, $"Storage has not been initialized. Waited: {(int)waited.TotalSeconds}s."));
+        }
+
+        public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken)
+        {
+            _storageReady = true;
+            return Task.CompletedTask;
+        }
+    }
+}

src/Microsoft.Health.Fhir.R4B.Api.UnitTests/Microsoft.Health.Fhir.R4B.Api.UnitTests.csproj

@@ -15,6 +15,9 @@
     <ProjectReference Include="..\Microsoft.Health.Fhir.R4B.Api\Microsoft.Health.Fhir.R4B.Api.csproj" />
     <ProjectReference Include="..\Microsoft.Health.Fhir.Tests.Common\Microsoft.Health.Fhir.Tests.Common.csproj" />
   </ItemGroup>
+  <ItemGroup Condition="'$(TargetFramework)' != 'net6.0'">
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
+  </ItemGroup>
   <Import Project="..\Microsoft.Health.Fhir.Shared.Tests\Microsoft.Health.Fhir.Shared.Tests.projitems" Label="Shared" />
   <Import Project="..\Microsoft.Health.Fhir.Shared.Api.UnitTests\Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems" Label="Shared" />
 </Project>

src/Microsoft.Health.Fhir.R5.Api.UnitTests/Microsoft.Health.Fhir.R5.Api.UnitTests.csproj

@@ -15,6 +15,9 @@
     <ProjectReference Include="..\Microsoft.Health.Fhir.R5.Api\Microsoft.Health.Fhir.R5.Api.csproj" />
     <ProjectReference Include="..\Microsoft.Health.Fhir.Tests.Common\Microsoft.Health.Fhir.Tests.Common.csproj" />
   </ItemGroup>
+  <ItemGroup Condition="'$(TargetFramework)' != 'net6.0'">
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
+  </ItemGroup>
   <Import Project="..\Microsoft.Health.Fhir.Shared.Tests\Microsoft.Health.Fhir.Shared.Tests.projitems" Label="Shared" />
   <Import Project="..\Microsoft.Health.Fhir.Shared.Api.UnitTests\Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems" Label="Shared" />
 </Project>

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Health/StorageInitializedHealthCheckTests.cs

@@ -0,0 +1,52 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Health.Fhir.Core.Extensions;
+using Microsoft.Health.Fhir.Core.Messages.Storage;
+using Microsoft.Health.Fhir.Tests.Common;
+using Microsoft.Health.Test.Utilities;
+using Xunit;
+
+namespace Microsoft.Health.Fhir.Api.Features.Health;
+
+[Trait(Traits.OwningTeam, OwningTeam.Fhir)]
+[Trait(Traits.Category, Categories.DataSourceValidation)]
+public class StorageInitializedHealthCheckTests
+{
+    private readonly StorageInitializedHealthCheck _sut = new();
+
+    [Fact]
+    public async Task GivenStorageInitialized_WhenCheckHealthAsync_ThenReturnsHealthy()
+    {
+        await _sut.Handle(new StorageInitializedNotification(), CancellationToken.None);
+
+        HealthCheckResult result = await _sut.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
+
+        Assert.Equal(HealthStatus.Healthy, result.Status);
+    }
+
+    [Fact]
+    public async Task GivenStorageInitializedHealthCheck_WhenCheckHealthAsync_ThenStartsAsDegraded()
+    {
+        HealthCheckResult result = await _sut.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
+        Assert.Equal(HealthStatus.Degraded, result.Status);
+    }
+
+#if NET8_0_OR_GREATER
+    [Fact]
+    public async Task GivenStorageInitializedHealthCheck_WhenCheckHealthAsync_ThenChangedToUnhealthyAfter5Minute()
+    {
+        using (Mock.Property(() => ClockResolver.TimeProvider, new Microsoft.Extensions.Time.Testing.FakeTimeProvider(DateTimeOffset.Now.AddMinutes(5).AddSeconds(1))))
+        {
+            HealthCheckResult result = await _sut.CheckHealthAsync(new HealthCheckContext(), CancellationToken.None);
+            Assert.Equal(HealthStatus.Unhealthy, result.Status);
+        }
+    }
+#endif
+}

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems

@@ -58,6 +58,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Features\Headers\ImportResultExtensionsTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Headers\ExportResultExtensionsTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Headers\FhirResultExtensionsTests.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Features\Health\StorageInitializedHealthCheckTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Operations\Import\ImportRequestExtensionsTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Operations\Import\InitialImportLockMiddlewareTests.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Features\Operations\OperationsCapabilityProviderTests.cs" />
@@ -80,7 +81,4 @@
       <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </EmbeddedResource>
   </ItemGroup>
-  <ItemGroup>
-    <Folder Include="$(MSBuildThisFileDirectory)Features\HealthCheck\" />
-  </ItemGroup>
 </Project>

src/Microsoft.Health.Fhir.Shared.Api/Modules/FhirModule.cs

@@ -35,6 +35,7 @@
 using Microsoft.Health.Fhir.Core.Features.Persistence;
 using Microsoft.Health.Fhir.Core.Features.Security;
 using Microsoft.Health.Fhir.Core.Messages.CapabilityStatement;
+using Microsoft.Health.Fhir.Core.Messages.Storage;
 using Microsoft.Health.Fhir.Core.Models;
 
 namespace Microsoft.Health.Fhir.Api.Modules
@@ -176,6 +177,16 @@ ResourceElement SetMetadata(Resource resource, string versionId, DateTimeOffset
 
             services.AddHealthChecks().AddCheck<ImproperBehaviorHealthCheck>(name: "BehaviorHealthCheck");
 
+            // Registers a health check to ensure storage gets initialized
+            services.RemoveServiceTypeExact<StorageInitializedHealthCheck, INotificationHandler<StorageInitializedNotification>>()
+                .Add<StorageInitializedHealthCheck>()
+                .Singleton()
+                .AsSelf()
+                .AsService<IHealthCheck>()
+                .AsService<INotificationHandler<StorageInitializedNotification>>();
+
+            services.AddHealthChecks().AddCheck<StorageInitializedHealthCheck>(name: "StorageInitializedHealthCheck");
+
             services.AddLazy();
             services.AddScoped();
 

src/Microsoft.Health.Fhir.Stu3.Api.UnitTests/Microsoft.Health.Fhir.Stu3.Api.UnitTests.csproj

@@ -17,6 +17,9 @@
     <ProjectReference Include="..\Microsoft.Health.Fhir.Stu3.Api\Microsoft.Health.Fhir.Stu3.Api.csproj" />
     <ProjectReference Include="..\Microsoft.Health.Fhir.Tests.Common\Microsoft.Health.Fhir.Tests.Common.csproj" />
   </ItemGroup>
+  <ItemGroup Condition="'$(TargetFramework)' != 'net6.0'">
+    <PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
+  </ItemGroup>
   <Import Project="..\Microsoft.Health.Fhir.Shared.Tests\Microsoft.Health.Fhir.Shared.Tests.projitems" Label="Shared" />
   <Import Project="..\Microsoft.Health.Fhir.Shared.Api.UnitTests\Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems" Label="Shared" />
 </Project>