Handle all BYOK errors from Cosmos DB (#1014)

* Handle all BYOK errors from Cosmos DB

* Addressed PR comments

* Moved CustomerManagedKeyException into Microsoft.Health.Fhir.CosmosDb

* Revert "Moved CustomerManagedKeyException into Microsoft.Health.Fhir.CosmosDb"

This reverts commit 0122a1e.

* Address PR comments

* Reverting code to commit #2 of this branch (792fcd6), and also resolve a couple PR comments

* Remove circular reference by moving KnownCosmosDbCmkSubStatusValue into Core.

* Fix build error

* - per offline feedback, move the code that determines the CMK error message from Core and into CosmosDb.  This removes any Cosmos-related knownledge from Core

- Cosmos DB has published documentation on BYOK substatus codes: https://docs.microsoft.com/en-us/rest/api/cosmos-db/http-status-codes-for-cosmosdb.  Updating Resources to better aligned with the Cosmos wording.

* Fix spacing changes accidentally included in previous commit.

* CMK exception should call base ctor

src/Microsoft.Health.Fhir.Core/Exceptions/CustomerManagedKeyInaccessibleException.cs → src/Microsoft.Health.Fhir.Core/Exceptions/CustomerManagedKeyException.cs

@@ -7,14 +7,15 @@
 
 namespace Microsoft.Health.Fhir.Core.Exceptions
 {
-    public class CustomerManagedKeyInaccessibleException : FhirException
+    public class CustomerManagedKeyException : FhirException
     {
-        public CustomerManagedKeyInaccessibleException()
+        public CustomerManagedKeyException(string message)
+            : base(message)
         {
             Issues.Add(new OperationOutcomeIssue(
                     OperationOutcomeConstants.IssueSeverity.Error,
                     OperationOutcomeConstants.IssueType.Forbidden,
-                    Resources.CustomerManagedKeyInaccessible));
+                    message));
         }
     }
 }

src/Microsoft.Health.Fhir.Core/Resources.resx

@@ -175,9 +175,6 @@
     <value>Found result with Id '{0}', which did not match the provided Id '{1}'.</value>
     <comment>{0} is an Id e.g. 123456</comment>
   </data>
-  <data name="CustomerManagedKeyInaccessible" xml:space="preserve">
-    <value>The customer managed key is not accessible.</value>
-  </data>
   <data name="CustomHeaderPrefixCannotBeEmpty" xml:space="preserve">
     <value>The prefix used to identify custom audit headers cannot be empty.</value>
   </data>
@@ -421,4 +418,4 @@
   <data name="WeakETagFormatRequired" xml:space="preserve">
     <value>WeakETag must be in the weak ETag format.</value>
   </data>
-</root>
+</root>

src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosResponseProcessorTests.cs

@@ -155,20 +155,31 @@ public async Task GivenADocumentClientExceptionWithSpecificMessage_WhenProcessin
             ValidateExecution(expectedSessionToken: null, 12.4, false);
         }
 
-        [Fact]
-        public async Task GivenADocumentClientExceptionWithCustomerManagedKeyInaccessibleSubStatus_WhenProcessing_ThenExceptionShouldThrow()
+        [Theory]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.AadClientCredentialsGrantFailure)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.AadServiceUnavailable)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultAuthenticationFailure)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultKeyNotFound)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultServiceUnavailable)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultWrapUnwrapFailure)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.InvalidKeyVaultKeyUri)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.InvalidInputBytes)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultInternalServerError)]
+        [InlineData(KnownCosmosDbCmkSubStatusValue.KeyVaultDnsNotResolved)]
+        public async Task GivenADocumentClientExceptionWithCmkSubStatus_WhenProcessing_ThenExceptionShouldThrow(KnownCosmosDbCmkSubStatusValue subStatusValue)
         {
-            DocumentClientException documentClientException = CreateDocumentClientException("12.4", "fail", HttpStatusCode.Forbidden, CosmosDbSubStatusValues.CustomerManagedKeyInaccessible.ToString());
+            DocumentClientException documentClientException = CreateDocumentClientException("12.4", "fail", HttpStatusCode.Forbidden, Convert.ToString((int)subStatusValue));
 
-            await Assert.ThrowsAsync<CustomerManagedKeyInaccessibleException>(async () => await _cosmosResponseProcessor.ProcessException(documentClientException));
+            await Assert.ThrowsAsync<CustomerManagedKeyException>(async () => await _cosmosResponseProcessor.ProcessException(documentClientException));
 
             ValidateExecution(expectedSessionToken: null, 12.4, false);
         }
 
         [Theory]
         [InlineData("")]
+        [InlineData(null)]
         [InlineData("3999")]
-        public async Task GivenADocumentClientExceptionWithForbiddenStatusCodeAndNotKeyInaccessibleSubStatus_WhenProcessing_ThenNothingElseShouldOccur(string subsStatusCode)
+        public async Task GivenADocumentClientExceptionWithForbiddenStatusCodeAndUnknownSubStatus_WhenProcessing_ThenNothingElseShouldOccur(string subsStatusCode)
         {
             DocumentClientException documentClientException = CreateDocumentClientException("12.4", "fail", HttpStatusCode.Forbidden, subsStatusCode);
 

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosResponseProcessor.cs

@@ -72,9 +72,13 @@ public async Task ProcessException(Exception ex)
                     // 2. Larger documents are rejected by CosmosDb with HttpStatusCode.RequestEntityTooLarge
                     throw new Core.Exceptions.RequestEntityTooLargeException();
                 }
-                else if (dce.StatusCode == HttpStatusCode.Forbidden && dce.GetSubStatusValue() == CosmosDbSubStatusValues.CustomerManagedKeyInaccessible)
+                else if (dce.StatusCode == HttpStatusCode.Forbidden)
                 {
-                    throw new Core.Exceptions.CustomerManagedKeyInaccessibleException();
+                    int? subStatusValue = dce.GetSubStatusValue();
+                    if (subStatusValue.HasValue && Enum.IsDefined(typeof(KnownCosmosDbCmkSubStatusValue), subStatusValue))
+                    {
+                        throw new Core.Exceptions.CustomerManagedKeyException(GetCustomerManagedKeyErrorMessage(subStatusValue.Value));
+                    }
                 }
             }
         }
@@ -149,5 +153,46 @@ private async Task AddRequestChargeToFhirRequestContext(double responseRequestCh
                 _logger.LogCritical(ex, "Unable to publish CosmosDB metric.");
             }
         }
+
+        private string GetCustomerManagedKeyErrorMessage(int subStatusCode)
+        {
+            string errorMessage = Resources.CmkDefaultError;
+
+            switch ((KnownCosmosDbCmkSubStatusValue)subStatusCode)
+            {
+                case KnownCosmosDbCmkSubStatusValue.AadClientCredentialsGrantFailure:
+                    errorMessage = Resources.AadClientCredentialsGrantFailure;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.AadServiceUnavailable:
+                    errorMessage = Resources.AadServiceUnavailable;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultAuthenticationFailure:
+                    errorMessage = Resources.KeyVaultAuthenticationFailure;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultKeyNotFound:
+                    errorMessage = Resources.KeyVaultKeyNotFound;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultServiceUnavailable:
+                    errorMessage = Resources.KeyVaultServiceUnavailable;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultWrapUnwrapFailure:
+                    errorMessage = Resources.KeyVaultWrapUnwrapFailure;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.InvalidKeyVaultKeyUri:
+                    errorMessage = Resources.InvalidKeyVaultKeyUri;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.InvalidInputBytes:
+                    errorMessage = Resources.InvalidInputBytes;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultInternalServerError:
+                    errorMessage = Resources.KeyVaultInternalServerError;
+                    break;
+                case KnownCosmosDbCmkSubStatusValue.KeyVaultDnsNotResolved:
+                    errorMessage = Resources.KeyVaultDnsNotResolved;
+                    break;
+            }
+
+            return errorMessage;
+        }
     }
 }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/KnownCosmosDbCmkSubStatusValue.cs

@@ -0,0 +1,25 @@
+// -------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
+// -------------------------------------------------------------------------------------------------
+
+namespace Microsoft.Health.Fhir.CosmosDb.Features.Storage
+{
+    /// <summary>
+    /// Cosmos DB customer-managed key (CMK) sub status values
+    /// </summary>
+    public enum KnownCosmosDbCmkSubStatusValue
+    {
+        // Customer-Managed Key (CMK) values
+        AadClientCredentialsGrantFailure = 4000,
+        AadServiceUnavailable = 4001,
+        KeyVaultAuthenticationFailure = 4002,
+        KeyVaultKeyNotFound = 4003,
+        KeyVaultServiceUnavailable = 4004,
+        KeyVaultWrapUnwrapFailure = 4005,
+        InvalidKeyVaultKeyUri = 4006,
+        InvalidInputBytes = 4007,
+        KeyVaultInternalServerError = 4008,
+        KeyVaultDnsNotResolved = 4009,
+    }
+}

src/Microsoft.Health.Fhir.CosmosDb/Resources.resx

@@ -132,4 +132,37 @@
   <data name="UnrecognizedConsistencyLevel" xml:space="preserve">
     <value>Invalid consistency level '{0}'. Valid values are: {1}.</value>
   </data>
-</root>
+  <data name="AadClientCredentialsGrantFailure" xml:space="preserve">
+    <value>Failed to get or access the Azure AD token to access the customer-managed key.</value>
+  </data>
+  <data name="AadServiceUnavailable" xml:space="preserve">
+    <value>Azure AD service is unavailable to get access to the customer-managed key.</value>
+  </data>
+  <data name="KeyVaultAuthenticationFailure" xml:space="preserve">
+    <value>Key Vault does not grant permission to the Azure AD, or the customer-managed key is disabled.</value>
+  </data>
+  <data name="KeyVaultKeyNotFound" xml:space="preserve">
+    <value>The customer-managed key is not found.</value>
+  </data>
+  <data name="KeyVaultServiceUnavailable" xml:space="preserve">
+    <value>The Key Vault Service is unavailable to access the customer-managed key.</value>
+  </data>
+  <data name="KeyVaultWrapUnwrapFailure" xml:space="preserve">
+    <value>Failure to Wrap or Unwrap the customer-managed key.</value>
+  </data>
+  <data name="InvalidKeyVaultKeyUri" xml:space="preserve">
+    <value>The customer-managed key URI is invalid.</value>
+  </data>
+  <data name="InvalidInputBytes" xml:space="preserve">
+    <value>Internal server error: the input bytes are not in base64 format.</value>
+  </data>
+  <data name="KeyVaultInternalServerError" xml:space="preserve">
+    <value>Key Vault internal service errors accessing the customer-managed key.</value>
+  </data>
+  <data name="KeyVaultDnsNotResolved" xml:space="preserve">
+    <value>Could not resolve the DNS name of the Key Vault of the customer-managed key.</value>
+  </data>
+  <data name="CmkDefaultError" xml:space="preserve">
+    <value>There was an error using the customer-managed key.</value>
+  </data>
+</root>

src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/OperationOutcomeExceptionFilterTests.cs

@@ -200,9 +200,9 @@ public void GivenARequestEntityTooLargeException_WhenExecutingAnAction_ThenTheRe
         }
 
         [Fact]
-        public void GivenACustomerManagedKeyInaccessibleException_WhenExecutingAnAction_ThenTheResponseShouldBeAnOperationOutcome()
+        public void GivenCustomerManagedKeyException_WhenExecutingAnAction_ThenTheResponseShouldBeAnOperationOutcome()
         {
-            ValidateOperationOutcome(new CustomerManagedKeyInaccessibleException(), HttpStatusCode.Forbidden);
+            ValidateOperationOutcome(new CustomerManagedKeyException("A customer-managed key error message"), HttpStatusCode.Forbidden);
         }
 
         [Fact]

src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/OperationOutcomeExceptionFilterAttribute.cs

@@ -113,7 +113,7 @@ public override void OnActionExecuted(ActionExecutedContext context)
                         break;
                     case InvalidSearchOperationException _:
                     case SearchOperationNotSupportedException _:
-                    case CustomerManagedKeyInaccessibleException _:
+                    case CustomerManagedKeyException _:
                         operationOutcomeResult.StatusCode = HttpStatusCode.Forbidden;
                         break;
                     case UnsupportedConfigurationException _: