diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs index 4986854aa1..d9a21f7c6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs @@ -730,6 +730,47 @@ public async Task GetOrUpdateEncryptionSettingsFromCacheAsyn cancellationToken: cancellationToken); } + /// + /// This function handles the scenario where a container is deleted(say from different Client) and recreated with same Id but with different client encryption policy. + /// The idea is to have the container Rid cached and sent out as part of RequestOptions with Container Rid set in "x-ms-cosmos-intended-collection-rid" header. + /// So when the container being referenced here gets recreated we would end up with a stale encryption settings and container Rid and this would result in BadRequest( and a substatus 1024). + /// This would allow us to refresh the encryption settings and Container Rid, on the premise that the container recreated could possibly be configured with a new encryption policy. + /// + /// Response message to validate. + /// Current cached encryption settings to refresh if required. + /// Encryption specific diagnostics. + /// Cancellation token. + internal async Task ThrowIfRequestNeedsARetryPostPolicyRefreshAsync( + ResponseMessage responseMessage, + EncryptionSettings encryptionSettings, + EncryptionDiagnosticsContext encryptionDiagnosticsContext, + CancellationToken cancellationToken) + { + if (responseMessage.StatusCode == HttpStatusCode.BadRequest && + string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) + { + // get the latest encryption settings. + await this.GetOrUpdateEncryptionSettingsFromCacheAsync( + obsoleteEncryptionSettings: encryptionSettings, + cancellationToken: cancellationToken); + + if (encryptionDiagnosticsContext == null) + { + throw new ArgumentNullException(nameof(encryptionDiagnosticsContext)); + } + + encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage); + + throw new EncryptionCosmosException( + "Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Retrying may fix the issue. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + responseMessage.ErrorMessage, + HttpStatusCode.BadRequest, + int.Parse(Constants.IncorrectContainerRidSubStatus), + responseMessage.Headers.ActivityId, + responseMessage.Headers.RequestCharge, + responseMessage.Diagnostics); + } + } + internal async Task> EncryptPatchOperationsAsync( IReadOnlyList patchOperations, EncryptionSettings encryptionSettings, @@ -808,7 +849,7 @@ internal async Task> EncryptPatchOperationsAsync( /// /// Original ItemRequestOptions. /// ItemRequestOptions. - private static ItemRequestOptions GetClonedItemRequestOptions(ItemRequestOptions itemRequestOptions) + private static ItemRequestOptions EncryptionContainerGetClonedItemRequestOptions(ItemRequestOptions itemRequestOptions) { ItemRequestOptions clonedRequestOptions = itemRequestOptions != null ? (ItemRequestOptions)itemRequestOptions.ShallowCopy() : new ItemRequestOptions(); @@ -819,8 +860,7 @@ private async Task CreateItemHelperAsync( Stream streamPayload, PartitionKey partitionKey, ItemRequestOptions requestOptions, - CancellationToken cancellationToken, - bool isRetry = false) + CancellationToken cancellationToken) { EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken); if (!encryptionSettings.PropertiesToEncrypt.Any()) @@ -839,13 +879,8 @@ private async Task CreateItemHelperAsync( encryptionDiagnosticsContext, cancellationToken); - ItemRequestOptions clonedRequestOptions = requestOptions; - - // Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers. - if (!isRetry) - { - clonedRequestOptions = GetClonedItemRequestOptions(requestOptions); - } + // Clone the request options since we modify it to set AddRequestHeaders to add additional headers. + ItemRequestOptions clonedRequestOptions = EncryptionContainerGetClonedItemRequestOptions(requestOptions); encryptionSettings.SetRequestHeaders(clonedRequestOptions); @@ -855,35 +890,7 @@ private async Task CreateItemHelperAsync( clonedRequestOptions, cancellationToken); - // This handles the scenario where a container is deleted(say from different Client) and recreated with same Id but with different client encryption policy. - // The idea is to have the container Rid cached and sent out as part of RequestOptions with Container Rid set in "x-ms-cosmos-intended-collection-rid" header. - // So when the container being referenced here gets recreated we would end up with a stale encryption settings and container Rid and this would result in BadRequest( and a substatus 1024). - // This would allow us to refresh the encryption settings and Container Rid, on the premise that the container recreated could possibly be configured with a new encryption policy. - if (!isRetry && - responseMessage.StatusCode == HttpStatusCode.BadRequest && - string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - // Even though the streamPayload position is expected to be 0, - // because for MemoryStream we just use the underlying buffer to send over the wire rather than using the Stream APIs - // resetting it 0 to be on a safer side. - streamPayload.Position = 0; - - // Now the streamPayload itself is not disposed off(and hence safe to use it in the below call) since the stream that is passed to CreateItemStreamAsync is a MemoryStream and not the original Stream - // that the user has passed. The call to EncryptAsync reads out the stream(and processes it) and returns a MemoryStream which is eventually cloned in the - // Cosmos SDK and then used. This stream however is to be disposed off as part of ResponseMessage when this gets returned. - streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync( - streamPayload, - encryptionSettings, - cancellationToken); - - // we try to recreate the item with the StreamPayload(to be encrypted) now that the encryptionSettings would have been updated with latest values if any. - return await this.CreateItemHelperAsync( - streamPayload, - partitionKey, - clonedRequestOptions, - cancellationToken, - isRetry: true); - } + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); responseMessage.Content = await EncryptionProcessor.DecryptAsync( responseMessage.Content, @@ -899,8 +906,7 @@ private async Task ReadItemHelperAsync( string id, PartitionKey partitionKey, ItemRequestOptions requestOptions, - CancellationToken cancellationToken, - bool isRetry = false) + CancellationToken cancellationToken) { EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken); if (!encryptionSettings.PropertiesToEncrypt.Any()) @@ -912,13 +918,8 @@ private async Task ReadItemHelperAsync( cancellationToken); } - ItemRequestOptions clonedRequestOptions = requestOptions; - - // Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers. - if (!isRetry) - { - clonedRequestOptions = GetClonedItemRequestOptions(requestOptions); - } + // Clone the request options since we modify it to set AddRequestHeaders to add additional headers. + ItemRequestOptions clonedRequestOptions = EncryptionContainerGetClonedItemRequestOptions(requestOptions); encryptionSettings.SetRequestHeaders(clonedRequestOptions); @@ -928,24 +929,10 @@ private async Task ReadItemHelperAsync( clonedRequestOptions, cancellationToken); - if (!isRetry && - responseMessage.StatusCode == HttpStatusCode.BadRequest && - string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - // get the latest encryption settings. - await this.GetOrUpdateEncryptionSettingsFromCacheAsync( - obsoleteEncryptionSettings: encryptionSettings, - cancellationToken: cancellationToken); + EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext(); - return await this.ReadItemHelperAsync( - id, - partitionKey, - clonedRequestOptions, - cancellationToken, - isRetry: true); - } + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); - EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext(); responseMessage.Content = await EncryptionProcessor.DecryptAsync( responseMessage.Content, encryptionSettings, @@ -961,8 +948,7 @@ private async Task ReplaceItemHelperAsync( string id, PartitionKey partitionKey, ItemRequestOptions requestOptions, - CancellationToken cancellationToken, - bool isRetry = false) + CancellationToken cancellationToken) { if (partitionKey == null) { @@ -989,11 +975,8 @@ private async Task ReplaceItemHelperAsync( ItemRequestOptions clonedRequestOptions = requestOptions; - // Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers. - if (!isRetry) - { - clonedRequestOptions = GetClonedItemRequestOptions(requestOptions); - } + // Clone the request options since we modify it to set AddRequestHeaders to add additional headers. + clonedRequestOptions = EncryptionContainerGetClonedItemRequestOptions(requestOptions); encryptionSettings.SetRequestHeaders(clonedRequestOptions); @@ -1004,24 +987,7 @@ private async Task ReplaceItemHelperAsync( clonedRequestOptions, cancellationToken); - if (!isRetry && - responseMessage.StatusCode == HttpStatusCode.BadRequest && - string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - streamPayload.Position = 0; - streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync( - streamPayload, - encryptionSettings, - cancellationToken); - - return await this.ReplaceItemHelperAsync( - streamPayload, - id, - partitionKey, - clonedRequestOptions, - cancellationToken, - isRetry: true); - } + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); responseMessage.Content = await EncryptionProcessor.DecryptAsync( responseMessage.Content, @@ -1037,8 +1003,7 @@ private async Task UpsertItemHelperAsync( Stream streamPayload, PartitionKey partitionKey, ItemRequestOptions requestOptions, - CancellationToken cancellationToken, - bool isRetry = false) + CancellationToken cancellationToken) { if (partitionKey == null) { @@ -1064,11 +1029,8 @@ private async Task UpsertItemHelperAsync( ItemRequestOptions clonedRequestOptions = requestOptions; - // Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers. - if (!isRetry) - { - clonedRequestOptions = GetClonedItemRequestOptions(requestOptions); - } + // Clone the request options since we modify it to set AddRequestHeaders to add additional headers. + clonedRequestOptions = EncryptionContainerGetClonedItemRequestOptions(requestOptions); encryptionSettings.SetRequestHeaders(clonedRequestOptions); @@ -1078,23 +1040,7 @@ private async Task UpsertItemHelperAsync( clonedRequestOptions, cancellationToken); - if (!isRetry && - responseMessage.StatusCode == HttpStatusCode.BadRequest && - string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - streamPayload.Position = 0; - streamPayload = await this.DecryptStreamPayloadAndUpdateEncryptionSettingsAsync( - streamPayload, - encryptionSettings, - cancellationToken); - - return await this.UpsertItemHelperAsync( - streamPayload, - partitionKey, - clonedRequestOptions, - cancellationToken, - isRetry: true); - } + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); responseMessage.Content = await EncryptionProcessor.DecryptAsync( responseMessage.Content, @@ -1143,6 +1089,8 @@ private async Task PatchItemHelperAsync( clonedRequestOptions, cancellationToken); + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); + responseMessage.Content = await EncryptionProcessor.DecryptAsync( responseMessage.Content, encryptionSettings, @@ -1153,35 +1101,6 @@ private async Task PatchItemHelperAsync( return responseMessage; } - /// - /// This method takes in an encrypted stream payload. - /// The streamPayload is decrypted with the same policy which was used to encrypt and then the original plain stream payload is - /// returned which can be used to re-encrypt after the latest encryption settings is retrieved. - /// The method also updates the cached Encryption Settings with the latest value if any. - /// - /// Data encrypted with wrong encryption policy. - /// EncryptionSettings which was used to encrypt the payload. - /// Cancellation token. - /// Returns the decrypted stream payload and diagnostics content. - private async Task DecryptStreamPayloadAndUpdateEncryptionSettingsAsync( - Stream streamPayload, - EncryptionSettings encryptionSettings, - CancellationToken cancellationToken) - { - streamPayload = await EncryptionProcessor.DecryptAsync( - streamPayload, - encryptionSettings, - operationDiagnostics: null, - cancellationToken); - - // get the latest encryption settings. - await this.GetOrUpdateEncryptionSettingsFromCacheAsync( - obsoleteEncryptionSettings: encryptionSettings, - cancellationToken: cancellationToken); - - return streamPayload; - } - private async Task> DecryptChangeFeedDocumentsAsync( IReadOnlyCollection documents, CancellationToken cancellationToken) @@ -1208,8 +1127,7 @@ private async Task> DecryptChangeFeedDocumentsAsync( private async Task ReadManyItemsHelperAsync( IReadOnlyList<(string id, PartitionKey partitionKey)> items, ReadManyRequestOptions readManyRequestOptions = null, - CancellationToken cancellationToken = default, - bool isRetry = false) + CancellationToken cancellationToken = default) { EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync( obsoleteEncryptionSettings: null, @@ -1223,13 +1141,8 @@ private async Task ReadManyItemsHelperAsync( cancellationToken); } - ReadManyRequestOptions clonedRequestOptions = readManyRequestOptions; - - // Clone(once) the request options since we modify it to set AddRequestHeaders to add additional headers. - if (!isRetry) - { - clonedRequestOptions = readManyRequestOptions != null ? (ReadManyRequestOptions)readManyRequestOptions.ShallowCopy() : new ReadManyRequestOptions(); - } + // Clone the request options since we modify it to set AddRequestHeaders to add additional headers. + ReadManyRequestOptions clonedRequestOptions = readManyRequestOptions != null ? (ReadManyRequestOptions)readManyRequestOptions.ShallowCopy() : new ReadManyRequestOptions(); encryptionSettings.SetRequestHeaders(clonedRequestOptions); @@ -1238,33 +1151,19 @@ private async Task ReadManyItemsHelperAsync( clonedRequestOptions, cancellationToken); - if (!isRetry && - responseMessage.StatusCode == HttpStatusCode.BadRequest && - string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - // get the latest encryption settings. - await this.GetOrUpdateEncryptionSettingsFromCacheAsync( - obsoleteEncryptionSettings: encryptionSettings, - cancellationToken: cancellationToken); + EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext(); - return await this.ReadManyItemsHelperAsync( - items, - clonedRequestOptions, - cancellationToken, - isRetry: true); - } + await this.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - EncryptionDiagnosticsContext decryptDiagnostics = new EncryptionDiagnosticsContext(); - Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( responseMessage.Content, encryptionSettings, - decryptDiagnostics, + encryptionDiagnosticsContext, cancellationToken); - decryptDiagnostics.AddEncryptionDiagnosticsToResponseMessage(responseMessage); + encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage); return new DecryptedResponseMessage(responseMessage, decryptedContent); } diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionCosmosException.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionCosmosException.cs new file mode 100644 index 0000000000..3d48a43f86 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionCosmosException.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption +{ + using System; + using System.Net; + + internal sealed class EncryptionCosmosException : CosmosException + { + private readonly CosmosDiagnostics encryptionCosmosDiagnostics; + + public EncryptionCosmosException( + string message, + HttpStatusCode statusCode, + int subStatusCode, + string activityId, + double requestCharge, + CosmosDiagnostics encryptionCosmosDiagnostics) + : base(message, statusCode, subStatusCode, activityId, requestCharge) + { + this.encryptionCosmosDiagnostics = encryptionCosmosDiagnostics ?? throw new ArgumentNullException(nameof(encryptionCosmosDiagnostics)); + } + + public override CosmosDiagnostics Diagnostics => this.encryptionCosmosDiagnostics; + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs index 0ffe1b75f1..c8d036e2e2 100644 --- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs +++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs @@ -35,33 +35,20 @@ public override async Task ReadNextAsync(CancellationToken canc ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken); - // check for Bad Request and Wrong RID intended and update the cached RID and Client Encryption Policy. - if (responseMessage.StatusCode == HttpStatusCode.BadRequest - && string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) - { - await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync( - obsoleteEncryptionSettings: encryptionSettings, - cancellationToken: cancellationToken); + EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext(); - throw new CosmosException( - "Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + responseMessage.ErrorMessage, - responseMessage.StatusCode, - int.Parse(Constants.IncorrectContainerRidSubStatus), - responseMessage.Headers.ActivityId, - responseMessage.Headers.RequestCharge); - } + // check for Bad Request and Wrong RID intended and update the cached RID and Client Encryption Policy. + await this.encryptionContainer.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken); if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - EncryptionDiagnosticsContext decryptDiagnostics = new EncryptionDiagnosticsContext(); - Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( responseMessage.Content, encryptionSettings, - decryptDiagnostics, + encryptionDiagnosticsContext, cancellationToken); - decryptDiagnostics.AddEncryptionDiagnosticsToResponseMessage(responseMessage); + encryptionDiagnosticsContext.AddEncryptionDiagnosticsToResponseMessage(responseMessage); return new DecryptedResponseMessage(responseMessage, decryptedContent); } diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettingForProperty.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettingForProperty.cs index f693e54d75..45b2a37c51 100644 --- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettingForProperty.cs +++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettingForProperty.cs @@ -125,9 +125,15 @@ private async Task ForceRefreshGatewayCacheAndBuildP if (ex.StatusCode == HttpStatusCode.NotModified) { // looks like the key was never rewrapped with a valid Key Encryption Key. - throw new InvalidOperationException($"The Client Encryption Key with key id:{this.ClientEncryptionKeyId} on database:{this.encryptionContainer.Database.Id} and container:{this.encryptionContainer.Id} , needs to be rewrapped with a valid Key Encryption Key using RewrapClientEncryptionKeyAsync. " + + throw new EncryptionCosmosException( + $"The Client Encryption Key with key id:{this.ClientEncryptionKeyId} on database:{this.encryptionContainer.Database.Id} and container:{this.encryptionContainer.Id} , needs to be rewrapped with a valid Key Encryption Key using RewrapClientEncryptionKeyAsync. " + $" The Key Encryption Key used to wrap the Client Encryption Key has been revoked: {ex.Message}." + - $" Please refer to https://aka.ms/CosmosClientEncryption for more details. "); + $" Please refer to https://aka.ms/CosmosClientEncryption for more details. ", + HttpStatusCode.BadRequest, + int.Parse(Constants.IncorrectContainerRidSubStatus), + ex.ActivityId, + ex.RequestCharge, + ex.Diagnostics); } else { diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs index b149048698..6b83df4c0b 100644 --- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs +++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs @@ -202,19 +202,27 @@ public override async Task ExecuteAsync( response = await this.transactionalBatch.ExecuteAsync(clonedRequestOptions, cancellationToken); } - // FIXME this should check for BadRequest StatusCode too, requires a service fix to return 400 instead of -1 which is currently returned. - if (string.Equals(response.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) + if (response.StatusCode == HttpStatusCode.BadRequest && string.Equals(response.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus)) { await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync( obsoleteEncryptionSettings: encryptionSettings, cancellationToken: cancellationToken); - throw new CosmosException( - "Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + response.ErrorMessage, + // no access to the encryption diagnostics. Just pass empty encryption diagnostics for now. + EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext(); + EncryptionCosmosDiagnostics encryptionDiagnostics = new EncryptionCosmosDiagnostics( + response.Diagnostics, + encryptionDiagnosticsContext.EncryptContent, + encryptionDiagnosticsContext.DecryptContent, + encryptionDiagnosticsContext.TotalProcessingDuration); + + throw new EncryptionCosmosException( + "Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container. Retrying may fix the issue. Please refer to https://aka.ms/CosmosClientEncryption for more details. " + response.ErrorMessage, HttpStatusCode.BadRequest, int.Parse(Constants.IncorrectContainerRidSubStatus), response.Headers.ActivityId, - response.Headers.RequestCharge); + response.Headers.RequestCharge, + encryptionDiagnostics); } return await this.DecryptTransactionalBatchResponseAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs index 2aba8ea39a..b44fe889e4 100644 --- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs @@ -1045,6 +1045,22 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk() ContainerResponse containerResponse = await database.CreateContainerAsync(containerProperties, 400); + try + { + await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + Assert.Fail("Create operation should have failed."); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Create operation should have failed with 1024 SubStatusCode. "); + } + + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 3); + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + TestDoc docToReplace = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); docToReplace.Sensitive_StringFormat = "docTobeReplace"; @@ -1059,6 +1075,29 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk() MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer), }; + try + { + await Task.WhenAll(tasks); + Assert.Fail("Bulk operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Bulk operation should have failed with 1024 SubStatusCode."); + } + + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0); + + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + + tasks = new List() + { + MdeEncryptionTests.MdeUpsertItemAsync(otherEncryptionContainer, docToUpsert, HttpStatusCode.OK), + MdeEncryptionTests.MdeReplaceItemAsync(otherEncryptionContainer, docToReplace), + MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer), + }; await Task.WhenAll(tasks); tasks = new List() @@ -1146,6 +1185,23 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy }; ContainerResponse containerResponse = await database.CreateContainerAsync(containerProperties, 400); + + // operation fails, policy gets updated. + try + { + await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + Assert.Fail("Create operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Create operation should have failed with 1024 SubStatusCode"); + } + + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0); + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); @@ -1169,6 +1225,11 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB } catch(CosmosException ex) { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("CreateTransactionalBatch operation should have failed with 1024 SubStatusCode"); + } + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); } @@ -1257,6 +1318,23 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteQuery() ContainerResponse containerResponse = await database.CreateContainerAsync(containerProperties, 400); + try + { + await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + Assert.Fail("Create operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Create operation should have failed with 1024 SubStatusCode ."); + } + + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0); + + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); // check w.r.t to query if we are able to fail and update the policy @@ -1287,6 +1365,179 @@ await MdeEncryptionTests.ValidateQueryResultsAsync( expectedPropertiesDecryptedCount: 2); } + [TestMethod] + public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch() + { + Collection paths = new Collection() + { + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_StringFormat", + ClientEncryptionKeyId = "key1", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_NestedObjectFormatL1", + ClientEncryptionKeyId = "key1", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + }; + + ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + + ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy }; + + Container encryptionContainerToDelete = await database.CreateContainerAsync(containerProperties, 400); + await encryptionContainerToDelete.InitializeEncryptionAsync(); + + CosmosClient otherClient = TestCommon.CreateCosmosClient(builder => builder + .Build()); + + CosmosClient otherEncryptionClient = otherClient.WithEncryption(new TestEncryptionKeyStoreProvider()); + Database otherDatabase = otherEncryptionClient.GetDatabase(MdeEncryptionTests.database.Id); + + Container otherEncryptionContainer = otherDatabase.GetContainer(encryptionContainerToDelete.Id); + + await MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer); + + // Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy + using (await database.GetContainer(encryptionContainerToDelete.Id).DeleteContainerStreamAsync()) + { } + + paths = new Collection() + { + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_IntArray", + ClientEncryptionKeyId = "key1", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_DecimalFormat", + ClientEncryptionKeyId = "key2", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_FloatFormat", + ClientEncryptionKeyId = "key1", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + + new ClientEncryptionIncludedPath() + { + Path = "/Sensitive_ArrayFormat", + ClientEncryptionKeyId = "key2", + EncryptionType = "Deterministic", + EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256", + }, + }; + + clientEncryptionPolicy = new ClientEncryptionPolicy(paths); + + containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy }; + + ContainerResponse containerResponse = await database.CreateContainerAsync(containerProperties, 400); + + // operation fails, policy gets updated. + try + { + await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + Assert.Fail("Create operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Create operation should have failed with 1024 SubStatusCode."); + } + + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0); + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + + TestDoc docPostPatching = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + + // request should fail and pick up new policy. + docPostPatching.NonSensitive = Guid.NewGuid().ToString(); + docPostPatching.NonSensitiveInt++; + docPostPatching.Sensitive_StringFormat = Guid.NewGuid().ToString(); + docPostPatching.Sensitive_DateFormat = new DateTime(2020, 02, 02); + docPostPatching.Sensitive_DecimalFormat = 11.11m; + docPostPatching.Sensitive_IntArray[1] = 19877; + docPostPatching.Sensitive_IntMultiDimArray[1, 0] = 19877; + docPostPatching.Sensitive_IntFormat = 2020; + docPostPatching.Sensitive_NestedObjectFormatL1 = new TestDoc.Sensitive_NestedObjectL1() + { + Sensitive_IntArrayL1 = new int[2] { 999, 100 }, + Sensitive_IntFormatL1 = 1999, + Sensitive_DecimalFormatL1 = 1999.1m, + Sensitive_ArrayFormatL1 = new TestDoc.Sensitive_ArrayData[] + { + new TestDoc.Sensitive_ArrayData + { + Sensitive_ArrayIntFormat = 0, + Sensitive_ArrayDecimalFormat = 0.1m + }, + new TestDoc.Sensitive_ArrayData + { + Sensitive_ArrayIntFormat = 1, + Sensitive_ArrayDecimalFormat = 2.1m + }, + new TestDoc.Sensitive_ArrayData + { + Sensitive_ArrayIntFormat = 2, + Sensitive_ArrayDecimalFormat = 3.1m + } + } + }; + + // Maximum 10 operations at a time (current limit) + List patchOperations = new List + { + PatchOperation.Increment("/NonSensitiveInt", 1), + PatchOperation.Replace("/NonSensitive", docPostPatching.NonSensitive), + PatchOperation.Replace("/Sensitive_StringFormat", docPostPatching.Sensitive_StringFormat), + PatchOperation.Replace("/Sensitive_DateFormat", docPostPatching.Sensitive_DateFormat), + PatchOperation.Replace("/Sensitive_DecimalFormat", docPostPatching.Sensitive_DecimalFormat), + PatchOperation.Set("/Sensitive_IntArray/1", docPostPatching.Sensitive_IntArray[1]), + PatchOperation.Set("/Sensitive_IntMultiDimArray/1/0", docPostPatching.Sensitive_IntMultiDimArray[1,0]), + PatchOperation.Replace("/Sensitive_IntFormat", docPostPatching.Sensitive_IntFormat), + PatchOperation.Remove("/Sensitive_NestedObjectFormatL1/Sensitive_NestedObjectFormatL2"), + PatchOperation.Set("/Sensitive_NestedObjectFormatL1/Sensitive_ArrayFormatL1/0", docPostPatching.Sensitive_NestedObjectFormatL1.Sensitive_ArrayFormatL1[0]) + }; + + try + { + await MdeEncryptionTests.MdePatchItemAsync(otherEncryptionContainer, patchOperations, docPostPatching, HttpStatusCode.OK); + Assert.Fail("Patch operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Patch operation should have failed with 1024 SubStatusCode. "); + } + + // stale policy has two path for encryption. + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0); + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + + // retry post policy refresh. + await MdeEncryptionTests.MdePatchItemAsync(otherEncryptionContainer, patchOperations, docPostPatching, HttpStatusCode.OK); + } + [TestMethod] public async Task EncryptionValidatePolicyRefreshPostDatabaseDelete() { @@ -1405,9 +1656,45 @@ public async Task EncryptionValidatePolicyRefreshPostDatabaseDelete() containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy }; ContainerResponse containerResponse = await mainDatabase.CreateContainerAsync(containerProperties, 400); - + + // container gets re-created with a new policy, hence an already referenced client/container would hold an obselete policy and should fail. + try + { + await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + Assert.Fail("Create operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Create operation should have failed with 1024 SubStatusCode. "); + } + + // only encrypt diags are logged since it fails at create. + VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation:false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0); + + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + + // retrying the operation should succeed. TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete); + // try from other container. Should fail due to policy mismatch. + try + { + await MdeEncryptionTests.VerifyItemByReadAsync(otherEncryptionContainer, testDoc); + Assert.Fail("Read operation should have failed. "); + } + catch (CosmosException ex) + { + if (ex.SubStatusCode != 1024) + { + Assert.Fail("Read operation should have failed with 1024 SubStatusCode. "); + } + + Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container.")); + } + // retry should be a success. await MdeEncryptionTests.VerifyItemByReadAsync(otherEncryptionContainer, testDoc); @@ -1530,8 +1817,9 @@ public async Task VerifyKekRevokeHandling() await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer); Assert.Fail("Create Item should have failed."); } - catch(InvalidOperationException) - { + catch(CosmosException ex) + { + Assert.IsTrue(ex.Message.Contains("needs to be rewrapped with a valid Key Encryption Key using RewrapClientEncryptionKeyAsync")); } // testing query read fail due to revoked access. @@ -1543,8 +1831,9 @@ await MdeEncryptionTests.ValidateQueryResultsAsync( testDoc1); Assert.Fail("Query should have failed, since property path /Sensitive_NestedObjectFormatL1 has been encrypted using Cek with revoked access. "); } - catch (InvalidOperationException) + catch (CosmosException ex) { + Assert.IsTrue(ex.Message.Contains("needs to be rewrapped with a valid Key Encryption Key using RewrapClientEncryptionKeyAsync")); } // for unwrap to succeed