Skip to content

Commit

Permalink
Client Encryption: Fixes System.Text custom serializer issue with Dat…
Browse files Browse the repository at this point in the history
…aEncryptionKeyContainer operations. (Azure#3386)

* Use Cosmos Base serializer for DEK serialization.

* Update DataEncryptionKeyFeedIterator.cs

* Update MdeCustomEncryptionTests.cs

* Update MdeCustomEncryptionTests.cs

* updated tests.

* Update DataEncryptionKeyContainerCore.cs

* updated changelogs and build props

* fixes as per review comment

* Update DataEncryptionKeyContainerCore.cs

* fixes.
  • Loading branch information
kr-santosh authored Aug 19, 2022
1 parent 1151f55 commit 0dcd4a8
Show file tree
Hide file tree
Showing 8 changed files with 505 additions and 19 deletions.
2 changes: 1 addition & 1 deletion Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<EncryptionOfficialVersion>2.0.0</EncryptionOfficialVersion>
<EncryptionPreviewVersion>2.0.0</EncryptionPreviewVersion>
<EncryptionPreviewSuffixVersion>preview</EncryptionPreviewSuffixVersion>
<CustomEncryptionVersion>1.0.0-preview03</CustomEncryptionVersion>
<CustomEncryptionVersion>1.0.0-preview04</CustomEncryptionVersion>
<HybridRowVersion>1.1.0-preview3</HybridRowVersion>
<LangVersion>10.0</LangVersion>
<AboveDirBuildProps>$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))</AboveDirBuildProps>
Expand Down
5 changes: 5 additions & 0 deletions Microsoft.Azure.Cosmos.Encryption.Custom/changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ Preview features are treated as a separate branch and will not be included in th
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

### <a name="1.0.0-preview04"/> [1.0.0-preview04](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview04) - 2022-08-16

#### Fixes
- [#3386](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3386) Fixes custom serializer issue with DataEncryptionKeyContainer operations.

### <a name="1.0.0-preview03"/> [1.0.0-preview03](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption.Custom/1.0.0-preview03) - 2022-04-15
- [#3145](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3145) Adds dependency on latest Microsoft.Azure.Cosmos preview (3.26.0-preview).

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
Expand All @@ -26,15 +27,27 @@ public override FeedIterator<T> GetDataEncryptionKeyQueryIterator<T>(
string continuationToken = null,
QueryRequestOptions requestOptions = null)
{
return this.DekProvider.Container.GetItemQueryIterator<T>(queryText, continuationToken, requestOptions);
return new DataEncryptionKeyFeedIterator<T>(
new DataEncryptionKeyFeedIterator(
this.DekProvider.Container.GetItemQueryStreamIterator(
queryText,
continuationToken,
requestOptions)),
this.DekProvider.Container.Database.Client.ResponseFactory);
}

public override FeedIterator<T> GetDataEncryptionKeyQueryIterator<T>(
QueryDefinition queryDefinition,
string continuationToken = null,
QueryRequestOptions requestOptions = null)
{
return this.DekProvider.Container.GetItemQueryIterator<T>(queryDefinition, continuationToken, requestOptions);
return new DataEncryptionKeyFeedIterator<T>(
new DataEncryptionKeyFeedIterator(
this.DekProvider.Container.GetItemQueryStreamIterator(
queryDefinition,
continuationToken,
requestOptions)),
this.DekProvider.Container.Database.Client.ResponseFactory);
}

public override async Task<ItemResponse<DataEncryptionKeyProperties>> CreateDataEncryptionKeyAsync(
Expand Down Expand Up @@ -86,19 +99,31 @@ public override async Task<ItemResponse<DataEncryptionKeyProperties>> CreateData
updatedMetadata,
DateTime.UtcNow);

ItemResponse<DataEncryptionKeyProperties> dekResponse = await this.DekProvider.Container.CreateItemAsync(
dekProperties,
// Since T is not exposed, a user passing System.Text based custom serializers would result in a failure, since DataEncryptionKeyProperties is based
// on Newtonsoft JSON serialization and is not compatible. So we just convert it to stream using cosmos's base serializer.
Stream dekpropertiesStream = EncryptionProcessor.BaseSerializer.ToStream(dekProperties);

ResponseMessage dekResponse = await this.DekProvider.Container.CreateItemStreamAsync(
dekpropertiesStream,
new PartitionKey(dekProperties.Id),
cancellationToken: cancellationToken);

this.DekProvider.DekCache.SetDekProperties(id, dekResponse.Resource);
if (!dekResponse.IsSuccessStatusCode)
{
string subStatusCode = dekResponse.Headers.GetValueOrDefault("x-ms-substatus") ?? "0";
throw new EncryptionCosmosException(dekResponse.ErrorMessage, dekResponse.StatusCode, int.Parse(subStatusCode), dekResponse.Headers.ActivityId, dekResponse.Headers.RequestCharge, dekResponse.Diagnostics);
}

dekProperties = EncryptionProcessor.BaseSerializer.FromStream<DataEncryptionKeyProperties>(dekResponse.Content);

this.DekProvider.DekCache.SetDekProperties(id, dekProperties);

if (string.Equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized))
{
this.DekProvider.DekCache.SetRawDek(id, inMemoryRawDek);
}

return dekResponse;
return new EncryptionItemResponse<DataEncryptionKeyProperties>(dekResponse, dekProperties);
}

/// <inheritdoc/>
Expand Down Expand Up @@ -192,18 +217,27 @@ public override async Task<ItemResponse<DataEncryptionKeyProperties>> RewrapData
EncryptionKeyWrapMetadata = updatedMetadata,
};

ItemResponse<DataEncryptionKeyProperties> response;
ResponseMessage dekResponse;

try
{
response = await this.DekProvider.Container.ReplaceItemAsync(
newDekProperties,
newDekProperties.Id,
new PartitionKey(newDekProperties.Id),
requestOptions,
cancellationToken);
Stream newDekpropertiesStream = EncryptionProcessor.BaseSerializer.ToStream(newDekProperties);

dekResponse = await this.DekProvider.Container.ReplaceItemStreamAsync(
newDekpropertiesStream,
newDekProperties.Id,
new PartitionKey(newDekProperties.Id),
requestOptions,
cancellationToken);

if (!dekResponse.IsSuccessStatusCode)
{
string subStatusCode = dekResponse.Headers.GetValueOrDefault("x-ms-substatus") ?? "0";
throw new EncryptionCosmosException(dekResponse.ErrorMessage, dekResponse.StatusCode, int.Parse(subStatusCode), dekResponse.Headers.ActivityId, dekResponse.Headers.RequestCharge, dekResponse.Diagnostics);
}

Debug.Assert(response.Resource != null);
dekProperties = EncryptionProcessor.BaseSerializer.FromStream<DataEncryptionKeyProperties>(dekResponse.Content);
Debug.Assert(dekProperties != null);
}
catch (CosmosException ex)
{
Expand All @@ -228,14 +262,14 @@ await this.ReadDataEncryptionKeyAsync(
cancellationToken);
}

this.DekProvider.DekCache.SetDekProperties(id, response.Resource);
this.DekProvider.DekCache.SetDekProperties(id, dekProperties);

if (string.Equals(newDekProperties.EncryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized))
{
this.DekProvider.DekCache.SetRawDek(id, updatedRawDek);
}

return response;
return new EncryptionItemResponse<DataEncryptionKeyProperties>(dekResponse, dekProperties);
}

internal async Task<DataEncryptionKeyProperties> FetchDataEncryptionKeyPropertiesAsync(
Expand Down Expand Up @@ -568,11 +602,21 @@ private async Task<ItemResponse<DataEncryptionKeyProperties>> ReadInternalAsync(
{
using (diagnosticsContext.CreateScope("ReadInternalAsync"))
{
return await this.DekProvider.Container.ReadItemAsync<DataEncryptionKeyProperties>(
ResponseMessage response = await this.DekProvider.Container.ReadItemStreamAsync(
id,
new PartitionKey(id),
requestOptions,
cancellationToken);
cancellationToken: cancellationToken);

if (!response.IsSuccessStatusCode)
{
string subStatusCode = response.Headers.GetValueOrDefault("x-ms-substatus") ?? "0";
throw new EncryptionCosmosException(response.ErrorMessage, response.StatusCode, int.Parse(subStatusCode), response.Headers.ActivityId, response.Headers.RequestCharge, response.Diagnostics);
}

DataEncryptionKeyProperties dekProperties = EncryptionProcessor.BaseSerializer.FromStream<DataEncryptionKeyProperties>(response.Content);

return new EncryptionItemResponse<DataEncryptionKeyProperties>(response, dekProperties);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------

namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System.Threading;
using System.Threading.Tasks;

internal sealed class DataEncryptionKeyFeedIterator : FeedIterator
{
private readonly FeedIterator feedIterator;

public DataEncryptionKeyFeedIterator(
FeedIterator feedIterator)
{
this.feedIterator = feedIterator;
}

public override bool HasMoreResults => this.feedIterator.HasMoreResults;

public override Task<ResponseMessage> ReadNextAsync(CancellationToken cancellationToken = default)
{
return this.feedIterator.ReadNextAsync(cancellationToken);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
//------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
//------------------------------------------------------------

namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;

internal sealed class DataEncryptionKeyFeedIterator<T> : FeedIterator<T>
{
private readonly FeedIterator feedIterator;
private readonly CosmosResponseFactory responseFactory;

public DataEncryptionKeyFeedIterator(
DataEncryptionKeyFeedIterator feedIterator,
CosmosResponseFactory responseFactory)
{
if (!(feedIterator is DataEncryptionKeyFeedIterator))
{
throw new ArgumentOutOfRangeException($"{nameof(feedIterator)} must be of type {nameof(DataEncryptionKeyFeedIterator)}.");
}

this.feedIterator = feedIterator;
this.responseFactory = responseFactory;
}

public override bool HasMoreResults => this.feedIterator.HasMoreResults;

public override async Task<FeedResponse<T>> ReadNextAsync(CancellationToken cancellationToken = default)
{
ResponseMessage responseMessage;

if (typeof(T) == typeof(DataEncryptionKeyProperties))
{
IReadOnlyCollection<T> resource;
(responseMessage, resource) = await this.ReadNextUsingCosmosBaseSerializerAsync(cancellationToken);

return DecryptableFeedResponse<T>.CreateResponse(
responseMessage,
resource);
}
else
{
responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken);
}

return this.responseFactory.CreateItemFeedResponse<T>(responseMessage);
}

public async Task<(ResponseMessage, List<T>)> ReadNextUsingCosmosBaseSerializerAsync(CancellationToken cancellationToken = default)
{
CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null);
using (diagnosticsContext.CreateScope("FeedIterator.ReadNextWithoutDecryption"))
{
ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken);
List<T> dataEncryptionKeyPropertiesList = null;

if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null)
{
dataEncryptionKeyPropertiesList = this.ConvertResponseToDataEncryptionKeyPropertiesList(
responseMessage.Content);

return (responseMessage, dataEncryptionKeyPropertiesList);
}

return (responseMessage, dataEncryptionKeyPropertiesList);
}
}

private List<T> ConvertResponseToDataEncryptionKeyPropertiesList(
Stream content)
{
JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(content);

if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents))
{
throw new InvalidOperationException("Feed Response body contract was violated. Feed Response did not have an array of Documents.");
}

List<T> dataEncryptionKeyPropertiesList = new List<T>(documents.Count);

foreach (JToken value in documents)
{
dataEncryptionKeyPropertiesList.Add(value.ToObject<T>());
}

return dataEncryptionKeyPropertiesList;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// ------------------------------------------------------------
// Copyright (c) Microsoft Corporation. All rights reserved.
// ------------------------------------------------------------

namespace Microsoft.Azure.Cosmos.Encryption.Custom
{
using System;
using System.Net;

internal sealed class EncryptionCosmosException : CosmosException
{
private readonly CosmosDiagnostics encryptionCosmosDiagnostics;

public EncryptionCosmosException(
string message,
HttpStatusCode statusCode,
int subStatusCode,
string activityId,
double requestCharge,
CosmosDiagnostics encryptionCosmosDiagnostics)
: base(message, statusCode, subStatusCode, activityId, requestCharge)
{
this.encryptionCosmosDiagnostics = encryptionCosmosDiagnostics ?? throw new ArgumentNullException(nameof(encryptionCosmosDiagnostics));
}

public override CosmosDiagnostics Diagnostics => this.encryptionCosmosDiagnostics;
}
}
Loading

0 comments on commit 0dcd4a8

Please sign in to comment.