diff --git a/Directory.Build.props b/Directory.Build.props
index b388ac0f73..d05fb8f3fa 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -5,8 +5,8 @@
3.29.0
preview
3.29.1
- 1.0.1
- 1.0.1
+ 2.0.0
+ 2.0.0
preview
1.0.0-preview03
1.1.0-preview3
diff --git a/Microsoft.Azure.Cosmos.Encryption/changelog.md b/Microsoft.Azure.Cosmos.Encryption/changelog.md
index 4c8f50595b..e3274bce7e 100644
--- a/Microsoft.Azure.Cosmos.Encryption/changelog.md
+++ b/Microsoft.Azure.Cosmos.Encryption/changelog.md
@@ -3,6 +3,15 @@ Preview features are treated as a separate branch and will not be included in th
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+### [2.0.0](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.0) - 2022-06-28
+
+#### Added
+- [#3241](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3241) Adds code to support PartitionKey and Id encryption.
+
+### [2.0.0-preview](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.0-preview) - 2022-06-28
+
+#### Added
+- [#3241](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3241) Adds code to support PartitionKey and Id encryption.
### [1.0.1](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/1.0.1) - 2022-06-01
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Constants.cs b/Microsoft.Azure.Cosmos.Encryption/src/Constants.cs
index 347c7c103e..57ef091bb4 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/Constants.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/Constants.cs
@@ -16,6 +16,7 @@ internal static class Constants
public const string DiagnosticsStartTime = "Start time";
public const string DocumentsResourcePropertyName = "Documents";
public const string IncorrectContainerRidSubStatus = "1024";
+ public const string PartitionKeyMismatch = "1001";
// TODO: Good to have constants available in the Cosmos SDK. Tracked via https://github.com/Azure/azure-cosmos-dotnet-v3/issues/2431
public const string IntendedCollectionHeader = "x-ms-cosmos-intended-collection-rid";
@@ -23,6 +24,6 @@ internal static class Constants
public const string AllowCachedReadsHeader = "x-ms-cosmos-allow-cachedreads";
public const string DatabaseRidHeader = "x-ms-cosmos-database-rid";
public const string SubStatusHeader = "x-ms-substatus";
- public const int SupportedClientEncryptionPolicyFormatVersion = 1;
+ public const int SupportedClientEncryptionPolicyFormatVersion = 2;
}
}
\ No newline at end of file
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
index 13d09d72c0..d32bfa1c94 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionContainer.cs
@@ -16,8 +16,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
internal sealed class EncryptionContainer : Container
{
- private readonly Container container;
-
private readonly AsyncCache encryptionSettingsByContainerName;
///
@@ -30,7 +28,7 @@ public EncryptionContainer(
Container container,
EncryptionCosmosClient encryptionCosmosClient)
{
- this.container = container ?? throw new ArgumentNullException(nameof(container));
+ this.Container = container ?? throw new ArgumentNullException(nameof(container));
this.EncryptionCosmosClient = encryptionCosmosClient ?? throw new ArgumentNullException(nameof(container));
this.ResponseFactory = this.Database.Client.ResponseFactory;
this.CosmosSerializer = this.Database.Client.ClientOptions.Serializer;
@@ -43,13 +41,15 @@ public EncryptionContainer(
public EncryptionCosmosClient EncryptionCosmosClient { get; }
- public override string Id => this.container.Id;
+ public override string Id => this.Container.Id;
+
+ public override Conflicts Conflicts => this.Container.Conflicts;
- public override Conflicts Conflicts => this.container.Conflicts;
+ public override Scripts.Scripts Scripts => this.Container.Scripts;
- public override Scripts.Scripts Scripts => this.container.Scripts;
+ public override Database Database => this.Container.Database;
- public override Database Database => this.container.Database;
+ internal Container Container { get; }
public override async Task> CreateItemAsync(
T item,
@@ -98,13 +98,18 @@ public override async Task CreateItemStreamAsync(
cancellationToken);
}
- public override Task> DeleteItemAsync(
+ public override async Task> DeleteItemAsync(
string id,
PartitionKey partitionKey,
ItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.DeleteItemAsync(
+ EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
+
+ id = await this.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken);
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+
+ return await this.Container.DeleteItemAsync(
id,
partitionKey,
requestOptions,
@@ -117,7 +122,7 @@ public override Task DeleteItemStreamAsync(
ItemRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.DeleteItemStreamAsync(
+ return this.Container.DeleteItemStreamAsync(
id,
partitionKey,
requestOptions,
@@ -265,8 +270,20 @@ public override async Task UpsertItemStreamAsync(
public override TransactionalBatch CreateTransactionalBatch(
PartitionKey partitionKey)
{
+ EncryptionSettings encryptionSettings = this.GetOrUpdateEncryptionSettingsFromCacheAsync(
+ obsoleteEncryptionSettings: null,
+ cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
+ (partitionKey, _) = this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey: partitionKey, encryptionSettings: encryptionSettings, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
return new EncryptionTransactionalBatch(
- this.container.CreateTransactionalBatch(partitionKey),
+ this.Container.CreateTransactionalBatch(partitionKey),
this,
this.CosmosSerializer);
}
@@ -275,7 +292,7 @@ public override Task DeleteContainerAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.DeleteContainerAsync(
+ return this.Container.DeleteContainerAsync(
requestOptions,
cancellationToken);
}
@@ -284,7 +301,7 @@ public override Task DeleteContainerStreamAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.DeleteContainerStreamAsync(
+ return this.Container.DeleteContainerStreamAsync(
requestOptions,
cancellationToken);
}
@@ -294,7 +311,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedEstimatorBuilder(
ChangesEstimationHandler estimationDelegate,
TimeSpan? estimationPeriod = null)
{
- return this.container.GetChangeFeedEstimatorBuilder(
+ return this.Container.GetChangeFeedEstimatorBuilder(
processorName,
estimationDelegate,
estimationPeriod);
@@ -306,7 +323,7 @@ public override IOrderedQueryable GetItemLinqQueryable(
QueryRequestOptions requestOptions = null,
CosmosLinqSerializerOptions linqSerializerOptions = null)
{
- return this.container.GetItemLinqQueryable(
+ return this.Container.GetItemLinqQueryable(
allowSynchronousQueryExecution,
continuationToken,
requestOptions,
@@ -343,7 +360,7 @@ public override Task ReadContainerAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReadContainerAsync(
+ return this.Container.ReadContainerAsync(
requestOptions,
cancellationToken);
}
@@ -352,7 +369,7 @@ public override Task ReadContainerStreamAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReadContainerStreamAsync(
+ return this.Container.ReadContainerStreamAsync(
requestOptions,
cancellationToken);
}
@@ -360,14 +377,14 @@ public override Task ReadContainerStreamAsync(
public override Task ReadThroughputAsync(
CancellationToken cancellationToken = default)
{
- return this.container.ReadThroughputAsync(cancellationToken);
+ return this.Container.ReadThroughputAsync(cancellationToken);
}
public override Task ReadThroughputAsync(
RequestOptions requestOptions,
CancellationToken cancellationToken = default)
{
- return this.container.ReadThroughputAsync(
+ return this.Container.ReadThroughputAsync(
requestOptions,
cancellationToken);
}
@@ -377,7 +394,7 @@ public override Task ReplaceContainerAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReplaceContainerAsync(
+ return this.Container.ReplaceContainerAsync(
containerProperties,
requestOptions,
cancellationToken);
@@ -388,7 +405,7 @@ public override Task ReplaceContainerStreamAsync(
ContainerRequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReplaceContainerStreamAsync(
+ return this.Container.ReplaceContainerStreamAsync(
containerProperties,
requestOptions,
cancellationToken);
@@ -399,7 +416,7 @@ public override Task ReplaceThroughputAsync(
RequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReplaceThroughputAsync(
+ return this.Container.ReplaceThroughputAsync(
throughput,
requestOptions,
cancellationToken);
@@ -413,11 +430,9 @@ public override FeedIterator GetItemQueryStreamIterator(
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
- this.container.GetItemQueryStreamIterator(
- queryDefinition,
- continuationToken,
- clonedRequestOptions),
this,
+ queryDefinition,
+ continuationToken,
clonedRequestOptions);
}
@@ -429,11 +444,9 @@ public override FeedIterator GetItemQueryStreamIterator(
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
- this.container.GetItemQueryStreamIterator(
- queryText,
- continuationToken,
- clonedRequestOptions),
this,
+ queryText,
+ continuationToken,
clonedRequestOptions);
}
@@ -442,7 +455,7 @@ public override Task ReplaceThroughputAsync(
RequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.ReplaceThroughputAsync(
+ return this.Container.ReplaceThroughputAsync(
throughputProperties,
requestOptions,
cancellationToken);
@@ -451,14 +464,14 @@ public override Task ReplaceThroughputAsync(
public override Task> GetFeedRangesAsync(
CancellationToken cancellationToken = default)
{
- return this.container.GetFeedRangesAsync(cancellationToken);
+ return this.Container.GetFeedRangesAsync(cancellationToken);
}
public override ChangeFeedEstimator GetChangeFeedEstimator(
string processorName,
Container leaseContainer)
{
- return this.container.GetChangeFeedEstimator(processorName, leaseContainer);
+ return this.Container.GetChangeFeedEstimator(processorName, leaseContainer);
}
public override FeedIterator GetChangeFeedStreamIterator(
@@ -471,11 +484,9 @@ public override FeedIterator GetChangeFeedStreamIterator(
: new ChangeFeedRequestOptions();
return new EncryptionFeedIterator(
- this.container.GetChangeFeedStreamIterator(
- changeFeedStartFrom,
- changeFeedMode,
- clonedchangeFeedRequestOptions),
this,
+ changeFeedStartFrom,
+ changeFeedMode,
clonedchangeFeedRequestOptions);
}
@@ -546,7 +557,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(
string processorName,
ChangesHandler onChangesDelegate)
{
- return this.container.GetChangeFeedProcessorBuilder(
+ return this.Container.GetChangeFeedProcessorBuilder(
processorName,
async (
IReadOnlyCollection documents,
@@ -565,7 +576,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(
string processorName,
ChangeFeedHandler onChangesDelegate)
{
- return this.container.GetChangeFeedProcessorBuilder(
+ return this.Container.GetChangeFeedProcessorBuilder(
processorName,
async (
ChangeFeedProcessorContext context,
@@ -585,7 +596,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu
string processorName,
ChangeFeedHandlerWithManualCheckpoint onChangesDelegate)
{
- return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint(
+ return this.Container.GetChangeFeedProcessorBuilderWithManualCheckpoint(
processorName,
async (
ChangeFeedProcessorContext context,
@@ -606,7 +617,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder(
string processorName,
ChangeFeedStreamHandler onChangesDelegate)
{
- return this.container.GetChangeFeedProcessorBuilder(
+ return this.Container.GetChangeFeedProcessorBuilder(
processorName,
async (
ChangeFeedProcessorContext context,
@@ -632,7 +643,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu
string processorName,
ChangeFeedStreamHandlerWithManualCheckpoint onChangesDelegate)
{
- return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint(
+ return this.Container.GetChangeFeedProcessorBuilderWithManualCheckpoint(
processorName,
async (
ChangeFeedProcessorContext context,
@@ -699,12 +710,10 @@ public override FeedIterator GetItemQueryStreamIterator(
QueryRequestOptions clonedRequestOptions = requestOptions != null ? (QueryRequestOptions)requestOptions.ShallowCopy() : new QueryRequestOptions();
return new EncryptionFeedIterator(
- this.container.GetItemQueryStreamIterator(
- feedRange,
- queryDefinition,
- continuationToken,
- clonedRequestOptions),
this,
+ feedRange,
+ queryDefinition,
+ continuationToken,
clonedRequestOptions);
}
@@ -724,12 +733,16 @@ public override FeedIterator GetItemQueryIterator(
}
#if ENCRYPTIONPREVIEW
- public override Task DeleteAllItemsByPartitionKeyStreamAsync(
+ public override async Task DeleteAllItemsByPartitionKeyStreamAsync(
Cosmos.PartitionKey partitionKey,
RequestOptions requestOptions = null,
CancellationToken cancellationToken = default)
{
- return this.container.DeleteAllItemsByPartitionKeyStreamAsync(
+ EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
+
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+
+ return await this.Container.DeleteAllItemsByPartitionKeyStreamAsync(
partitionKey,
requestOptions,
cancellationToken);
@@ -739,7 +752,7 @@ public override Task> GetPartitionKeyRangesAsync(
FeedRange feedRange,
CancellationToken cancellationToken = default)
{
- return this.container.GetPartitionKeyRangesAsync(feedRange, cancellationToken);
+ return this.Container.GetPartitionKeyRangesAsync(feedRange, cancellationToken);
}
#endif
@@ -759,13 +772,53 @@ internal async Task ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(
EncryptionDiagnosticsContext encryptionDiagnosticsContext,
CancellationToken cancellationToken)
{
- if (responseMessage.StatusCode == HttpStatusCode.BadRequest &&
- string.Equals(responseMessage.Headers.Get(Constants.SubStatusHeader), Constants.IncorrectContainerRidSubStatus))
+ string subStatusCode = responseMessage.Headers.Get(Constants.SubStatusHeader);
+ bool isPartitionKeyMismatch = string.Equals(subStatusCode, Constants.PartitionKeyMismatch);
+ bool isContainerRidIncorrect = string.Equals(subStatusCode, Constants.IncorrectContainerRidSubStatus);
+
+ // if the partition key check is done before container rid check.
+ if (responseMessage.StatusCode == HttpStatusCode.BadRequest && (isContainerRidIncorrect || isPartitionKeyMismatch))
{
+ // The below code avoids unneccessary force refresh of encryption settings if wrong partition key was passed and the PartitionKeyMismatch was not
+ // due to us not encrypting the partition key because of incorrect cached policy.
+ if (isPartitionKeyMismatch && encryptionSettings.PartitionKeyPaths.Any())
+ {
+ EncryptionSettingForProperty encryptionSettingForProperty = null;
+ foreach (string path in encryptionSettings.PartitionKeyPaths)
+ {
+ string partitionKeyPath = path.Split('/')[1];
+ encryptionSettingForProperty = encryptionSettings.GetEncryptionSettingForProperty(partitionKeyPath);
+
+ // break on first path encountered.
+ if (encryptionSettingForProperty != null)
+ {
+ break;
+ }
+ }
+
+ // if none of the paths were part of encryption policy
+ if (encryptionSettingForProperty == null)
+ {
+ return;
+ }
+ }
+
+ string currentContainerRid = encryptionSettings.ContainerRidValue;
+
+ // either way we cannot be sure if PartitionKeyMismatch was to due us using an invalid setting or we did not encrypt it.
// get the latest encryption settings.
- await this.GetOrUpdateEncryptionSettingsFromCacheAsync(
- obsoleteEncryptionSettings: encryptionSettings,
- cancellationToken: cancellationToken);
+ EncryptionSettings updatedEncryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(
+ obsoleteEncryptionSettings: encryptionSettings,
+ cancellationToken: cancellationToken);
+
+ string containerRidPostSettingsUpdate = updatedEncryptionSettings.ContainerRidValue;
+
+ // gets returned back due to PartitionKeyMismatch.(in case of batch looks like the container rid check gets done first)
+ // if the container was not recreated, so policy has not changed, just return the original response.
+ if (currentContainerRid == containerRidPostSettingsUpdate)
+ {
+ return;
+ }
if (encryptionDiagnosticsContext == null)
{
@@ -808,11 +861,11 @@ internal async Task> EncryptPatchOperationsAsync(
}
// get the top level path's encryption setting.
- EncryptionSettingForProperty settingforProperty = encryptionSettings.GetEncryptionSettingForProperty(
+ EncryptionSettingForProperty encryptionSettingForProperty = encryptionSettings.GetEncryptionSettingForProperty(
patchOperation.Path.Split('/')[1]);
// non-encrypted path
- if (settingforProperty == null)
+ if (encryptionSettingForProperty == null)
{
encryptedPatchOperations.Add(patchOperation);
continue;
@@ -828,9 +881,10 @@ internal async Task> EncryptPatchOperationsAsync(
}
Stream encryptedPropertyValue = await EncryptionProcessor.EncryptValueStreamAsync(
- valueParam,
- settingforProperty,
- cancellationToken);
+ valueStreamToEncrypt: valueParam,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: patchOperation.Path.Split('/')[1] == "id",
+ cancellationToken: cancellationToken);
propertiesEncryptedCount++;
@@ -857,6 +911,128 @@ internal async Task> EncryptPatchOperationsAsync(
return encryptedPatchOperations;
}
+ internal async Task CheckIfIdIsEncryptedAndGetEncryptedIdAsync(
+ string id,
+ EncryptionSettings encryptionSettings,
+ CancellationToken cancellationToken)
+ {
+ if (!encryptionSettings.PropertiesToEncrypt.Any() || string.IsNullOrEmpty(id))
+ {
+ return id;
+ }
+
+ EncryptionSettingForProperty encryptionSettingForProperty = encryptionSettings.GetEncryptionSettingForProperty("id");
+
+ if (encryptionSettingForProperty == null)
+ {
+ return id;
+ }
+
+ Stream valueStream = this.CosmosSerializer.ToStream(id);
+
+ Stream encryptedIdStream = await EncryptionProcessor.EncryptValueStreamAsync(
+ valueStreamToEncrypt: valueStream,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: true,
+ cancellationToken: cancellationToken);
+ using (StreamReader reader = new StreamReader(encryptedIdStream))
+ {
+ string encryptedId = await reader.ReadToEndAsync();
+ return JToken.Parse(encryptedId).ToString();
+ }
+ }
+
+ internal async Task<(PartitionKey, bool)> CheckIfPkIsEncryptedAndGetEncryptedPkAsync(
+ PartitionKey partitionKey,
+ EncryptionSettings encryptionSettings,
+ CancellationToken cancellationToken)
+ {
+ if (!encryptionSettings.PartitionKeyPaths.Any() || !encryptionSettings.PropertiesToEncrypt.Any() || partitionKey == null || (partitionKey != null && (partitionKey == PartitionKey.None || partitionKey == PartitionKey.Null)))
+ {
+ return (partitionKey, false);
+ }
+
+ EncryptionSettingForProperty encryptionSettingForProperty;
+
+ JArray jArray = JArray.Parse(partitionKey.ToString());
+
+#if ENCRYPTIONPREVIEW
+ if (jArray.Count > 1)
+ {
+ int i = 0;
+ PartitionKeyBuilder partitionKeyBuilder = new PartitionKeyBuilder();
+
+ bool isPkEncrypted = false;
+ // partitionKeyBuilder expects the paths and values to be in same order.
+ foreach (string path in encryptionSettings.PartitionKeyPaths)
+ {
+ // case: partition key path is /a/b/c and the client encryption policy has /a in path.
+ // hence encrypt the partition key value with using its top level path /a since /c would have been encrypted in the document using /a's policy.
+ string partitionKeyPath = path.Split('/')[1];
+
+ encryptionSettingForProperty = encryptionSettings.GetEncryptionSettingForProperty(
+ partitionKeyPath);
+
+ if (encryptionSettingForProperty == null)
+ {
+ partitionKeyBuilder.Add(jArray[i++].ToString());
+ continue;
+ }
+
+ isPkEncrypted = true;
+ Stream valueStream = EncryptionProcessor.BaseSerializer.ToStream(jArray[i++]);
+
+ Stream encryptedPartitionKey = await EncryptionProcessor.EncryptValueStreamAsync(
+ valueStreamToEncrypt: valueStream,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: partitionKeyPath == "id",
+ cancellationToken: cancellationToken);
+
+ string encryptedPK = null;
+ using (StreamReader reader = new StreamReader(encryptedPartitionKey))
+ {
+ encryptedPK = await reader.ReadToEndAsync();
+ }
+
+ JToken encryptedKey = JToken.Parse(encryptedPK);
+
+ partitionKeyBuilder.Add(encryptedKey.ToString());
+ }
+
+ return (partitionKeyBuilder.Build(), isPkEncrypted);
+ }
+ else
+#endif
+ {
+ string partitionKeyPath = encryptionSettings.PartitionKeyPaths.Single().Split('/')[1];
+ encryptionSettingForProperty = encryptionSettings.GetEncryptionSettingForProperty(
+ partitionKeyPath);
+
+ if (encryptionSettingForProperty == null)
+ {
+ return (partitionKey, false);
+ }
+
+ Stream valueStream = EncryptionProcessor.BaseSerializer.ToStream(jArray[0]);
+
+ Stream encryptedPartitionKey = await EncryptionProcessor.EncryptValueStreamAsync(
+ valueStreamToEncrypt: valueStream,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: partitionKeyPath == "id",
+ cancellationToken: cancellationToken);
+
+ string encryptedPK = null;
+ using (StreamReader reader = new StreamReader(encryptedPartitionKey))
+ {
+ encryptedPK = await reader.ReadToEndAsync();
+ }
+
+ JToken encryptedKey = JToken.Parse(encryptedPK);
+
+ return (new PartitionKey(encryptedKey.ToString()), true);
+ }
+ }
+
///
/// Returns a cloned copy of the passed RequestOptions if passed else creates a new ItemRequestOptions.
///
@@ -878,7 +1054,7 @@ private async Task CreateItemHelperAsync(
EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
- return await this.container.CreateItemStreamAsync(
+ return await this.Container.CreateItemStreamAsync(
streamPayload,
partitionKey,
requestOptions,
@@ -897,7 +1073,8 @@ private async Task CreateItemHelperAsync(
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
- ResponseMessage responseMessage = await this.container.CreateItemStreamAsync(
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+ ResponseMessage responseMessage = await this.Container.CreateItemStreamAsync(
streamPayload,
partitionKey,
clonedRequestOptions,
@@ -924,7 +1101,7 @@ private async Task ReadItemHelperAsync(
EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
- return await this.container.ReadItemStreamAsync(
+ return await this.Container.ReadItemStreamAsync(
id,
partitionKey,
requestOptions,
@@ -936,7 +1113,10 @@ private async Task ReadItemHelperAsync(
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
- ResponseMessage responseMessage = await this.container.ReadItemStreamAsync(
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+ id = await this.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken);
+
+ ResponseMessage responseMessage = await this.Container.ReadItemStreamAsync(
id,
partitionKey,
clonedRequestOptions,
@@ -971,7 +1151,7 @@ private async Task ReplaceItemHelperAsync(
EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
- return await this.container.ReplaceItemStreamAsync(
+ return await this.Container.ReplaceItemStreamAsync(
streamPayload,
id,
partitionKey,
@@ -993,7 +1173,9 @@ private async Task ReplaceItemHelperAsync(
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
- ResponseMessage responseMessage = await this.container.ReplaceItemStreamAsync(
+ id = await this.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken);
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+ ResponseMessage responseMessage = await this.Container.ReplaceItemStreamAsync(
streamPayload,
id,
partitionKey,
@@ -1026,7 +1208,7 @@ private async Task UpsertItemHelperAsync(
EncryptionSettings encryptionSettings = await this.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
- return await this.container.UpsertItemStreamAsync(
+ return await this.Container.UpsertItemStreamAsync(
streamPayload,
partitionKey,
requestOptions,
@@ -1034,6 +1216,7 @@ private async Task UpsertItemHelperAsync(
}
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
+
streamPayload = await EncryptionProcessor.EncryptAsync(
streamPayload,
encryptionSettings,
@@ -1046,8 +1229,8 @@ private async Task UpsertItemHelperAsync(
clonedRequestOptions = EncryptionContainerGetClonedItemRequestOptions(requestOptions);
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
-
- ResponseMessage responseMessage = await this.container.UpsertItemStreamAsync(
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+ ResponseMessage responseMessage = await this.Container.UpsertItemStreamAsync(
streamPayload,
partitionKey,
clonedRequestOptions,
@@ -1095,7 +1278,9 @@ private async Task PatchItemHelperAsync(
encryptionDiagnosticsContext,
cancellationToken);
- ResponseMessage responseMessage = await this.container.PatchItemStreamAsync(
+ (partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(partitionKey, encryptionSettings, cancellationToken);
+ id = await this.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken);
+ ResponseMessage responseMessage = await this.Container.PatchItemStreamAsync(
id,
partitionKey,
encryptedPatchOperations,
@@ -1148,7 +1333,7 @@ private async Task ReadManyItemsHelperAsync(
if (!encryptionSettings.PropertiesToEncrypt.Any())
{
- return await this.container.ReadManyItemsStreamAsync(
+ return await this.Container.ReadManyItemsStreamAsync(
items,
readManyRequestOptions,
cancellationToken);
@@ -1159,8 +1344,17 @@ private async Task ReadManyItemsHelperAsync(
encryptionSettings.SetRequestHeaders(clonedRequestOptions);
- ResponseMessage responseMessage = await this.container.ReadManyItemsStreamAsync(
- items,
+ List<(string, PartitionKey)> encryptedItemList = new List<(string, PartitionKey)>();
+
+ for (int i = 0; i < items.Count; i++)
+ {
+ string id = await this.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(items[i].id, encryptionSettings, cancellationToken);
+ (PartitionKey partitionKey, _) = await this.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(items[i].partitionKey, encryptionSettings, cancellationToken);
+ encryptedItemList.Add((id, partitionKey));
+ }
+
+ ResponseMessage responseMessage = await this.Container.ReadManyItemsStreamAsync(
+ encryptedItemList,
clonedRequestOptions,
cancellationToken);
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs
index c8d036e2e2..3c51357a9e 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionFeedIterator.cs
@@ -6,38 +6,128 @@ namespace Microsoft.Azure.Cosmos.Encryption
{
using System;
using System.IO;
- using System.Net;
using System.Threading;
using System.Threading.Tasks;
internal sealed class EncryptionFeedIterator : FeedIterator
{
- private readonly FeedIterator feedIterator;
+ private readonly SemaphoreSlim singleIteratorInitSema = new SemaphoreSlim(1, 1);
private readonly EncryptionContainer encryptionContainer;
private readonly RequestOptions requestOptions;
+ private readonly QueryDefinition queryDefinition;
+ private readonly string queryText;
+ private readonly string continuationToken;
+ private readonly FeedRange feedRange;
+ private readonly QueryType queryType;
+
+ private bool isIteratorInitialized = false;
public EncryptionFeedIterator(
FeedIterator feedIterator,
EncryptionContainer encryptionContainer,
RequestOptions requestOptions)
{
- this.feedIterator = feedIterator ?? throw new ArgumentNullException(nameof(feedIterator));
+ this.FeedIterator = feedIterator ?? throw new ArgumentNullException(nameof(feedIterator));
this.encryptionContainer = encryptionContainer ?? throw new ArgumentNullException(nameof(encryptionContainer));
this.requestOptions = requestOptions ?? throw new ArgumentNullException(nameof(requestOptions));
}
- public override bool HasMoreResults => this.feedIterator.HasMoreResults;
+ public EncryptionFeedIterator(
+ EncryptionContainer encryptionContainer,
+ QueryDefinition queryDefinition,
+ string continuationToken = null,
+ QueryRequestOptions requestOptions = null)
+ {
+ this.encryptionContainer = encryptionContainer ?? throw new ArgumentNullException(nameof(encryptionContainer));
+ this.queryDefinition = queryDefinition;
+ this.continuationToken = continuationToken;
+ this.requestOptions = requestOptions;
+
+ this.FeedIterator = encryptionContainer.Container.GetItemQueryStreamIterator(
+ queryDefinition,
+ continuationToken,
+ requestOptions);
+
+ this.queryType = QueryType.QueryDefinitionType;
+ }
+
+ public EncryptionFeedIterator(
+ EncryptionContainer encryptionContainer,
+ string queryText = null,
+ string continuationToken = null,
+ QueryRequestOptions requestOptions = null)
+ {
+ this.encryptionContainer = encryptionContainer ?? throw new ArgumentNullException(nameof(encryptionContainer));
+ this.queryText = queryText;
+ this.continuationToken = continuationToken;
+ this.requestOptions = requestOptions;
+
+ this.FeedIterator = encryptionContainer.Container.GetItemQueryStreamIterator(
+ queryText,
+ continuationToken,
+ requestOptions);
+
+ this.queryType = QueryType.QueryTextType;
+ }
+
+ public EncryptionFeedIterator(
+ EncryptionContainer encryptionContainer,
+ ChangeFeedStartFrom changeFeedStartFrom,
+ ChangeFeedMode changeFeedMode,
+ ChangeFeedRequestOptions changeFeedRequestOptions = null)
+ {
+ this.encryptionContainer = encryptionContainer ?? throw new ArgumentNullException(nameof(encryptionContainer));
+ this.requestOptions = changeFeedRequestOptions;
+ this.FeedIterator = encryptionContainer.Container.GetChangeFeedStreamIterator(
+ changeFeedStartFrom,
+ changeFeedMode,
+ changeFeedRequestOptions);
+ }
+
+ public EncryptionFeedIterator(
+ EncryptionContainer encryptionContainer,
+ FeedRange feedRange,
+ QueryDefinition queryDefinition,
+ string continuationToken,
+ QueryRequestOptions requestOptions = null)
+ {
+ this.encryptionContainer = encryptionContainer ?? throw new ArgumentNullException(nameof(encryptionContainer));
+ this.feedRange = feedRange;
+ this.queryDefinition = queryDefinition;
+ this.continuationToken = continuationToken;
+ this.requestOptions = requestOptions;
+
+ this.FeedIterator = encryptionContainer.Container.GetItemQueryStreamIterator(
+ feedRange,
+ queryDefinition,
+ continuationToken,
+ requestOptions);
+
+ this.queryType = QueryType.QueryDefinitionWithFeedRangeType;
+ }
+
+ private enum QueryType
+ {
+ QueryWithoutEncryptedPk = 0,
+ QueryTextType = 1,
+ QueryDefinitionType = 2,
+ QueryDefinitionWithFeedRangeType = 3,
+ }
+
+ public override bool HasMoreResults => this.FeedIterator.HasMoreResults;
+
+ private FeedIterator FeedIterator { get; set; }
public override async Task ReadNextAsync(CancellationToken cancellationToken = default)
{
EncryptionSettings encryptionSettings = await this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: cancellationToken);
- encryptionSettings.SetRequestHeaders(this.requestOptions);
+ await this.GetIteratorWithEncryptionHeaderAndEncryptPartitionKeyIfRequiredAsync(encryptionSettings);
- ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken);
+ ResponseMessage responseMessage = await this.FeedIterator.ReadNextAsync(cancellationToken);
EncryptionDiagnosticsContext encryptionDiagnosticsContext = new EncryptionDiagnosticsContext();
- // check for Bad Request and Wrong RID intended and update the cached RID and Client Encryption Policy.
+ // check for Bad Request and Wrong intended RID and update the cached RID and Client Encryption Policy.
await this.encryptionContainer.ThrowIfRequestNeedsARetryPostPolicyRefreshAsync(responseMessage, encryptionSettings, encryptionDiagnosticsContext, cancellationToken);
if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null)
@@ -55,5 +145,71 @@ public override async Task ReadNextAsync(CancellationToken canc
return responseMessage;
}
+
+ private async Task GetIteratorWithEncryptionHeaderAndEncryptPartitionKeyIfRequiredAsync(EncryptionSettings encryptionSettings)
+ {
+ encryptionSettings.SetRequestHeaders(this.requestOptions);
+
+ // should be fine, flag is set at the end of init
+ if (this.isIteratorInitialized || this.queryType == QueryType.QueryWithoutEncryptedPk)
+ {
+ return;
+ }
+
+ if (await this.singleIteratorInitSema.WaitAsync(-1))
+ {
+ if (!this.isIteratorInitialized)
+ {
+ try
+ {
+ if (this.requestOptions is QueryRequestOptions queryRequestOptions)
+ {
+ if (queryRequestOptions != null && queryRequestOptions.PartitionKey.HasValue)
+ {
+ (queryRequestOptions.PartitionKey, bool isPkEncrypted) = await this.encryptionContainer.CheckIfPkIsEncryptedAndGetEncryptedPkAsync(
+ queryRequestOptions.PartitionKey.Value,
+ encryptionSettings,
+ cancellationToken: default);
+
+ if (!isPkEncrypted)
+ {
+ this.isIteratorInitialized = true;
+ return;
+ }
+ }
+
+ // we rebuild iterators which take in request options with partiton key and if partition key was encrypted.
+ this.FeedIterator = this.queryType switch
+ {
+ QueryType.QueryTextType => this.encryptionContainer.Container.GetItemQueryStreamIterator(
+ this.queryText,
+ this.continuationToken,
+ queryRequestOptions),
+ QueryType.QueryDefinitionType => this.encryptionContainer.Container.GetItemQueryStreamIterator(
+ this.queryDefinition,
+ this.continuationToken,
+ queryRequestOptions),
+ QueryType.QueryDefinitionWithFeedRangeType => this.encryptionContainer.Container.GetItemQueryStreamIterator(
+ this.feedRange,
+ this.queryDefinition,
+ this.continuationToken,
+ queryRequestOptions),
+ _ => this.FeedIterator,
+ };
+ }
+
+ this.isIteratorInitialized = true;
+ }
+ finally
+ {
+ this.singleIteratorInitSema.Release(1);
+ }
+ }
+ else
+ {
+ this.singleIteratorInitSema.Release(1);
+ }
+ }
+ }
}
}
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionProcessor.cs
index 4cd2c94a9c..af42fe338b 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionProcessor.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionProcessor.cs
@@ -18,13 +18,6 @@ namespace Microsoft.Azure.Cosmos.Encryption
internal static class EncryptionProcessor
{
- private static readonly CosmosJsonDotNetSerializer BaseSerializer = new CosmosJsonDotNetSerializer(
- new JsonSerializerSettings()
- {
- DateParseHandling = DateParseHandling.None,
- MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr
- });
-
private static readonly SqlSerializerFactory SqlSerializerFactory = SqlSerializerFactory.Default;
// UTF-8 Encoding
@@ -39,6 +32,13 @@ private enum TypeMarker : byte
String = 5,
}
+ public static CosmosJsonDotNetSerializer BaseSerializer { get; } = new CosmosJsonDotNetSerializer(
+ new JsonSerializerSettings()
+ {
+ DateParseHandling = DateParseHandling.None,
+ MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr
+ });
+
///
/// If there isn't any PathsToEncrypt, input stream will be returned without any modification.
/// Else input stream will be disposed, and a new stream is returned.
@@ -79,6 +79,7 @@ public static async Task EncryptAsync(
await EncryptJTokenAsync(
propertyToEncrypt.Value,
settingforProperty,
+ propertyName == "id",
cancellationToken);
propertiesEncryptedCount++;
@@ -182,30 +183,40 @@ internal static async Task DeserializeAndDecryptResponseAsync(
}
internal static async Task EncryptValueStreamAsync(
- Stream valueStream,
- EncryptionSettingForProperty settingsForProperty,
+ Stream valueStreamToEncrypt,
+ EncryptionSettingForProperty encryptionSettingForProperty,
+ bool shouldEscape,
CancellationToken cancellationToken)
{
- if (valueStream == null)
+ if (valueStreamToEncrypt == null)
{
- throw new ArgumentNullException(nameof(valueStream));
+ throw new ArgumentNullException(nameof(valueStreamToEncrypt));
}
- if (settingsForProperty == null)
+ if (encryptionSettingForProperty == null)
{
- throw new ArgumentNullException(nameof(settingsForProperty));
+ throw new ArgumentNullException(nameof(encryptionSettingForProperty));
}
- JToken propertyValueToEncrypt = EncryptionProcessor.BaseSerializer.FromStream(valueStream);
+ JToken propertyValueToEncrypt = EncryptionProcessor.BaseSerializer.FromStream(valueStreamToEncrypt);
JToken encryptedPropertyValue = propertyValueToEncrypt;
+
if (propertyValueToEncrypt.Type == JTokenType.Object || propertyValueToEncrypt.Type == JTokenType.Array)
{
- await EncryptJTokenAsync(encryptedPropertyValue, settingsForProperty, cancellationToken);
+ await EncryptJTokenAsync(
+ jTokenToEncrypt: encryptedPropertyValue,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: shouldEscape,
+ cancellationToken: cancellationToken);
}
else
{
- encryptedPropertyValue = await SerializeAndEncryptValueAsync(propertyValueToEncrypt, settingsForProperty, cancellationToken);
+ encryptedPropertyValue = await SerializeAndEncryptValueAsync(
+ jTokenToEncrypt: propertyValueToEncrypt,
+ encryptionSettingForProperty: encryptionSettingForProperty,
+ shouldEscape: shouldEscape,
+ cancellationToken: cancellationToken);
}
return EncryptionProcessor.BaseSerializer.ToStream(encryptedPropertyValue);
@@ -240,6 +251,7 @@ private static JToken DeserializeAndAddProperty(
private static async Task EncryptJTokenAsync(
JToken jTokenToEncrypt,
EncryptionSettingForProperty encryptionSettingForProperty,
+ bool shouldEscape,
CancellationToken cancellationToken)
{
// Top Level can be an Object
@@ -250,6 +262,7 @@ private static async Task EncryptJTokenAsync(
await EncryptJTokenAsync(
jProperty.Value,
encryptionSettingForProperty,
+ shouldEscape,
cancellationToken);
}
}
@@ -262,30 +275,44 @@ await EncryptJTokenAsync(
await EncryptJTokenAsync(
jTokenToEncrypt[i],
encryptionSettingForProperty,
+ shouldEscape,
cancellationToken);
}
}
}
else
{
- jTokenToEncrypt.Replace(await SerializeAndEncryptValueAsync(jTokenToEncrypt, encryptionSettingForProperty, cancellationToken));
+ jTokenToEncrypt.Replace(await SerializeAndEncryptValueAsync(
+ jTokenToEncrypt,
+ encryptionSettingForProperty,
+ shouldEscape,
+ cancellationToken));
}
return;
}
private static async Task SerializeAndEncryptValueAsync(
- JToken jToken,
+ JToken jTokenToEncrypt,
EncryptionSettingForProperty encryptionSettingForProperty,
+ bool shouldEscape,
CancellationToken cancellationToken)
{
- JToken propertyValueToEncrypt = jToken;
+ JToken propertyValueToEncrypt = jTokenToEncrypt;
if (propertyValueToEncrypt.Type == JTokenType.Null)
{
return propertyValueToEncrypt;
}
+ if (shouldEscape)
+ {
+ if (jTokenToEncrypt.Type != JTokenType.String)
+ {
+ throw new ArgumentException("Unsupported argument type. The value to escape has to be string type. Please refer to https://aka.ms/CosmosClientEncryption for more details.");
+ }
+ }
+
(TypeMarker typeMarker, byte[] plainText) = Serialize(propertyValueToEncrypt);
AeadAes256CbcHmac256EncryptionAlgorithm aeadAes256CbcHmac256EncryptionAlgorithm = await encryptionSettingForProperty.BuildEncryptionAlgorithmForSettingAsync(cancellationToken: cancellationToken);
@@ -299,15 +326,37 @@ private static async Task SerializeAndEncryptValueAsync(
byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1];
cipherTextWithTypeMarker[0] = (byte)typeMarker;
Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length);
+
+ if (shouldEscape)
+ {
+ // case: id does not support '/','\','?','#'. Convert Base64 string to Uri safe string
+ return ConvertToBase64UriSafeString(cipherTextWithTypeMarker);
+ }
+
return cipherTextWithTypeMarker;
}
private static async Task DecryptAndDeserializeValueAsync(
JToken jToken,
EncryptionSettingForProperty encryptionSettingForProperty,
+ bool isEscaped,
CancellationToken cancellationToken)
{
- byte[] cipherTextWithTypeMarker = jToken.ToObject();
+ byte[] cipherTextWithTypeMarker = null;
+
+ if (isEscaped)
+ {
+ if (jToken.Type == JTokenType.Null)
+ {
+ return null;
+ }
+
+ cipherTextWithTypeMarker = ConvertFromBase64UriSafeString(jToken.ToObject());
+ }
+ else
+ {
+ cipherTextWithTypeMarker = jToken.ToObject();
+ }
if (cipherTextWithTypeMarker == null)
{
@@ -333,6 +382,7 @@ private static async Task DecryptAndDeserializeValueAsync(
private static async Task DecryptJTokenAsync(
JToken jTokenToDecrypt,
EncryptionSettingForProperty encryptionSettingForProperty,
+ bool isEscaped,
CancellationToken cancellationToken)
{
if (jTokenToDecrypt.Type == JTokenType.Object)
@@ -342,6 +392,7 @@ private static async Task DecryptJTokenAsync(
await DecryptJTokenAsync(
jProperty.Value,
encryptionSettingForProperty,
+ isEscaped,
cancellationToken);
}
}
@@ -354,6 +405,7 @@ await DecryptJTokenAsync(
await DecryptJTokenAsync(
jTokenToDecrypt[i],
encryptionSettingForProperty,
+ isEscaped,
cancellationToken);
}
}
@@ -363,6 +415,7 @@ await DecryptJTokenAsync(
jTokenToDecrypt.Replace(await DecryptAndDeserializeValueAsync(
jTokenToDecrypt,
encryptionSettingForProperty,
+ isEscaped,
cancellationToken));
}
}
@@ -388,6 +441,7 @@ private static async Task DecryptObjectAsync(
await DecryptJTokenAsync(
propertyToDecrypt.Value,
settingsForProperty,
+ propertyName == "id",
cancellationToken);
propertiesDecryptedCount++;
@@ -417,5 +471,21 @@ private static JObject RetrieveItem(
return itemJObj;
}
+
+ private static string ConvertToBase64UriSafeString(byte[] bytesToProcess)
+ {
+ string base64String = Convert.ToBase64String(bytesToProcess);
+
+ // Base 64 Encoding with URL and Filename Safe Alphabet https://datatracker.ietf.org/doc/html/rfc4648#section-5
+ // https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-item-limits, due to base64 conversion and encryption
+ // the permissible size of the property will further reduce.
+ return new StringBuilder(base64String, base64String.Length).Replace("/", "_").Replace("+", "-").ToString();
+ }
+
+ private static byte[] ConvertFromBase64UriSafeString(string uriSafeBase64String)
+ {
+ StringBuilder fromUriSafeBase64String = new StringBuilder(uriSafeBase64String, uriSafeBase64String.Length).Replace("_", "/").Replace("-", "+");
+ return Convert.FromBase64String(fromUriSafeBase64String.ToString());
+ }
}
}
\ No newline at end of file
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettings.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettings.cs
index 33f76b8920..6f8ebf18b1 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettings.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionSettings.cs
@@ -11,21 +11,23 @@ namespace Microsoft.Azure.Cosmos.Encryption
using System.Net;
using System.Threading;
using System.Threading.Tasks;
- using Microsoft.Data.Encryption.Cryptography;
internal sealed class EncryptionSettings
{
private readonly Dictionary encryptionSettingsDictByPropertyName;
- private EncryptionSettings(string containerRidValue)
+ private EncryptionSettings(string containerRidValue, IReadOnlyList partitionKeyPath)
{
this.ContainerRidValue = containerRidValue;
+ this.PartitionKeyPaths = partitionKeyPath;
this.encryptionSettingsDictByPropertyName = new Dictionary();
this.PropertiesToEncrypt = this.encryptionSettingsDictByPropertyName.Keys;
}
public string ContainerRidValue { get; }
+ public IReadOnlyList PartitionKeyPaths { get; }
+
public IEnumerable PropertiesToEncrypt { get; }
public static Task CreateAsync(EncryptionContainer encryptionContainer, CancellationToken cancellationToken)
@@ -74,10 +76,23 @@ private static async Task InitializeEncryptionSettingsAsync(
// set the Container Rid.
string containerRidValue = containerResponse.Resource.SelfLink.Split('/').ElementAt(3);
+ IReadOnlyList partitionKeyPaths = null;
+
+#if ENCRYPTIONPREVIEW
+ if (containerResponse.Resource.PartitionKeyPaths.Any())
+ {
+ partitionKeyPaths = containerResponse.Resource.PartitionKeyPaths;
+ }
+ else
+#endif
+ {
+ partitionKeyPaths = new List { containerResponse.Resource.PartitionKeyPath };
+ }
+
// set the ClientEncryptionPolicy for the Settings.
ClientEncryptionPolicy clientEncryptionPolicy = containerResponse.Resource.ClientEncryptionPolicy;
- EncryptionSettings encryptionSettings = new EncryptionSettings(containerRidValue);
+ EncryptionSettings encryptionSettings = new EncryptionSettings(containerRidValue, partitionKeyPaths);
if (clientEncryptionPolicy != null)
{
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs
index 6b83df4c0b..0ff7e7defa 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/EncryptionTransactionalBatch.cs
@@ -54,11 +54,11 @@ public override TransactionalBatch CreateItemStream(
if (encryptionSettings.PropertiesToEncrypt.Any())
{
- streamPayload = EncryptionProcessor.EncryptAsync(
- streamPayload,
- encryptionSettings,
- operationDiagnostics: null,
- cancellationToken: default)
+ streamPayload = EncryptionProcessor.EncryptAsync(
+ streamPayload,
+ encryptionSettings,
+ operationDiagnostics: null,
+ cancellationToken: default)
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
@@ -75,6 +75,16 @@ public override TransactionalBatch DeleteItem(
string id,
TransactionalBatchItemRequestOptions requestOptions = null)
{
+ EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
+ id = this.encryptionContainer.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
this.transactionalBatch = this.transactionalBatch.DeleteItem(
id,
requestOptions);
@@ -86,6 +96,16 @@ public override TransactionalBatch ReadItem(
string id,
TransactionalBatchItemRequestOptions requestOptions = null)
{
+ EncryptionSettings encryptionSettings = this.encryptionContainer.GetOrUpdateEncryptionSettingsFromCacheAsync(obsoleteEncryptionSettings: null, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
+ id = this.encryptionContainer.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
this.transactionalBatch = this.transactionalBatch.ReadItem(
id,
requestOptions);
@@ -125,6 +145,11 @@ public override TransactionalBatch ReplaceItemStream(
.ConfigureAwait(false)
.GetAwaiter()
.GetResult();
+
+ id = this.encryptionContainer.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
}
this.transactionalBatch = this.transactionalBatch.ReplaceItemStream(
@@ -264,6 +289,11 @@ public override TransactionalBatch PatchItem(
.GetAwaiter()
.GetResult();
+ id = this.encryptionContainer.CheckIfIdIsEncryptedAndGetEncryptedIdAsync(id, encryptionSettings, cancellationToken: default)
+ .ConfigureAwait(false)
+ .GetAwaiter()
+ .GetResult();
+
this.transactionalBatch = this.transactionalBatch.PatchItem(
id,
encryptedPatchOperations,
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj b/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
index 9d8db401ec..6aa1f85932 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
+++ b/Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj
@@ -28,11 +28,11 @@
-
+
-
+
diff --git a/Microsoft.Azure.Cosmos.Encryption/src/QueryDefinitionExtensions.cs b/Microsoft.Azure.Cosmos.Encryption/src/QueryDefinitionExtensions.cs
index 549dad8055..7df68dbd4c 100644
--- a/Microsoft.Azure.Cosmos.Encryption/src/QueryDefinitionExtensions.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/src/QueryDefinitionExtensions.cs
@@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Encryption
{
using System;
using System.IO;
+ using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -96,7 +97,13 @@ public static async Task AddParameterAsync(
}
Stream valueStream = encryptionContainer.CosmosSerializer.ToStream(value);
- Stream encryptedValueStream = await EncryptionProcessor.EncryptValueStreamAsync(valueStream, settingsForProperty, cancellationToken);
+
+ Stream encryptedValueStream = await EncryptionProcessor.EncryptValueStreamAsync(
+ valueStream,
+ settingsForProperty,
+ path == "/id",
+ cancellationToken);
+
queryDefinitionwithEncryptedValues.WithParameterStream(name, encryptedValueStream);
return queryDefinitionwithEncryptedValues;
diff --git a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
index 20a97619d2..6ab6d8c333 100644
--- a/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
+++ b/Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs
@@ -10,7 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.EmulatorTests
using System.IO;
using System.Linq;
using System.Net;
- using System.Net.Http.Headers;
using System.Reflection;
using System.Text;
using System.Threading;
@@ -76,7 +75,7 @@ await database.CreateClientEncryptionKeyAsync(
{
new ClientEncryptionIncludedPath()
{
- Path = "/Sensitive_StringFormat",
+ Path = "/PK",
ClientEncryptionKeyId = "key1",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
@@ -100,7 +99,7 @@ await database.CreateClientEncryptionKeyAsync(
new ClientEncryptionIncludedPath()
{
- Path = "/Sensitive_IntArray",
+ Path = "/id",
ClientEncryptionKeyId = "key2",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
@@ -172,15 +171,16 @@ await database.CreateClientEncryptionKeyAsync(
};
- clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
+
containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
ContainerProperties containerPropertiesForChangeFeed = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
- encryptionContainer = await database.CreateContainerAsync(containerProperties, 20000);
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 15000);
await encryptionContainer.InitializeEncryptionAsync();
- encryptionContainerForChangeFeed = await database.CreateContainerAsync(containerPropertiesForChangeFeed, 20000);
+ encryptionContainerForChangeFeed = await database.CreateContainerAsync(containerPropertiesForChangeFeed, 15000);
await encryptionContainerForChangeFeed.InitializeEncryptionAsync();
}
@@ -262,6 +262,7 @@ public async Task EncryptionBulkCrud()
};
await Task.WhenAll(tasks);
+ encryptionCosmosClientWithBulk.Dispose();
}
[TestMethod]
@@ -419,6 +420,7 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
VerifyExpectedDocResponse(testDoc, createResponse.Resource);
testKeyEncryptionKeyResolver.RevokeAccessSet = false;
+ encryptionCosmosClient.Dispose();
}
@@ -469,6 +471,8 @@ public async Task EncryptionResourceTokenAuthRestricted()
catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{
}
+
+ encryptedclientForRestrictedUser.Dispose();
}
[TestMethod]
@@ -516,17 +520,34 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
query: null,
expectedDocList: new List { expectedDoc });
+ expectedDoc = new TestDoc(testDoc);
await MdeEncryptionTests.ValidateQueryResultsAsync(
MdeEncryptionTests.encryptionContainer,
"SELECT * FROM c",
expectedDocList: new List { expectedDoc });
+ QueryDefinition withEncryptedParameter = MdeEncryptionTests.encryptionContainer.CreateQueryDefinition(
+ "select * from c where c.id = @theId and c.PK = @thePK");
+
+ await withEncryptedParameter.AddParameterAsync(
+ "@theId",
+ expectedDoc.Id,
+ "/id");
+
+ await withEncryptedParameter.AddParameterAsync(
+ "@thePK",
+ expectedDoc.PK,
+ "/PK");
+
+ await MdeEncryptionTests.ValidateQueryResultsAsync(
+ MdeEncryptionTests.encryptionContainer,
+ queryDefinition: withEncryptedParameter,
+ expectedDocList: new List { expectedDoc });
+
await MdeEncryptionTests.ValidateQueryResultsAsync(
MdeEncryptionTests.encryptionContainer,
string.Format(
- "SELECT * FROM c where c.PK = '{0}' and c.id = '{1}' and c.NonSensitive = '{2}'",
- expectedDoc.PK,
- expectedDoc.Id,
+ "SELECT * FROM c where c.NonSensitive = '{0}'",
expectedDoc.NonSensitive),
expectedDocList: new List { expectedDoc });
@@ -535,14 +556,6 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
string.Format("SELECT * FROM c where c.Sensitive_IntFormat = '{0}'", testDoc.Sensitive_IntFormat),
expectedDocList: null);
- await MdeEncryptionTests.ValidateQueryResultsAsync(
- MdeEncryptionTests.encryptionContainer,
- queryDefinition: new QueryDefinition(
- "select * from c where c.id = @theId and c.PK = @thePK")
- .WithParameter("@theId", expectedDoc.Id)
- .WithParameter("@thePK", expectedDoc.PK),
- expectedDocList: new List { expectedDoc });
-
expectedDoc.Sensitive_IntMultiDimArray = null;
expectedDoc.Sensitive_ArrayMultiTypes = null;
expectedDoc.Sensitive_NestedObjectFormatL1 = null;
@@ -853,7 +866,7 @@ public async Task EncryptionDecryptQueryResultMultipleDocs()
// test GetItemLinqQueryable
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, null, expectedPropertiesDecryptedCount: 12);
- string query = $"SELECT * FROM c WHERE c.PK in ('{testDoc1.PK}', '{testDoc2.PK}')";
+ string query = $"SELECT * FROM c WHERE c.NonSensitive in ('{testDoc1.NonSensitive}', '{testDoc2.NonSensitive}')";
await MdeEncryptionTests.ValidateQueryResultsMultipleDocumentsAsync(MdeEncryptionTests.encryptionContainer, testDoc1, testDoc2, query, expectedPropertiesDecryptedCount: 12);
// ORDER BY query
@@ -904,50 +917,8 @@ await withEncryptedParameter.AddParameterAsync(
}
[TestMethod]
- public async Task EncryptionRestrictedProperties()
+ public void EncryptionRestrictedProperties()
{
- // restricted path id
- ClientEncryptionIncludedPath restrictedPathId = new ClientEncryptionIncludedPath()
- {
- Path = "/id",
- ClientEncryptionKeyId = "key1",
- EncryptionType = "Deterministic",
- EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
- };
-
- Collection paths = new Collection { restrictedPathId };
- try
- {
- ClientEncryptionPolicy clientEncryptionPolicyId = new ClientEncryptionPolicy(paths);
- }
- catch (ArgumentException ex)
- {
- Assert.AreEqual("Invalid path '/id'.", ex.Message);
- }
-
- // restricted path PK
- ClientEncryptionIncludedPath restrictedPathPk = new ClientEncryptionIncludedPath()
- {
- Path = "/PK",
- ClientEncryptionKeyId = "key2",
- EncryptionType = "Deterministic",
- EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
- };
-
- Collection pathsRestrictedPathPk = new Collection { restrictedPathPk };
- ClientEncryptionPolicy clientEncryptionPolicyPk = new ClientEncryptionPolicy(pathsRestrictedPathPk);
-
- ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicyPk };
-
- try
- {
- Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
- Assert.Fail("CreateContainerAsync operation with PK specified to be encrypted should have failed. ");
- }
- catch (ArgumentException)
- {
- }
-
// duplicate paths in policy.
ClientEncryptionIncludedPath pathdup1 = new ClientEncryptionIncludedPath()
{
@@ -1006,18 +977,25 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DoubleFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
- ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
+ ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/Sensitive_DoubleFormat") { ClientEncryptionPolicy = clientEncryptionPolicy };
Container encryptionContainerToDelete = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainerToDelete.InitializeEncryptionAsync();
- // FIXME Set WithBulkExecution to true post SDK/Backend fix.
CosmosClient otherClient = TestCommon.CreateCosmosClient(builder => builder
- .WithBulkExecution(false)
+ .WithBulkExecution(true)
.Build());
CosmosClient otherEncryptionClient = otherClient.WithEncryption(new TestKeyEncryptionKeyResolver(), TestKeyEncryptionKeyResolver.Id);
@@ -1025,7 +1003,12 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
Container otherEncryptionContainer = otherDatabase.GetContainer(encryptionContainerToDelete.Id);
- await MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer);
+ TestDoc testDoc = TestDoc.Create();
+ ItemResponse createResponse = await otherEncryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
// Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy
using (await database.GetContainer(encryptionContainerToDelete.Id).DeleteContainerStreamAsync())
@@ -1056,9 +1039,17 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/PK",
+ ClientEncryptionKeyId = "key2",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
@@ -1076,7 +1067,7 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
Assert.Fail("Create operation should have failed with 1024 SubStatusCode. ");
}
- VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 3);
+ VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 4, expectedPropertiesDecryptedCount: 3);
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
}
@@ -1087,15 +1078,16 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
TestDoc docToUpsert = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
docToUpsert.Sensitive_StringFormat = "docTobeUpserted";
- List tasks = new List()
- {
- MdeEncryptionTests.MdeUpsertItemAsync(otherEncryptionContainer, docToUpsert, HttpStatusCode.OK),
- MdeEncryptionTests.MdeReplaceItemAsync(otherEncryptionContainer, docToReplace),
- MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer),
- };
-
+ List tasks;
try
{
+ tasks = new List()
+ {
+ MdeEncryptionTests.MdeUpsertItemAsync(otherEncryptionContainer, docToUpsert, HttpStatusCode.OK),
+ MdeEncryptionTests.MdeReplaceItemAsync(otherEncryptionContainer, docToReplace),
+ MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer)
+ };
+
await Task.WhenAll(tasks);
Assert.Fail("Bulk operation should have failed. ");
}
@@ -1106,7 +1098,7 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
Assert.Fail("Bulk operation should have failed with 1024 SubStatusCode.");
}
- VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0);
+ VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 4, expectedPropertiesDecryptedCount: 0);
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
}
@@ -1130,11 +1122,12 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteWithBulk()
// validate if the right policy was used, by reading them all back.
FeedIterator queryResponseIterator = otherEncryptionContainer.GetItemQueryIterator("select * from c");
-
while (queryResponseIterator.HasMoreResults)
{
FeedResponse readDocs = await queryResponseIterator.ReadNextAsync();
}
+
+ otherEncryptionClient.Dispose();
}
[TestMethod]
@@ -1157,11 +1150,19 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DoubleFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
- ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
+ ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/Sensitive_DoubleFormat") { ClientEncryptionPolicy = clientEncryptionPolicy };
Container encryptionContainerToDelete = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainerToDelete.InitializeEncryptionAsync();
@@ -1174,7 +1175,12 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB
Container otherEncryptionContainer = otherDatabase.GetContainer(encryptionContainerToDelete.Id);
- await MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer);
+ TestDoc testDoc = TestDoc.Create();
+ ItemResponse createResponse = await otherEncryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
// Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy
using (await database.GetContainer(encryptionContainerToDelete.Id).DeleteContainerStreamAsync())
@@ -1197,9 +1203,17 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/PK",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ }
};
- clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
@@ -1218,11 +1232,11 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB
Assert.Fail("Create operation should have failed with 1024 SubStatusCode");
}
- VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0);
+ VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0);
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
}
- TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
+ testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
string partitionKey = "thePK";
@@ -1267,6 +1281,8 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteTransactionB
TransactionalBatchOperationResult doc2 = batchResponse.GetOperationResultAtIndex(1);
VerifyExpectedDocResponse(doc2ToCreate, doc2.Resource);
+
+ otherEncryptionClient.Dispose();
}
[TestMethod]
@@ -1289,11 +1305,19 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteQuery()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DoubleFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
- ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
+ ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/Sensitive_DoubleFormat") { ClientEncryptionPolicy = clientEncryptionPolicy };
Container encryptionContainerToDelete = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainerToDelete.InitializeEncryptionAsync();
@@ -1306,9 +1330,14 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteQuery()
Container otherEncryptionContainer = otherDatabase.GetContainer(encryptionContainerToDelete.Id);
- await MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer);
+ TestDoc testDoc = TestDoc.Create();
+ ItemResponse createResponse = await otherEncryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
- // Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy
+ // Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy, with new Pk path
using (await database.GetContainer(encryptionContainerToDelete.Id).DeleteContainerStreamAsync())
{ }
@@ -1329,32 +1358,48 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeleteQuery()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/PK",
+ ClientEncryptionKeyId = "key2",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
+
};
- clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
ContainerResponse containerResponse = await database.CreateContainerAsync(containerProperties, 400);
+ testDoc = TestDoc.Create();
try
- {
- await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
+ {
+ await encryptionContainerToDelete.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.PK));
Assert.Fail("Create operation should have failed. ");
}
catch (CosmosException ex)
{
if (ex.SubStatusCode != 1024)
{
- Assert.Fail("Create operation should have failed with 1024 SubStatusCode .");
+ Assert.Fail($"Create operation should have failed with 1024 SubStatusCode. Failed with substatus code {ex.SubStatusCode}");
}
- VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0);
+ VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0);
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
}
- TestDoc testDoc = await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
+ createResponse = await encryptionContainerToDelete.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.PK));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
// check w.r.t to query if we are able to fail and update the policy
try
@@ -1370,7 +1415,7 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
{
if (ex.SubStatusCode != 1024)
{
- Assert.Fail("Query should have failed with 1024 SubStatusCode. ");
+ Assert.Fail($"Query should have failed with 1024 SubStatusCode. Failed with substatus code {ex.SubStatusCode} ");
}
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
@@ -1382,6 +1427,8 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
"SELECT * FROM c",
expectedDocList: new List { testDoc },
expectedPropertiesDecryptedCount: 2);
+
+ otherEncryptionClient.Dispose();
}
[TestMethod]
@@ -1404,11 +1451,19 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DoubleFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ ClientEncryptionPolicy clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
- ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
+ ContainerProperties containerProperties = new ContainerProperties(Guid.NewGuid().ToString(), "/Sensitive_DoubleFormat") { ClientEncryptionPolicy = clientEncryptionPolicy };
Container encryptionContainerToDelete = await database.CreateContainerAsync(containerProperties, 400);
await encryptionContainerToDelete.InitializeEncryptionAsync();
@@ -1421,7 +1476,12 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch()
Container otherEncryptionContainer = otherDatabase.GetContainer(encryptionContainerToDelete.Id);
- await MdeEncryptionTests.MdeCreateItemAsync(otherEncryptionContainer);
+ TestDoc testDoc = TestDoc.Create();
+ ItemResponse createResponse = await otherEncryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
// Client 1 Deletes the Container referenced in Client 2 and Recreate with different policy
using (await database.GetContainer(encryptionContainerToDelete.Id).DeleteContainerStreamAsync())
@@ -1460,9 +1520,17 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch()
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
},
+
+ new ClientEncryptionIncludedPath()
+ {
+ Path = "/PK",
+ ClientEncryptionKeyId = "key2",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ },
};
- clientEncryptionPolicy = new ClientEncryptionPolicy(paths);
+ clientEncryptionPolicy = new ClientEncryptionPolicy(paths, 2);
containerProperties = new ContainerProperties(encryptionContainerToDelete.Id, "/PK") { ClientEncryptionPolicy = clientEncryptionPolicy };
@@ -1471,17 +1539,20 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch()
// operation fails, policy gets updated.
try
{
- await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainerToDelete);
+ testDoc = TestDoc.Create();
+ createResponse = await encryptionContainerToDelete.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.PK));
Assert.Fail("Create operation should have failed. ");
}
catch (CosmosException ex)
{
if (ex.SubStatusCode != 1024)
{
- Assert.Fail("Create operation should have failed with 1024 SubStatusCode.");
+ Assert.Fail($"Create operation should have failed with 1024 SubStatusCode. Failed with substatus code {ex.SubStatusCode}");
}
- VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 2, expectedPropertiesDecryptedCount: 0);
+ VerifyDiagnostics(ex.Diagnostics, encryptOperation: true, decryptOperation: false, expectedPropertiesEncryptedCount: 3, expectedPropertiesDecryptedCount: 0);
Assert.IsTrue(ex.Message.Contains("Operation has failed due to a possible mismatch in Client Encryption Policy configured on the container."));
}
@@ -1555,6 +1626,8 @@ public async Task EncryptionValidatePolicyRefreshPostContainerDeletePatch()
// retry post policy refresh.
await MdeEncryptionTests.MdePatchItemAsync(otherEncryptionContainer, patchOperations, docPostPatching, HttpStatusCode.OK);
+
+ otherEncryptionClient.Dispose();
}
[TestMethod]
@@ -1716,14 +1789,14 @@ public async Task EncryptionValidatePolicyRefreshPostDatabaseDelete()
// and here we would not hit the incorrect container rid issue.
ClientEncryptionIncludedPath newModifiedPath2 = new ClientEncryptionIncludedPath()
{
- Path = "/Sensitive_StringFormat",
+ Path = "/PK",
ClientEncryptionKeyId = "myCek",
EncryptionType = "Deterministic",
EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
};
Container otherEncryptionContainer2 = await otherDatabase.DefineContainer("otherContainer2", "/PK")
- .WithClientEncryptionPolicy()
+ .WithClientEncryptionPolicy(policyFormatVersion: 2)
.WithIncludedPath(newModifiedPath2)
.Attach()
.CreateAsync(throughput: 1000);
@@ -1782,6 +1855,10 @@ public async Task EncryptionValidatePolicyRefreshPostDatabaseDelete()
}
await mainClient.GetDatabase("databaseToBeDeleted").DeleteStreamAsync();
+
+ encryptionCosmosClient.Dispose();
+ otherEncryptionClient.Dispose();
+ otherEncryptionClient2.Dispose();
}
[TestMethod]
@@ -1887,6 +1964,8 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
// Should fail but will try to fetch the lastest from the Backend and updates the cache.
await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
testKeyEncryptionKeyResolver.RevokeAccessSet = false;
+
+ encryptionCosmosClient.Dispose();
}
[TestMethod]
@@ -1934,25 +2013,283 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
[TestMethod]
public async Task CreateAndDeleteDatabaseWithoutKeys()
{
- Database database = await MdeEncryptionTests.encryptionCosmosClient.CreateDatabaseAsync("NoCEKDatabase");
- ContainerProperties containerProperties = new ContainerProperties("NoCEPContainer", "/PK");
+ Database database = null;
+ try
+ {
+ database = await MdeEncryptionTests.encryptionCosmosClient.CreateDatabaseAsync("NoCEKDatabase");
+ ContainerProperties containerProperties = new ContainerProperties("NoCEPContainer", "/PK");
- Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ Container encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ TestDoc testDoc = TestDoc.Create();
+ ItemResponse createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.PK));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+ }
+ finally
+ {
+ await database.DeleteStreamAsync();
+ }
+ }
+
+ [TestMethod]
+ public async Task ValidatePkAndIdEncryptionSupport()
+ {
+ // encrypt String type PK
+ ClientEncryptionIncludedPath cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/PK",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ ClientEncryptionIncludedPath cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/id",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ Collection paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ ClientEncryptionPolicy clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ ContainerProperties containerProperties = new ContainerProperties("StringPkEncContainer", "/id") { ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek };
+
+ Container encryptionContainer = await MdeEncryptionTests.database.CreateContainerAsync(containerProperties, 400);
await encryptionContainer.InitializeEncryptionAsync();
TestDoc testDoc = TestDoc.Create();
ItemResponse createResponse = await encryptionContainer.CreateItemAsync(
testDoc,
- new PartitionKey(testDoc.PK));
+ new PartitionKey(testDoc.Id));
Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
VerifyExpectedDocResponse(testDoc, createResponse.Resource);
- await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
- await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
- await MdeEncryptionTests.MdeCreateItemAsync(encryptionContainer);
+ ItemResponse readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ new PartitionKey(testDoc.Id));
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ // encrypt Float type PK
+ cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_FloatFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/id",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ containerProperties = new ContainerProperties("FloatPkEncContainer", "/Sensitive_FloatFormat") { ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek };
+
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(double.Parse(testDoc.Sensitive_FloatFormat.ToString())));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // readback
+ readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ new PartitionKey(double.Parse(testDoc.Sensitive_FloatFormat.ToString())));
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ // encrypt bool type PK
+ cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_BoolFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/id",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ containerProperties = new ContainerProperties("BoolPkEncContainer", "/Sensitive_BoolFormat") { ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek };
+
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_BoolFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ new PartitionKey(testDoc.Sensitive_BoolFormat));
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+ // encrypt Decimal type PK
+ cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DecimalFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/id",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ containerProperties = new ContainerProperties("DecimalPkEncContainer", "/Sensitive_DecimalFormat") { ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek };
+
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(Double.Parse(testDoc.Sensitive_DecimalFormat.ToString())));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ new PartitionKey(Double.Parse(testDoc.Sensitive_DecimalFormat.ToString())));
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+
+ // encrypt double type PK
+ cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_DoubleFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/id",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ containerProperties = new ContainerProperties("DoublePkEncContainer", "/Sensitive_DoubleFormat") { ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek };
- await database.DeleteStreamAsync();
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ new PartitionKey(testDoc.Sensitive_DoubleFormat));
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+
+#if ENCRYPTIONTESTPREVIEW
+ // hierarchical
+ cepWithPKIdPath1 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_LongFormat",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ cepWithPKIdPath2 = new ClientEncryptionIncludedPath()
+ {
+ Path = "/Sensitive_NestedObjectFormatL1",
+ ClientEncryptionKeyId = "key1",
+ EncryptionType = "Deterministic",
+ EncryptionAlgorithm = "AEAD_AES_256_CBC_HMAC_SHA256",
+ };
+
+ paths = new Collection { cepWithPKIdPath1, cepWithPKIdPath2 };
+
+ clientEncryptionPolicyWithRevokedKek = new ClientEncryptionPolicy(paths, 2);
+
+ containerProperties = new ContainerProperties()
+ {
+ Id = "HierarchicalPkEncContainer",
+ PartitionKeyPaths = new List { "/Sensitive_StringFormat", "/Sensitive_NestedObjectFormatL1/Sensitive_NestedObjectFormatL2/Sensitive_StringFormatL2" },
+ ClientEncryptionPolicy = clientEncryptionPolicyWithRevokedKek
+ };
+
+ encryptionContainer = await database.CreateContainerAsync(containerProperties, 400);
+ await encryptionContainer.InitializeEncryptionAsync();
+
+ PartitionKey hirarchicalPk = new PartitionKeyBuilder()
+ .Add(testDoc.Sensitive_StringFormat)
+ .Add(testDoc.Sensitive_NestedObjectFormatL1.Sensitive_NestedObjectFormatL2.Sensitive_StringFormatL2)
+ .Build();
+
+ createResponse = await encryptionContainer.CreateItemAsync(
+ testDoc,
+ partitionKey: hirarchicalPk);
+ Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, createResponse.Resource);
+
+ // read back
+ readResponse = await encryptionContainer.ReadItemAsync(
+ testDoc.Id,
+ hirarchicalPk);
+
+ Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode);
+ VerifyExpectedDocResponse(testDoc, readResponse.Resource);
+#endif
}
[TestMethod]
@@ -2066,7 +2403,7 @@ public async Task EncryptionPatchItem()
docPostPatching,
HttpStatusCode.OK);
- VerifyDiagnostics(patchResponse.Diagnostics, expectedPropertiesEncryptedCount: 8);
+ VerifyDiagnostics(patchResponse.Diagnostics, expectedPropertiesEncryptedCount: 6);
docPostPatching.Sensitive_ArrayFormat = new TestDoc.Sensitive_ArrayData[]
{
@@ -2218,6 +2555,8 @@ await MdeEncryptionTests.ValidateQueryResultsAsync(
encryptionContainerWithCustomSerializer,
queryDefinition: withEncryptedParameter,
expectedDocList: new List { expectedDoc });
+
+ encryptionCosmosClientWithCustomSerializer.Dispose();
}
[TestMethod]
@@ -2742,7 +3081,7 @@ private static async Task> MdeUpsertItemAsync(
{
ItemResponse upsertResponse = await container.UpsertItemAsync(
testDoc,
- new PartitionKey(testDoc.PK));
+ new PartitionKey(testDoc.PK));
Assert.AreEqual(expectedStatusCode, upsertResponse.StatusCode);
VerifyExpectedDocResponse(testDoc, upsertResponse.Resource);
return upsertResponse;
@@ -3001,7 +3340,7 @@ public class TestDoc
public string NonSensitive { get; set; }
- public int NonSensitiveInt { get; set; }
+ public int NonSensitiveInt { get; set; }
public string Sensitive_StringFormat { get; set; }
@@ -3013,8 +3352,12 @@ public class TestDoc
public int Sensitive_IntFormat { get; set; }
+ public long Sensitive_LongFormat { get; set; }
+
public float Sensitive_FloatFormat { get; set; }
+ public double Sensitive_DoubleFormat { get; set; }
+
public int[] Sensitive_IntArray { get; set; }
public int[,] Sensitive_IntMultiDimArray { get; set; }
@@ -3093,8 +3436,10 @@ public TestDoc(TestDoc other)
this.Sensitive_DateFormat = other.Sensitive_DateFormat;
this.Sensitive_DecimalFormat = other.Sensitive_DecimalFormat;
this.Sensitive_IntFormat = other.Sensitive_IntFormat;
+ this.Sensitive_LongFormat = other.Sensitive_LongFormat;
this.Sensitive_BoolFormat = other.Sensitive_BoolFormat;
this.Sensitive_FloatFormat = other.Sensitive_FloatFormat;
+ this.Sensitive_DoubleFormat = other.Sensitive_DoubleFormat;
this.Sensitive_ArrayFormat = other.Sensitive_ArrayFormat;
this.Sensitive_IntArray = other.Sensitive_IntArray;
this.Sensitive_ObjectArrayType = other.Sensitive_ObjectArrayType;
@@ -3113,9 +3458,11 @@ public override bool Equals(object obj)
&& this.Sensitive_DateFormat == doc.Sensitive_DateFormat
&& this.Sensitive_DecimalFormat == doc.Sensitive_DecimalFormat
&& this.Sensitive_IntFormat == doc.Sensitive_IntFormat
+ && this.Sensitive_LongFormat == doc.Sensitive_LongFormat
&& this.Sensitive_ArrayFormat == doc.Sensitive_ArrayFormat
&& this.Sensitive_BoolFormat == doc.Sensitive_BoolFormat
&& this.Sensitive_FloatFormat == doc.Sensitive_FloatFormat
+ && this.Sensitive_DoubleFormat == doc.Sensitive_DoubleFormat
&& this.Sensitive_IntArray == doc.Sensitive_IntArray
&& this.Sensitive_ObjectArrayType == doc.Sensitive_ObjectArrayType
&& this.Sensitive_NestedObjectFormatL1 != doc.Sensitive_NestedObjectFormatL1
@@ -3142,10 +3489,12 @@ public override int GetHashCode()
hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_DateFormat);
hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_DecimalFormat);
hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_IntFormat);
+ hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_LongFormat);
hashCode = (hashCode * -1521134295) + EqualityComparer
-
-
-
+
+
@@ -67,7 +66,7 @@
true
-
- $(DefineConstants);SDKPROJECTREF
+
+ $(DefineConstants);ENCRYPTIONTESTPREVIEW
diff --git a/templates/build-preview.yml b/templates/build-preview.yml
index f3e74663d9..05387989c6 100644
--- a/templates/build-preview.yml
+++ b/templates/build-preview.yml
@@ -60,6 +60,37 @@ jobs:
arguments: -p:Optimize=true -p:IsPreview=true;SdkProjectRef=true
versioningScheme: OFF
+- job:
+ displayName: Preview Encryption EmulatorTests ${{ parameters.BuildConfiguration }}
+ timeoutInMinutes: 90
+ condition: and(succeeded(), eq('${{ parameters.OS }}', 'Windows'))
+ pool:
+ vmImage: ${{ parameters.VmImage }}
+
+ steps:
+ - checkout: self # self represents the repo where the initial Pipelines YAML file was found
+ clean: true # if true, execute `execute git clean -ffdx && git reset --hard HEAD` before fetching
+
+ # Add this Command to Include the .NET 6 SDK
+ - task: UseDotNet@2
+ displayName: Use .NET 6.0
+ inputs:
+ packageType: 'sdk'
+ version: '6.x'
+
+ - template: emulator-setup.yml
+
+ - task: DotNetCoreCLI@2
+ displayName: PREVIEW Microsoft.Azure.Cosmos.Encryption.EmulatorTests
+ condition: succeeded()
+ inputs:
+ command: test
+ projects: 'Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/*.csproj'
+ arguments: ${{ parameters.Arguments }} --configuration ${{ parameters.BuildConfiguration }} /p:IsPreview=true /p:SdkProjectRef=false /p:OS=${{ parameters.OS }}
+ nugetConfigPath: NuGet.config
+ publishTestResults: true
+ testRunTitle: Microsoft.Azure.Cosmos.Encryption.EmulatorTests
+
- job:
displayName: Encryption.Custom Project Ref SDK Preview ${{ parameters.BuildConfiguration }}
pool: