Merge branch 'users/ezrahaleva/ParserSupportForFIRSTLAST' of https://…

…github.com/Azure/azure-cosmos-dotnet-v3 into users/ezrahaleva/ParserSupportForFIRSTLAST

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: needs-investigation
 assignees: ''
 
 ---

.github/fabricbot.json

@@ -6,7 +6,6 @@
       "capabilityId": "ScheduledSearch",
       "subCapability": "ScheduledSearch",
       "version": "1.1",
-      "id": "a-PNOJWK3Jbr7QY7vkc2O",
       "config": {
         "frequency": [
           {
@@ -50,7 +49,60 @@
         ]
       },
       "disabled": true
+    },
+    {
+      "taskType": "trigger",
+      "capabilityId": "IssueResponder",
+      "subCapability": "IssuesOnlyResponder",
+      "version": "1.0",
+      "config": {
+        "conditions": {
+          "operator": "and",
+          "operands": [
+            {
+              "operator": "not",
+              "operands": [
+                {
+                  "name": "isAssignedToSomeone",
+                  "parameters": {}
+                }
+              ]
+            },
+            {
+              "name": "isAction",
+              "parameters": {
+                "action": "opened"
+              }
+            },
+            {
+              "operator": "not",
+              "operands": [
+                {
+                  "name": "hasLabel",
+                  "parameters": {
+                    "label": "needs-investigation"
+                  }
+                }
+              ]
+            }
+          ]
+        },
+        "eventType": "issue",
+        "eventNames": [
+          "issues",
+          "project_card"
+        ],
+        "taskName": "Add needs-investigation",
+        "actions": [
+          {
+            "name": "addLabel",
+            "parameters": {
+              "label": "needs-investigation"
+            }
+          }
+        ]
+      }
     }
   ],
   "userGroups": []
-}
+}

Directory.Build.props

@@ -1,12 +1,11 @@
-<?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 	<PropertyGroup>
-		<ClientOfficialVersion>3.31.2</ClientOfficialVersion>
-		<ClientPreviewVersion>3.31.2</ClientPreviewVersion>
+		<ClientOfficialVersion>3.32.0</ClientOfficialVersion>
+		<ClientPreviewVersion>3.32.0</ClientPreviewVersion>
 		<ClientPreviewSuffixVersion>preview</ClientPreviewSuffixVersion>
-		<DirectVersion>3.29.4</DirectVersion>
-		<EncryptionOfficialVersion>2.0.0</EncryptionOfficialVersion>
-		<EncryptionPreviewVersion>2.0.0</EncryptionPreviewVersion>
+		<DirectVersion>3.30.2</DirectVersion>
+		<EncryptionOfficialVersion>2.0.1</EncryptionOfficialVersion>
+		<EncryptionPreviewVersion>2.0.1</EncryptionPreviewVersion>
 		<EncryptionPreviewSuffixVersion>preview</EncryptionPreviewSuffixVersion>
 		<CustomEncryptionVersion>1.0.0-preview04</CustomEncryptionVersion>
 		<HybridRowVersion>1.1.0-preview3</HybridRowVersion>

Microsoft.Azure.Cosmos.Encryption/changelog.md

@@ -3,6 +3,16 @@ Preview features are treated as a separate branch and will not be included in th
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+### <a name="2.0.1"/> [2.0.1](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.1) - 2023-03-11
+
+#### Added
+- [#3642](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3642) Adds validation code to check if the Key Vault URI provided in wrap metadata is a valid key identifier.
+
+### <a name="2.0.1-preview"/> [2.0.1-preview](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.1-preview) - 2023-01-11
+
+#### Added
+- [#3642](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/3642) Adds validation code to check if the Key Vault URI provided in wrap metadata is a valid key identifier.
+
 ### <a name="2.0.0"/> [2.0.0](https://www.nuget.org/packages/Microsoft.Azure.Cosmos.Encryption/2.0.0) - 2022-06-28
 
 #### Added

Microsoft.Azure.Cosmos.Encryption/src/EncryptionDatabaseExtensions.cs

@@ -91,6 +91,17 @@ public static async Task<ClientEncryptionKeyResponse> CreateClientEncryptionKeyA
                     + " Please refer to https://aka.ms/CosmosClientEncryption for more details.");
             }
 
+            if (string.Equals(encryptionCosmosClient.KeyEncryptionKeyResolverName, KeyEncryptionKeyResolverName.AzureKeyVault))
+            {
+                // https://KEYVAULTNAME.vault.azure.net/keys/KEYNAME/KEYVERSION
+                string[] keyVaultUriSegments = new Uri(encryptionKeyWrapMetadata.Value).Segments;
+
+                if (keyVaultUriSegments.Length != 4 || !string.Equals(keyVaultUriSegments[1], "keys/", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    throw new ArgumentException($"Invalid Key Vault URI'{encryptionKeyWrapMetadata.Value}' passed. Pass the complete Azure keyvault key identifier. Please refer to https://aka.ms/CosmosClientEncryption for more details.");
+                }
+            }
+
             KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate(
                 encryptionKeyWrapMetadata.Name,
                 encryptionKeyWrapMetadata.Value,
@@ -192,6 +203,17 @@ public static async Task<ClientEncryptionKeyResponse> RewrapClientEncryptionKeyA
                     + " Please refer to https://aka.ms/CosmosClientEncryption for more details.");
             }
 
+            if (string.Equals(encryptionCosmosClient.KeyEncryptionKeyResolverName, KeyEncryptionKeyResolverName.AzureKeyVault))
+            {
+                // https://KEYVAULTNAME.vault.azure.net/keys/KEYNAME/KEYVERSION
+                string[] keyVaultUriSegments = new Uri(newEncryptionKeyWrapMetadata.Value).Segments;
+
+                if (keyVaultUriSegments.Length != 4 || !string.Equals(keyVaultUriSegments[1], "keys/", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    throw new ArgumentException($"Invalid Key Vault URI'{newEncryptionKeyWrapMetadata.Value}' passed. Pass the complete Azure keyvault key identifier. Please refer to https://aka.ms/CosmosClientEncryption for more details.");
+                }
+            }
+
             ClientEncryptionKeyProperties clientEncryptionKeyProperties = await clientEncryptionKey.ReadAsync(cancellationToken: cancellationToken);
 
             RequestOptions requestOptions = new RequestOptions

Microsoft.Azure.Cosmos.Encryption/src/Microsoft.Azure.Cosmos.Encryption.csproj

@@ -28,11 +28,11 @@
   </ItemGroup>
 
   <ItemGroup Condition=" '$(SdkProjectRef)' != 'True' AND '$(IsPreview)' != 'True' ">
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="[3.28.0,3.29)" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="[3.31.0,3.32)" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(SdkProjectRef)' != 'True' AND '$(IsPreview)' == 'True' ">
-    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.28.0-preview" />
+    <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.31.2-preview" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(SdkProjectRef)' == 'True' ">

Microsoft.Azure.Cosmos.Encryption/tests/EmulatorTests/MdeEncryptionTests.cs

@@ -296,6 +296,69 @@ await MdeEncryptionTests.CreateClientEncryptionKeyAsync(
                 if (ex is CosmosException cosmosException)
                     Assert.AreEqual(HttpStatusCode.Conflict, cosmosException.StatusCode);
             }
+
+            cekId = "testAkvKid";
+            CosmosClient client = TestCommon.CreateCosmosClient();
+            TestKeyEncryptionKeyResolver testKeyEncryptionKeyResolver = new TestKeyEncryptionKeyResolver();
+
+            EncryptionKeyWrapMetadata metadata = MdeEncryptionTests.CreateEncryptionKeyWrapMetadata(KeyEncryptionKeyResolverName.AzureKeyVault, "key1", "https://testkeyvault.vault.azure.net/keys/testkey/12345678");
+
+            CosmosClient encryptionCosmosClient = client.WithEncryption(
+                testKeyEncryptionKeyResolver,
+                KeyEncryptionKeyResolverName.AzureKeyVault,
+                TimeSpan.Zero);
+
+            Database database = await encryptionCosmosClient.CreateDatabaseAsync(Guid.NewGuid().ToString());
+
+            ClientEncryptionKeyResponse clientEncrytionKeyResponse = await database.CreateClientEncryptionKeyAsync(
+                    cekId,
+                    DataEncryptionAlgorithm.AeadAes256CbcHmacSha256,
+                    metadata);
+
+            Assert.AreEqual(HttpStatusCode.Created, clientEncrytionKeyResponse.StatusCode);
+
+            metadata = MdeEncryptionTests.CreateEncryptionKeyWrapMetadata(KeyEncryptionKeyResolverName.AzureKeyVault, "key1", "https://testkeyvault.vault.azure.net/keys/testkey/9101112");
+
+            clientEncrytionKeyResponse = await database.RewrapClientEncryptionKeyAsync(
+                  cekId,
+                  metadata);
+
+            Assert.AreEqual(HttpStatusCode.OK, clientEncrytionKeyResponse.StatusCode);
+
+            // complete key identifier not passed
+            metadata = MdeEncryptionTests.CreateEncryptionKeyWrapMetadata(KeyEncryptionKeyResolverName.AzureKeyVault, "key1", "https://testkeyvault.vault.azure.net/keys/testkey");
+
+            try
+            {
+                clientEncrytionKeyResponse = await database.CreateClientEncryptionKeyAsync(
+                   cekId,
+                   DataEncryptionAlgorithm.AeadAes256CbcHmacSha256,
+                   metadata);
+
+                Assert.Fail("Key creation should have failed.");
+
+            }
+            catch(Exception ex)
+            {
+                Assert.AreEqual(true, ex.Message.Contains("Invalid Key Vault URI"));
+            }
+
+            // rewrap old key with new key vault uri without complete key identifier
+            try
+            {
+                clientEncrytionKeyResponse = await database.RewrapClientEncryptionKeyAsync(
+                  cekId,
+                  metadata);
+
+                Assert.Fail("Key rewrap should have failed.");
+
+            }
+            catch (Exception ex)
+            {
+                Assert.AreEqual(true, ex.Message.Contains("Invalid Key Vault URI"));
+            }
+
+            encryptionCosmosClient.Dispose();
         }
 
         [TestMethod]

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs

@@ -143,7 +143,9 @@ internal void Print()
                 Utility.TeeTraceInformation($"{nameof(BenchmarkConfig)} arguments");
                 Utility.TeeTraceInformation($"IsServerGC: {GCSettings.IsServerGC}");
                 Utility.TeeTraceInformation("--------------------------------------------------------------------- ");
-                Utility.TeeTraceInformation(JsonHelper.ToString(this));
+                Utility.TeeTraceInformation(JsonHelper.ToString(
+                    input: this, 
+                    capacity: 2048));
                 Utility.TeeTraceInformation("--------------------------------------------------------------------- ");
                 Utility.TeeTraceInformation(string.Empty);
             }
@@ -222,7 +224,7 @@ internal Microsoft.Azure.Cosmos.CosmosClient CreateCosmosClient(string accountKe
                 clientOptions.ConsistencyLevel = (Microsoft.Azure.Cosmos.ConsistencyLevel)Enum.Parse(typeof(Microsoft.Azure.Cosmos.ConsistencyLevel), this.ConsistencyLevel, ignoreCase: true);
             }
 
-            clientOptions.EnableDistributedTracing = this.EnableDistributedTracing;
+            clientOptions.IsDistributedTracingEnabled = this.EnableDistributedTracing;
 
             return new Microsoft.Azure.Cosmos.CosmosClient(
                         this.EndPoint,

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/JsonHelper.cs

@@ -17,9 +17,9 @@ internal static class JsonHelper
                 });
         private const int DefaultCapacity = 1024;
 
-        public static string ToString<T>(T input)
+        public static string ToString<T>(T input, int capacity = JsonHelper.DefaultCapacity)
         {
-            using (MemoryStream stream = JsonHelper.ToStream(input))
+            using (MemoryStream stream = JsonHelper.ToStream(input, capacity))
             using (StreamReader sr = new StreamReader(stream))
             {
                 return sr.ReadToEnd();
@@ -31,15 +31,15 @@ public static T Deserialize<T>(string payload)
             return JsonConvert.DeserializeObject<T>(payload);
         }
 
-        public static MemoryStream ToStream<T>(T input)
+        public static MemoryStream ToStream<T>(T input, int capacity = JsonHelper.DefaultCapacity)
         {
-            byte[] blob = System.Buffers.ArrayPool<byte>.Shared.Rent(JsonHelper.DefaultCapacity);
-            MemoryStream memStreamPayload = new MemoryStream(blob, 0, JsonHelper.DefaultCapacity, writable: true, publiclyVisible: true);
+            byte[] blob = System.Buffers.ArrayPool<byte>.Shared.Rent(capacity);
+            MemoryStream memStreamPayload = new MemoryStream(blob, 0, capacity, writable: true, publiclyVisible: true);
             memStreamPayload.SetLength(0);
             memStreamPayload.Position = 0;
             using (StreamWriter streamWriter = new StreamWriter(memStreamPayload,
                 encoding: JsonHelper.DefaultEncoding,
-                bufferSize: JsonHelper.DefaultCapacity,
+                bufferSize: capacity,
                 leaveOpen: true))
             {
                 using (JsonWriter writer = new JsonTextWriter(streamWriter))

Microsoft.Azure.Cosmos.Samples/Tools/CTL/CTLConfig.cs

@@ -11,6 +11,7 @@ namespace CosmosCTL
     using CommandLine.Text;
     using Microsoft.Azure.Cosmos;
     using Newtonsoft.Json;
+    using static CosmosCTL.ReservoirProvider;
 
     public class CTLConfig
     {
@@ -76,7 +77,7 @@ public string DiagnosticsThresholdDuration
         }
 
         [Option("ctl_content_response_on_write", Required = false, HelpText = "Should return content response on writes")]
-        public bool IsContentResponseOnWriteEnabled { get; set; } = true;
+        public bool? IsContentResponseOnWriteEnabled { get; set; } = true;
 
         [Option("ctl_output_event_traces", Required = false, HelpText = "Outputs TraceSource to console")]
         public bool OutputEventTraces { get; set; } = false;
@@ -102,6 +103,12 @@ public string DiagnosticsThresholdDuration
         [Option("ctl_telemetry_schedule_in_sec", Required = false, HelpText = "telemetry task schedule time in sec")]
         public string TelemetryScheduleInSeconds { get; set; }
 
+        [Option("ctl_reservoir_type", Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")]
+        public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow;
+
+        [Option("ctl_reservoir_sample_size", Required = false, HelpText = "The reservoir sample size.")]
+        public int ReservoirSampleSize { get; set; } = 1028;
+
         internal TimeSpan RunningTimeDurationAsTimespan { get; private set; } = TimeSpan.FromHours(10);
         internal TimeSpan DiagnosticsThresholdDurationAsTimespan { get; private set; } = TimeSpan.FromSeconds(60);
 

Microsoft.Azure.Cosmos.Samples/Tools/CTL/ReservoirProvider.cs

@@ -0,0 +1,49 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosCTL
+{
+    using System;
+    using App.Metrics.ReservoirSampling;
+
+    /// <summary>
+    /// Returns the <see cref="IReservoir"/> based on the CTL configuration.
+    /// </summary>
+    public class ReservoirProvider
+    {
+        /// <summary>
+        /// Create and returns a new instance of the <see cref="IReservoir"/> based on the CTL configuration.
+        /// </summary>
+        /// <param name="ctlConfig">An instance of <see cref="CTLConfig"/> containing the CTL config params.</param>
+        /// <returns>An implementation of <see cref="IReservoir"/>.</returns>
+        public static IReservoir GetReservoir(CTLConfig ctlConfig)
+        {
+            return ctlConfig.ReservoirType switch
+            {
+                ReservoirTypes.Uniform => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir(
+                    sampleSize: ctlConfig.ReservoirSampleSize),
+
+                ReservoirTypes.SlidingWindow => new App.Metrics.ReservoirSampling.SlidingWindow.DefaultSlidingWindowReservoir(
+                    sampleSize: ctlConfig.ReservoirSampleSize),
+
+                ReservoirTypes.ExponentialDecay => new App.Metrics.ReservoirSampling.ExponentialDecay.DefaultForwardDecayingReservoir(
+                    sampleSize: ctlConfig.ReservoirSampleSize,
+                    alpha: 0.015),
+
+                _ => throw new ArgumentException(
+                    message: "Invalid ReservoirType Specified."),
+            };
+        }
+
+        /// <summary>
+        /// An enum containing different reservoir types.
+        /// </summary>
+        public enum ReservoirTypes
+        {
+            Uniform,
+            SlidingWindow,
+            ExponentialDecay
+        }
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/CTL/Scenarios/ChangeFeedPullScenario.cs

@@ -64,7 +64,7 @@ public async Task RunAsync(
                 DurationUnit = TimeUnit.Milliseconds,
                 RateUnit = TimeUnit.Seconds,
                 Context = loggingContextIdentifier,
-                Reservoir = () => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir()
+                Reservoir = () => ReservoirProvider.GetReservoir(config)
             };
 
             Container container = cosmosClient.GetContainer(config.Database, config.Collection);

Microsoft.Azure.Cosmos.Samples/Tools/CTL/Scenarios/ReadManyScenario.cs

@@ -76,7 +76,7 @@ public async Task RunAsync(
                     DurationUnit = TimeUnit.Milliseconds,
                     RateUnit = TimeUnit.Seconds,
                     Context = loggingContextIdentifier,
-                    Reservoir = () => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir()
+                    Reservoir = () => ReservoirProvider.GetReservoir(config)
                 };
 
                 Container container = cosmosClient.GetContainer(config.Database, config.Collection);