Internal Gates: Fixes Tests and update pipelines (Azure#2427)

Fixes the read many test which is only used in preview runs Fixes the pipelines to all use the same windows version. Hoping this might be the cause the perf gates are failing for the preview build but not others. Updated the perf tests to only update the values if it has a change greater than 2% and to list the tests that exceeded the 5% in a separate list to make it easy to see which ones changed.
juraj-blazek · Apr 27, 2021 · b75fcf2 · b75fcf2
1 parent 96db331
commit b75fcf2
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 23 deletions.
diff --git a/...osoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs b/...osoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosReadManyItemsTests.cs
@@ -363,7 +363,7 @@ public async Task ReadManyMultiplePK()
         {
             IReadOnlyList<string> pkPaths = new List<string> { "/pk", "/description" };
             ContainerProperties containerSettings = new ContainerProperties(id: Guid.NewGuid().ToString(), partitionKeyPaths: pkPaths);
-            Container container = await this.database.CreateContainerAsync(this.containerSettings);
+            Container container = await this.database.CreateContainerAsync(containerSettings);
 
             for (int i = 0; i < 5; i++)
             {

diff --git a/...ure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json b/...ure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/BenchmarkResults.json
@@ -19,7 +19,7 @@
   "MockedItemBenchmark.ReadFeed;[Type=Stream]": 42546.0,
   "MockedItemBenchmark.ReadItemExists;[Type=OfT]": 41814.0,
   "MockedItemBenchmark.ReadItemExists;[Type=OfTCustom]": 41806.0,
-  "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 70752.0,
+  "MockedItemBenchmark.ReadItemExists;[Type=OfTWithDiagnosticsToString]": 74898.0,
   "MockedItemBenchmark.ReadItemExists;[Type=Stream]": 34464.0,
   "MockedItemBenchmark.ReadItemNotExists;[Type=OfT]": 52826.0,
   "MockedItemBenchmark.ReadItemNotExists;[Type=OfTCustom]": 52864.0,

diff --git a/....Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs b/....Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Contracts/PerformanceValidation.cs
@@ -72,40 +72,60 @@ public static int ValidateSummaryResultsAgainstBaseline(SortedDictionary<string,
 
             currentDirectory += PerformanceValidation.DirectoryPath;
 
-            // Always write the updated version. This will change with each run.
-            string currentBenchmarkResults = JsonConvert.SerializeObject(operationToMemoryAllocated, Formatting.Indented);
-            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
-
             string baselineJson = File.ReadAllText(currentDirectory + PerformanceValidation.BaselineBenchmarkResultsFileName);
             Dictionary<string, double> baselineBenchmarkResults = JsonConvert.DeserializeObject<Dictionary<string, double>>(baselineJson);
 
-            if (baselineBenchmarkResults.Count != operationToMemoryAllocated.Count)
-            {
-                Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                return 1;
-            }
-
-            foreach(KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
+            List<string> failures = new List<string>();
+            SortedDictionary<string, double> updatedBaseline = new SortedDictionary<string, double>();
+            foreach (KeyValuePair<string, double> currentResult in operationToMemoryAllocated)
             {
-                double baselineResult = baselineBenchmarkResults[currentResult.Key];
+                if(!baselineBenchmarkResults.TryGetValue(
+                    currentResult.Key, 
+                    out double baselineResult))
+                {
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    continue;
+                }
 
                 // Add 5% buffer to avoid minor variation between test runs
-                double diff = currentResult.Value - baselineResult;
+                double diff = Math.Abs(currentResult.Value - baselineResult);
                 double maxAllowedDiff = baselineResult * .05;
+                double minDiffToUpdatebaseLine = baselineResult * .02;
                 if (diff > maxAllowedDiff)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                    failures.Add($"{currentResult.Key}: {currentResult.Value}");
                 }
-                else if (-diff > maxAllowedDiff)
+                else if(diff > minDiffToUpdatebaseLine)
                 {
-                    Console.WriteLine(PerformanceValidation.UpdateMessage + currentBenchmarkResults);
-                    return 1;
+                    // Update the value if it is greater than 2% difference.
+                    // This reduces the noise and make it easier to see which values actually changed
+                    updatedBaseline.Add(currentResult.Key, currentResult.Value);
+                }
+                else
+                {
+                    // Use the baseline if the value didn't change by more than 2% to avoid updating values unnecessarily
+                    // This makes it easier to see which values actually need to be updated.
+                    updatedBaseline.Add(currentResult.Key, baselineResult);
                 }
             }
 
+            // Always write the updated version. This will change with each run.
+            string currentBenchmarkResults = JsonConvert.SerializeObject(updatedBaseline, Formatting.Indented);
+            File.WriteAllText(currentDirectory + PerformanceValidation.CurrentBenchmarkResultsFileName, currentBenchmarkResults);
             Console.WriteLine("Current benchmark results: " + currentBenchmarkResults);
 
+            if (failures.Any())
+            {
+                Console.WriteLine(PerformanceValidation.UpdateMessage);
+                foreach(string failure in failures)
+                {
+                    Console.WriteLine(failure);
+                }
+
+                return 1;
+            }
+
             return 0;
         }
     }

diff --git a/azure-pipelines-functional.yml b/azure-pipelines-functional.yml
@@ -14,7 +14,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
 
 
 jobs: 

diff --git a/azure-pipelines-nightly.yml b/azure-pipelines-nightly.yml
@@ -14,7 +14,7 @@ schedules:
 
 variables:
   ReleaseArguments: ' --filter "TestCategory!=Quarantine" --verbosity normal ' 
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops 
   BuildConfiguration: Release 
   IsNightly: true 
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -13,7 +13,7 @@ pr:
 variables:
   DebugArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
   ReleaseArguments: ' --filter "TestCategory!=Quarantine & TestCategory!=Functional" --verbosity normal '
-  VmImage: vs2017-win2016 # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
+  VmImage: windows-latest # https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops
 
 
 jobs: