From 76d49cf82f6d95dc7394a8926d3e24053396cb25 Mon Sep 17 00:00:00 2001 From: Andrew Smith Date: Fri, 16 Apr 2021 15:54:57 -0700 Subject: [PATCH] TestResultCoordinator: Explicitly fail for duplicate expected results (#4684) (#4863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … (#4684) We need to account for duplicate expected results in our tests. The test report operates with an assumption that expected results are unique and actual results can be duplicated. When duplicate expected results happen (due to product issues), the test report makes it seem like we are missing some actual results. But we need to make it explicit that there are duplicate expected results, which should not be happening in our tests. Here is what I mean. Imagine the following scenario: ``` Message 0 sent Message 1 sent Message 1 sent Message 2 sent ~ time passes ~ Message 0 received Message 1 received Message 1 received Message 2 received ``` In the TRC here are the expected / actual datastructures: ``` Expected: 0->1->1->2 Actual: 0->1->1->->2 ``` When the TRC is processing the report it does not account for duplicate actual results. So it is the same scenario as this: ``` Expected: 0->1->1->2 Actual: 0->1->2 ``` In processing the counting report, we match results 0 and 1. We see that there are duplicate actual results for 1 so we eliminate them. Then end up in this state which is the problem. Now the counting report will view the situation as we have an unmatched expected result with sequence number 1. This is not the case. ``` Expected: 1->2 Actual: 2 ``` The report looks indicates there is one unmatched result. But really all results are matched. It is just that there is a duplicate expected result. The test report will now reflect this by giving us the number of duplicate expected results that were seen. --- .../Reports/CountingReportGeneratorTest.cs | 45 +++++++++------- .../Reports/CountingReportTest.cs | 9 +++- .../TwinCountingReportGeneratorTest.cs | 2 +- .../TestReportUtilTest.cs | 3 +- .../Reports/CountingReport.cs | 30 ++++++++--- .../Reports/CountingReportGenerator.cs | 54 +++++++++++++------ 6 files changed, 98 insertions(+), 45 deletions(-) diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs index a061e4761b3..62b88157452 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs @@ -19,23 +19,25 @@ public class CountingReportGeneratorTest public static IEnumerable GetCreateReportData => new List { - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 0, 0 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 10, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 10, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 10, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 10, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 2, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 3, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "1", "2", "3", "4", "5", "6", "6" }, 10, 7, 6, 2, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 4, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 1 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 4, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 4, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 4, 7, 5, 0, 2 }, - new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7", "7" }, 4, 7, 5, 1, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 0, 0, 0 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 10, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 10, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 10, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 10, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 0, 2, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 0, 3, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "1", "2", "3", "4", "5", "6", "6" }, 10, 7, 6, 0, 2, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 4, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 0, 1 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 4, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 4, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 4, 7, 5, 0, 0, 2 }, + new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7", "7" }, 4, 7, 5, 0, 1, 2 }, + new object[] { new[] { "1", "2", "3", "4", "5", "6", "7", "7" }, Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 1, 0, 0 }, + new object[] { new[] { "1", "1", "2", "3", "4", "5", "6", "7", "7" }, Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 2, 0, 0 }, }; static readonly string TestDescription = "dummy description"; static readonly ushort UnmatchedResultsMaxSize = 10; @@ -307,7 +309,8 @@ public async Task TestCreateReportAsyncWithEmptyResults() Assert.Equal(0UL, report.TotalExpectCount); Assert.Equal(0UL, report.TotalMatchCount); - Assert.Equal(0UL, report.TotalDuplicateResultCount); + Assert.Equal(0UL, report.TotalDuplicateExpectedResultCount); + Assert.Equal(0UL, report.TotalDuplicateActualResultCount); Assert.Equal(0, report.UnmatchedResults.Count); } @@ -319,7 +322,8 @@ public async Task TestCreateReportAsync( int batchSize, ulong expectedTotalExpectedCount, ulong expectedTotalMatchCount, - ulong expectedTotalDuplicateResultCount, + ulong expectedTotalDuplicateExpectedResultCount, + ulong expectedTotalDuplicateActualResultCount, int expectedMissingResultsCount) { string expectedSource = "expectedSource"; @@ -361,7 +365,8 @@ public async Task TestCreateReportAsync( Assert.Equal(expectedTotalExpectedCount, report.TotalExpectCount); Assert.Equal(expectedTotalMatchCount, report.TotalMatchCount); - Assert.Equal(expectedTotalDuplicateResultCount, report.TotalDuplicateResultCount); + Assert.Equal(expectedTotalDuplicateExpectedResultCount, report.TotalDuplicateExpectedResultCount); + Assert.Equal(expectedTotalDuplicateActualResultCount, report.TotalDuplicateActualResultCount); Assert.Equal(expectedMissingResultsCount, report.UnmatchedResults.Count); } diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs index 67f0e915849..ae6af4142bd 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs @@ -26,6 +26,7 @@ public void TestConstructorSuccess() 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), @@ -41,7 +42,8 @@ public void TestConstructorSuccess() Assert.Equal("resultType1", report.ResultType); Assert.Equal(945UL, report.TotalExpectCount); Assert.Equal(923UL, report.TotalMatchCount); - Assert.Equal(33UL, report.TotalDuplicateResultCount); + Assert.Equal(33UL, report.TotalDuplicateExpectedResultCount); + Assert.Equal(34UL, report.TotalDuplicateActualResultCount); Assert.Equal("expectedSource", report.UnmatchedResults[0].Source); Assert.Equal("resultType1", report.UnmatchedResults[0].Type); @@ -69,6 +71,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), @@ -95,6 +98,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId) 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), @@ -121,6 +125,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), @@ -147,6 +152,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), @@ -173,6 +179,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType) 945, 923, 33, + 34, new List { new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)), diff --git a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs index 06213b8bbb4..71e2802f8fd 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs @@ -266,7 +266,7 @@ public async Task TestCreateReportAsyncWithEmptyResults() Assert.Equal(0UL, report.TotalExpectCount); Assert.Equal(0UL, report.TotalMatchCount); - Assert.Equal(0UL, report.TotalDuplicateResultCount); + Assert.Equal(0UL, report.TotalDuplicateActualResultCount); Assert.Equal(0, report.UnmatchedResults.Count); } diff --git a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs index a38d8709575..71cc18a0d91 100644 --- a/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs +++ b/test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs @@ -6,7 +6,6 @@ namespace Modules.Test.TestResultCoordinator using System.Threading.Tasks; using global::TestResultCoordinator; using global::TestResultCoordinator.Reports; - using global::TestResultCoordinator.Reports.DirectMethod; using global::TestResultCoordinator.Reports.DirectMethod.Connectivity; using global::TestResultCoordinator.Reports.DirectMethod.LongHaul; using global::TestResultCoordinator.Reports.EdgeHubRestartTest; @@ -541,7 +540,7 @@ private Task MockTestResultReport(bool throwException) { if (!throwException) { - return Task.FromResult(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, new List(), Option.None(), Option.None())); + return Task.FromResult(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, 0, new List(), Option.None(), Option.None())); } return Task.FromException(new ApplicationException("Inject exception for testing")); diff --git a/test/modules/TestResultCoordinator/Reports/CountingReport.cs b/test/modules/TestResultCoordinator/Reports/CountingReport.cs index 3eed4d1ec29..62a192c4b85 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReport.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReport.cs @@ -9,7 +9,21 @@ namespace TestResultCoordinator.Reports using Newtonsoft.Json; /// - /// This is a counting report to show test result counts, e.g. expect and match counts; and contains a list of unmatched test results. + /// This is a counting report to show test result counts. It tracks a number + /// of result counts in order to give full context of test operation. + /// + /// This counting report will fail the test for the following reasons: + /// 1: Duplicate expected results (not congruent with the design of the TRC) + /// 2: Unmatched results + /// + /// It also supports a special mode if the counting report is tracking event hub + /// results (i.e. upstream telemetry). This is needed because there are large + /// delays reading messages from eventhub, so we don't want to fail the tests + /// for messages that just take a long time to come in. Specifically, this will + /// allow unmatched results if: + /// 1: All missing actual result sequence numbers are higher than the last + /// received actual result + /// 2: We are still receiving messages from eventhub /// class CountingReport : TestResultReportBase { @@ -21,7 +35,8 @@ public CountingReport( string resultType, ulong totalExpectCount, ulong totalMatchCount, - ulong totalDuplicateResultCount, + ulong totalDuplicateExpectedResultCount, + ulong totalDuplicateActualResultCount, IReadOnlyList unmatchedResults, Option eventHubSpecificReportComponents, Option lastActualResultTimestamp) @@ -31,7 +46,8 @@ public CountingReport( this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource)); this.TotalExpectCount = totalExpectCount; this.TotalMatchCount = totalMatchCount; - this.TotalDuplicateResultCount = totalDuplicateResultCount; + this.TotalDuplicateExpectedResultCount = totalDuplicateExpectedResultCount; + this.TotalDuplicateActualResultCount = totalDuplicateActualResultCount; this.UnmatchedResults = unmatchedResults ?? new List(); this.EventHubSpecificReportComponents = eventHubSpecificReportComponents; this.LastActualResultTimestamp = lastActualResultTimestamp; @@ -45,7 +61,9 @@ public CountingReport( public ulong TotalMatchCount { get; } - public ulong TotalDuplicateResultCount { get; } + public ulong TotalDuplicateExpectedResultCount { get; } + + public ulong TotalDuplicateActualResultCount { get; } public IReadOnlyList UnmatchedResults { get; } @@ -64,14 +82,14 @@ public CountingReport( public bool IsPassedHelper() { - return this.EventHubSpecificReportComponents.Match( + return this.TotalExpectCount > 0 && this.TotalDuplicateExpectedResultCount == 0 && this.EventHubSpecificReportComponents.Match( eh => { return eh.AllActualResultsMatch && eh.StillReceivingFromEventHub; }, () => { - return this.TotalExpectCount == this.TotalMatchCount && this.TotalExpectCount > 0; + return this.TotalExpectCount == this.TotalMatchCount; }); } diff --git a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs index 6126dbddb99..0b5a72c3bbf 100644 --- a/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs +++ b/test/modules/TestResultCoordinator/Reports/CountingReportGenerator.cs @@ -70,10 +70,12 @@ public async Task CreateReportAsync() { Logger.LogInformation($"Start to generate report by {nameof(CountingReportGenerator)} for Sources [{this.ExpectedSource}] and [{this.ActualSource}]"); - var lastLoadedResult = default(TestOperationResult); + var lastLoadedExpectedResult = default(TestOperationResult); + var lastLoadedActualResult = default(TestOperationResult); ulong totalExpectCount = 0; ulong totalMatchCount = 0; - ulong totalDuplicateResultCount = 0; + ulong totalDuplicateExpectedResultCount = 0; + ulong totalDuplicateActualResultCount = 0; var unmatchedResults = new Queue(); bool allActualResultsMatch = false; Option eventHubSpecificReportComponents = Option.None(); @@ -87,11 +89,23 @@ public async Task CreateReportAsync() this.ValidateResult(this.ExpectedTestResults.Current, this.ExpectedSource); this.ValidateResult(this.ActualTestResults.Current, this.ActualSource); + if (this.TestResultComparer.Matches(lastLoadedExpectedResult, this.ExpectedTestResults.Current)) + { + totalDuplicateExpectedResultCount++; + + // If we encounter a duplicate expected result, we have already + // accounted for corresponding actual results in prev iteration + hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync(); + continue; + } + + lastLoadedExpectedResult = this.ExpectedTestResults.Current; + // Skip any duplicate actual value - while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current)) + while (hasActualResult && this.TestResultComparer.Matches(lastLoadedActualResult, this.ActualTestResults.Current)) { - totalDuplicateResultCount++; - lastLoadedResult = this.ActualTestResults.Current; + totalDuplicateActualResultCount++; + lastLoadedActualResult = this.ActualTestResults.Current; hasActualResult = await this.ActualTestResults.MoveNextAsync(); } @@ -99,7 +113,7 @@ public async Task CreateReportAsync() if (this.TestResultComparer.Matches(this.ExpectedTestResults.Current, this.ActualTestResults.Current)) { - lastLoadedResult = this.ActualTestResults.Current; + lastLoadedActualResult = this.ActualTestResults.Current; hasActualResult = await this.ActualTestResults.MoveNextAsync(); hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync(); totalMatchCount++; @@ -112,10 +126,10 @@ public async Task CreateReportAsync() } // Check duplicates at the end of actual results - while (hasActualResult && this.TestResultComparer.Matches(lastLoadedResult, this.ActualTestResults.Current)) + while (hasActualResult && this.TestResultComparer.Matches(lastLoadedActualResult, this.ActualTestResults.Current)) { - totalDuplicateResultCount++; - lastLoadedResult = this.ActualTestResults.Current; + totalDuplicateActualResultCount++; + lastLoadedActualResult = this.ActualTestResults.Current; hasActualResult = await this.ActualTestResults.MoveNextAsync(); } @@ -123,8 +137,17 @@ public async Task CreateReportAsync() while (hasExpectedResult) { - totalExpectCount++; - TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize); + if (this.TestResultComparer.Matches(lastLoadedExpectedResult, this.ExpectedTestResults.Current)) + { + totalDuplicateExpectedResultCount++; + } + else + { + totalExpectCount++; + TestReportUtil.EnqueueAndEnforceMaxSize(unmatchedResults, this.ExpectedTestResults.Current, this.unmatchedResultsMaxSize); + } + + lastLoadedExpectedResult = this.ExpectedTestResults.Current; hasExpectedResult = await this.ExpectedTestResults.MoveNextAsync(); } @@ -142,7 +165,7 @@ public async Task CreateReportAsync() .Expect( () => throw new ArgumentException("TRC must be in long haul mode to be generating an EventHubLongHaul CountingReport")) .EventHubDelayTolerance; - if (lastLoadedResult == null || lastLoadedResult.CreatedAt < DateTime.UtcNow - eventHubDelayTolerance) + if (lastLoadedActualResult == null || lastLoadedActualResult.CreatedAt < DateTime.UtcNow - eventHubDelayTolerance) { stillReceivingFromEventHub = false; } @@ -169,9 +192,9 @@ public async Task CreateReportAsync() hasActualResult = await this.ActualTestResults.MoveNextAsync(); } - if (lastLoadedResult != null) + if (lastLoadedActualResult != null) { - lastLoadedResultCreatedAt = Option.Some(lastLoadedResult.CreatedAt); + lastLoadedResultCreatedAt = Option.Some(lastLoadedActualResult.CreatedAt); } return new CountingReport( @@ -182,7 +205,8 @@ public async Task CreateReportAsync() this.ResultType, totalExpectCount, totalMatchCount, - totalDuplicateResultCount, + totalDuplicateExpectedResultCount, + totalDuplicateActualResultCount, new List(unmatchedResults).AsReadOnly(), eventHubSpecificReportComponents, lastLoadedResultCreatedAt);