Skip to content

Commit

Permalink
TestResultCoordinator: Explicitly fail for duplicate expected results (…
Browse files Browse the repository at this point in the history 
…#4684) (#4863)

… (#4684)

We need to account for duplicate expected results in our tests. The test report operates with an assumption that expected results are unique and actual results can be duplicated. When duplicate expected results happen (due to product issues), the test report makes it seem like we are missing some actual results. But we need to make it explicit that there are duplicate expected results, which should not be happening in our tests.

Here is what I mean. Imagine the following scenario:
```
Message 0 sent
Message 1 sent
Message 1 sent
Message 2 sent

~ time passes ~

Message 0 received
Message 1 received
Message 1 received
Message 2 received
``` 

In the TRC here are the expected / actual datastructures:
```
Expected: 0->1->1->2
Actual: 0->1->1->->2
```

When the TRC is processing the report it does not account for duplicate actual results. So it is the same scenario as this:
```
Expected: 0->1->1->2
Actual: 0->1->2

```

In processing the counting report, we match results 0 and 1. We see that there are duplicate actual results for 1 so we eliminate them. Then end up in this state which is the problem. Now the counting report will view the situation as we have an unmatched expected result with sequence number 1. This is not the case. 
```
Expected: 1->2
Actual: 2

```

The report looks indicates there is one unmatched result. But really all results are matched. It is just that there is a duplicate expected result. The test report will now reflect this by giving us the number of duplicate expected results that were seen.
  • Loading branch information
@and-rewsmith
and-rewsmith authored Apr 16, 2021
1 parent 944f909 commit 3393473
Show file tree
Hide file tree
Showing 6 changed files with 98 additions and 45 deletions.
45 changes: 25 additions & 20 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,25 @@ public class CountingReportGeneratorTest
public static IEnumerable<object[]> GetCreateReportData =>
new List<object[]>
{
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 0, 0 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 10, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 10, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 10, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 10, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 2, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 3, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "1", "2", "3", "4", "5", "6", "6" }, 10, 7, 6, 2, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 4, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 4, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 4, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 4, 7, 5, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7", "7" }, 4, 7, 5, 1, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 0, 0, 0 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 10, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 10, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 10, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 10, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 10, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 0, 2, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "2", "2", "3", "4", "4", "5", "6" }, 10, 7, 6, 0, 3, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "1", "2", "3", "4", "5", "6", "6" }, 10, 7, 6, 0, 2, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "2", "3", "4", "5", "6" }, 4, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6", "7" }, 4, 7, 6, 0, 0, 1 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "6", "7" }, 4, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "1", "3", "4", "5", "6" }, 4, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7" }, 4, 7, 5, 0, 0, 2 },
new object[] { Enumerable.Range(1, 7).Select(v => v.ToString()), new[] { "2", "3", "4", "5", "7", "7" }, 4, 7, 5, 0, 1, 2 },
new object[] { new[] { "1", "2", "3", "4", "5", "6", "7", "7" }, Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 1, 0, 0 },
new object[] { new[] { "1", "1", "2", "3", "4", "5", "6", "7", "7" }, Enumerable.Range(1, 7).Select(v => v.ToString()), 10, 7, 7, 2, 0, 0 },
};
static readonly string TestDescription = "dummy description";
static readonly ushort UnmatchedResultsMaxSize = 10;
Expand Down Expand Up @@ -307,7 +309,8 @@ public async Task TestCreateReportAsyncWithEmptyResults()

Assert.Equal(0UL, report.TotalExpectCount);
Assert.Equal(0UL, report.TotalMatchCount);
Assert.Equal(0UL, report.TotalDuplicateResultCount);
Assert.Equal(0UL, report.TotalDuplicateExpectedResultCount);
Assert.Equal(0UL, report.TotalDuplicateActualResultCount);
Assert.Equal(0, report.UnmatchedResults.Count);
}

Expand All @@ -319,7 +322,8 @@ public async Task TestCreateReportAsync(
int batchSize,
ulong expectedTotalExpectedCount,
ulong expectedTotalMatchCount,
ulong expectedTotalDuplicateResultCount,
ulong expectedTotalDuplicateExpectedResultCount,
ulong expectedTotalDuplicateActualResultCount,
int expectedMissingResultsCount)
{
string expectedSource = "expectedSource";
Expand Down Expand Up @@ -361,7 +365,8 @@ public async Task TestCreateReportAsync(

Assert.Equal(expectedTotalExpectedCount, report.TotalExpectCount);
Assert.Equal(expectedTotalMatchCount, report.TotalMatchCount);
Assert.Equal(expectedTotalDuplicateResultCount, report.TotalDuplicateResultCount);
Assert.Equal(expectedTotalDuplicateExpectedResultCount, report.TotalDuplicateExpectedResultCount);
Assert.Equal(expectedTotalDuplicateActualResultCount, report.TotalDuplicateActualResultCount);
Assert.Equal(expectedMissingResultsCount, report.UnmatchedResults.Count);
}

Expand Down
9 changes: 8 additions & 1 deletion test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public void TestConstructorSuccess()
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand All @@ -41,7 +42,8 @@ public void TestConstructorSuccess()
Assert.Equal("resultType1", report.ResultType);
Assert.Equal(945UL, report.TotalExpectCount);
Assert.Equal(923UL, report.TotalMatchCount);
Assert.Equal(33UL, report.TotalDuplicateResultCount);
Assert.Equal(33UL, report.TotalDuplicateExpectedResultCount);
Assert.Equal(34UL, report.TotalDuplicateActualResultCount);

Assert.Equal("expectedSource", report.UnmatchedResults[0].Source);
Assert.Equal("resultType1", report.UnmatchedResults[0].Type);
Expand Down Expand Up @@ -69,6 +71,7 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand All @@ -95,6 +98,7 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId)
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand All @@ -121,6 +125,7 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand All @@ -147,6 +152,7 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand All @@ -173,6 +179,7 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
945,
923,
33,
34,
new List<TestOperationResult>
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
Expand Down
2 changes: 1 addition & 1 deletion test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ public async Task TestCreateReportAsyncWithEmptyResults()

Assert.Equal(0UL, report.TotalExpectCount);
Assert.Equal(0UL, report.TotalMatchCount);
Assert.Equal(0UL, report.TotalDuplicateResultCount);
Assert.Equal(0UL, report.TotalDuplicateActualResultCount);
Assert.Equal(0, report.UnmatchedResults.Count);
}

Expand Down
3 changes: 1 addition & 2 deletions test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ namespace Modules.Test.TestResultCoordinator
using System.Threading.Tasks;
using global::TestResultCoordinator;
using global::TestResultCoordinator.Reports;
using global::TestResultCoordinator.Reports.DirectMethod;
using global::TestResultCoordinator.Reports.DirectMethod.Connectivity;
using global::TestResultCoordinator.Reports.DirectMethod.LongHaul;
using global::TestResultCoordinator.Reports.EdgeHubRestartTest;
Expand Down Expand Up @@ -541,7 +540,7 @@ private Task<ITestResultReport> MockTestResultReport(bool throwException)
{
if (!throwException)
{
return Task.FromResult<ITestResultReport>(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, new List<TestOperationResult>(), Option.None<EventHubSpecificReportComponents>(), Option.None<DateTime>()));
return Task.FromResult<ITestResultReport>(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, 0, new List<TestOperationResult>(), Option.None<EventHubSpecificReportComponents>(), Option.None<DateTime>()));
}

return Task.FromException<ITestResultReport>(new ApplicationException("Inject exception for testing"));
Expand Down
30 changes: 24 additions & 6 deletions test/modules/TestResultCoordinator/Reports/CountingReport.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,21 @@ namespace TestResultCoordinator.Reports
using Newtonsoft.Json;

/// <summary>
/// This is a counting report to show test result counts, e.g. expect and match counts; and contains a list of unmatched test results.
/// This is a counting report to show test result counts. It tracks a number
/// of result counts in order to give full context of test operation.
///
/// This counting report will fail the test for the following reasons:
/// 1: Duplicate expected results (not congruent with the design of the TRC)
/// 2: Unmatched results
///
/// It also supports a special mode if the counting report is tracking event hub
/// results (i.e. upstream telemetry). This is needed because there are large
/// delays reading messages from eventhub, so we don't want to fail the tests
/// for messages that just take a long time to come in. Specifically, this will
/// allow unmatched results if:
/// 1: All missing actual result sequence numbers are higher than the last
/// received actual result
/// 2: We are still receiving messages from eventhub
/// </summary>
class CountingReport : TestResultReportBase
{
Expand All @@ -21,7 +35,8 @@ public CountingReport(
string resultType,
ulong totalExpectCount,
ulong totalMatchCount,
ulong totalDuplicateResultCount,
ulong totalDuplicateExpectedResultCount,
ulong totalDuplicateActualResultCount,
IReadOnlyList<TestOperationResult> unmatchedResults,
Option<EventHubSpecificReportComponents> eventHubSpecificReportComponents,
Option<DateTime> lastActualResultTimestamp)
Expand All @@ -31,7 +46,8 @@ public CountingReport(
this.ActualSource = Preconditions.CheckNonWhiteSpace(actualSource, nameof(actualSource));
this.TotalExpectCount = totalExpectCount;
this.TotalMatchCount = totalMatchCount;
this.TotalDuplicateResultCount = totalDuplicateResultCount;
this.TotalDuplicateExpectedResultCount = totalDuplicateExpectedResultCount;
this.TotalDuplicateActualResultCount = totalDuplicateActualResultCount;
this.UnmatchedResults = unmatchedResults ?? new List<TestOperationResult>();
this.EventHubSpecificReportComponents = eventHubSpecificReportComponents;
this.LastActualResultTimestamp = lastActualResultTimestamp;
Expand All @@ -45,7 +61,9 @@ public CountingReport(

public ulong TotalMatchCount { get; }

public ulong TotalDuplicateResultCount { get; }
public ulong TotalDuplicateExpectedResultCount { get; }

public ulong TotalDuplicateActualResultCount { get; }

public IReadOnlyList<TestOperationResult> UnmatchedResults { get; }

Expand All @@ -64,14 +82,14 @@ public CountingReport(

public bool IsPassedHelper()
{
return this.EventHubSpecificReportComponents.Match(
return this.TotalExpectCount > 0 && this.TotalDuplicateExpectedResultCount == 0 && this.EventHubSpecificReportComponents.Match(
eh =>
{
return eh.AllActualResultsMatch && eh.StillReceivingFromEventHub;
},
() =>
{
return this.TotalExpectCount == this.TotalMatchCount && this.TotalExpectCount > 0;
return this.TotalExpectCount == this.TotalMatchCount;
});
}

Expand Down
Loading

0 comments on commit 3393473

Please sign in to comment.