Skip to content

Commit

Permalink
Adding event hub long haul mode to trc tests (#4507)
Browse files Browse the repository at this point in the history 
Add EventHubLongHaul mode to TRC tests so that we can pass on tests that involve using EventHub. We've seen these tests have incredibly long delays. 

This PR makes the change to implement this solution: ignore expected messages (== ones sent successfully from loadGen) that do not match, only match on actual messages (== ones received from eventHub), and also ensure that we are still receiving from EventHub by 1) Using our own timestamps in the TRC for messages we receive and 2) making sure the last message received for the actual store was within an hour (we can define that tolerance however we want)
  • Loading branch information
@dylanbronson
dylanbronson authored Mar 17, 2021
1 parent 1029360 commit 51f1102
Show file tree
Hide file tree
Showing 15 changed files with 197 additions and 61 deletions.
2 changes: 2 additions & 0 deletions e2e_deployment_files/long_haul_deployment.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer1.send",
"ActualSource": "relayer1.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | amqp"
},
"reportMetadata3": {
Expand All @@ -536,6 +537,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer2.send",
"ActualSource": "relayer2.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | mqtt"
},
"reportMetadata5": {
Expand Down
2 changes: 2 additions & 0 deletions e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_amqp.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer1.send",
"ActualSource": "relayer1.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | amqp"
},
"reportMetadata3": {
Expand All @@ -533,6 +534,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer2.send",
"ActualSource": "relayer2.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | mqtt"
},
"reportMetadata5": {
Expand Down
3 changes: 3 additions & 0 deletions e2e_deployment_files/nestededge_bottomLayerBaseDeployment_long_haul_mqtt.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer1.send",
"ActualSource": "relayer1.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | amqp"
},
"reportMetadata3": {
Expand All @@ -599,6 +600,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "relayer2.send",
"ActualSource": "relayer2.eventHub",
"LongHaulEventHubMode": "true",
"TestDescription": "messages | upstream | mqtt"
},
"reportMetadata5": {
Expand Down Expand Up @@ -628,6 +630,7 @@
"TestOperationResultType": "Messages",
"ExpectedSource": "genericMqttTester.send",
"ActualSource": "genericMqttTester.receive",
"LongHaulEventHubMode": "false",
"TestDescription": "messages | local | mqtt | generic"
},
"reportMetadata10": {
Expand Down
37 changes: 25 additions & 12 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ namespace Modules.Test.TestResultCoordinator.Reports
using global::TestResultCoordinator.Reports;
using Microsoft.Azure.Devices.Edge.ModuleUtil;
using Microsoft.Azure.Devices.Edge.Storage;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Azure.Devices.Edge.Util.Test.Common;
using Moq;
using Xunit;
Expand Down Expand Up @@ -61,7 +62,8 @@ public void TestConstructorSuccess()
actualResults.GetAsyncEnumerator(),
resultType,
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize);
UnmatchedResultsMaxSize,
false);

Assert.Equal(TestDescription, reportGenerator.TestDescription);
Assert.Equal(actualSource, reportGenerator.ActualSource);
Expand Down Expand Up @@ -90,7 +92,8 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("testDescription", ex.Message);
}
Expand All @@ -113,7 +116,8 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId)
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("trackingId", ex.Message);
}
Expand All @@ -136,7 +140,8 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("expectedSource", ex.Message);
}
Expand All @@ -156,7 +161,8 @@ public void TestConstructorThrowsWhenExpectedStoreIsNotProvided()
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.Equal("expectedTestResults", ex.ParamName);
}
Expand All @@ -179,7 +185,8 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("actualSource", ex.Message);
}
Expand All @@ -199,7 +206,8 @@ public void TestConstructorThrowsWhenActualStoreIsNotProvided()
null,
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.Equal("actualTestResults", ex.ParamName);
}
Expand All @@ -222,7 +230,8 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
mockActualStore.Object,
resultType,
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("resultType", ex.Message);
}
Expand All @@ -243,7 +252,8 @@ public void TestConstructorThrowsWhenTestResultComparerIsNotProvided()
mockActualStore.Object,
"resultType1",
null,
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.Equal("testResultComparer", ex.ParamName);
}
Expand All @@ -265,7 +275,8 @@ public void TestConstructorThrowsWhenUnmatchedResultsMaxSizeIsNonPositive(ushort
mockActualStore.Object,
"resultType1",
new SimpleTestOperationResultComparer(),
unmatchedResultsMaxSize));
unmatchedResultsMaxSize,
false));
}

[Fact]
Expand All @@ -289,7 +300,8 @@ public async Task TestCreateReportAsyncWithEmptyResults()
actualResults.GetAsyncEnumerator(),
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize);
UnmatchedResultsMaxSize,
false);

var report = (CountingReport)await reportGenerator.CreateReportAsync();

Expand Down Expand Up @@ -328,7 +340,8 @@ public async Task TestCreateReportAsync(
actualResults.GetAsyncEnumerator(),
resultType,
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize);
UnmatchedResultsMaxSize,
false);

var expectedStoreData = GetStoreData(expectedSource, resultType, expectedStoreValues);
for (int i = 0; i < expectedStoreData.Count; i += batchSize)
Expand Down
25 changes: 19 additions & 6 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/CountingReportTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ namespace Modules.Test.TestResultCoordinator.Reports
using System.Collections.Generic;
using global::TestResultCoordinator.Reports;
using Microsoft.Azure.Devices.Edge.ModuleUtil;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Azure.Devices.Edge.Util.Test.Common;
using Xunit;

Expand All @@ -29,7 +30,9 @@ public void TestConstructorSuccess()
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
});
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>());

Assert.Equal(TestDescription, report.TestDescription);
Assert.Equal("trackingId123", report.TrackingId);
Expand Down Expand Up @@ -70,7 +73,9 @@ public void TestConstructorThrowsWhenTestDescriptionIsNotProvided(string testDes
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
}));
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>()));

Assert.StartsWith("testDescription", ex.Message);
}
Expand All @@ -94,7 +99,9 @@ public void TestConstructorThrowsWhenTrackingIdIsNotProvided(string trackingId)
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
}));
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>()));

Assert.StartsWith("trackingId", ex.Message);
}
Expand All @@ -118,7 +125,9 @@ public void TestConstructorThrowsWhenExpectedSourceIsNotProvided(string expected
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
}));
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>()));

Assert.StartsWith("expectedSource", ex.Message);
}
Expand All @@ -142,7 +151,9 @@ public void TestConstructorThrowsWhenActualSourceIsNotProvided(string actualSour
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
}));
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>()));

Assert.StartsWith("actualSource", ex.Message);
}
Expand All @@ -166,7 +177,9 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
{
new TestOperationResult("expectedSource", "resultType1", "332", new DateTime(2019, 12, 4, 10, 15, 15)),
new TestOperationResult("expectedSource", "resultType1", "734", new DateTime(2019, 12, 4, 10, 15, 18)),
}));
},
Option.None<EventHubSpecificReportComponents>(),
Option.None<DateTime>()));

Assert.StartsWith("resultType", ex.Message);
}
Expand Down
6 changes: 4 additions & 2 deletions test/modules/Modules.Test/TestResultCoordinator/Reports/TwinCountingReportGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ public void TestConstructorThrowsWhenResultTypeIsNotProvided(string resultType)
mockActualStore.Object,
resultType,
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize));
UnmatchedResultsMaxSize,
false));

Assert.StartsWith("resultType", ex.Message);
}
Expand Down Expand Up @@ -258,7 +259,8 @@ public async Task TestCreateReportAsyncWithEmptyResults()
actualResults.GetAsyncEnumerator(),
"resultType1",
new SimpleTestOperationResultComparer(),
UnmatchedResultsMaxSize);
UnmatchedResultsMaxSize,
false);

var report = (CountingReport)await reportGenerator.CreateReportAsync();

Expand Down
34 changes: 32 additions & 2 deletions test/modules/Modules.Test/TestResultCoordinator/TestReportUtilTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ namespace Modules.Test.TestResultCoordinator
using global::TestResultCoordinator.Reports.DirectMethod.LongHaul;
using global::TestResultCoordinator.Reports.EdgeHubRestartTest;
using Microsoft.Azure.Devices.Edge.ModuleUtil;
using Microsoft.Azure.Devices.Edge.Util;
using Microsoft.Azure.Devices.Edge.Util.Test.Common;
using Microsoft.Extensions.Logging;
using Moq;
Expand Down Expand Up @@ -78,7 +79,7 @@ public async Task TestGenerateTestResultReportsAsync_ReportGeneration(
var mockTestReportGeneratorFactory = new Mock<ITestReportGeneratorFactory>();

string trackingId = "fakeTrackingId";
var countingReportMetadata = new CountingReportMetadata(TestDescription, "CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport);
var countingReportMetadata = new CountingReportMetadata(TestDescription, "CountingExpectedSource", "CountingAcutalSource", TestOperationResultType.Messages, TestReportType.CountingReport, false);
var twinCountingReportMetadata = new TwinCountingReportMetadata(TestDescription, "TwinExpectedSource", "TwinActualSource", TestReportType.TwinCountingReport, TwinTestPropertyType.Desired);
var deploymentReportMetadata = new DeploymentTestReportMetadata(TestDescription, "DeploymentExpectedSource", "DeploymentActualSource");
var directMethodConnectivityReportMetadata = new DirectMethodConnectivityReportMetadata(TestDescription, "DirectMethodSenderSource", new TimeSpan(0, 0, 0, 0, 5), "DirectMethodReceiverSource");
Expand Down Expand Up @@ -228,6 +229,35 @@ public void ParseReportMetadataList_ParseCountingReportMetadata()
Assert.Equal(TestReportType.CountingReport, reportMetadata.TestReportType);
Assert.Equal("loadGen1.send", reportMetadata.ExpectedSource);
Assert.Equal("relayer1.receive", reportMetadata.ActualSource);
Assert.False(reportMetadata.LongHaulEventHubMode);
}

[Fact]
public void ParseReportMetadataList_ParseCountingReportEventHubMetadata()
{
const string testDataJson =
@"{
""reportMetadata"": {
""TestDescription"": ""messages | local | amqp"",
""TestReportType"": ""CountingReport"",
""LongHaulEventHubMode"": ""true"",
""TestOperationResultType"": ""Messages"",
""ExpectedSource"": ""loadGen1.send"",
""ActualSource"": ""relayer1.receive""
}
}";

List<ITestReportMetadata> results = TestReportUtil.ParseReportMetadataJson(testDataJson, new Mock<ILogger>().Object);

Assert.Single(results);
var reportMetadata = results[0] as CountingReportMetadata;
Assert.NotNull(reportMetadata);
Assert.Equal("messages | local | amqp", reportMetadata.TestDescription);
Assert.Equal(TestOperationResultType.Messages, reportMetadata.TestOperationResultType);
Assert.Equal(TestReportType.CountingReport, reportMetadata.TestReportType);
Assert.Equal("loadGen1.send", reportMetadata.ExpectedSource);
Assert.Equal("relayer1.receive", reportMetadata.ActualSource);
Assert.True(reportMetadata.LongHaulEventHubMode);
}

[Fact]
Expand Down Expand Up @@ -511,7 +541,7 @@ private Task<ITestResultReport> MockTestResultReport(bool throwException)
{
if (!throwException)
{
return Task.FromResult<ITestResultReport>(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, new List<TestOperationResult>()));
return Task.FromResult<ITestResultReport>(new CountingReport("mock", "mock", "mock", "mock", "mock", 23, 21, 12, new List<TestOperationResult>(), Option.None<EventHubSpecificReportComponents>(), Option.None<DateTime>()));
}

return Task.FromException<ITestResultReport>(new ApplicationException("Inject exception for testing"));
Expand Down
Loading

0 comments on commit 51f1102

Please sign in to comment.