Skip to content

Fix flakiness/timeout in gRPC template tests #19982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Mar 25, 2020
Merged

Fix flakiness/timeout in gRPC template tests #19982

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
dd80b39
Fix flakiness/timeout in gRPC template tests
Mar 19, 2020
6c130c1
Debug add test logging for hangs
Mar 23, 2020
b6f7b73
Log to relative dir
Mar 23, 2020
cdf7de3
Revert "Log to relative dir"
Mar 23, 2020
051bfc5
Maybe it's exiting too early
Mar 23, 2020
00a4267
Test theory that the hang was caused by runtime changes
Mar 25, 2020
b6c16df
Revert "Test theory that the hang was caused by runtime changes"
Mar 25, 2020
108eff3
Test theory that the hang was actually fixed
Mar 25, 2020
a8d6b78
Test again
Mar 25, 2020
dd93d2e
Test again
Mar 25, 2020
3bf6852
Add fix back
Mar 25, 2020
b50b337
Copy pasta
Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions src/ProjectTemplates/test/GrpcTemplateTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Testing;
Expand All @@ -24,11 +25,16 @@ public GrpcTemplateTest(ProjectFactoryFixture projectFactory, ITestOutputHelper
public ProjectFactoryFixture ProjectFactory { get; }
public ITestOutputHelper Output { get; }

[ConditionalFact(Skip = "This test run for over an hour")]
[ConditionalFact]
[SkipOnHelix("Not supported queues", Queues = "Windows.7.Amd64;Windows.7.Amd64.Open;OSX.1014.Amd64;OSX.1014.Amd64.Open")]
[QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/19716")]
public async Task GrpcTemplate()
{
// Setup AssemblyTestLog
var assemblyLog = AssemblyTestLog.Create(Assembly.GetExecutingAssembly(), baseDirectory: Project.ArtifactsLogDir);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we doing this here instead of the normal way?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Glad you asked. Normally, we would be using LoggedTest but that requires the use of AspNetTestFramework xunit test framework. This cannot be used in template tests since they already use their own test framework:

aspnetcore/src/ProjectTemplates/test/AssemblyInfo.AssemblyFixtures.cs

Line 8 in 9f1a978

[assembly: TestFramework("Microsoft.AspNetCore.E2ETesting.XunitTestFrameworkWithAssemblyFixture", "ProjectTemplates.Tests")]
and so we have to do it the oldschool way.

I think it's possible to combine the logic of the two xunit frameworks but that's a rather large undertaking that doesn't seem appropriate here.

using var testLog = assemblyLog.StartTestLog(Output, nameof(GrpcTemplateTest), out var loggerFactory);
var logger = loggerFactory.CreateLogger("TestLogger");

Project = await ProjectFactory.GetOrCreateProject("grpc", Output);

var createResult = await Project.RunDotNetNewAsync("grpc");
Expand All @@ -40,18 +46,24 @@ public async Task GrpcTemplate()
var buildResult = await Project.RunDotNetBuildAsync();
Assert.True(0 == buildResult.ExitCode, ErrorMessages.GetFailedProcessMessage("build", Project, buildResult));

using (var serverProcess = Project.StartBuiltProjectAsync())
var isOsx = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
var isWindowsOld = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2);
var unsupported = isOsx || isWindowsOld;

using (var serverProcess = Project.StartBuiltProjectAsync(hasListeningUri: !unsupported, logger: logger))
{
// These templates are HTTPS + HTTP/2 only which is not supported on Mac due to missing ALPN support.
// https://github.com/dotnet/aspnetcore/issues/11061
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
if (isOsx)
{
serverProcess.Process.WaitForExit(assertSuccess: false);
Assert.True(serverProcess.Process.HasExited, "built");
Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on macOS due to missing ALPN support.",
ErrorMessages.GetFailedProcessMessageOrEmpty("Run built service", Project, serverProcess.Process));
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2))
else if (isWindowsOld)
{
serverProcess.Process.WaitForExit(assertSuccess: false);
Assert.True(serverProcess.Process.HasExited, "built");
Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on Windows 7 due to missing ALPN support.",
ErrorMessages.GetFailedProcessMessageOrEmpty("Run built service", Project, serverProcess.Process));
Expand All @@ -64,18 +76,20 @@ public async Task GrpcTemplate()
}
}

using (var aspNetProcess = Project.StartPublishedProjectAsync())
using (var aspNetProcess = Project.StartPublishedProjectAsync(hasListeningUri: !unsupported))
{
// These templates are HTTPS + HTTP/2 only which is not supported on Mac due to missing ALPN support.
// https://github.com/dotnet/aspnetcore/issues/11061
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
if (isOsx)
{
aspNetProcess.Process.WaitForExit(assertSuccess: false);
Assert.True(aspNetProcess.Process.HasExited, "published");
Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on macOS due to missing ALPN support.",
ErrorMessages.GetFailedProcessMessageOrEmpty("Run published service", Project, aspNetProcess.Process));
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && Environment.OSVersion.Version < new Version(6, 2))
else if (isWindowsOld)
{
aspNetProcess.Process.WaitForExit(assertSuccess: false);
Assert.True(aspNetProcess.Process.HasExited, "published");
Assert.Contains("System.NotSupportedException: HTTP/2 over TLS is not supported on Windows 7 due to missing ALPN support.",
ErrorMessages.GetFailedProcessMessageOrEmpty("Run published service", Project, aspNetProcess.Process));
Expand Down
12 changes: 11 additions & 1 deletion src/ProjectTemplates/test/Helpers/AspNetProcess.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using Microsoft.AspNetCore.Internal;
using Microsoft.AspNetCore.Server.IntegrationTesting;
using Microsoft.Extensions.CommandLineUtils;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using OpenQA.Selenium;
using OpenQA.Selenium.Edge;
Expand All @@ -38,7 +39,8 @@ public AspNetProcess(
string dllPath,
IDictionary<string, string> environmentVariables,
bool published = true,
bool hasListeningUri = true)
bool hasListeningUri = true,
ILogger logger = null)
{
_output = output;
_httpClient = new HttpClient(new HttpClientHandler()
Expand All @@ -57,10 +59,18 @@ public AspNetProcess(
output.WriteLine("Running ASP.NET application...");

var arguments = published ? $"exec {dllPath}" : "run";

logger?.LogInformation($"AspNetProcess - process: {DotNetMuxer.MuxerPathOrDefault()} arguments: {arguments}");

Process = ProcessEx.Run(output, workingDirectory, DotNetMuxer.MuxerPathOrDefault(), arguments, envVars: environmentVariables);

logger?.LogInformation("AspNetProcess - process started");

if (hasListeningUri)
{
logger?.LogInformation("AspNetProcess - Getting listening uri");
ListeningUri = GetListeningUri(output) ?? throw new InvalidOperationException("Couldn't find the listening URL.");
logger?.LogInformation($"AspNetProcess - Got {ListeningUri.ToString()}");
}
}

Expand Down
5 changes: 3 additions & 2 deletions src/ProjectTemplates/test/Helpers/Project.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using System.Threading.Tasks;
using Microsoft.AspNetCore.Internal;
using Microsoft.Extensions.CommandLineUtils;
using Microsoft.Extensions.Logging;
using Xunit;
using Xunit.Abstractions;
using Xunit.Sdk;
Expand Down Expand Up @@ -207,7 +208,7 @@ internal AspNetProcess StartPublishedClientAsync()
return new AspNetProcess(Output, TemplateClientReleaseDir, projectDll, environment);
}

internal AspNetProcess StartBuiltProjectAsync(bool hasListeningUri = true)
internal AspNetProcess StartBuiltProjectAsync(bool hasListeningUri = true, ILogger logger = null)
{
var environment = new Dictionary<string, string>
{
Expand All @@ -220,7 +221,7 @@ internal AspNetProcess StartBuiltProjectAsync(bool hasListeningUri = true)
};

var projectDll = Path.Combine(TemplateBuildDir, $"{ProjectName}.dll");
return new AspNetProcess(Output, TemplateOutputDir, projectDll, environment, hasListeningUri: hasListeningUri);
return new AspNetProcess(Output, TemplateOutputDir, projectDll, environment, hasListeningUri: hasListeningUri, logger: logger);
}

internal AspNetProcess StartPublishedProjectAsync(bool hasListeningUri = true)
Expand Down
10 changes: 10 additions & 0 deletions src/ProjectTemplates/test/ProjectTemplates.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@
<ProjectReference Include="../Web.Spa.ProjectTemplates/Microsoft.DotNet.Web.Spa.ProjectTemplates.csproj" ReferenceOutputAssembly="false" />
</ItemGroup>

<PropertyGroup>
<PreserveExistingLogsInOutput Condition="'$(PreserveExistingLogsInOutput)' == '' AND '$(ContinuousIntegrationBuild)' == 'true'">true</PreserveExistingLogsInOutput>
<PreserveExistingLogsInOutput Condition="'$(PreserveExistingLogsInOutput)' == ''">false</PreserveExistingLogsInOutput>
</PropertyGroup>

<ItemGroup>
<AssemblyAttribute Include="System.Reflection.AssemblyMetadataAttribute">
<_Parameter1>DotNetEfFullPath</_Parameter1>
Expand All @@ -69,6 +74,11 @@
<_Parameter1>ContinuousIntegrationBuild</_Parameter1>
<_Parameter2>true</_Parameter2>
</AssemblyAttribute>
<AssemblyAttribute Include="Microsoft.AspNetCore.Testing.TestFrameworkFileLoggerAttribute">
<_Parameter1>$(PreserveExistingLogsInOutput)</_Parameter1>
<_Parameter2>$(TargetFramework)</_Parameter2>
<_Parameter3></_Parameter3>
</AssemblyAttribute>
</ItemGroup>

<Target Name="PrepareForTest" BeforeTargets="CoreCompile" Condition="$(DesignTimeBuild) != true">
Expand Down