Skip to content

Commit f97c18b

Browse files
authored
Tool to stress tiered compilation (and PGO) (#57839)
Add a test wrapper that executes a test's `Main` method so that more methods are able to reach Tier1. In particular this is useful in trying to increase test coverage for dynamic PGO. Uses various tricks to tolerate tests that are not amenable to being run in this manner: * failures on the first or second iteration are ignored * long running tests are run for fewer iterations * some tests that become flaky after 2 iterations are excluded Also add support for running tests in this mode to various CI scripts.
1 parent 07b13f4 commit f97c18b

File tree

12 files changed

+335
-0
lines changed

12 files changed

+335
-0
lines changed

eng/Subsets.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@
216216

217217
<ItemGroup Condition="$(_subset.Contains('+clr.tools+'))">
218218
<ProjectToBuild Include="$(CoreClrProjectRoot)tools\runincontext\runincontext.csproj;
219+
$(CoreClrProjectRoot)tools\tieringtest\tieringtest.csproj;
219220
$(CoreClrProjectRoot)tools\r2rdump\R2RDump.csproj;
220221
$(CoreClrProjectRoot)tools\dotnet-pgo\dotnet-pgo.csproj;
221222
$(CoreClrProjectRoot)tools\r2rtest\R2RTest.csproj" Category="clr" Condition="'$(DotNetBuildFromSource)' != 'true'"/>

eng/pipelines/common/templates/runtimes/run-test-job.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ parameters:
1616
stagedBuild: false
1717
displayNameArgs: ''
1818
runInUnloadableContext: false
19+
tieringTest: false
1920
runtimeVariant: ''
2021
variables: {}
2122
pool: ''
@@ -359,6 +360,7 @@ jobs:
359360

360361
compositeBuildMode: ${{ parameters.compositeBuildMode }}
361362
runInUnloadableContext: ${{ parameters.runInUnloadableContext }}
363+
tieringTest: ${{ parameters.tieringTest }}
362364

363365
${{ if eq(variables['System.TeamProject'], 'internal') }}:
364366
# Access token variable for internal project from the

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ parameters:
2121
compositeBuildMode: false
2222
helixProjectArguments: ''
2323
runInUnloadableContext: ''
24+
tieringTest: ''
2425
longRunningGcTests: ''
2526
gcSimulatorTests: ''
2627
runtimeFlavorDisplayName: 'CoreCLR'
@@ -49,6 +50,7 @@ steps:
4950
_RunCrossGen2: ${{ parameters.runCrossGen2 }}
5051
_CompositeBuildMode: ${{ parameters.compositeBuildMode }}
5152
_RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
53+
_TieringTest: ${{ parameters.runInUnloadableContext }}
5254
_LongRunningGcTests: ${{ parameters.longRunningGcTests }}
5355
_GcSimulatorTests: ${{ parameters.gcSimulatorTests }}
5456
_Scenarios: ${{ join(',', parameters.scenarios) }}
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
trigger: none
2+
3+
schedules:
4+
- cron: "0 18 * * 6,0"
5+
displayName: Sat and Sun at 10:00 AM (UTC-8:00)
6+
branches:
7+
include:
8+
- main
9+
always: true
10+
11+
jobs:
12+
13+
- template: /eng/pipelines/common/platform-matrix.yml
14+
parameters:
15+
jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml
16+
buildConfig: checked
17+
platforms:
18+
- Linux_x64
19+
- windows_x64
20+
- windows_x86
21+
- CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
22+
jobParameters:
23+
testGroup: outerloop
24+
25+
- template: /eng/pipelines/common/platform-matrix.yml
26+
parameters:
27+
jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
28+
buildConfig: checked
29+
platforms:
30+
- CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
31+
jobParameters:
32+
testGroup: outerloop
33+
liveLibrariesBuildConfig: Release
34+
35+
- template: /eng/pipelines/common/platform-matrix.yml
36+
parameters:
37+
jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
38+
buildConfig: checked
39+
platforms:
40+
- Linux_x64
41+
- windows_x64
42+
- windows_x86
43+
helixQueueGroup: ci
44+
helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
45+
jobParameters:
46+
testGroup: outerloop
47+
tieringTest: true
48+
displayNameArgs: TieringTest
49+
liveLibrariesBuildConfig: Release
Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Diagnostics;
6+
using System.IO;
7+
using System.Reflection;
8+
using System.Runtime.Loader;
9+
using System.Threading;
10+
11+
class Program
12+
{
13+
// Normalybehavior is to not print anything on success.
14+
//
15+
static bool verbose = false;
16+
17+
// Repeatedly execute a test case's Main method so that methods jitted
18+
// by the test can get rejitted at Tier1.
19+
//
20+
static int Main(string[] args)
21+
{
22+
string testAssemblyName = args[0];
23+
24+
// We'll stop iterating if total test time exceeds this value (in ms).
25+
//
26+
int timeout = 10_000;
27+
28+
// Some tests return zero for success.
29+
//
30+
int expectedResult = 100;
31+
32+
string[][] zeroReturnValuePatterns = {
33+
new string[] { "JIT", "jit64", "regress", "vsw", "102754", "test1"},
34+
new string[] { "JIT", "Regression", "CLR-x86-JIT", "V1-M09", "b16102", "b16102"},
35+
};
36+
37+
foreach (string[] pattern in zeroReturnValuePatterns)
38+
{
39+
if (testAssemblyName.IndexOf(Path.Join(pattern)) > 0)
40+
{
41+
expectedResult = 0;
42+
break;
43+
}
44+
}
45+
46+
// Exclude tests that seem to be incompatible.
47+
// Todo: root cause these and fix tests if possible.
48+
//
49+
// With Full PGO:
50+
// RngchkStress2_o can hit a jit assert: '!m_failedToConverge' in 'SimpleArray_01.Test:Test1()' during 'Profile incorporation'
51+
// GitHub_25027 can hit a jit assert: 'verCurrentState.esStackDepth == 0' in 'X:Main():int' during 'Morph - Inlining'
52+
//
53+
string[][] exclusionPatterns = {
54+
new string[] { "JIT", "jit64", "opt", "cse", "VolatileTest_op" },
55+
new string[] { "JIT", "jit64", "opt", "rngchk", "ArrayWithThread_o" },
56+
new string[] { "baseservices", "threading", "threadstatic", "ThreadStatic01" },
57+
new string[] { "GC", "Scenarios", "ReflectObj", "reflectobj"},
58+
new string[] { "baseservices", "threading", "mutex", "openexisting", "openmutexpos4"},
59+
new string[] { "GC", "Scenarios", "NDPin", "ndpinfinal"},
60+
new string[] { "JIT", "Regression", "JitBlue", "GitHub_4044", "GitHub_4044"},
61+
new string[] { "JIT", "HardwareIntrinsics", "X86", "Regression", "GitHub_21666", "GitHub_21666_ro"},
62+
new string[] { "Interop", "NativeLibrary", "API", "NativeLibraryTests"},
63+
new string[] { "baseservices", "compilerservices", "FixedAddressValueType", "FixedAddressValueType"},
64+
new string[] { "GC", "LargeMemory", "API", "gc", "gettotalmemory" },
65+
66+
new string[] { "JIT", "jit64", "opt", "rngchk", "RngchkStress2_o" },
67+
new string[] { "JIT", "Regression", "JitBlue", "GitHub_25027", "GitHub_25027" },
68+
};
69+
70+
foreach (string[] pattern in exclusionPatterns)
71+
{
72+
if (testAssemblyName.IndexOf(Path.Join(pattern)) > 0)
73+
{
74+
if (verbose)
75+
{
76+
Console.WriteLine($"Test {Path.Join(pattern)} excluded; marked as incompatible");
77+
}
78+
return expectedResult;
79+
}
80+
}
81+
82+
AssemblyLoadContext alc = AssemblyLoadContext.Default;
83+
Assembly testAssembly = alc.LoadFromAssemblyPath(testAssemblyName);
84+
MethodInfo main = testAssembly.EntryPoint;
85+
86+
if (main == null)
87+
{
88+
Console.WriteLine($"Can't find entry point in {Path.GetFileName(args[0])}");
89+
return -1;
90+
}
91+
92+
string[] mainArgs = new string[args.Length - 1];
93+
Array.Copy(args, 1, mainArgs, 0, mainArgs.Length);
94+
95+
// Console.WriteLine($"Found entry point {main.Name} in {Path.GetFileName(args[0])}");
96+
97+
// See if main wants any args.
98+
//
99+
var mainParams = main.GetParameters();
100+
101+
int result = 0;
102+
103+
// Repeatedly invoke main to get things to pass through Tier0 and rejit at Tier1
104+
//
105+
int warmup = 40;
106+
int final = 5;
107+
int total = warmup + final;
108+
int i = 0;
109+
int sleepInterval = 5;
110+
Stopwatch s = new Stopwatch();
111+
s.Start();
112+
113+
// We might decide to give up on this test, for reasons.
114+
//
115+
// If the test fails at iteration 0, assume it's incompatible with the way we're running it
116+
// and don't report as a failure.
117+
//
118+
// If the test fails at iteration 1, assume it's got some bit of state that carries over
119+
// from one call to main to the next, and so don't report it as failure.
120+
//
121+
// If the test takes too long, just give up on iterating it.
122+
//
123+
bool giveUp = false;
124+
125+
try
126+
{
127+
128+
for (; i < warmup && !giveUp; i++)
129+
{
130+
if (mainParams.Length == 0)
131+
{
132+
result = (int)main.Invoke(null, new object[] { });
133+
}
134+
else
135+
{
136+
result = (int)main.Invoke(null, new object[] { mainArgs });
137+
}
138+
139+
if (result != expectedResult)
140+
{
141+
if (i < 2)
142+
{
143+
Console.WriteLine($"[tieringtest] test failed at iteration {i} with result {result}. Test is likely incompatible.");
144+
result = expectedResult;
145+
}
146+
else
147+
{
148+
Console.WriteLine($"[tieringtest] test failed at iteration {i}: {result} (expected {expectedResult})");
149+
}
150+
giveUp = true;
151+
break;
152+
}
153+
154+
// Don't iterate if test is running long.
155+
//
156+
if (s.ElapsedMilliseconds > timeout)
157+
{
158+
Console.WriteLine($"[tieringtest] test running long ({s.ElapsedMilliseconds / (i + 1)} ms/iteration). Giving up at iteration {i}");
159+
giveUp = true;
160+
break;
161+
}
162+
163+
// Give TC a chance to jit some things.
164+
//
165+
Thread.Sleep(sleepInterval);
166+
}
167+
168+
for (; i < total && !giveUp; i++)
169+
{
170+
if (mainParams.Length == 0)
171+
{
172+
result = (int)main.Invoke(null, new object[] { });
173+
}
174+
else
175+
{
176+
result = (int)main.Invoke(null, new object[] { mainArgs });
177+
}
178+
179+
if (result != expectedResult)
180+
{
181+
Console.WriteLine($"[tieringtest] failed at iteration {i}: {result} (expected {expectedResult})");
182+
giveUp = true;
183+
break;
184+
}
185+
186+
// Don't iterate if test is running long.
187+
//
188+
if (s.ElapsedMilliseconds > timeout)
189+
{
190+
Console.WriteLine($"[tieringtest] test running long ({s.ElapsedMilliseconds / (i + 1)} ms/iteration). Giving up at iteration {i}");
191+
giveUp = true;
192+
break;
193+
}
194+
}
195+
196+
if (result == expectedResult)
197+
{
198+
if (verbose)
199+
{
200+
Console.WriteLine($"[tieringtest] ran {total} test iterations sucessfully");
201+
}
202+
}
203+
}
204+
catch (Exception e)
205+
{
206+
if (i < 2)
207+
{
208+
if (verbose)
209+
{
210+
Console.WriteLine($"[tieringtest] test failed at iteration {i} with exception {e.Message}. Test is likely incompatible.");
211+
}
212+
result = expectedResult;
213+
}
214+
else
215+
{
216+
Console.WriteLine($"[tieringtest] test failed at iteration {i} with exception {e.Message}");
217+
result = -1;
218+
}
219+
}
220+
221+
return result;
222+
}
223+
}
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
<TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
5+
<RuntimeFrameworkVersion>$(MicrosoftNETCoreAppVersion)</RuntimeFrameworkVersion>
6+
<UseAppHost>false</UseAppHost>
7+
<CLRTestKind>BuildOnly</CLRTestKind>
8+
<OutputPath>$(RuntimeBinDir)</OutputPath>
9+
</PropertyGroup>
10+
</Project>

src/tests/Common/helixpublishwitharcade.proj

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
LongRunningGCTests=$(_LongRunningGCTests);
3232
GcSimulatorTests=$(_GcSimulatorTests);
3333
RunInUnloadableContext=$(_RunInUnloadableContext);
34+
TieringTest=$(_TieringTest);
3435
TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes);
3536
TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes);
3637
RuntimeVariant=$(_RuntimeVariant);
@@ -197,6 +198,8 @@
197198
<Copy SourceFiles="@(_XUnitConsoleRunnerFiles)" DestinationFolder="$(CoreRootDirectory)\xunit" />
198199
<Copy SourceFiles="$(MSBuildThisFileDirectory)scripts\runincontext.cmd" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' == 'true' and '$(_RunInUnloadableContext)' == 'true'" />
199200
<Copy SourceFiles="$(MSBuildThisFileDirectory)scripts/runincontext.sh" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' != 'true' and '$(_RunInUnloadableContext)' == 'true'" />
201+
<Copy SourceFiles="$(MSBuildThisFileDirectory)scripts\tieringtest.cmd" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' == 'true' and '$(_TieringTest)' == 'true'" />
202+
<Copy SourceFiles="$(MSBuildThisFileDirectory)scripts/tieringtest.sh" DestinationFolder="$(CoreRootDirectory)" Condition=" '$(TestWrapperTargetsWindows)' != 'true' and '$(_TieringTest)' == 'true'" />
200203
</Target>
201204

202205
<Target Name="CreateTestEnvFiles">
@@ -273,6 +276,7 @@
273276
<HelixPreCommand Include="set RunningGCSimulatorTests=1" Condition=" '$(GcSimulatorTests)' == 'true' " />
274277
<HelixPreCommand Include="set RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />
275278
<HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\runincontext.cmd" Condition=" '$(RunInUnloadableContext)' == 'true' " />
279+
<HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\tieringtest.cmd" Condition=" '$(TieringTest)' == 'true' " />
276280
<HelixPreCommand Include="set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\$(TestEnvFileName)" />
277281
<HelixPreCommand Include="set __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
278282
<HelixPreCommand Include="set __CollectDumps=1" />
@@ -293,6 +297,7 @@
293297
<HelixPreCommand Include="export RunningGCSimulatorTests=1" Condition=" '$(GcSimulatorTests)' == 'true' " />
294298
<HelixPreCommand Include="export RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />
295299
<HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/runincontext.sh" Condition=" '$(RunInUnloadableContext)' == 'true' " />
300+
<HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/tieringtest.sh" Condition=" '$(TieringTest)' == 'true' " />
296301
<HelixPreCommand Include="export __TestEnv=$HELIX_WORKITEM_PAYLOAD/$(TestEnvFileName)" />
297302
<HelixPreCommand Include="export __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
298303
<HelixPreCommand Include="export __CollectDumps=1" />
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
@rem This script is a bridge that allows .cmd files of individual tests to run repeatedly so methods
2+
@rem can tier up.
3+
@rem
4+
@rem To use this script, set the CLRCustomTestLauncher environment variable to the full path of this script.
5+
6+
set CORE_LIBRARIES=%1
7+
%_DebuggerFullPath% "%CORE_ROOT%\corerun.exe" "%CORE_ROOT%\tieringtest.dll" %1%2 %3 %4 %5 %6 %7 %8 %9
8+
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
# This script is a bridge that allows .cmd files of individual tests to run the respective test executables
3+
# repeatedly so that more methods get rejitted at Tier1
4+
#
5+
# To use this script, set the CLRCustomTestLauncher environment variable to the full path of this script.
6+
7+
export CORE_LIBRARIES=$1
8+
$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/tieringtest.dll" $1$2 "${@:3}"

0 commit comments

Comments
 (0)