Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Improve Windows test build performance by building projects in groups - 4x less memory 326% faster build. #17161

Merged
merged 1 commit into from
Mar 28, 2018
Merged

Improve Windows test build performance by building projects in groups - 4x less memory 326% faster build. #17161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 42 additions & 11 deletions build-test.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ set __ZipTests=
set __TargetsWindows=1
set __DoCrossgen=

@REM CMD has a nasty habit of eating "=" on the argument list, so passing:
@REM -priority=1
@REM appears to CMD parsing as "-priority 1". Handle -priority specially to avoid problems,
@REM and allow the "-priority=1" syntax.
set __Priority=0

:Arg_Loop
if "%1" == "" goto ArgsDone

Expand All @@ -75,6 +81,7 @@ if /i "%1" == "crossgen" (set __DoCrossgen=1&set processedArgs=!pro
if /i "%1" == "runtimeid" (set __RuntimeId=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
if /i "%1" == "targetsNonWindows" (set __TargetsWindows=0&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
if /i "%1" == "Exclude" (set __Exclude=%2&set processedArgs=!processedArgs! %1 %2&shift&shift&goto Arg_Loop)
if /i "%1" == "-priority" (set __Priority=%2&shift&set processedArgs=!processedArgs! %1=%2&shift&goto Arg_Loop)

if [!processedArgs!]==[] (
set __UnprocessedBuildArgs=%__args%
Expand All @@ -87,6 +94,9 @@ if [!processedArgs!]==[] (

:ArgsDone

@REM Special handling for -priority=N argument.
if %__Priority% GTR 0 (set "__UnprocessedBuildArgs=!__UnprocessedBuildArgs! -priority=%__Priority%")

if defined __BuildAgainstPackagesArg (
if not defined __RuntimeID (
echo %__MsgPrefix%Error: When building against packages, you must supply a target Runtime ID.
Expand Down Expand Up @@ -267,22 +277,43 @@ if not defined VSINSTALLDIR (
echo %__MsgPrefix%Error: build-test.cmd should be run from a Visual Studio Command Prompt. Please see https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md for build instructions.
exit /b 1
)

set __AppendToLog=false
set __BuildLogRootName=Tests_Managed
set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.log
set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.wrn
set __BuildErr=%__LogsDir%\%__BuildLogRootName%_%__BuildOS%__%__BuildArch%__%__BuildType%.err
set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%"
set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%"
set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%"

call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs%
if errorlevel 1 (
echo %__MsgPrefix%Error: build failed. Refer to the build log files for details:
echo %__BuildLog%
echo %__BuildWrn%
echo %__BuildErr%
exit /b 1
REM Execute msbuild test build in stages - workaround for excessive data retention in MSBuild ConfigCache
REM See https://github.com/Microsoft/msbuild/issues/2993

set __SkipPackageRestore=false
set __SkipTargetingPackBuild=false
set __BuildLoopCount=2
set __TestGroupToBuild=1

if %__Priority% GTR 0 (set __BuildLoopCount=16&set __TestGroupToBuild=2)
echo %__MsgPrefix%Building tests group %__TestGroupToBuild% with %__BuildLoopCount% subgroups

for /l %%G in (1, 1, %__BuildLoopCount%) do (

set __msbuildLog=/flp:Verbosity=normal;LogFile="%__BuildLog%";Append=!__AppendToLog!
set __msbuildWrn=/flp1:WarningsOnly;LogFile="%__BuildWrn%";Append=!__AppendToLog!
set __msbuildErr=/flp2:ErrorsOnly;LogFile="%__BuildErr%";Append=!__AppendToLog!

set TestBuildSlice=%%G
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any reason you aren't passing this as a property instead of depending on environment variable?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Both are possible but for me it is more clear code if it remains environment variable which like property passed on MSBuild commandline will be available during project build, however, commandline is already 233 characters long so I decided to not pass it that way while having very same effect.

If that style is not expected in build scripts I am open to any changes which will suit the team

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was looking at the command line to try and figure out how you split these and I didn't find it and then realized it was happening because of the env variable. I generally try to avoid env variables in build scripts/projects because it is harder to reproduce and diagnose. In this case I don't have a strong opinion though it is just a general principle.

Copy link
Author

@4creators 4creators Mar 27, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If it is general principle and you have had problem decoding usage of variable than I think it should be passed via command line. Would it be fine to fixup this in follow PR?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would it be fine to fixup this in follow PR?

Yes that is fine with me.

call "%__ProjectDir%\run.cmd" build -Project=%__ProjectDir%\tests\build.proj -MsBuildLog=!__msbuildLog! -MsBuildWrn=!__msbuildWrn! -MsBuildErr=!__msbuildErr! %__RunArgs% %__BuildAgainstPackagesArg% %__unprocessedBuildArgs%

if errorlevel 1 (
echo %__MsgPrefix%Error: build failed. Refer to the build log files for details:
echo %__BuildLog%
echo %__BuildWrn%
echo %__BuildErr%
exit /b 1
)

set __SkipPackageRestore=true
set __SkipTargetingPackBuild=true
set __AppendToLog=true
)

REM Prepare the Test Drop
Expand Down
10 changes: 5 additions & 5 deletions tests/build.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
</ItemGroup>

<Import Project="dir.targets" />

<Import Project="dir.traversal.targets" />

<PropertyGroup>
<TraversalBuildDependsOn>
BatchRestorePackages;
Expand All @@ -34,15 +34,15 @@
<RestoreProjects Include="$(MSBuildThisFileDirectory)src\TestWrappersConfig\TestWrappersConfig.csproj" />
</ItemGroup>

<Target Name="BuildTargetingPack" AfterTargets="BatchRestorePackages">
<Target Name="BuildTargetingPack" AfterTargets="BatchRestorePackages" Condition="$(__SkipTargetingPackBuild) != 'true'">
<Message Text="Building Targeting Pack" Importance="High" />
<Error Text="BuildOS has not been specified. Please do that then run build again." Condition="'$(BuildOS)' == 'AnyOS'" />
<MSBuild Projects="$(MSBuildThisFileDirectory)\src\Common\external\external.depproj" />
</Target>

<Target Name="BatchRestorePackages">
<Target Name="BatchRestorePackages" Condition="$(__SkipPackageRestore) != 'true'">
<Message Importance="High" Text="[$([System.DateTime]::Now.ToString('HH:mm:ss.ff'))] Restoring all packages..." />

<!-- restore all csproj's with PackageReferences in one pass -->
<MSBuild Projects="build.proj"
Properties="RestoreProj=%(RestoreProjects.Identity)"
Expand Down
214 changes: 212 additions & 2 deletions tests/src/dirs.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,225 @@
<DisabledProjects Include="Loader\classloader\generics\regressions\DD117522\Test.csproj" />
<DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
</ItemGroup>

<ItemGroup>

<!-- Unix builds do not support subgroups -->
<ItemGroup Condition="$(__BuildOS) != 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<!-- Test build is divided in slices which can be created within Test Group
Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->
<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<!-- Test build is divided in slices which can be created within Test Group
Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->
<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
<Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Common\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="baseservices\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Common\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
<Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="CoreMangLib\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
<Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="GC\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Interop\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="E*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="GC\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="Interop\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
<Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\C*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Directed\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
<Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\C*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Directed\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
<Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\*Intrinsics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Generics\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\*Intrinsics\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
<Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\IL_Conformance\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
<Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\jit64\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
<Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
<Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
<Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Performance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\S*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\opt\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\Performance\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="JIT\S*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
<Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
<Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
<Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
<Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
<Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="p*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="r*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="s*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="t*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="m*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="p*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="r*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="s*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="t*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

</Target>

<Import Project="..\dir.traversal.targets" />
Expand Down