Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Split unix test builds in slices #17785

Merged
merged 2 commits into from
May 4, 2018
Merged

Split unix test builds in slices #17785

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e04ea31
Split unix test builds in slices
sdmaclea Apr 25, 2018
223750c
Address review feedback
sdmaclea May 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 70 additions & 17 deletions build-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,28 +295,75 @@ build_Tests_internal()
__BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
__BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
__BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""

# Generate build command
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
# Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
# See https://github.com/Microsoft/msbuild/issues/2993

echo "Building step '$stepName' via $buildCommand"
# __SkipPackageRestore and __SkipTargetingPackBuild used to control build by tests/src/dirs.proj
export __SkipPackageRestore=false
export __SkipTargetingPackBuild=false
export __BuildLoopCount=2
export __TestGroupToBuild=1
__AppendToLog=false

# Invoke MSBuild
eval $buildCommand
if [ -n __priority1 ]; then
export __BuildLoopCount=16
export __TestGroupToBuild=2
fi

# Invoke MSBuild
# $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
do
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""

# Make sure everything is OK
if [ $? -ne 0 ]; then
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
echo " $__BuildLog"
echo " $__BuildWrn"
echo " $__BuildErr"
exit 1
export TestBuildSlice=$slice

# Generate build command
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"

echo "Building step '$stepName' slice=$slice via $buildCommand"

# Invoke MSBuild
eval $buildCommand

# Make sure everything is OK
if [ $? -ne 0 ]; then
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
echo " $__BuildLog"
echo " $__BuildWrn"
echo " $__BuildErr"
exit 1
fi
export __SkipPackageRestore=true
export __SkipTargetingPackBuild=true
__AppendToLog=true
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you initialize __AppendToLog=false before the slice loop? Even if passing Append= (empty) to msbuild works, it seems like being explicit would be better (it looks like build-test.cmd does this).

done
else
__msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
__msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
__msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""

# Generate build command
buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} -MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"

echo "Building step '$stepName' via $buildCommand"

# Invoke MSBuild
eval $buildCommand

# Invoke MSBuild
# $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs

# Make sure everything is OK
if [ $? -ne 0 ]; then
echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
echo " $__BuildLog"
echo " $__BuildWrn"
echo " $__BuildErr"
exit 1
fi
fi
}

Expand All @@ -342,6 +389,7 @@ usage()
echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
echo "priority1 - include priority=1 tests in the build"
exit 1
}

Expand Down Expand Up @@ -461,6 +509,7 @@ __RunTests=0
__RebuildTests=0
__BuildTestWrappers=0
__GenerateLayoutOnly=
__priority1=
CORE_ROOT=

while :; do
Expand Down Expand Up @@ -615,6 +664,10 @@ while :; do
msbuildonunsupportedplatform)
__msbuildonunsupportedplatform=1
;;
priority1)
__priority1=1
__UnprocessedBuildArgs="$__UnprocessedBuildArgs -priority=1"
;;
*)
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
;;
Expand Down
46 changes: 18 additions & 28 deletions tests/src/dirs.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,26 +33,16 @@
<DisabledProjects Include="Loader\classloader\generics\GenericMethods\VSW491668.csproj" /> <!-- issue 5501 -->
</ItemGroup>

<!-- Unix builds do not support subgroups -->
<ItemGroup Condition="$(__BuildOS) != 'Windows_NT'" >
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<!-- Test build is divided in slices which can be created within Test Group
Priority 0 tests are build using Test Group 1 with 2 subgroups or slices -->

<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '1'">
<Project Include="*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="$(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
<ItemGroup Condition="$(__TestGroupToBuild) == '1' And $(TestBuildSlice) == '2'">
<Project Include="*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -61,7 +51,7 @@
<!-- Test build is divided in slices which can be created within Test Group
Priority 1 or higher tests are build using Test Group 2 with 16 subgroups or slices -->

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '1')">
<Project Include="baseservices\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -76,7 +66,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '2')">
<Project Include="CoreMangLib\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -85,7 +75,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '3')">
<Project Include="E*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -112,7 +102,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '4')">
<Project Include="JIT\B*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -124,7 +114,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '5')">
<Project Include="JIT\B*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -136,7 +126,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '6')">
<Project Include="JIT\Generics\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -151,7 +141,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '7')">
<Project Include="JIT\IL_Conformance\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -160,7 +150,7 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '8')">
<Project Include="JIT\jit64\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -169,19 +159,19 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '9')">
<Project Include="JIT\Methodical\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '10')">
<Project Include="JIT\Methodical\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '11')">
<Project Include="JIT\opt\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand All @@ -202,31 +192,31 @@
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '12')">
<Project Include="JIT\R*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '13')">
<Project Include="JIT\R*\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '14')">
<Project Include="Loader\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '15')">
<Project Include="Loader\**\*.ilproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
</ItemGroup>

<ItemGroup Condition="($(__BuildOS) == 'Windows_NT' And $(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
<ItemGroup Condition="($(__TestGroupToBuild) == '2' And $(TestBuildSlice) == '16')">
<Project Include="m*\**\*.csproj" Exclude="@(DisabledProjects)">
<AdditionalProperties>OSGroup=$(OSGroup)</AdditionalProperties>
</Project>
Expand Down