Add basic check that the correct number of tests is built

build-test.cmd

@@ -314,6 +314,15 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do (
     set __AppendToLog=true
 )
 
+REM Check that we've built about as many tests as we expect. This is primarily intended to prevent accidental changes that cause us to build
+REM drastically fewer Pri-1 tests than expected.
+echo %__MsgPrefix%Check the managed tests build
+call %__DotnetHost% msbuild %__ProjectDir%\tests\runtest.proj /t:CheckTestBuild /p:CLRTestPriorityToBuild=%__Priority% %__msbuildArgs% %__unprocessedBuildArgs%
+if errorlevel 1 (
+    echo %__MsgPrefix%Error: build failed.
+    exit /b 1
+)
+
 :SkipManagedBuild
 REM Prepare the Test Drop
 REM Cleans any NI from the last run

build-test.sh

@@ -252,6 +252,20 @@ build_Tests()
         echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
         exit 1
     else
+        echo "Checking the Managed Tests Build..."
+
+        if [ -n __priority1 ]; then
+            __Priority=1
+        else
+            __Priority=0
+        fi
+        build_Tests_internal "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild /p:CLRTestPriorityToBuild=$__Priority"
+
+        if [ $? -ne 0 ]; then
+            echo "${__MsgPrefix}Error: Check Test Build failed."
+            exit 1
+        fi
+
         echo "Managed tests build success!"
     fi
 

tests/runtest.proj

@@ -27,7 +27,7 @@
   <Target Name="FindCmdDirectories" DependsOnTargets="GetListOfTestCmds">
 
     <Error Condition="!Exists('$(XunitTestBinBase)')"
-           Text="$(XunitTestBinBase) does not exist. Please run buildtest.cmd from the (repo root)\tests at least once to get the tests built." />
+           Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." />
 
     <ItemGroup>
 
@@ -54,6 +54,29 @@
 
   </Target>
 
+  <!-- Target to check the test build, to see if it looks ok. We've had several cases where a change inadvertently and drastically changes
+       the set of tests that are built, and that change is unnoticed. The most common case is for a build of the Priority 1 tests
+       to only build the Priority 0 tests. This target is run after a test build to verify that the basic number of tests that were
+       built is basically what was expected. When this was written, there were about 2500 Priority 0 tests and about 12270 Priority 1
+       tests (differing slightly based on platform). We currently check that the number of Priority 0 tests is greater than 2000 and
+       less than 3000, and the number of Priority 1 tests is greater than 12000.
+  -->
+  <Target Name="CheckTestBuild" DependsOnTargets="GetListOfTestCmds">
+    <Error Condition="!Exists('$(XunitTestBinBase)')"
+        Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." />
+
+    <PropertyGroup>
+        <TestCount>@(AllRunnableTestPaths->Count())</TestCount>
+    </PropertyGroup>
+
+    <Message Text="Found $(TestCount) built tests"/>
+
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &lt;= 2000" Text="Unexpected test count. Expected &gt; 2000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &gt;= 3000" Text="Unexpected test count. Expected &lt; 3000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '1' and '$(TestCount)' &lt;= 12000" Text="Unexpected test count. Expected &gt; 12000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' != '0' and '$(CLRTestPriorityToBuild)' != '1'" Text="Unknown priority $(CLRTestPriorityToBuild)" />
+  </Target>
+
   <Import Project="$(__Exclude)" Condition="'$(__Exclude)' != '' AND '$(XunitTestBinBase)' != ''" />
   <PropertyGroup>
     <HaveExcludes>False</HaveExcludes>