Skip to content

Commit a8bb9f1

Browse files
author
Steve Pfister
committed
Alter amount of tests that run per PR for mono mobile configurations
In an effort to better utilize CI resources, this change will only run the System.Runtime library test per PR for Android and Wasm configurations. The full libraries and runtime tests will move to only run on the post-PR validation pipeline. Note: iOS/tvOS/MacCatalyst are excluded until dotnet#59503 lands
1 parent ed9ab1b commit a8bb9f1

File tree

4 files changed

+61
-46
lines changed

4 files changed

+61
-46
lines changed

‎eng/pipelines/libraries/base-job.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ jobs:
5454
- _extraHelixArguments: ''
5555
- _crossBuildPropertyArg: ''
5656
- _testRunNamePrefixSuffix: ''
57+
- _runSmokeTestsOnlyArg: ''
5758

5859
- librariesBuildArtifactName: ${{ format('libraries_bin_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
5960

@@ -87,6 +88,9 @@ jobs:
8788
- ${{ if eq(parameters.isOfficialBuild, 'true') }}:
8889
- _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber)
8990

91+
- ${{ if eq(variables['isFullMatrix'], false) }}:
92+
- _runSmokeTestsOnlyArg: /p:RunSmokeTestsOnly=true
93+
9094
- _runtimeArtifactName: ''
9195
- _runtimeDownloadPath: ''
9296
- _runtimeArtifactsPathArg: ''

‎eng/pipelines/runtime-staging.yml‎

Lines changed: 35 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ jobs:
212212
jobParameters:
213213
testGroup: innerloop
214214
nameSuffix: AllSubsets_Mono
215-
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true
215+
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg)
216216
timeoutInMinutes: 180
217217
condition: >-
218218
or(
@@ -249,16 +249,15 @@ jobs:
249249
jobParameters:
250250
testGroup: innerloop
251251
nameSuffix: AllSubsets_Mono
252-
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true
252+
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg)
253253
timeoutInMinutes: 180
254254
condition: >-
255255
or(
256256
eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true),
257257
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
258258
eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true),
259259
eq(variables['isFullMatrix'], true))
260-
261-
# don't run tests on PRs until we can get significantly more devices
260+
# Device tests are rolling build only
262261
${{ if eq(variables['isFullMatrix'], true) }}:
263262
# extra steps, run tests
264263
extraStepsTemplate: /eng/pipelines/libraries/helix.yml
@@ -320,7 +319,7 @@ jobs:
320319
buildConfig: Release
321320
runtimeFlavor: mono
322321
platforms:
323-
- Android_x64
322+
- Android_x64
324323
variables:
325324
- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
326325
- name: _HelixSource
@@ -343,11 +342,13 @@ jobs:
343342
eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
344343
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
345344
eq(variables['isFullMatrix'], true))
346-
# extra steps, run tests
347-
extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
348-
extraStepsParameters:
349-
creator: dotnet-bot
350-
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
345+
# NOTE: Per PR test execution is not recommended for runtime tests
346+
${{ if eq(variables['isFullMatrix'], true) }}:
347+
# extra steps, run tests
348+
extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
349+
extraStepsParameters:
350+
creator: dotnet-bot
351+
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
351352

352353
#
353354
# Build the whole product using Mono and run runtime tests with the JIT.
@@ -359,7 +360,7 @@ jobs:
359360
buildConfig: Release
360361
runtimeFlavor: mono
361362
platforms:
362-
- iOSSimulator_x64
363+
- iOSSimulator_x64
363364
variables:
364365
- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
365366
- name: _HelixSource
@@ -381,15 +382,14 @@ jobs:
381382
eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
382383
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
383384
eq(variables['isFullMatrix'], true))
384-
# Test execution is temporarily disabled because test apps no longer launch
385-
# and the test suite times out after two hours, even if xharness cannot
386-
# successfully launch any tests. Re-enable once these issues have been fixed.
385+
# NOTE: Per PR test execution is not recommended for runtime tests
387386
#
388-
# extra steps, run tests
389-
# extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
390-
# extraStepsParameters:
391-
# creator: dotnet-bot
392-
# testRunNamePrefixSuffix: Mono_$(_BuildConfig)
387+
# ${{ if eq(variables['isFullMatrix'], true) }}:
388+
# extra steps, run tests
389+
# extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
390+
# extraStepsParameters:
391+
# creator: dotnet-bot
392+
# testRunNamePrefixSuffix: Mono_$(_BuildConfig)
393393

394394
#
395395
# Build the whole product using Mono for Android and run runtime tests with Android devices
@@ -401,7 +401,7 @@ jobs:
401401
buildConfig: Release
402402
runtimeFlavor: mono
403403
platforms:
404-
- Android_arm64
404+
- Android_arm64
405405
variables:
406406
- ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}:
407407
- name: _HelixSource
@@ -460,20 +460,21 @@ jobs:
460460
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
461461
eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true),
462462
eq(variables['isFullMatrix'], true))
463-
# extra steps, run tests
464-
extraStepsTemplate: /eng/pipelines/libraries/helix.yml
465-
extraStepsParameters:
466-
creator: dotnet-bot
467-
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
468-
extraHelixArguments: /p:BrowserHost=windows
469-
scenarios:
470-
- normal
471-
- wasmtestonbrowser
472-
condition: >-
473-
or(
474-
eq(variables['librariesContainsChange'], true),
475-
eq(variables['monoContainsChange'], true),
476-
eq(variables['isFullMatrix'], true))
463+
${{ if eq(variables['isFullMatrix'], true) }}:
464+
# extra steps, run tests
465+
extraStepsTemplate: /eng/pipelines/libraries/helix.yml
466+
extraStepsParameters:
467+
creator: dotnet-bot
468+
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
469+
extraHelixArguments: /p:BrowserHost=windows
470+
scenarios:
471+
- normal
472+
- wasmtestonbrowser
473+
condition: >-
474+
or(
475+
eq(variables['librariesContainsChange'], true),
476+
eq(variables['monoContainsChange'], true),
477+
eq(variables['isFullMatrix'], true))
477478
478479
#
479480
# CoreCLR Build for running Apple Silicon libraries-innerloop

‎eng/pipelines/runtime.yml‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -291,7 +291,7 @@ jobs:
291291
jobParameters:
292292
testGroup: innerloop
293293
nameSuffix: AllSubsets_Mono
294-
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true
294+
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg)
295295
timeoutInMinutes: 180
296296
condition: >-
297297
or(
@@ -323,7 +323,9 @@ jobs:
323323
buildConfig: Release
324324
runtimeFlavor: mono
325325
platforms:
326-
- Browser_wasm
326+
# BuildWasmApps should only happen on the rolling build. No need to duplicate the build on PR's
327+
${{ if eq(variables['isFullMatrix'], true) }}:
328+
- Browser_wasm
327329
variables:
328330
# map dependencies variables to local variables
329331
- name: monoContainsChange
@@ -373,7 +375,7 @@ jobs:
373375
jobParameters:
374376
testGroup: innerloop
375377
nameSuffix: AllSubsets_Mono_EAT
376-
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:EnableAggressiveTrimming=true /p:BuildAOTTestsOnHelix=true /p:RunAOTCompilation=false
378+
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:EnableAggressiveTrimming=true /p:BuildAOTTestsOnHelix=true /p:RunAOTCompilation=false
377379
timeoutInMinutes: 180
378380
condition: >-
379381
or(
@@ -415,7 +417,7 @@ jobs:
415417
jobParameters:
416418
testGroup: innerloop
417419
nameSuffix: AllSubsets_Mono_AOT
418-
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:EnableAggressiveTrimming=true /p:BuildAOTTestsOnHelix=true /p:RunAOTCompilation=true
420+
buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) $(_runSmokeTestsOnlyArg) /p:ArchiveTests=true /p:EnableAggressiveTrimming=true /p:BuildAOTTestsOnHelix=true /p:RunAOTCompilation=true
419421
timeoutInMinutes: 180
420422
condition: >-
421423
or(
@@ -491,11 +493,13 @@ jobs:
491493
eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
492494
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
493495
eq(variables['isFullMatrix'], true))
494-
# extra steps, run tests
495-
extraStepsTemplate: /eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml
496-
extraStepsParameters:
497-
creator: dotnet-bot
498-
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
496+
# NOTE: Per PR test execution is not recommended for mobile runtime tests
497+
${{ if eq(variables['isFullMatrix'], true) }}:
498+
# extra steps, run tests
499+
extraStepsTemplate: /eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml
500+
extraStepsParameters:
501+
creator: dotnet-bot
502+
testRunNamePrefixSuffix: Mono_$(_BuildConfig)
499503

500504
#
501505
# Build the whole product using Mono for Android and run runtime tests with Android emulator
@@ -529,6 +533,8 @@ jobs:
529533
eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true),
530534
eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true),
531535
eq(variables['isFullMatrix'], true))
536+
# NOTE: Per PR test execution is not recommended for mobile runtime tests
537+
${{ if eq(variables['isFullMatrix'], true) }}:
532538
# extra steps, run tests
533539
extraStepsTemplate: /eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml
534540
extraStepsParameters:

‎src/libraries/tests.proj‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,11 @@
285285
<ProjectExclusions Remove="$(MSBuildThisFileDirectory)System.IO.IsolatedStorage\tests\System.IO.IsolatedStorage.Tests.csproj" />
286286
</ItemGroup>
287287

288-
<ItemGroup>
288+
<ItemGroup Condition="'$(RunSmokeTestsOnly)' == 'true'">
289+
<ProjectReference Include="$(RepoRoot)\src\libraries\System.Runtime\tests\System.Runtime.Tests.csproj" />
290+
</ItemGroup>
291+
292+
<ItemGroup Condition="'$(RunSmokeTestsOnly)' != 'true'">
289293
<ProjectReference Include="$(MSBuildThisFileDirectory)*\tests\**\*.Tests.csproj"
290294
Exclude="@(ProjectExclusions)"
291295
Condition="'$(TestAssemblies)' == 'true'" />
@@ -344,7 +348,7 @@
344348
BuildInParallel="false" />
345349
</ItemGroup>
346350

347-
<ItemGroup Condition="'$(ArchiveTests)' == 'true' and '$(TargetOS)' == 'Android'">
351+
<ItemGroup Condition="'$(ArchiveTests)' == 'true' and '$(RunSmokeTestsOnly)' != 'true' and '$(TargetOS)' == 'Android'">
348352
<ProjectReference Include="$(MonoProjectRoot)sample\Android\AndroidSampleApp.csproj"
349353
BuildInParallel="false" />
350354
<ProjectReference Include="$(RepoRoot)\src\tests\FunctionalTests\Android\**\*.Test.csproj"
@@ -353,7 +357,7 @@
353357
</ItemGroup>
354358

355359
<!-- Don't build samples, and functional tests on EAT, AOT, and WBT lanes -->
356-
<ItemGroup Condition="'$(ArchiveTests)' == 'true' and '$(TargetOS)' == 'Browser' and '$(BuildAOTTestsOnHelix)' != 'true' and '$(TestWasmBuildTests)' != 'true'">
360+
<ItemGroup Condition="'$(ArchiveTests)' == 'true' and '$(RunSmokeTestsOnly)' != 'true' and '$(TargetOS)' == 'Browser' and '$(BuildAOTTestsOnHelix)' != 'true' and '$(TestWasmBuildTests)' != 'true'">
357361
<ProjectReference Include="$(MonoProjectRoot)sample\wasm\**\*.Sample.csproj"
358362
Exclude="@(ProjectExclusions)"
359363
BuildInParallel="true" />

0 commit comments

Comments
 (0)