Skip to content

Commit

Permalink
Helix Integration. (#5837)
Browse files Browse the repository at this point in the history
* Helix working. Squashed local commits

* Updates from PR comments.

* Updates from PR comments.

* Update Directory.Build.targets

Co-authored-by: Santiago Fernandez Madero <safern@microsoft.com>

* Fixes from pr comments.

* Switched tabs to spaces.

* Helix changes from pr comments.

* yml changes.

* yml changes.

* yml changes.

* yml changes.

* yml changes.

* Indentation fixes.

* Indentation fixes.

Co-authored-by: Santiago Fernandez Madero <safern@microsoft.com>
  • Loading branch information
michaelgsharp and safern committed Jun 23, 2021
1 parent 7f888c2 commit ce7f91a
Show file tree
Hide file tree
Showing 54 changed files with 466 additions and 287 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@
*.sln.docstates
*.DS_Store

# Helix payload files
*.payload

# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs

Expand Down
7 changes: 7 additions & 0 deletions .vsts-dotnet-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ jobs:
innerLoop: true
pool:
name: Hosted Ubuntu 1604
helixQueue: Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix-20210528183707-dde38af

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -39,6 +40,7 @@ jobs:
innerLoop: true
pool:
name: Hosted Ubuntu 1604
helixQueue: Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-helix-20210528184647-dde38af

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -47,6 +49,7 @@ jobs:
innerLoop: true
pool:
name: Hosted macOS
helixQueue: OSX.1015.Amd64.Open

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -67,6 +70,7 @@ jobs:
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0"
pool:
name: Hosted VS2017
helixQueue: Windows.10.Amd64.Open

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -76,6 +80,7 @@ jobs:
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
pool:
name: Hosted VS2017
helixQueue: Windows.10.Amd64.Open

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -96,6 +101,7 @@ jobs:
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0"
pool:
name: Hosted VS2017
helixQueue: Windows.10.Amd64.Open

- template: /build/ci/job-template.yml
parameters:
Expand All @@ -106,3 +112,4 @@ jobs:
vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
pool:
name: Hosted VS2017
helixQueue: Windows.10.Amd64.Open
23 changes: 20 additions & 3 deletions Directory.Build.targets
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<PackageReference Include="coverlet.msbuild" Version="2.9.0" PrivateAssets="all" />
</ItemGroup>

<Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory">
<Target Name="SetCopyProperties">
<PropertyGroup>
<LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
<LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
Expand All @@ -33,7 +33,23 @@
<NativeAssemblyReference Remove="FastTreeNative"/>
<NativeAssemblyReference Remove="SymSgdNative"/>
<NativeAssemblyReference Remove="MklProxyNative"/>
<NativeAssemblyReference Remove="libiomp5md"/>
</ItemGroup>
</Target>

<Target Name="CopyNativeAssembliesPublish" AfterTargets="Publish" DependsOnTargets="SetCopyProperties">
<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
DestinationFolder="$(OutDir)\publish"
OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
Retries="$(CopyRetryCount)"
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
</Copy>
</Target>

<Target Name="CopyNativeAssembiles" AfterTargets="CopyFilesToOutputDirectory" DependsOnTargets="SetCopyProperties">

<Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
DestinationFolder="$(OutDir)"
Expand All @@ -42,9 +58,10 @@
RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
<Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
</Copy>
</Target>

<PropertyGroup Condition="'$(Coverage)' == 'true'">
<!-- https://github.com/tonerdo/coverlet/issues/363 -->
<DeterministicSourcePaths>false</DeterministicSourcePaths>
Expand All @@ -54,7 +71,7 @@

<CollectCoverage>true</CollectCoverage>
<SingleHit>true</SingleHit>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
<CopyLocalLockFileAssemblies>false</CopyLocalLockFileAssemblies>
<CoverletOutputFormat>opencover</CoverletOutputFormat>
<CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\coverage.opencover.xml</CoverletOutput>
<Include></Include>
Expand Down
51 changes: 39 additions & 12 deletions build/ci/job-template.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#TODO: Need to update build documentation.

parameters:
name: ''
architecture: x64
Expand All @@ -11,6 +12,7 @@ parameters:
runSpecific: false
container: ''
useVSTestTask: false
helixQueue: ''

jobs:
- job: ${{ parameters.name }}
Expand Down Expand Up @@ -68,7 +70,7 @@ jobs:
- ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
- bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
- script: ${{ parameters.buildScript }} -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:Coverage=${{ parameters.codeCoverage }}
displayName: Build
- ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
- task: Bash@3
Expand All @@ -85,11 +87,12 @@ jobs:
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
- script: dir /s "$(Build.SourcesDirectory)"
displayName: show bin folder disk usage
- task: PowerShell@2
inputs:
targetType: inline
script: Get-ChildItem -Path '$(Build.SourcesDirectory)\packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
# - ${{ if eq(parameters.codeCoverage, 'true') }}:
# - task: PowerShell@2
# inputs:
# targetType: inline
# script: Get-ChildItem -Path '$(Build.SourcesDirectory)\.packages\*\runtimes\*' -Recurse | Select -ExpandProperty FullName | Where {$_ -notlike '*\win-*'} | sort length -Descending | Remove-Item -Recurse -Confirm:$false -Force
# displayName: Clean up non-Windows runtime folders of NuGet Packages to save disk space
- ${{ if eq(parameters.nightlyBuild, 'true') }}:
- script: $(dotnetPath) restore $(nightlyBuildProjPath)
displayName: Restore nightly build project
Expand All @@ -113,11 +116,10 @@ jobs:
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
- ${{ if eq(parameters.innerLoop, 'false') }}:
- ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
# TODO: Code coverage needs to be fixed.
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:Coverage=${{ parameters.codeCoverage }}
displayName: Run All Tests.
- ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-trait$(spaceValue)Category=RunSpecificTest' /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-trait$(spaceValue)Category=RunSpecificTest' /p:Coverage=${{ parameters.codeCoverage }}
displayName: Run Specific Tests.
- ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
- task: VSTest@2
Expand All @@ -141,8 +143,21 @@ jobs:
collectDumpOn: onAbortOnly
publishRunAttachments: true
- ${{ if eq(parameters.innerLoop, 'true') }}:
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages /p:Coverage=${{ parameters.codeCoverage }}
displayName: Run CI Tests.
- ${{ if eq(parameters.codeCoverage, True) }}:
- script: ${{ parameters.buildScript }} /p:Build=false -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} -test -integrationTest -ci /p:TestRunnerAdditionalArguments='-notrait$(spaceValue)Category=SkipInCI' /p:Coverage=${{ parameters.codeCoverage }}
displayName: Run CI Tests.
- ${{ if eq(parameters.codeCoverage, False) }}:
- template: /build/ci/send-to-helix.yml
parameters:
HelixTargetQueues: ${{ parameters.helixQueue }}
Configuration: $(_configuration)
Architecture: ${{ parameters.architecture }}
${{ if eq(parameters.buildScript, 'build.cmd') }}:
MsBuildScript: 'powershell $(Build.SourcesDirectory)/eng/common/msbuild.ps1'
WarnAsError: '-warnAsError 0'
${{ if eq(parameters.buildScript, './build.sh') }}:
MsBuildScript: '$(Build.SourcesDirectory)/eng/common/msbuild.sh'
WarnAsError: '--warnAsError false'
- script: $(dotnetPath) msbuild -restore build/Codecoverage.proj
displayName: Upload coverage to codecov.io
condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
Expand Down Expand Up @@ -193,6 +208,18 @@ jobs:
pathToPublish: $(Build.ArtifactStagingDirectory)
artifactName: ${{ parameters.name }} $(_config_short)
artifactType: container
- ${{ if eq(parameters.buildScript, './build.sh') }}:
- task: Bash@3
inputs:
targetType: inline
script: find ./artifacts/bin -type d -path "*/runtimes" -exec rm -rv {} +;
displayName: Clean up runtime folder for package (Unix)
- ${{ if eq(parameters.buildScript, 'build.cmd') }}:
- task: PowerShell@2
inputs:
targetType: inline
script: Get-ChildItem -Path '.\artifacts\bin' -Recurse | Where-Object {$_.FullName -like "*runtimes*"} | Remove-Item -Recurse -Confirm:$false -Force
displayName: Clean up runtime folder for package (Unix)
- ${{ if eq(parameters.nightlyBuild, 'false') }}:
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }} /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
- script: ${{ parameters.buildScript }} /p:Build=false -pack -ci -configuration $(_configuration) /p:TargetArchitecture=${{ parameters.architecture }} /p:TestArchitectures=${{ parameters.architecture }}
displayName: Build Packages
46 changes: 46 additions & 0 deletions build/ci/send-to-helix.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Please remember to update the documentation if you make changes to these parameters!
parameters:
HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
HelixType: 'tests/default' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
Configuration: 'Debug'
Architecture: 'x64'
MsBuildScript: ''
WarnAsError: ''
HelixConfiguration: '' # optional -- additional property attached to a job
IncludeDotNetCli: true # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
EnableXUnitReporter: true # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
HelixBaseUri: 'https://helix.dot.net' # optional -- sets the Helix API base URI (allows targeting int)
Creator: 'ML.NET' # optional -- if the build is external, use this to specify who is sending the job
DisplayNamePrefix: 'Run Helix Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()

steps:
- script: ${{ parameters.MsBuildScript}}
$(Build.SourcesDirectory)/eng/helix.proj
/t:Test
/bl:$(Build.SourcesDirectory)/artifacts/log/${{ parameters.Configuration }}/SendToHelix.binlog
/p:Configuration=${{ parameters.Configuration }}
/p:TargetArchitecture=${{ parameters.Architecture }}
/p:BuildConfig=${{ parameters.Configuration }}
/p:BuildArchitecture=${{ parameters.Architecture }}
/p:HelixSource=${{ parameters.HelixSource }}
/p:HelixType=${{ parameters.HelixType }}
/p:HelixBuild=${{ parameters.HelixBuild }}
/p:HelixConfiguration="${{ parameters.HelixConfiguration }}"
/p:HelixAccessToken="${{ parameters.HelixAccessToken }}"
/p:IncludeDotNetCli=${{ parameters.IncludeDotNetCli }}
/p:EnableXUnitReporter=${{ parameters.EnableXUnitReporter }}
/p:WaitForWorkItemCompletion=${{ parameters.WaitForWorkItemCompletion }}
/p:HelixBaseUri=${{ parameters.HelixBaseUri }}
${{ parameters.WarnAsError }}
displayName: ${{ parameters.DisplayNamePrefix }}
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
Creator: ${{ parameters.Creator }}
HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
continueOnError: ${{ parameters.continueOnError }}
6 changes: 3 additions & 3 deletions build/publish.proj
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<Project>

<Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />

<PropertyGroup>
Expand All @@ -8,8 +8,8 @@
<NuGetPushTimeoutSeconds Condition="'$(NuGetPushTimeoutSeconds)' == ''">600</NuGetPushTimeoutSeconds>
</PropertyGroup>

<Import Project="$(NuGetPackageRoot)\$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />
<Import Project="$(NuGetPackageRoot)$(PublishSymbolsPackage.ToLower())\$(MicrosoftSymbolUploaderBuildTaskVersion)\build\PublishSymbols.targets" />

<Target Name="PublishSymbolPackages"
Condition="'$(EnablePublishSymbols)'=='true'"
DependsOnTargets="SetupPublishSymbols">
Expand Down
8 changes: 4 additions & 4 deletions build/vsts-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,10 @@ phases:
# For our nightly builds we want it to be empty, and when creating the official nugets, we want it to be "release"
# the value of the version kind is set when queuing the publishing job on AzureDevOps by adding a VERSIONKIND variable
# See more info in: https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Versioning.md#package-version
- script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
- script: ./build.cmd -configuration $(BuildConfig) -pack -ci /p:OfficialBuildId=$(OfficialBuildId) /p:DotNetFinalVersionKind=$(DotnetVersionKind)
displayName: Build Packages

- script: ./sign.cmd /p:SignNugetPackages=true /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
- script: ./sign.cmd /p:SignNugetPackages=true
displayName: sign packages
continueOnError: false

Expand Down Expand Up @@ -267,15 +267,15 @@ phases:
displayName: Publish Symbols to SymWeb Symbol Server
inputs:
solution: build/publish.proj
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_SymwebSymbolServerPath) /p:SymbolServerPAT=$(SymwebSymbolServerPAT)
msbuildVersion: 15.0
continueOnError: true

- task: MSBuild@1
displayName: Publish Symbols to Msdl Symbol Server
inputs:
solution: build/publish.proj
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT) /p:RestorePackagesPath=$(Build.SourcesDirectory)\packages /p:NUGET_PACKAGES=$(Build.SourcesDirectory)\packages
msbuildArguments: /t:PublishSymbolPackages /p:SymbolServerPath=$(_MsdlSymbolServerPath) /p:SymbolServerPAT=$(MsdlSymbolServerPAT)
msbuildVersion: 15.0
continueOnError: true

Expand Down
Loading

0 comments on commit ce7f91a

Please sign in to comment.