Correct queues

azure-pipelines.yml

@@ -1,6 +1,25 @@
 variables:
   DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
 
+resources:
+  containers:
+  - container: ubuntu_1404_arm_cross_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180426002420
+
+  - container: ubuntu_1604_arm64_cross_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921
+
+  - container: ubuntu_1604_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-c103199-20180628134544
+
+  - container: musl_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:alpine-3.6-e2521f8-20180716231200
+
+  - container: centos7_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424
+
+  - container: centos6_x64_build_image
+    image: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331
 
 jobs:
 
@@ -58,30 +77,30 @@ jobs:
 ##                                            |  (passed-in jobTemplate)  |                    (arcade)
 ##                                            \------> test-job.yml ------/
 
-
+# TODO: simplify logic surrounding official build/ci. See
+# https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information
 
 #
-# Debug build
+# Debug build (CI)
 #
-
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: debug
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: build-job.yml
+      buildConfig: debug
 
 #
-# Checked build
+# Checked build (CI)
 #
-
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: build-job.yml
-    buildConfig: checked
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: build-job.yml
+      buildConfig: checked
 
 #
-# Release build
+# Release build (Official Build)
 #
-
 - template: eng/platform-matrix.yml
   parameters:
     jobTemplate: build-job.yml
@@ -91,53 +110,58 @@ jobs:
 # Checked test builds
 #
 
-# Pri0
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 0
-
-# Pri1
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 1
-      scenarios: 'normal;jitstress2'
-
-# Pri1 crossgen
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: checked
-    jobParameters:
-      priority: 1
-      crossgen: true
-      scenarios: 'normal;jitstress2'
+# Pri0 (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 0
+
+# Pri1 (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 1
+        scenarios: 'normal;jitstress2'
+
+# Pri1 crossgen (CI)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: checked
+      jobParameters:
+        priority: 1
+        crossgen: true
+        scenarios: 'normal;jitstress2'
 
 #
-# Release test builds
+# Release test builds (Official Build)
 #
 
 # Pri1
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: release
-    jobParameters:
-      priority: 1
-
-# Pri1 crossgen
-- template: eng/platform-matrix.yml
-  parameters:
-    jobTemplate: test-job.yml
-    buildConfig: release
-    jobParameters:
-      priority: 1
-      crossgen: true
+- ${{ if ne(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: release
+      jobParameters:
+        priority: 1
+
+# Pri1 crossgen (Official Build)
+- ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  - template: eng/platform-matrix.yml
+    parameters:
+      jobTemplate: test-job.yml
+      buildConfig: release
+      jobParameters:
+        priority: 1
+        crossgen: true
 
 
 # Publish build information to Build Assets Registry

build-test.sh

@@ -11,12 +11,7 @@ initHostDistroRid()
 
     if [ "$__HostOS" == "Linux" ]; then
         if [ -e /etc/redhat-release ]; then
-            local redhatRelease=$(</etc/redhat-release)
-            if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
-                __HostDistroRid="rhel.6-$__HostArch"
-            else
-                __PortableBuild=1
-            fi
+            __PortableBuild=1
         elif [ -e /etc/os-release ]; then
             source /etc/os-release
             if [[ $ID == "alpine" ]]; then

eng/build-job.yml

@@ -2,6 +2,8 @@ parameters:
   buildConfig: ''
   archType: ''
   osGroup: ''
+  osGroupName: ''
+  containerName: ''
 
 ### Product build
 jobs:
@@ -10,35 +12,40 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
+    osGroupName: ${{ parameters.osGroupName }}
 
     # Compute job name from template parameters
-    name: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
-    displayName: ${{ format('Build {0} {1} {2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+    name: ${{ format('build_{0}_{1}_{2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+    displayName: ${{ format('Build {0} {1} {2}', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
+
+    # Run all steps in the container.
+    # Note that the containers are resources defined in azure-pipelines.yml
+    containerName: ${{ parameters.containerName }}
 
     steps:
 
     # Install native dependencies
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    #
+    # This is only required for non-docker builds.
+    - ${{ if eq(parameters.osGroup, 'OSX') }}:
       - script: sh eng/install-native-dependencies.sh $(osGroup)
         displayName: Install native dependencies
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # Necessary to install python
       - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
         displayName: Install native dependencies
 
-
     # Run init-tools (pre-arcade dependency bootstrapping)
     # TODO: replace this with an arcade equivalent
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - script: ./init-tools.sh
         displayName: Init tools
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - script: .\init-tools.cmd
         displayName: Init tools
 
-
     # Sync
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - script: ./Tools/dotnetcli/dotnet msbuild build.proj /p:RestoreDuringBuild=true /t:Sync
         displayName: Sync
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -47,21 +54,27 @@ jobs:
 
 
     # Build
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if and(and(ne(parameters.archType, 'arm'), ne(parameters.archType, 'arm64')), ne(parameters.osGroup, 'Windows_NT')) }}:
       - script: ./build.sh $(buildConfig) $(archType) -skipnuget -skiprestore
         displayName: Build product
+    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm')) }}:
+      - script: ROOTFS_DIR=$(rootfsDir) CAC_ROOTFS_DIR=$(cacRootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore
+        displayName: Build product
+    - ${{ if and(ne(parameters.osGroup, 'Windows_NT'), eq(parameters.archType, 'arm64')) }}:
+      - script: ROOTFS_DIR=$(rootfsDir) ./build.sh $(buildConfig) $(archType) -cross -skipnuget -skiprestore
+        displayName: Build product
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?
       - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages -skiprestore
         displayName: Build product
 
 
     # Upload build as pipeline artifact
-    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
       - task: PublishPipelineArtifact@0
         displayName: Save product build as pipeline artifact
         inputs:
-          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroupName, parameters.archType, parameters.buildConfig) }}
           targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
       - task: PublishPipelineArtifact@0

eng/platform-matrix.yml

@@ -5,13 +5,68 @@ parameters:
 
 jobs:
 
+# TODO: simplify osGroupName by adding osGroup and osSubGroup. See
+# https://github.com/Microsoft/azure-pipelines-yaml/pull/46 for more information
+
+# Linux arm
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm
+    osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1404_arm_cross_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Linux arm64
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm64
+    osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1604_arm64_cross_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Linux musl
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_musl
+    containerName: musl_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# RHEL 6
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_rhel6
+    containerName: centos6_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# RHEL 7
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    osGroupName: Linux_rhel7
+    containerName: centos7_x64_build_image
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
 # Linux x64
 
 - template: ${{ parameters.jobTemplate }}
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: Linux
+    osGroupName: Linux
+    containerName: ubuntu_1604_x64_build_image
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # macOS x64
@@ -21,6 +76,7 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: OSX
+    osGroupName: OSX
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 # Windows x64/x86/arm/arm64
@@ -30,32 +86,29 @@ jobs:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x64
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: x86
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: arm
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
 
 - template: ${{ parameters.jobTemplate }}
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: arm64
     osGroup: Windows_NT
+    osGroupName: Windows_NT
     ${{ insert }}: ${{ parameters.jobParameters }}
-
-# TODO for official build:
-# RedHat x64
-# Linux crossbuild arm
-# Linux crossbuild arm64
-# Linux musl x64
-