Support OfficialBuildId for SB

eng/pipelines/templates/jobs/source-only-build-and-validate.yml

@@ -0,0 +1,96 @@
+parameters:
+- name: buildNamePrefix
+  type: string
+
+- name: container
+  type: object
+  default:
+    image: ''
+    name: ''
+
+# Allow downloading artifacts from the internet during the build
+- name: runOnline
+  type: boolean
+  default: true
+
+- name: targetArchitecture
+  type: string
+  default: ''
+
+- name: pool_Linux
+  type: object
+  default:
+    name: $(defaultPoolName)
+    image: $(poolImage_Linux)
+    demands: ImageOverride -equals $(poolImage_Linux)
+    os: linux
+
+jobs:
+# Source Build Leg
+- template: ../jobs/vmr-build.yml
+  parameters:
+    buildName: ${{ format('{0}_SourceBuild_Online_MsftSdk', parameters.buildNamePrefix) }}
+    isBuiltFromVmr: true
+    vmrBranch: ${{ variables.VmrBranch }}
+    targetArchitecture: x64
+    pool: ${{ parameters.pool_Linux }}
+    container: ${{ parameters.container }}
+    buildSourceOnly: true
+    runOnline: ${{ parameters.runOnline }}
+    runTests: false
+    extraProperties: "/p:EnableDevBuildAsDefaultForSourceOnly=true"
+
+# Source Build Validation
+- template: ../jobs/vmr-build.yml
+  parameters:
+    buildName: ${{ format('{0}_SourceBuild_Online_MsftSdk', parameters.buildNamePrefix) }}_Validation
+    isBuiltFromVmr: true
+    vmrBranch: ${{ variables.VmrBranch }}
+    targetArchitecture: ${{ parameters.targetArchitecture }}
+    pool: ${{ parameters.pool_Linux }}
+    container: ${{ parameters.container }}
+    buildSourceOnly: true
+    runOnline: ${{ parameters.runOnline }}
+    runTests: true
+    skipBuild: true
+    extraProperties: >
+      /p:DotNetSourceOnlyTestOnly=true
+      /p:ExtraRestoreSourcePath=$(Pipeline.Workspace)/msft-pkgs
+      /p:RestoreAdditionalProjectSources=$(Pipeline.Workspace)/msft-pkgs
+    dependsOn:
+    - AzureLinux_x64_Cross_x64
+    - Browser_Shortstack_wasm
+    - Windows_x64
+    - ${{ format('{0}_SourceBuild_Online_MsftSdk_x64', parameters.buildNamePrefix) }}
+    testInitSteps:
+    # Ensure the artifacts staging directory exists so that even if no files get placed there, it won't fail
+    # when publishing the artifacts.
+    - script: mkdir -p $(artifactsStagingDir)
+      displayName: Create Artifacts Staging Directory
+    - template: ../steps/download-artifacts.yml
+      parameters:
+        artifactDescription: Microsoft Linux Packages
+        artifactName: AzureLinux_x64_Cross_x64_Artifacts
+        downloadFilePatterns: packages/Release/Shipping/**
+        copyDestination: $(Pipeline.Workspace)/msft-pkgs
+        flattenDirs: true
+    - template: ../steps/download-artifacts.yml
+      parameters:
+        artifactDescription: Microsoft WASM Packages
+        artifactName: Browser_Shortstack_wasm_Artifacts
+        downloadFilePatterns: packages/Release/Shipping/**
+        copyDestination: $(Pipeline.Workspace)/msft-pkgs
+        flattenDirs: true
+    - template: ../steps/download-artifacts.yml
+      parameters:
+        artifactDescription: Microsoft Windows Packages
+        artifactName: Windows_x64_Artifacts
+        downloadFilePatterns: packages/Release/Shipping/**
+        copyDestination: $(Pipeline.Workspace)/msft-pkgs
+        flattenDirs: true
+    - template: ../steps/download-artifacts.yml
+      parameters:
+        artifactDescription: Source Build Assets
+        artifactName: ${{ format('{0}_SourceBuild_Online_MsftSdk_x64', parameters.buildNamePrefix) }}_Artifacts
+        downloadFilePatterns: assets/Release/**/*.tar.gz
+        copyDestination: $(Build.SourcesDirectory)/artifacts

eng/pipelines/templates/jobs/vmr-build.yml

@@ -63,6 +63,18 @@ parameters:
   type: boolean
   default: false
 
+# Name of previous job(s) (from the same template as this) whose output will be used to build this job
+# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
+- name: reuseBuildArtifactsFrom
+  type: object
+  default: ''
+
+# Names of jobs that this job depends on. This only provides a job dependency and does not automatically
+# download artifacts from the other job. Use reuseBuildArtifactsFrom for that.
+- name: dependsOn
+  type: object
+  default: ''
+
 #### SOURCE-ONLY parameters ####
 
 # Instead of building the VMR directly, exports the sources into a tarball and builds from that
@@ -83,12 +95,6 @@ parameters:
   type: boolean
   default: false
 
-# Name of previous job(s) (from the same template as this) whose output will be used to build this job
-# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
-- name: reuseBuildArtifactsFrom
-  type: object
-  default: ''
-
 # Allow downloading artifacts from the internet during the build
 - name: runOnline
   type: boolean
@@ -108,6 +114,16 @@ parameters:
   type: boolean
   default: false
 
+# Skip the build step (this would be used when wanting to run tests only)
+- name: skipBuild
+  type: boolean
+  default: false
+
+# Custom steps to run before running tests
+- name: testInitSteps
+  type: stepList
+  default: []
+
 #### sdk parameters ####
 
 - name: isBuiltFromVmr
@@ -138,7 +154,7 @@ jobs:
         image: ${{ parameters.container.image }}
         options: $(defaultContainerOptions)
 
-  ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
+  ${{ if or(ne(parameters.reuseBuildArtifactsFrom, ''), ne(parameters.dependsOn, '')) }}:
     ${{ if eq(parameters.buildPass, '') }}:
       # For PR builds, skip the stage 2 build if the stage 1 build fails.
       # Otherwise, run the stage 2 build even if the stage 1 build fails so that we can get a complete assessment of the build status.
@@ -149,7 +165,11 @@ jobs:
         condition: succeededOrFailed()
     ${{ else }}:
       condition: succeeded()
-    dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}
+    dependsOn:
+    - ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
+      - ${{ parameters.reuseBuildArtifactsFrom }}
+    - ${{ if ne(parameters.dependsOn, '') }}:
+      - ${{ parameters.dependsOn }}
   variables:
   - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
     - group: AzureDevOps-Artifact-Feeds-Pats
@@ -344,20 +364,15 @@ jobs:
   - ${{ if ne(parameters.reuseBuildArtifactsFrom,'') }}:
     - ${{ each reuseBuildArtifacts in parameters.reuseBuildArtifactsFrom }}:
       - ${{ if eq(parameters.buildSourceOnly, true) }}:
-        - download: current
-          artifact: ${{ reuseBuildArtifacts }}_Artifacts
-          patterns: |
-            **/Private.SourceBuilt.Artifacts.*.tar.gz
-            **/dotnet-sdk-*.tar.gz
-            **/*.nupkg
-          displayName: Download Previous Build (${{ reuseBuildArtifacts }})
-
-        - task: CopyFiles@2
-          displayName: Copy Previous Build (${{ reuseBuildArtifacts }} - Source Build artifacts)
-          inputs:
-            SourceFolder: $(Pipeline.Workspace)/${{ reuseBuildArtifacts }}_Artifacts/assets/Release
-            Contents: '*.tar.gz'
-            TargetFolder: $(sourcesPath)/prereqs/packages/archive/
+        - template: ../steps/download-artifacts.yml@self
+          parameters:
+            artifactDescription: Previous Build (${{ reuseBuildArtifacts }} - Source Build artifacts)
+            artifactName: ${{ reuseBuildArtifacts }}_Artifacts
+            downloadFilePatterns: |
+              **/Private.SourceBuilt.Artifacts.*.tar.gz
+              **/dotnet-sdk-*.tar.gz
+            copyDestination: $(sourcesPath)/prereqs/packages/archive/
+            flattenDirs: true
 
       - ${{ else }}:
         - task: DownloadPipelineArtifact@2
@@ -502,61 +517,62 @@ jobs:
         displayName: Prep the Build
         workingDirectory: $(sourcesPath)
 
-    - script: |
-        set -ex
-        df -h
+    - ${{ if ne(parameters.skipBuild, 'true') }}:
 
-        customEnvVars=""
-        customPreBuildArgs=""
-        customBuildArgs="$(baseArguments) $(signArguments) $(_SignDiagnosticFilesArgs)"
-        extraBuildProperties="$(baseProperties) /p:ArtifactsStagingDir=$(Build.ArtifactStagingDirectory) $(targetProperties) $(signProperties) $(buildPassProperties) ${{ parameters.extraProperties }}"
+      - script: |
+          set -ex
+          df -h
 
-        if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
-          customBuildArgs="$customBuildArgs --online"
-        else
-          customPreBuildArgs="$customPreBuildArgs sudo unshare -n"
-        fi
+          customEnvVars=""
+          customPreBuildArgs=""
+          customBuildArgs="$(baseArguments) $(signArguments) $(_SignDiagnosticFilesArgs)"
+          extraBuildProperties="$(baseProperties) /p:ArtifactsStagingDir=$(Build.ArtifactStagingDirectory) $(targetProperties) $(signProperties) $(buildPassProperties) ${{ parameters.extraProperties }}"
 
-        if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
-          customBuildArgs="$customBuildArgs --poison"
-        fi
+          if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --online"
+          else
+            customPreBuildArgs="$customPreBuildArgs sudo unshare -n"
+          fi
 
-        if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
-          customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
-          customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
-        fi
+          if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --poison"
+          fi
 
-        if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
-          customBuildArgs="$customBuildArgs --source-only"
-          extraBuildProperties="$extraBuildProperties /p:ReportSbrpUsage=true"
-        fi
+          if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
+            customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
+          fi
 
-        if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
-          customBuildArgs="$customBuildArgs --use-mono-runtime"
-        fi
+          if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --source-only"
+            extraBuildProperties="$extraBuildProperties /p:ReportSbrpUsage=true"
+          fi
 
-        if [[ -n "${{ parameters.targetRid }}" ]]; then
-          customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
-        fi
+          if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
+            customBuildArgs="$customBuildArgs --use-mono-runtime"
+          fi
 
-        if [[ -n "${{ parameters.crossRootFs }}" ]]; then
-          customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
-          if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then
-            extraBuildProperties="$extraBuildProperties /p:CrossBuild=true"
+          if [[ -n "${{ parameters.targetRid }}" ]]; then
+            customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
+          fi
+
+          if [[ -n "${{ parameters.crossRootFs }}" ]]; then
+            customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
+            if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then
+              extraBuildProperties="$extraBuildProperties /p:CrossBuild=true"
+            fi
           fi
-        fi
 
-        buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
-        buildArgs=$(echo $buildArgs | xargs) # Remove extra spaces
+          buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
 
-        for envVar in $customEnvVars; do
-          customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
-        done
+          for envVar in $customEnvVars; do
+            customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
+          done
 
-        eval $customEnvVarsWithBashSyntax
-        $customPreBuildArgs ./build.sh $buildArgs
-      displayName: Build
-      workingDirectory: $(sourcesPath)
+          eval $customEnvVarsWithBashSyntax
+          $customPreBuildArgs ./build.sh $buildArgs
+        displayName: Build
+        workingDirectory: $(sourcesPath)
 
     - ${{ if ne(parameters.runOnline, 'True' )}}:
       - script: |
@@ -567,6 +583,8 @@ jobs:
 
     # Only run tests if enabled
     - ${{ if eq(parameters.runTests, 'True') }}:
+      - ${{ parameters.testInitSteps }}
+
       # Setup the NuGet sources used by the tests to use private feeds. This is necessary when testing internal-only product
       # builds where the packages are only available in the private feeds. This allows the tests to restore from those feeds.
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:

eng/pipelines/templates/stages/vmr-verticals.yml

@@ -995,3 +995,12 @@ stages:
             - Windows_x64
             - Windows_x86
             - Windows_arm64
+
+        - template: ../jobs/source-only-build-and-validate.yml
+          parameters:
+            buildNamePrefix: ${{ variables.centOSStreamName }}
+            runOnline: true
+            targetArchitecture: x64
+            container:
+              name: ${{ variables.centOSStreamContainerName }}
+              image: ${{ variables.centOSStreamContainerImage }}

eng/pipelines/templates/steps/download-artifacts.yml

@@ -0,0 +1,25 @@
+parameters:
+- name: artifactDescription
+  type: string
+- name: artifactName
+  type: string
+- name: downloadFilePatterns
+  type: string
+- name: copyDestination
+  type: string
+- name: flattenDirs
+  type: boolean
+  default: false
+
+steps:
+- download: current
+  artifact: ${{ parameters.artifactName }}
+  patterns: ${{ parameters.downloadFilePatterns }}
+  displayName: Download ${{ parameters.artifactDescription }}
+- task: CopyFiles@2
+  displayName: Copy ${{ parameters.artifactDescription }}
+  inputs:
+    SourceFolder: $(Pipeline.Workspace)/${{ parameters.artifactName }}
+    Contents: '**'
+    FlattenFolders: ${{ parameters.flattenDirs }}
+    TargetFolder: ${{ parameters.copyDestination }}

src/Layout/redist/targets/RestoreLayout.targets

@@ -12,7 +12,7 @@
 
     <!-- Blob storage directories are not stabilized, so these must refer to a package that does not stabilize -->
     <!-- In unified build, the layout does match, so use the runtime package versions rather than the VS redist versions -->
-    <AspNetCoreBlobVersion>$(MicrosoftAspNetCoreAppRefInternalPackageVersion)</AspNetCoreBlobVersion>
+    <AspNetCoreBlobVersion>$(MicrosoftAspNetCoreAppRuntimePackageVersion)</AspNetCoreBlobVersion>
     <NetRuntimeBlobVersion>$(MicrosoftNETCorePlatformsPackageVersion)</NetRuntimeBlobVersion>
     <WindowsDesktopBlobVersion>$(VSRedistCommonWindowsDesktopSharedFrameworkx64100PackageVersion)</WindowsDesktopBlobVersion>
     <WindowsDesktopBlobVersion Condition="'$(DotNetBuildOrchestrator)' == 'true'">$(MicrosoftWindowsDesktopAppRuntimePackageVersion)</WindowsDesktopBlobVersion>