Add a job for experimental Framework MSBuild insertion to a pipeline (#9309)

AR-May · MichalPavlik · commit 98ac0d87a470 · 2023-10-17T12:48:26.000+02:00
Context
We need to add an extra job to .exp-insertions.yml pipeline for creating experimental Framework MSBuild.

Changes Made
Pipeline refactored, a job added
Fixed bug in deploy script code path for Framework MSBuild
Testing
Manual run of the pipeline
diff --git a/.exp-insertions.yml b/.exp-insertions.yml
@@ -1,13 +1,19 @@
-# Pipeline creates a dotnet with experimental msbuild dlls.
+# Pipeline creates experimental msbuild insertions.
 
 trigger: none # Prevents this pipeline from triggering on check-ins
 pr: none # don't run this on PR as well
 
 parameters:
-  # Dotnet installer channel from where to take the latest dotnet bits.
+  # Dotnet installer channel from which to take the latest dotnet bits.
   - name: DotnetInstallerChannel
     displayName: Dotnet installer channel
+    type: string  
+    default: 'none'
+  # VS version for which to take the latest Retail MSBuild bits.
+  - name: VSVersionName
+    displayName: VS Version
     type: string
+    default: 'none'
   # Branch from the MSBuild Build CI pipeline. Default: main
   # Top run for the branch would be used to create an experimental insertion. 
   - name: MSBuildBranch
@@ -23,105 +29,192 @@ parameters:
 variables:
   - name: _MsBuildCiPipelineId
     value: 9434
+  - name: _MSBuildConfigFilePath
+    value: "config/batmon/Q-Prod-Co3/Coordinator/ToolsReleaseConfig-GeneralPublic.json"
+  - name: VSVersion
+    value: ${{parameters.VSVersionName}}
 
 pool:
   vmImage: windows-latest
 
-steps:
-- powershell: |
-    mkdir '$(System.ArtifactsDirectory)/installer'
-
-    $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
-    $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
-
-    foreach ($sdk in $sdks)
-    {
-      Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
-      Invoke-WebRequest `
-        -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
-        -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
-    }
-    mkdir '$(Pipeline.Workspace)/artifacts'
-    
-  displayName: Download latest dotnet sdks
+resources:
+  repositories: 
+    - repository: CloudBuildConfig
+      type: git
+      name: CloudBuild/CloudBuildConfig
+      endpoint: CloudBuild_Test
+      ref: refs/heads/main 
 
-# Download latest build artifacts for a branch from MSBuild Build CI
-- ${{ if eq(parameters.MSBuildBuildID, 'default') }}:
-  - task: DownloadBuildArtifacts@1
-    inputs:
-      buildType: specific
-      project: DevDiv
-      pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: latestFromBranch
-      branchName: '${{parameters.MSBuildBranch}}'  
-      artifactName: bin
-      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"  
-    displayName: Download latest msbuild from branch
+jobs:
+- job: CreateExpDotnet
+  displayName: Create Experimental Dotnet
+  condition: ne('${{ parameters.DotnetInstallerChannel }}', 'none')
+  steps:
+  - powershell: |
+      mkdir '$(System.ArtifactsDirectory)/installer'
+
+      $dotnetChannel = '${{parameters.DotnetInstallerChannel}}'
+      $sdks = "dotnet-sdk-win-x64.zip", "dotnet-sdk-linux-x64.tar.gz"
+
+      foreach ($sdk in $sdks)
+      {
+        Write-Host "Downloading dotnet $sdk from channel $dotnetChannel"
+        Invoke-WebRequest `
+          -Uri "https://aka.ms/dotnet/$dotnetChannel/daily/$sdk" `
+          -OutFile "$(System.ArtifactsDirectory)/installer/$sdk"
+      }
+      mkdir '$(Pipeline.Workspace)/artifacts'
+    displayName: Download latest dotnet sdks
 
-# Download build artifacts for MSBuild Build CI specific build
-- ${{ if ne(parameters.MSBuildBuildID, 'default') }}:
   - task: DownloadBuildArtifacts@1
     inputs:
       buildType: specific
       project: DevDiv
       pipeline: $(_MsBuildCiPipelineId) 
-      buildVersionToDownload: specific
-      buildId: ${{parameters.MSBuildBuildID}} 
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}: 
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}} 
       artifactName: bin
       downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
-      itemPattern: "MSBuild.Bootstrap/**"
-    displayName: Download specified msbuild build
-    
-- powershell: |
-    $sdk = "dotnet-sdk-win-x64"
+    displayName: Download msbuild artifacts
 
-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
-    Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+  - powershell: |
+      $sdk = "dotnet-sdk-win-x64"
+
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.zip"
+      Expand-Archive "$(System.ArtifactsDirectory)/installer/$sdk.zip" -DestinationPath "$(Pipeline.Workspace)/exp-dotnet/$sdk"
     
-    $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
+      $dotnetDirectory = Get-ChildItem -Directory -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk"
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
     
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
     
-    Write-Host "Compressing dotnet sdk files"
-    Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
+      Write-Host "Compressing dotnet sdk files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/exp-dotnet/$sdk" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/$sdk.zip"
 
-  displayName: Dogfood msbuild dlls to dotnet sdk win-x64
+    displayName: Dogfood msbuild dlls to dotnet sdk win-x64
 
-- powershell: |
-    $sdk = "dotnet-sdk-linux-x64"
+  - powershell: |
+      $sdk = "dotnet-sdk-linux-x64"
     
-    mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+      mkdir "$(Pipeline.Workspace)/exp-dotnet/$sdk"
 
-    Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
-    tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
+      Write-Host "Extracting $(System.ArtifactsDirectory)/installer/$sdk.tar.gz"
+      tar -xzvf "$(System.ArtifactsDirectory)/installer/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk"
 
-    $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
-    $dotnetVersion = $dotnetDirectory.Name
-    Write-Host "Detected dotnet version: $dotnetVersion"
+      $dotnetDirectory = Get-ChildItem -Directory -Path $(Pipeline.Workspace)/exp-dotnet/$sdk/sdk
+      $dotnetVersion = $dotnetDirectory.Name
+      Write-Host "Detected dotnet version: $dotnetVersion"
     
-    Write-Host "Updating MSBuild dlls."
-    $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
-      -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
-      -bootstrapDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin/MSBuild.Bootstrap" `
-      -configuration Release `
-      -makeBackup $false
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/exp-dotnet/$sdk/sdk/$dotnetVersion" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
     
-    Write-Host "Compressing dotnet sdk files"
-    tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
-  displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
-
-- task: PublishPipelineArtifact@1
-  inputs:
-    targetPath: '$(Pipeline.Workspace)/artifacts'
-    artifactName: ExperimentalDotnet
-    parallel: true
-  condition: always()
-  displayName: Publish crank assests artifacts
+      Write-Host "Compressing dotnet sdk files"
+      tar -czvf "$(Pipeline.Workspace)/artifacts/$sdk.tar.gz" -C "$(Pipeline.Workspace)/exp-dotnet/$sdk" .
+    displayName: Dogfood msbuild dlls to dotnet sdk linux-x64
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalDotnet
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts
+
+
+- job: CreateExpMSBuild
+  displayName: "Create Experimental MSBuild"
+  condition: ne('${{ parameters.VSVersionName }}', 'none')
+  steps:
+  - checkout: self
+
+  - checkout: CloudBuildConfig
+
+  - powershell: |
+      $json = (Get-Content "$(Build.SourcesDirectory)/CloudBuildConfig/$(_MSBuildConfigFilePath)" -Raw) | ConvertFrom-Json 
+      $MSBuildDropPath = $json.Tools.MSBuild.Locations
+      Write-Host "##vso[task.setvariable variable=MSBuildDropPath]$MSBuildDropPath"
+      Write-Host "MSBuild Drop Path directory: $MSBuildDropPath"
+    displayName: Get Retail MSBuild Drop Path
+  
+  - task: NuGetToolInstaller@1
+    displayName: 'Install NuGet.exe'
+
+  - task: NuGetCommand@2
+    displayName: Restore internal tools
+    inputs:
+      command: restore
+      feedsToUse: config
+      restoreSolution: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\eng\common\internal\Tools.csproj'
+      nugetConfigPath: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\NuGet.config'
+      restoreDirectory: '$(Build.SourcesDirectory)\DotNet-msbuild-Trusted\.packages'
+  
+  - powershell: |
+      mkdir "$(Pipeline.Workspace)/artifacts"
+      
+      $dropAppDirectory = Get-ChildItem -Directory -Path "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app"
+      $dropAppVersion = $dropAppDirectory.Name
+      Write-Host "Detected drop.exe version: $dropAppVersion"
+
+      $dropExePath = "$(Build.SourcesDirectory)/DotNet-msbuild-Trusted/.packages/drop.app/$dropAppVersion/lib/net45/drop.exe"
+      Write-Host "Detected drop.exe path: $dropExePath"
+
+      Write-Host "Downloading VS msbuild"
+      & "$dropExePath" get --patAuthEnvVar 'cloudbuild-token' -u "$(MSBuildDropPath)\$(VSVersion)" -d "$(System.ArtifactsDirectory)/VSMSBuildDrop"
+      Write-Host "Download of VS msbuild finished"
+
+      Write-Host "Copying VS msbuild to $(Pipeline.Workspace)/VSMSBuild"
+      Copy-Item -Path "$(System.ArtifactsDirectory)/VSMSBuildDrop/*" -Destination "$(Pipeline.Workspace)/VSMSBuild" -Recurse
+      Write-Host "Copy of VS msbuild finished"
+    displayName: Download msbuild vs drop
+    env:
+      cloudbuild-token: $(cloudbuild-token)
+      
+  - task: DownloadBuildArtifacts@1
+    inputs:
+      buildType: specific
+      project: DevDiv
+      pipeline: $(_MsBuildCiPipelineId) 
+      ${{ if eq(parameters.MSBuildBuildID, 'default') }}: 
+        buildVersionToDownload: latestFromBranch
+        branchName: '${{parameters.MSBuildBranch}}'
+      ${{ else }}:
+        buildVersionToDownload: specific
+        buildId: ${{parameters.MSBuildBuildID}} 
+      artifactName: bin
+      downloadPath: '$(System.ArtifactsDirectory)/msbuild/artifacts/bin'
+    displayName: Download msbuild artifacts
+
+  - powershell: |
+      Write-Host "Updating MSBuild dlls."
+      $(Build.SourcesDirectory)/DotNet-msbuild-Trusted/scripts/Deploy-MSBuild.ps1 `
+        -destination "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)/MSBuild/Current/Bin" `
+        -binDirectory "$(System.ArtifactsDirectory)/msbuild/artifacts/bin" `
+        -configuration Release `
+        -makeBackup $false
+
+      ls "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)"
+      Write-Host "Compressing msbuild files"
+      Get-ChildItem -Path "$(Pipeline.Workspace)/VSMSBuild/$(VSVersion)" | Compress-Archive -DestinationPath "$(Pipeline.Workspace)/artifacts/MSBuild.zip"
+    displayName: Dogfood msbuild dlls 
+
+  - task: PublishPipelineArtifact@1
+    inputs:
+      targetPath: '$(Pipeline.Workspace)/artifacts'
+      artifactName: ExperimentalMSBuild
+      parallel: true
+    condition: always()
+    displayName: Publish crank assests artifacts
diff --git a/scripts/Deploy-MSBuild.ps1 b/scripts/Deploy-MSBuild.ps1
@@ -6,7 +6,7 @@ Param(
   [string] $configuration = "Debug",
   [ValidateSet('Core','Desktop', 'Detect', 'Full')]
   [string] $runtime = "Detect",
-  [string] $bootstrapDirectory = "",
+  [string] $binDirectory = "",
   [bool] $makeBackup = $true
 )
 
@@ -80,11 +80,11 @@ if ($runtime -eq "Desktop") {
 }
 
 # If bootstrap directory is not defined in parameters, use the default location
-if ($bootstrapDirectory -eq "") {
-    $bootstrapDirectory = "artifacts\bin\MSBuild.Bootstrap" 
+if ($binDirectory -eq "") {
+    $binDirectory = "artifacts\bin" 
 }
 
-$bootstrapBinDirectory = "$bootstrapDirectory\$configuration\$targetFramework"
+$bootstrapBinDirectory = "$binDirectory\MSBuild.Bootstrap\$configuration\$targetFramework"
 
 $filesToCopyToBin = @(
     FileToCopy "$bootstrapBinDirectory\Microsoft.Build.dll"
@@ -116,8 +116,8 @@ $filesToCopyToBin = @(
 
 if ($runtime -eq "Desktop") {
     $runtimeSpecificFiles = @(
-        FileToCopy "artifacts\bin\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll"
-        FileToCopy "artifacts\bin\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll"
+        FileToCopy "$binDirectory\Microsoft.Build.Conversion\$configuration\$targetFramework\Microsoft.Build.Conversion.Core.dll"
+        FileToCopy "$binDirectory\Microsoft.Build.Engine\$configuration\$targetFramework\Microsoft.Build.Engine.dll"
 
         FileToCopy "$bootstrapBinDirectory\Microsoft.Bcl.AsyncInterfaces.dll"
         FileToCopy "$bootstrapBinDirectory\Microsoft.Data.Entity.targets"
@@ -152,14 +152,14 @@ if ($runtime -eq "Desktop") {
     $x86files = @(
         FileToCopy "$bootstrapBinDirectory\MSBuild.exe"
         FileToCopy "$bootstrapBinDirectory\MSBuild.exe.config"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb"
+        FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.exe"
+        FileToCopy "$binDirectory\MSBuildTaskHost\$configuration\net35\MSBuildTaskHost.pdb"
     )
     $amd64files = @(
-        FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe"
-        FileToCopy "artifacts\bin\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe"
-        FileToCopy "artifacts\bin\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb"
+        FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe"
+        FileToCopy "$binDirectory\MSBuild\x64\$configuration\$targetFramework\MSBuild.exe.config"
+        FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.exe"
+        FileToCopy "$binDirectory\MSBuildTaskHost\x64\$configuration\net35\MSBuildTaskHost.pdb"
     )
 }
 
