Port SBOM generation to v3.x/ps6 (#726)

* Add SBOM generation as part of the build process (#716)

* Add SBOM task to generate manifest

* Update pipeline to generate manifest

* Update VM pool and image name (#718)

* Generate SBOM only for release builds

* Publish the test results

* Copy package to ArtifactStagingDirectory

* Simplify logic to upload the nuget package for integration testing

* Install .Net 3.1 which is required by the Microsoft.ManifestTool.dll tool for release builds

* Update AspNetVNext value from https://dotnet.myget.org/ to https://www.myget.org/

* Fix CosmosDB E2E test

* Obtain the branch name from BuildSourceBranch (#728)

* Update task condition boolean values to include single quotes (#731)

* Update task condition boolean value to include single quotes

Author: Francisco-Gamino

NuGet.config

@@ -5,6 +5,6 @@
     <add key="azure_app_service" value="https://www.myget.org/F/azure-appservice/api/v2" />
     <add key="azure_app_service_staging" value="https://www.myget.org/F/azure-appservice-staging/api/v2" />
     <add key="buildTools" value="https://www.myget.org/F/30de4ee06dd54956a82013fa17a3accb/" />
-    <add key="AspNetVNext" value="https://dotnet.myget.org/F/aspnetcore-dev/api/v3/index.json" />
+    <add key="AspNetVNext" value="https://www.myget.org/F/aspnetcore-dev/api/v3/index.json" />
   </packageSources>
 </configuration>

azure-pipelines.yml

@@ -8,20 +8,50 @@
 #- dev
 
 pool:
-    vmImage: 'vs2017-win2016'
+    name: '1ES-Hosted-AzFunc'
+    vmImage: 'MMS2019TLS'
 
 variables:
   Configuration: Release
   buildNumber: $[ counter('build', 400) ] # Start higher than our AppVeyor versions. Every build (pr or branch) will increment.
 
 steps:
+- pwsh: |
+    $releaseBranches = @('v4.x/ps7.2', 'v4.x/ps7.0', 'v3.x/ps7', 'v3.x/ps6', 'v2.x')
+
+    Write-Host "BuildSourceBranch: $($env:BuildSourceBranch)"
+    $branchName = $env:BuildSourceBranch.Replace("refs/heads/", "")
+    Write-Host "BranchName: $branchName"
+
+    $isReleaseBuild = ($releaseBranches -contains $branchName)
+    Write-Host "##vso[task.setvariable variable=IsReleaseBuild]$isReleaseBuild"
+    Write-Host "IsReleaseBuild: $isReleaseBuild"
+  displayName: 'Set IsReleaseBuild variable'
+  env:
+    BuildSourceBranch: $(Build.SourceBranch)
+
+- pwsh: |
+    Invoke-WebRequest 'https://raw.githubusercontent.com/dotnet/cli/master/scripts/obtain/dotnet-install.ps1' -OutFile 'dotnet-install.ps1'
+    ./dotnet-install.ps1 -InstallDir "$env:ProgramFiles/dotnet" -Version "3.1.415" -Channel 'release'
+  displayName: 'Install .Net 3.1 which is required by the Microsoft.ManifestTool.dll tool'
+  condition: eq(variables['IsReleaseBuild'], 'true')
+
 - pwsh: ./build.ps1 -NoBuild -Bootstrap
   displayName: 'Running ./build.ps1 -NoBuild -Bootstrap'
 
 - pwsh: |
       $ErrorActionPreference = "Stop"
-      ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)"
+      if ($isReleaseBuild)
+      {
+        ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)" -AddSBOM -SBOMUtilSASUrl $env:SBOMUtilSASUrl
+      }
+      else
+      {
+        ./build.ps1 -Clean -Configuration Release -BuildNumber "$(buildNumber)"
+      }
   displayName: 'Build worker code'
+  env:
+    SBOMUtilSASUrl: $(SBOMUtilSASUrl)
 
 - pwsh: ./build.ps1 -NoBuild -Test
   displayName: 'Running UnitTest'
@@ -36,41 +66,23 @@ steps:
     PSWorkerEnableExperimentalDurableFunctions: "true"
   displayName: 'Running E2ETest'
 
-- task: CopyFiles@2
+- task: PublishTestResults@2
   inputs:
-    SourceFolder: '$(System.DefaultWorkingDirectory)/testResults'
-    Contents: '*.trx'
-    TargetFolder: '$(Build.ArtifactStagingDirectory)'
-  displayName: 'Copying test result file for artifacts'
+    testResultsFormat: 'VSTest'
+    testResultsFiles: '**/*.trx'
+    failTaskOnFailedTests: true
+  condition: succeededOrFailed()
+  displayName: 'Publish tests results'
 
-- task: PublishBuildArtifacts@1
+- task: CopyFiles@2
   inputs:
-    PathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    ArtifactName: 'drop'
-    publishLocation: 'Container'
-  displayName: 'Publishing build and test result artifacts'
+    SourceFolder: '$(System.DefaultWorkingDirectory)/package'
+    Contents: '**/*.nupkg'
+    TargetFolder: '$(Build.ArtifactStagingDirectory)'
+  displayName: 'Copy package to artifacts directory'
 
-- pwsh: |
-      $sourcePath  = (ls package/**/*/Microsoft.Azure.Functions.PowerShellWorker*.nupkg).FullName
-      if (-not (test-path $sourcePath))
-      {
-        throw 'Unable to find Microsoft.Azure.Functions.PowerShellWorker*.nupkg at ./package'
-      }
-      Copy-Item -Path $sourcePath -Destination $(Build.ArtifactStagingDirectory) -ErrorAction Stop -Verbose -Force
-  displayName: 'Copy package to ArtifactStagingDirectory'
-
-- pwsh: |
-    $uploadPackage = $null
-    if (-not ([bool]::TryParse($env:UPLOADPACKAGETOPRERELEASEFEED, [ref] $uploadPackage)))
-    {
-      throw "UploadPackageToPreReleaseFeed can only be set to True or False. Current value is set to $env:UPLOADPACKAGETOPRERELEASEFEED"
-    }
-    Write-Host "##vso[task.setvariable variable=UploadPackage]$uploadPackage"
-    Write-Host "UploadPackage: $uploadPackage"
-  displayName: 'Set UploadPackage variable'
-
 - task: NuGetCommand@2
-  condition: and(ne(variables['Build.Reason'], 'PullRequest'), in(variables['Build.SourceBranch'], 'refs/heads/v3.x/ps7', 'refs/heads/v3.x/ps6', 'refs/heads/v2.x'), eq(variables.UploadPackage, false))
+  condition: and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['IsReleaseBuild'], 'true'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], 'false'))
   inputs:
     command: 'push'
     packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'
@@ -80,7 +92,7 @@ steps:
   displayName: 'Push NuGet package'
 
 - task: NuGetCommand@2
-  condition: eq(variables.UploadPackage, true)
+  condition: eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], 'true')
   inputs:
     command: 'push'
     packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'

build.ps1

@@ -22,11 +22,46 @@ param(
     $Configuration = "Debug",
 
     [string]
-    $BuildNumber = '0'
+    $BuildNumber = '0',
+
+    [switch]
+    $AddSBOM,
+
+    [string]
+    $SBOMUtilSASUrl
 )
 
 #Requires -Version 6.0
 
+function Install-SBOMUtil
+{
+    if ([string]::IsNullOrEmpty($SBOMUtilSASUrl))
+    {
+        throw "The `$SBOMUtilSASUrl parameter cannot be null or empty when specifying the `$AddSBOM switch"
+    }
+
+    $MANIFESTOOLNAME = "ManifestTool"
+    Write-Host "Installing $MANIFESTOOLNAME..."
+
+    $MANIFESTOOL_DIRECTORY = Join-Path $PSScriptRoot $MANIFESTOOLNAME
+    Remove-Item -Recurse -Force $MANIFESTOOL_DIRECTORY -ErrorAction Ignore
+
+    Invoke-RestMethod -Uri $SBOMUtilSASUrl -OutFile "$MANIFESTOOL_DIRECTORY.zip"
+    Expand-Archive "$MANIFESTOOL_DIRECTORY.zip" -DestinationPath $MANIFESTOOL_DIRECTORY
+
+    $dllName = "Microsoft.ManifestTool.dll"
+    $manifestToolPath = "$MANIFESTOOL_DIRECTORY/$dllName"
+
+    if (-not (Test-Path $manifestToolPath))
+    {
+        throw "$MANIFESTOOL_DIRECTORY does not contain '$dllName'"
+    }
+
+    Write-Host 'Done.'
+
+    return $manifestToolPath
+}
+
 Import-Module "$PSScriptRoot/tools/helper.psm1" -Force
 
 # Bootstrap step
@@ -87,6 +122,29 @@ if(!$NoBuild.IsPresent) {
         -OutFile "$PSScriptRoot/src/Modules/Microsoft.PowerShell.Management/Microsoft.PowerShell.Management.psd1" 
 
     dotnet publish -c $Configuration "/p:BuildNumber=$BuildNumber" $PSScriptRoot
+
+    if ($AddSBOM)
+    {
+        # Install manifest tool
+        $manifestTool = Install-SBOMUtil
+        Write-Log "manifestTool: $manifestTool "
+
+        # Generate manifest
+        $buildPath = "$PSScriptRoot/src/bin/$Configuration/$TargetFramework/publish"
+        $telemetryFilePath = Join-Path $PSScriptRoot ((New-Guid).Guid + ".json")
+        $packageName = "Microsoft.Azure.Functions.PowerShellWorker.nuspec"
+
+        # Delete the manifest folder if it exists
+        $manifestFolderPath = Join-Path $buildPath "_manifest"
+        if (Test-Path $manifestFolderPath)
+        {
+            Remove-Item $manifestFolderPath -Recurse -Force -ErrorAction Ignore
+        }
+
+        Write-Log "Running: dotnet $manifestTool generate -BuildDropPath $buildPath -BuildComponentPath $buildPath -Verbosity Information -t $telemetryFilePath"
+        & { dotnet $manifestTool generate -BuildDropPath $buildPath -BuildComponentPath $buildPath -Verbosity Information -t $telemetryFilePath -PackageName $packageName }
+    }
+
     dotnet pack -c $Configuration "/p:BuildNumber=$BuildNumber" "$PSScriptRoot/package"
 }
 

test/E2E/Azure.Functions.PowerShellWorker.E2E/Azure.Functions.PowerShellWorker.E2E/Helpers/CosmosDBHelpers.cs

@@ -57,7 +57,7 @@ await Utilities.RetryAsync(async () =>
                     retrievedDocument = await _docDbClient.ReadDocumentAsync(docUri);
                     return true;
                 }
-                catch (DocumentClientException ex) when (ex.Error.Code == "NotFound")
+                catch (DocumentClientException ex) when (ex.Error.Code == "NotFound" || ex.Error.Code == "Not Found")
                 {
                     return false;
                 }