From 7fd4b166ea46c8fa4b3a7f0954c9f0f99b6467ac Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 22 Jan 2024 22:36:03 -0700 Subject: [PATCH 01/86] Bump xunit from 2.6.4 to 2.6.6 --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 41167d99..1438548b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,7 +8,7 @@ - + From 050805e88f48ff2954489c7a795dda28c5282fb5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 25 Jan 2024 14:04:06 -0700 Subject: [PATCH 02/86] Add switch to avoid creating symbolic links Symbolic links present a problem, at least for "1ES PT". --- azure-pipelines/artifacts/_pipelines.ps1 | 5 +++-- azure-pipelines/artifacts/_stage_all.ps1 | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/artifacts/_pipelines.ps1 b/azure-pipelines/artifacts/_pipelines.ps1 index 2d3338b2..47321ed5 100644 --- a/azure-pipelines/artifacts/_pipelines.ps1 +++ b/azure-pipelines/artifacts/_pipelines.ps1 @@ -7,7 +7,8 @@ [CmdletBinding()] param ( [string]$ArtifactNameSuffix, - [switch]$StageOnly + [switch]$StageOnly, + [switch]$AvoidSymbolicLinks ) Function Set-PipelineVariable($name, $value) { @@ -24,7 +25,7 @@ Function Test-ArtifactUploaded($artifactName) { Test-Path "env:$varName" } -& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix |% { +& "$PSScriptRoot/_stage_all.ps1" -ArtifactNameSuffix $ArtifactNameSuffix -AvoidSymbolicLinks:$AvoidSymbolicLinks |% { # Set a variable which will out-live this script so that a subsequent attempt to collect and upload artifacts # will skip this one from a check in the _all.ps1 script. Set-PipelineVariable "ARTIFACTSTAGED_$($_.Name.ToUpper())" 'true' diff --git a/azure-pipelines/artifacts/_stage_all.ps1 b/azure-pipelines/artifacts/_stage_all.ps1 index b077931c..74d7a38d 100644 --- a/azure-pipelines/artifacts/_stage_all.ps1 +++ b/azure-pipelines/artifacts/_stage_all.ps1 @@ -7,7 +7,8 @@ [CmdletBinding()] param ( - [string]$ArtifactNameSuffix + [string]$ArtifactNameSuffix, + [switch]$AvoidSymbolicLinks ) $ArtifactStagingFolder = & "$PSScriptRoot/../Get-ArtifactsStagingDirectory.ps1" -CleanIfLocal @@ -48,7 +49,12 @@ $Artifacts |% { if (-not (Test-Path $DestinationFolder)) { New-Item -ItemType Directory -Path $DestinationFolder | Out-Null } if (Test-Path -PathType Leaf $_.Source) { # skip folders - Create-SymbolicLink -Link (Join-Path $DestinationFolder $Name) -Target $_.Source + $TargetPath = Join-Path $DestinationFolder $Name + if ($AvoidSymbolicLinks) { + Copy-Item -Path $_.Source -Destination $TargetPath + } else { + Create-SymbolicLink -Link $TargetPath -Target $_.Source + } } } From 9466f8ed67a86e5fa195c12beb501e2abf0f1446 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 26 Jan 2024 11:05:24 -0700 Subject: [PATCH 03/86] Clarify parameter type in AzP template --- azure-pipelines/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 5d1835d5..3d53ed83 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -4,6 +4,7 @@ parameters: default: vmImage: windows-2022 - name: includeMacOS + type: boolean - name: RunTests type: boolean default: true From 89ada64c57cb0dc31794c4c479f702eec86978ce Mon Sep 17 00:00:00 2001 From: trevors20 <49179298+trevors20@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:33:00 -0800 Subject: [PATCH 04/86] MicroBuild to 1ES PT template transition (#246) Co-authored-by: Andrew Arnott --- .vscode/settings.json | 1 + azure-pipelines/build.yml | 62 +++++++++++++- azure-pipelines/dotnet.yml | 15 +++- azure-pipelines/microbuild.after.yml | 4 - azure-pipelines/official.yml | 89 +++++++++++++++----- azure-pipelines/secure-development-tools.yml | 26 +----- 6 files changed, 141 insertions(+), 56 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ce72437c..1b99a955 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "files.trimFinalNewlines": true, + "azure-pipelines.1ESPipelineTemplatesSchemaFile": true, "omnisharp.enableEditorConfigSupport": true, "omnisharp.enableRoslynAnalyzers": true, "dotnet.completion.showCompletionItemsFromUnimportedNamespaces": true, diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index ecf2f7c4..64e97cf7 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -3,6 +3,14 @@ parameters: type: object default: vmImage: windows-2022 +- name: linuxPool + type: object + default: + vmImage: ubuntu-20.04 +- name: macOSPool + type: object + default: + vmImage: macOS-12 - name: includeMacOS type: boolean - name: RunTests @@ -14,6 +22,17 @@ parameters: - name: EnableAPIScan type: boolean default: false +- name: artifact_names + type: object + default: + - build_logs + - coverageResults + - deployables + - projectAssetsJson + - symbols + - testResults + - test_symbols + - Variables jobs: - job: Windows @@ -23,6 +42,25 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline - group: VSCloudServices-APIScan # Expected to provide ApiScanClientId, ApiScanSecret, ApiScanTenant + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Windows + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows + artifactName: ${{ artifact_name }}-Windows + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish VSInsertion-Windows + targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows + artifactName: VSInsertion-Windows + # This is useful when false positives appear so we can copy some of the output into the suppressions file. + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish Guardian failures + targetPath: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions + artifactName: guardian_failures_as_suppressions + condition: failed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. @@ -52,8 +90,16 @@ jobs: - template: expand-template.yml - job: Linux - pool: - vmImage: Ubuntu 20.04 + pool: ${{ parameters.linuxPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux + artifactName: ${{ artifact_name }}-Linux steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. @@ -68,8 +114,16 @@ jobs: - job: macOS condition: ${{ parameters.includeMacOS }} - pool: - vmImage: macOS-12 + pool: ${{ parameters.macOSPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS + artifactName: ${{ artifact_name }}-macOS steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 51241549..61c96a35 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -15,10 +15,17 @@ steps: displayName: โš™ Update pipeline variables based on build outputs condition: succeededOrFailed() -- powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose - failOnStderr: true - displayName: ๐Ÿ“ข Publish artifacts - condition: succeededOrFailed() +- ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + - powershell: azure-pipelines/artifacts/_pipelines.ps1 -StageOnly -AvoidSymbolicLinks -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose + failOnStderr: true + displayName: ๐Ÿ“ข Publish artifacts + condition: succeededOrFailed() + +- ${{ if ne(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose + failOnStderr: true + displayName: ๐Ÿ“ข Publish artifacts + condition: succeededOrFailed() - ${{ if and(ne(variables['codecov_token'], ''), parameters.RunTests) }}: - powershell: | diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 3dbbad9c..7531cf54 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -10,10 +10,6 @@ steps: $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration)/NuGet condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- task: MicroBuildCleanup@1 - condition: succeededOrFailed() - displayName: โš™๏ธ MicroBuild Cleanup - - task: Ref12Analyze@0 displayName: ๐Ÿ“‘ Ref12 (Codex) Analyze inputs: diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 6df6f28b..915c7e76 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -42,24 +42,75 @@ parameters: type: boolean default: true -stages: +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release -- stage: Build - variables: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - BuildConfiguration: Release - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages - SignTypeSelection: ${{ parameters.SignTypeSelection }} - Packaging.EnableSBOMSigning: false - Codeql.Enabled: true - - jobs: - - template: build.yml +extends: + ${{ if parameters.EnableCompliance }}: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - EnableCompliance: ${{ parameters.EnableCompliance }} - EnableAPIScan: ${{ parameters.EnableAPIScan }} - windowsPool: VSEngSS-MicroBuild2022-1ES - includeMacOS: ${{ parameters.includeMacOS }} - RunTests: ${{ parameters.RunTests }} - -- template: prepare-insertion-stages.yml + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + stages: + - stage: Build + variables: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + BuildConfiguration: Release + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + SignTypeSelection: ${{ parameters.SignTypeSelection }} + Packaging.EnableSBOMSigning: false + Codeql.Enabled: true + jobs: + - template: /azure-pipelines/build.yml@self + parameters: + EnableCompliance: ${{ parameters.EnableCompliance }} + EnableAPIScan: ${{ parameters.EnableAPIScan }} + windowsPool: VSEngSS-MicroBuild2022-1ES + linuxPool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + os: Linux + macOSPool: + name: Azure Pipelines + vmImage: macOS-12 + os: macOS + includeMacOS: ${{ parameters.includeMacOS }} + RunTests: ${{ parameters.RunTests }} + - template: /azure-pipelines/prepare-insertion-stages.yml@self + ${{ else }}: + template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + stages: + - stage: Build + variables: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + BuildConfiguration: Release + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + SignTypeSelection: ${{ parameters.SignTypeSelection }} + Packaging.EnableSBOMSigning: false + Codeql.Enabled: true + jobs: + - template: /azure-pipelines/build.yml@self + parameters: + EnableCompliance: ${{ parameters.EnableCompliance }} + EnableAPIScan: ${{ parameters.EnableAPIScan }} + windowsPool: VSEngSS-MicroBuild2022-1ES + linuxPool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + os: Linux + macOSPool: + name: Azure Pipelines + vmImage: macOS-12 + os: macOS + includeMacOS: ${{ parameters.includeMacOS }} + RunTests: ${{ parameters.RunTests }} + - template: /azure-pipelines/prepare-insertion-stages.yml@self diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index ce8ac94c..1bbacefe 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -6,18 +6,8 @@ steps: - powershell: echo "##vso[build.addbuildtag]compliance" displayName: ๐Ÿท๏ธ Tag run with 'compliance' -- task: CredScan@3 - displayName: ๐Ÿ” Run CredScan - -- task: PoliCheck@2 - displayName: ๐Ÿ” Run PoliCheck - inputs: - targetType: F - targetArgument: $(System.DefaultWorkingDirectory) - optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml - - task: CopyFiles@2 - displayName: ๐Ÿ“‚ Collect APIScan/BinSkim inputs + displayName: ๐Ÿ“‚ Collect APIScan inputs inputs: SourceFolder: $(Build.ArtifactStagingDirectory)/Symbols-$(Agent.JobName) # Exclude any patterns from the Contents (e.g. `!**/git2*`) that we have symbols for but do not need to run APIScan on. @@ -29,14 +19,6 @@ steps: !**/osx*/** TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs -- task: BinSkim@4 - displayName: ๐Ÿ” Run BinSkim - inputs: - InputType: Basic - Function: analyze - TargetPattern: guardianGlob - AnalyzeTargetGlob: $(Build.ArtifactStagingDirectory)/APIScanInputs/**/*.dll;$(Build.ArtifactStagingDirectory)/APIScanInputs/**/*.exe - - task: APIScan@2 displayName: ๐Ÿ” Run APIScan inputs: @@ -72,9 +54,3 @@ steps: GdnBreakSuppressionSets: falsepositives GdnBreakOutputSuppressionFile: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions/ GdnBreakOutputSuppressionSet: falsepositives - -# This is useful when false positives appear so we can copy some of the output into the suppressions file. -- publish: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions - artifact: guardian_failures_as_suppressions - displayName: ๐Ÿ” Publish Guardian failures - condition: failed() From 5c15e00ac2c63b6ad724d36b5356ee2fef5dad31 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 26 Jan 2024 12:24:30 -0700 Subject: [PATCH 05/86] Update the other 4 entrypoints for 1ES PT --- azure-pipelines/archive-sourcecode.yml | 83 ++++++---- azure-pipelines/prepare-insertion-stages.yml | 2 - azure-pipelines/release.yml | 109 +++++++------ azure-pipelines/vs-insertion.yml | 97 ++++++------ azure-pipelines/vs-validation.yml | 153 ++++++++++--------- 5 files changed, 249 insertions(+), 195 deletions(-) diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index db270eb9..afdf0f6b 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -11,6 +11,13 @@ schedules: include: - main +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + parameters: - name: notes displayName: Notes to include in the SCA request @@ -24,35 +31,47 @@ parameters: variables: - group: VS Core team # Expected to provide ManagerAlias, SourceCodeArchivalUri -pool: - name: AzurePipelines-EO - vmImage: AzurePipelinesUbuntu20.04compliant - -steps: -- checkout: self - clean: true - fetchDepth: 0 -- powershell: tools/Install-DotNetSdk.ps1 - displayName: โš™ Install .NET SDK -- task: NuGetAuthenticate@1 - displayName: ๐Ÿ” Authenticate NuGet feeds - inputs: - forceReinstallCredentialProvider: true -- script: dotnet tool restore - displayName: โš™๏ธ Restore CLI tools -- powershell: azure-pipelines/variables/_pipelines.ps1 - failOnStderr: true - displayName: โš™ Set pipeline variables based on source -- powershell: > - $TeamAlias = '$(TeamEmail)'.Substring(0, '$(TeamEmail)'.IndexOf('@')) - - azure-pipelines/Archive-SourceCode.ps1 - -ManagerAlias '$(ManagerAlias)' - -TeamAlias $TeamAlias - -BusinessGroupName '$(BusinessGroupName)' - -ProductName '$(SymbolsFeatureName)' - -ProductLanguage English - -Notes '${{ parameters.notes }}' - -Verbose - -WhatIf:$${{ parameters.whatif }} - displayName: ๐Ÿ—ƒ๏ธ Submit archival request +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + + stages: + - stage: archive + jobs: + - job: archive + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + os: Linux + + steps: + - checkout: self + clean: true + fetchDepth: 0 + - powershell: tools/Install-DotNetSdk.ps1 + displayName: โš™ Install .NET SDK + - task: NuGetAuthenticate@1 + displayName: ๐Ÿ” Authenticate NuGet feeds + inputs: + forceReinstallCredentialProvider: true + - script: dotnet tool restore + displayName: โš™๏ธ Restore CLI tools + - powershell: azure-pipelines/variables/_pipelines.ps1 + failOnStderr: true + displayName: โš™ Set pipeline variables based on source + - powershell: > + $TeamAlias = '$(TeamEmail)'.Substring(0, '$(TeamEmail)'.IndexOf('@')) + + azure-pipelines/Archive-SourceCode.ps1 + -ManagerAlias '$(ManagerAlias)' + -TeamAlias $TeamAlias + -BusinessGroupName '$(BusinessGroupName)' + -ProductName '$(SymbolsFeatureName)' + -ProductLanguage English + -Notes '${{ parameters.notes }}' + -Verbose + -WhatIf:$${{ parameters.whatif }} + displayName: ๐Ÿ—ƒ๏ธ Submit archival request diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index a41be032..f805607e 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -29,8 +29,6 @@ stages: SymbolsProject: VS SymbolsAgentPath: $(Pipeline.Workspace)/symbols-legacy azureSubscription: Symbols Upload (DevDiv) - - task: MicroBuildCleanup@1 - displayName: โ˜Ž๏ธ Send Telemetry # - job: push # displayName: azure-public/vs-impl feed diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 175c37a9..d949958f 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -2,6 +2,11 @@ trigger: none # We only want to trigger manually or based on resources pr: none resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release pipelines: - pipeline: CI source: Library # TODO: This should match the name of your CI pipeline @@ -12,50 +17,60 @@ resources: variables: - group: VS SDK feeds # Expected to provide NuGetOrgApiKey -jobs: -- job: release - pool: - name: AzurePipelines-EO - vmImage: AzurePipelinesUbuntu20.04compliant - steps: - - checkout: none - - powershell: | - Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" - if ('$(resources.pipeline.CI.runName)'.Contains('-')) { - Write-Host "##vso[task.setvariable variable=IsPrerelease]true" - } else { - Write-Host "##vso[task.setvariable variable=IsPrerelease]false" - } - displayName: โš™ Set up pipeline - - task: UseDotNet@2 - displayName: โš™ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - - download: CI - artifact: deployables-Windows - displayName: ๐Ÿ”ป Download deployables-Windows artifact - patterns: 'deployables-Windows/NuGet/*' - - task: GitHubRelease@1 - displayName: ๐Ÿ“ข GitHub release (create) - inputs: - gitHubConnection: # TODO: fill in service connection here - repositoryName: $(Build.Repository.Name) - target: $(resources.pipeline.CI.sourceCommit) - tagSource: userSpecifiedTag - tag: v$(resources.pipeline.CI.runName) - title: v$(resources.pipeline.CI.runName) - isDraft: true # After running this step, visit the new draft release, edit, and publish. - isPreRelease: $(IsPrerelease) - assets: $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg - changeLogCompareToRelease: lastNonDraftRelease - changeLogType: issueBased - changeLogLabels: | - [ - { "label" : "breaking change", "displayName" : "Breaking changes", "state" : "closed" }, - { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, - { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } - ] - - script: dotnet nuget push $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg -s https://api.nuget.org/v3/index.json --api-key $(NuGetOrgApiKey) --skip-duplicate - displayName: ๐Ÿ“ฆ Push packages to nuget.org - condition: and(succeeded(), ne(variables['NuGetOrgApiKey'], '')) +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + + stages: + - stage: release + jobs: + - job: release + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + os: Linux + steps: + - checkout: none + - powershell: | + Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" + if ('$(resources.pipeline.CI.runName)'.Contains('-')) { + Write-Host "##vso[task.setvariable variable=IsPrerelease]true" + } else { + Write-Host "##vso[task.setvariable variable=IsPrerelease]false" + } + displayName: โš™ Set up pipeline + - task: UseDotNet@2 + displayName: โš™ Install .NET SDK + inputs: + packageType: sdk + version: 6.x + - download: CI + artifact: deployables-Windows + displayName: ๐Ÿ”ป Download deployables-Windows artifact + patterns: 'deployables-Windows/NuGet/*' + - task: GitHubRelease@1 + displayName: ๐Ÿ“ข GitHub release (create) + inputs: + gitHubConnection: # TODO: fill in service connection here + repositoryName: $(Build.Repository.Name) + target: $(resources.pipeline.CI.sourceCommit) + tagSource: userSpecifiedTag + tag: v$(resources.pipeline.CI.runName) + title: v$(resources.pipeline.CI.runName) + isDraft: true # After running this step, visit the new draft release, edit, and publish. + isPreRelease: $(IsPrerelease) + assets: $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg + changeLogCompareToRelease: lastNonDraftRelease + changeLogType: issueBased + changeLogLabels: | + [ + { "label" : "breaking change", "displayName" : "Breaking changes", "state" : "closed" }, + { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, + { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } + ] + - script: dotnet nuget push $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg -s https://api.nuget.org/v3/index.json --api-key $(NuGetOrgApiKey) --skip-duplicate + displayName: ๐Ÿ“ฆ Push packages to nuget.org + condition: and(succeeded(), ne(variables['NuGetOrgApiKey'], '')) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index c7524559..37257bb2 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -2,6 +2,11 @@ trigger: none # We only want to trigger manually or based on resources pr: none resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release pipelines: - pipeline: CI source: Library # TODO: This should match the name of your CI pipeline @@ -12,46 +17,52 @@ resources: - Real signed - auto-insertion -jobs: -- job: insertion - displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES - steps: - - checkout: none - - powershell: Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" - displayName: โš™๏ธ Set pipeline name - - task: UseDotNet@2 - displayName: โš™๏ธ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - - task: NuGetAuthenticate@1 - displayName: ๐Ÿ” Authenticate NuGet feeds - inputs: - forceReinstallCredentialProvider: true - - template: release-deployment-prep.yml - - download: CI - artifact: VSInsertion-Windows - displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact - - script: dotnet nuget push $(Pipeline.Workspace)\CI\VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate - displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed - - task: MicroBuildInsertVsPayload@4 - displayName: ๐Ÿญ Insert VS Payload - inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) - InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) - InsertionBuildPolicy: Request Perf DDRITs - AutoCompletePR: true - AutoCompleteMergeStrategy: Squash - - task: MicroBuildCleanup@1 - displayName: โ˜Ž๏ธ Send Telemetry - - powershell: | - $contentType = 'application/json'; - $headers = @{ Authorization = 'Bearer $(System.AccessToken)' }; - $rawRequest = @{ daysValid = 365 * 2; definitionId = $(resources.pipeline.CI.pipelineID); ownerId = 'User:$(Build.RequestedForId)'; protectPipeline = $false; runId = $(resources.pipeline.CI.runId) }; - $request = ConvertTo-Json @($rawRequest); - Write-Host $request - $uri = "$(System.CollectionUri)$(System.TeamProject)/_apis/build/retention/leases?api-version=6.0-preview.1"; - Invoke-RestMethod -uri $uri -method POST -Headers $headers -ContentType $contentType -Body $request; - displayName: ๐Ÿ—ป Retain inserted builds +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + + stages: + - stage: insertion + jobs: + - job: insertion + displayName: VS insertion + pool: VSEngSS-MicroBuild2022-1ES + steps: + - checkout: none + - powershell: Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" + displayName: โš™๏ธ Set pipeline name + - task: UseDotNet@2 + displayName: โš™๏ธ Install .NET SDK + inputs: + packageType: sdk + version: 6.x + - task: NuGetAuthenticate@1 + displayName: ๐Ÿ” Authenticate NuGet feeds + inputs: + forceReinstallCredentialProvider: true + - template: azure-pipelines/release-deployment-prep.yml@self + - download: CI + artifact: VSInsertion-Windows + displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact + - script: dotnet nuget push $(Pipeline.Workspace)\CI\VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate + displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed + - task: MicroBuildInsertVsPayload@4 + displayName: ๐Ÿญ Insert VS Payload + inputs: + TeamName: $(TeamName) + TeamEmail: $(TeamEmail) + InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) + InsertionBuildPolicy: Request Perf DDRITs + AutoCompletePR: true + AutoCompleteMergeStrategy: Squash + - powershell: | + $contentType = 'application/json'; + $headers = @{ Authorization = 'Bearer $(System.AccessToken)' }; + $rawRequest = @{ daysValid = 365 * 2; definitionId = $(resources.pipeline.CI.pipelineID); ownerId = 'User:$(Build.RequestedForId)'; protectPipeline = $false; runId = $(resources.pipeline.CI.runId) }; + $request = ConvertTo-Json @($rawRequest); + Write-Host $request + $uri = "$(System.CollectionUri)$(System.TeamProject)/_apis/build/retention/leases?api-version=6.0-preview.1"; + Invoke-RestMethod -uri $uri -method POST -Headers $headers -ContentType $contentType -Body $request; + displayName: ๐Ÿ—ป Retain inserted builds diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index cb9a2072..d60b67cb 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -5,77 +5,88 @@ trigger: none # We only want to trigger manually or based on resources pr: none -stages: -- stage: Build - variables: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages - SignTypeSelection: Real - BuildConfiguration: Release - ValidationBuild: true +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release - jobs: - - template: build.yml - parameters: - windowsPool: VSEngSS-MicroBuild2022-1ES - includeMacOS: false - RunTests: false - -- template: prepare-insertion-stages.yml +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - ArchiveSymbols: false + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + + stages: + - stage: Build + variables: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + SignTypeSelection: Real + BuildConfiguration: Release + ValidationBuild: true + + jobs: + - template: /azure-pipelines/build.yml@self + parameters: + windowsPool: VSEngSS-MicroBuild2022-1ES + includeMacOS: false + RunTests: false + + - template: /azure-pipelines/prepare-insertion-stages.yml@self + parameters: + ArchiveSymbols: false -- stage: insertion - displayName: VS insertion - jobs: - - job: insertion - displayName: VS insertion - pool: VSEngSS-MicroBuild2022-1ES - steps: - - checkout: self - clean: true - fetchDepth: 1 - - task: UseDotNet@2 - displayName: โš™๏ธ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - - task: NuGetAuthenticate@1 - displayName: ๐Ÿ” Authenticate NuGet feeds - inputs: - forceReinstallCredentialProvider: true - - download: current - artifact: Variables-Windows - displayName: ๐Ÿ”ป Download Variables-Windows artifact - - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 - displayName: โš™๏ธ Set pipeline variables based on artifacts - - download: current - artifact: VSInsertion-Windows - displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact - - script: dotnet nuget push VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate - displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed - workingDirectory: $(Pipeline.Workspace) - - task: MicroBuildInsertVsPayload@4 - displayName: ๐Ÿญ Insert VS Payload - inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) - InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] - InsertionDescription: | - This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. - CustomScriptExecutionCommand: src/VSSDK/NuGet/AllowUnstablePackages.ps1 - InsertionBuildPolicy: Request Perf DDRITs - InsertionReviewers: $(Build.RequestedForEmail) - AutoCompletePR: false - - powershell: | - $insertionPRId = azure-pipelines/Get-InsertionPRId.ps1 - $Markdown = @" - Validation insertion pull request created: !$insertionPRId - Please check status there before proceeding to merge this PR. - Remember to Abandon and (if allowed) to Delete Source Branch on that insertion PR when validation is complete. - "@ - azure-pipelines/PostPRMessage.ps1 -AccessToken '$(System.AccessToken)' -Markdown $Markdown -Verbose - displayName: โœ๏ธ Comment on pull request - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - - task: MicroBuildCleanup@1 - displayName: โ˜Ž๏ธ Send Telemetry + - stage: insertion + displayName: VS insertion + jobs: + - job: insertion + displayName: VS insertion + pool: VSEngSS-MicroBuild2022-1ES + steps: + - checkout: self + clean: true + fetchDepth: 1 + - task: UseDotNet@2 + displayName: โš™๏ธ Install .NET SDK + inputs: + packageType: sdk + version: 6.x + - task: NuGetAuthenticate@1 + displayName: ๐Ÿ” Authenticate NuGet feeds + inputs: + forceReinstallCredentialProvider: true + - download: current + artifact: Variables-Windows + displayName: ๐Ÿ”ป Download Variables-Windows artifact + - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 + displayName: โš™๏ธ Set pipeline variables based on artifacts + - download: current + artifact: VSInsertion-Windows + displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact + - script: dotnet nuget push VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate + displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed + workingDirectory: $(Pipeline.Workspace) + - task: MicroBuildInsertVsPayload@4 + displayName: ๐Ÿญ Insert VS Payload + inputs: + TeamName: $(TeamName) + TeamEmail: $(TeamEmail) + InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] + InsertionDescription: | + This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. + CustomScriptExecutionCommand: src/VSSDK/NuGet/AllowUnstablePackages.ps1 + InsertionBuildPolicy: Request Perf DDRITs + InsertionReviewers: $(Build.RequestedForEmail) + AutoCompletePR: false + - powershell: | + $insertionPRId = azure-pipelines/Get-InsertionPRId.ps1 + $Markdown = @" + Validation insertion pull request created: !$insertionPRId + Please check status there before proceeding to merge this PR. + Remember to Abandon and (if allowed) to Delete Source Branch on that insertion PR when validation is complete. + "@ + azure-pipelines/PostPRMessage.ps1 -AccessToken '$(System.AccessToken)' -Markdown $Markdown -Verbose + displayName: โœ๏ธ Comment on pull request + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) From 220142fcadc255a686839652b2f11f5c69204095 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 26 Jan 2024 12:47:22 -0700 Subject: [PATCH 06/86] Switch vs-validation pipeline to Unofficial template --- azure-pipelines/vs-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index d60b67cb..137acbe4 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -13,7 +13,7 @@ resources: ref: refs/tags/release extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES From 4d1473c2924f40f02578e70a053ba666e6531ae1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:16:00 -0700 Subject: [PATCH 07/86] Bump dotnet-coverage from 17.9.6 to 17.10.1Bumps [dotnet-coverage](https://github.com/microsoft/codecoverage) from 17.9.6 to 17.10.1.- [Commits](https://github.com/microsoft/codecoverage/commits)---updated-dependencies:- dependency-name: dotnet-coverage dependency-type: direct:production update-type: version-update:semver-minor...Signed-off-by: dependabot[bot] * Bump dotnet-coverage from 17.9.6 to 17.10.1 Bumps [dotnet-coverage](https://github.com/microsoft/codecoverage) from 17.9.6 to 17.10.1. - [Commits](https://github.com/microsoft/codecoverage/commits) --- updated-dependencies: - dependency-name: dotnet-coverage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 589f0604..5941f4c8 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -9,7 +9,7 @@ ] }, "dotnet-coverage": { - "version": "17.9.6", + "version": "17.10.1", "commands": [ "dotnet-coverage" ] From ebb0607dd933ae5b44ee473eec4ff8716cd211d6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 29 Jan 2024 17:05:36 -0700 Subject: [PATCH 08/86] Format init.ps1 --- init.ps1 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/init.ps1 b/init.ps1 index 00a9589b..e2849855 100755 --- a/init.ps1 +++ b/init.ps1 @@ -84,8 +84,7 @@ try { $HeaderColor = 'Green' $RestoreArguments = @() - if ($Interactive) - { + if ($Interactive) { $RestoreArguments += '--interactive' } @@ -98,10 +97,10 @@ try { } if (!$NoToolRestore -and $PSCmdlet.ShouldProcess("dotnet tool", "restore")) { - dotnet tool restore @RestoreArguments - if ($lastexitcode -ne 0) { - throw "Failure while restoring dotnet CLI tools." - } + dotnet tool restore @RestoreArguments + if ($lastexitcode -ne 0) { + throw "Failure while restoring dotnet CLI tools." + } } & "$PSScriptRoot/tools/Set-EnvVars.ps1" -Variables $EnvVars -PrependPath $PrependPath | Out-Null From 72b8ba01196c0fcb51db7cf36335d2ce04f15be5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 31 Jan 2024 12:21:32 -0700 Subject: [PATCH 09/86] Fix NUGET_PACKAGES path in pipelines By default the NUGET_PACKAGES path has a trailing slash. In our override, we should do this as well or the msbuild property that gets generated is missing the slash, which can break some builds. This follows the pattern in 9805678248d0fbda63f97982f37e35de71890d2f, which only applied to one of the locations that in the microbuild branch we must change. --- azure-pipelines/official.yml | 4 ++-- azure-pipelines/vs-validation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 915c7e76..a57cdc34 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -60,7 +60,7 @@ extends: variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ SignTypeSelection: ${{ parameters.SignTypeSelection }} Packaging.EnableSBOMSigning: false Codeql.Enabled: true @@ -92,7 +92,7 @@ extends: variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ SignTypeSelection: ${{ parameters.SignTypeSelection }} Packaging.EnableSBOMSigning: false Codeql.Enabled: true diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 137acbe4..8ca0b33a 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -22,7 +22,7 @@ extends: - stage: Build variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ SignTypeSelection: Real BuildConfiguration: Release ValidationBuild: true From 9f902a1a59452d115c99edf232fbd7635530a755 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 1 Feb 2024 17:06:13 -0700 Subject: [PATCH 10/86] Fix the pool for pushing real-signed packages to be 1ES PT compliant --- azure-pipelines/prepare-insertion-stages.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index f805607e..6b0a91b4 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -36,7 +36,9 @@ stages: # dependsOn: symbol_archive # pool: # name: AzurePipelines-EO - # vmImage: AzurePipelinesUbuntu20.04compliant + # demands: + # - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + # os: Linux # steps: # - checkout: none # - download: current From 65f2106cda0b367d5471e820458ef73e2206ff68 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 2 Feb 2024 12:14:01 -0700 Subject: [PATCH 11/86] Set TeamName variable in `push` job This avoids a pipeline break due to a Microbuild pipeline template injected telemetry task. --- azure-pipelines/prepare-insertion-stages.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 6b0a91b4..f1d5ab89 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -42,6 +42,11 @@ stages: # steps: # - checkout: none # - download: current + # artifact: Variables-Windows + # displayName: ๐Ÿ”ป Download Variables-Windows artifact + # - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 + # displayName: โš™๏ธ Set pipeline variables based on artifacts + # - download: current # artifact: deployables-Windows # displayName: ๐Ÿ”ป Download deployables-Windows artifact # - task: UseDotNet@2 From bb38a7296be221aa5b81d54cbd28377ae5973d20 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 2 Feb 2024 10:59:02 -0700 Subject: [PATCH 12/86] Fix break when the RunTests parameter is false --- azure-pipelines/build.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 64e97cf7..481447b2 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -47,10 +47,11 @@ jobs: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Windows - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows - artifactName: ${{ artifact_name }}-Windows + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Windows + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows + artifactName: ${{ artifact_name }}-Windows - output: pipelineArtifact displayName: ๐Ÿ“ข Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows From 83d4a12c8b03fa109e7608875ee2225c467fb631 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 12:12:53 -0700 Subject: [PATCH 13/86] Push nuget packages in 1ES compliant way --- azure-pipelines/prepare-insertion-stages.yml | 22 +++++++++----------- azure-pipelines/release.yml | 21 +++++++++---------- azure-pipelines/vs-insertion.yml | 16 ++++++-------- azure-pipelines/vs-validation.yml | 17 ++++++--------- 4 files changed, 32 insertions(+), 44 deletions(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index f1d5ab89..98e5232c 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -39,6 +39,16 @@ stages: # demands: # - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT # os: Linux + # templateContext: + # outputParentDirectory: $(Pipeline.Workspace) + # outputs: + # - output: nuget + # displayName: ๐Ÿ“ฆ Push nuget packages + # packagesToPush: '(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg' + # packageParentPath: (Pipeline.Workspace)/deployables-Windows/NuGet + # allowPackageConflicts: true + # nuGetFeedType: external + # publishFeedCredentials: azure-public/vs-impl # steps: # - checkout: none # - download: current @@ -49,15 +59,3 @@ stages: # - download: current # artifact: deployables-Windows # displayName: ๐Ÿ”ป Download deployables-Windows artifact - # - task: UseDotNet@2 - # displayName: โš™๏ธ Install .NET SDK - # inputs: - # packageType: sdk - # version: 6.x - # - task: NuGetAuthenticate@1 - # displayName: ๐Ÿ” Authenticate NuGet feeds - # inputs: - # nuGetServiceConnections: azure-public/vs-impl - # forceReinstallCredentialProvider: true - # - script: dotnet nuget push $(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg -s https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl/nuget/v3/index.json --api-key azdo --skip-duplicate - # displayName: ๐Ÿ“ฆ Push nuget packages diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index d949958f..8b371771 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -14,9 +14,6 @@ resources: tags: - auto-release -variables: -- group: VS SDK feeds # Expected to provide NuGetOrgApiKey - extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: @@ -32,6 +29,16 @@ extends: demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT os: Linux + templateContext: + outputParentDirectory: $(Pipeline.Workspace) + outputs: + - output: nuget + displayName: ๐Ÿ“ฆ Push packages to nuget.org + packagesToPush: '$(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg' + packageParentPath: $(Pipeline.Workspace)/CI/deployables-Windows/NuGet + allowPackageConflicts: true + nuGetFeedType: external + publishFeedCredentials: VisualStudioExtensibility (nuget.org) steps: - checkout: none - powershell: | @@ -42,11 +49,6 @@ extends: Write-Host "##vso[task.setvariable variable=IsPrerelease]false" } displayName: โš™ Set up pipeline - - task: UseDotNet@2 - displayName: โš™ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - download: CI artifact: deployables-Windows displayName: ๐Ÿ”ป Download deployables-Windows artifact @@ -71,6 +73,3 @@ extends: { "label" : "bug", "displayName" : "Fixes", "state" : "closed" }, { "label" : "enhancement", "displayName": "Enhancements", "state" : "closed" } ] - - script: dotnet nuget push $(Pipeline.Workspace)/CI/deployables-Windows/NuGet/*.nupkg -s https://api.nuget.org/v3/index.json --api-key $(NuGetOrgApiKey) --skip-duplicate - displayName: ๐Ÿ“ฆ Push packages to nuget.org - condition: and(succeeded(), ne(variables['NuGetOrgApiKey'], '')) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 37257bb2..b958134b 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -33,21 +33,17 @@ extends: - checkout: none - powershell: Write-Host "##vso[build.updatebuildnumber]$(resources.pipeline.CI.runName)" displayName: โš™๏ธ Set pipeline name - - task: UseDotNet@2 - displayName: โš™๏ธ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - - task: NuGetAuthenticate@1 - displayName: ๐Ÿ” Authenticate NuGet feeds - inputs: - forceReinstallCredentialProvider: true - template: azure-pipelines/release-deployment-prep.yml@self - download: CI artifact: VSInsertion-Windows displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact - - script: dotnet nuget push $(Pipeline.Workspace)\CI\VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate + - task: 1ES.PublishNuget@1 displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed + inputs: + packagesToPush: '$(Pipeline.Workspace)/CI/VSInsertion-Windows/*.nupkg' + packageParentPath: $(Pipeline.Workspace)/CI/VSInsertion-Windows + allowPackageConflicts: true + publishVstsFeed: VS - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 8ca0b33a..176ec1bb 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -48,15 +48,6 @@ extends: - checkout: self clean: true fetchDepth: 1 - - task: UseDotNet@2 - displayName: โš™๏ธ Install .NET SDK - inputs: - packageType: sdk - version: 6.x - - task: NuGetAuthenticate@1 - displayName: ๐Ÿ” Authenticate NuGet feeds - inputs: - forceReinstallCredentialProvider: true - download: current artifact: Variables-Windows displayName: ๐Ÿ”ป Download Variables-Windows artifact @@ -65,9 +56,13 @@ extends: - download: current artifact: VSInsertion-Windows displayName: ๐Ÿ”ป Download VSInsertion-Windows artifact - - script: dotnet nuget push VSInsertion-windows\*.nupkg -s https://pkgs.dev.azure.com/devdiv/_packaging/VS/nuget/v3/index.json -k azdo --skip-duplicate + - task: 1ES.PublishNuget@1 displayName: ๐Ÿ“ฆ Push CoreXT packages to VS feed - workingDirectory: $(Pipeline.Workspace) + inputs: + packagesToPush: '$(Pipeline.Workspace)/VSInsertion-Windows/*.nupkg' + packageParentPath: $(Pipeline.Workspace)/VSInsertion-Windows + allowPackageConflicts: true + publishVstsFeed: VS - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: From fa2fa4a1717ac1f7dfcb3bad56fd24e95a3b7184 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 20:27:23 -0700 Subject: [PATCH 14/86] Fix pipeline failures when tests are turned off Finishes what bb38a7296be221 started --- azure-pipelines/build.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 481447b2..89605d92 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -97,10 +97,11 @@ jobs: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux - artifactName: ${{ artifact_name }}-Linux + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux + artifactName: ${{ artifact_name }}-Linux steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. @@ -121,10 +122,11 @@ jobs: outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS - artifactName: ${{ artifact_name }}-macOS + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS + artifactName: ${{ artifact_name }}-macOS steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. From 20d7eb1dec040f5a086e783674d1a6c6f76e8070 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 19:55:23 -0700 Subject: [PATCH 15/86] Fix name of stage task --- azure-pipelines/dotnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 61c96a35..546e9083 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -18,7 +18,7 @@ steps: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - powershell: azure-pipelines/artifacts/_pipelines.ps1 -StageOnly -AvoidSymbolicLinks -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose failOnStderr: true - displayName: ๐Ÿ“ข Publish artifacts + displayName: ๐Ÿ“ข Stage artifacts condition: succeededOrFailed() - ${{ if ne(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: From aab283cf8435ed07bab3310201f528dbcd49fd36 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 20:08:30 -0700 Subject: [PATCH 16/86] Migrate Microbuild-required variables to a yaml file These need to be set before we have a chance to run our variable ps1 scripts. --- azure-pipelines/GlobalVariables.yml | 4 ++++ azure-pipelines/official.yml | 3 +++ azure-pipelines/variables/TeamEmail.ps1 | 1 - azure-pipelines/variables/TeamName.ps1 | 2 -- azure-pipelines/vs-insertion.yml | 5 +++-- azure-pipelines/vs-validation.yml | 5 +++-- 6 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 azure-pipelines/GlobalVariables.yml delete mode 100644 azure-pipelines/variables/TeamEmail.ps1 delete mode 100644 azure-pipelines/variables/TeamName.ps1 diff --git a/azure-pipelines/GlobalVariables.yml b/azure-pipelines/GlobalVariables.yml new file mode 100644 index 00000000..b2d1d532 --- /dev/null +++ b/azure-pipelines/GlobalVariables.yml @@ -0,0 +1,4 @@ +variables: + # These variables are required for MicroBuild tasks + TeamName: VS IDE + TeamEmail: vsidemicrobuild@microsoft.com diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index a57cdc34..1769431e 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -49,6 +49,9 @@ resources: name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release +variables: +- template: GlobalVariables.yml + extends: ${{ if parameters.EnableCompliance }}: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate diff --git a/azure-pipelines/variables/TeamEmail.ps1 b/azure-pipelines/variables/TeamEmail.ps1 deleted file mode 100644 index 7cf66982..00000000 --- a/azure-pipelines/variables/TeamEmail.ps1 +++ /dev/null @@ -1 +0,0 @@ -'vsidemicrobuild@microsoft.com' diff --git a/azure-pipelines/variables/TeamName.ps1 b/azure-pipelines/variables/TeamName.ps1 deleted file mode 100644 index ba9c8231..00000000 --- a/azure-pipelines/variables/TeamName.ps1 +++ /dev/null @@ -1,2 +0,0 @@ -# This value is used as an input to the MicroBuild Insert VS task. -'VS IDE' diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index b958134b..c948b713 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -17,6 +17,9 @@ resources: - Real signed - auto-insertion +variables: +- template: GlobalVariables.yml + extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: @@ -47,8 +50,6 @@ extends: - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicy: Request Perf DDRITs AutoCompletePR: true diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 176ec1bb..062a5abf 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -12,6 +12,9 @@ resources: name: 1ESPipelineTemplates/MicroBuildTemplate ref: refs/tags/release +variables: +- template: GlobalVariables.yml + extends: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: @@ -66,8 +69,6 @@ extends: - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: - TeamName: $(TeamName) - TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] InsertionDescription: | This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. From 315a51ed3ed287e09d4d72cd4b4e48a285c96b9d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 18:24:12 -0700 Subject: [PATCH 17/86] Migrate SBOM plugin installation to the new template style --- azure-pipelines/build.yml | 3 +++ azure-pipelines/microbuild.before.yml | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 89605d92..64daed4f 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -44,6 +44,9 @@ jobs: - group: VSCloudServices-APIScan # Expected to provide ApiScanClientId, ApiScanSecret, ApiScanTenant ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: + mb: + sbom: + enabled: true outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index adb5ccc8..f5e1047f 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -16,7 +16,3 @@ steps: zipSources: false displayName: ๐Ÿ”ง Install MicroBuild Signing Plugin condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['SignType'], 'real'))) - -- task: MicroBuildSbomPlugin@1 - displayName: ๐Ÿ”ง Install MicroBuild Sbom Plugin - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) From e4fed75c0386011cd6d4d177ecd1bcc22612c3d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 05:49:52 -0700 Subject: [PATCH 18/86] Bump dotnet-coverage from 17.10.1 to 17.10.2 (#250) Bumps [dotnet-coverage](https://github.com/microsoft/codecoverage) from 17.10.1 to 17.10.2. - [Commits](https://github.com/microsoft/codecoverage/commits) --- updated-dependencies: - dependency-name: dotnet-coverage dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5941f4c8..6eff0445 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -9,7 +9,7 @@ ] }, "dotnet-coverage": { - "version": "17.10.1", + "version": "17.10.2", "commands": [ "dotnet-coverage" ] From 9374605db8f41fb296a391c1b73c0c2adbb06c98 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 18:24:31 -0700 Subject: [PATCH 19/86] Add placeholder for localization plugin It's disabled by default, but is easy to turn on. --- azure-pipelines/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 64daed4f..2ef058bf 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -47,6 +47,10 @@ jobs: mb: sbom: enabled: true + localization: + enabled: false + ${{ if eq(variables['Build.Reason'], 'pullRequest') }}: + languages: ENU,JPN outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: From 1c3dadb97af9d7211a927c4229ea07db8b7f42cf Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 5 Feb 2024 19:53:56 -0700 Subject: [PATCH 20/86] Migrate signing plugin installation to the new template style By requiring opt-in to sign on mac/linux, we remove the need for signing re-approval on pipelines that are already signing. It also avoids signing binaries that typically will not ship anyway. --- azure-pipelines/build.yml | 20 ++++++++++++++++++++ azure-pipelines/microbuild.before.yml | 7 ------- azure-pipelines/official.yml | 17 ++++++++++------- azure-pipelines/prepare-insertion-stages.yml | 5 ++++- azure-pipelines/variables/SignType.ps1 | 11 ----------- azure-pipelines/vs-validation.yml | 3 ++- 6 files changed, 36 insertions(+), 27 deletions(-) delete mode 100644 azure-pipelines/variables/SignType.ps1 diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 2ef058bf..d110df1d 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -1,4 +1,7 @@ parameters: +- name: RealSign + type: boolean + default: false - name: windowsPool type: object default: @@ -45,6 +48,13 @@ jobs: ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: mb: + signing: + enabled: true + zipSources: false + ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + signType: real + ${{ else }}: + signType: test sbom: enabled: true localization: @@ -101,6 +111,11 @@ jobs: pool: ${{ parameters.linuxPool }} ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: + mb: + ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: @@ -126,6 +141,11 @@ jobs: pool: ${{ parameters.macOSPool }} ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: + mb: + ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index f5e1047f..8e9c8c6a 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -9,10 +9,3 @@ steps: outputformat: text condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded - -- task: MicroBuildSigningPlugin@4 - inputs: - signType: $(SignType) - zipSources: false - displayName: ๐Ÿ”ง Install MicroBuild Signing Plugin - condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['SignType'], 'real'))) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 1769431e..3a506b91 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -20,11 +20,10 @@ trigger: # - microbuild parameters: -- name: SignTypeSelection - displayName: Sign type - type: string - default: Test - values: [ 'Test', 'Real' ] +- name: RealSign + displayName: Real sign? + type: boolean + default: false - name: includeMacOS displayName: Build on macOS type: boolean @@ -64,12 +63,12 @@ extends: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - SignTypeSelection: ${{ parameters.SignTypeSelection }} Packaging.EnableSBOMSigning: false Codeql.Enabled: true jobs: - template: /azure-pipelines/build.yml@self parameters: + RealSign: ${{ parameters.RealSign }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -85,6 +84,8 @@ extends: includeMacOS: ${{ parameters.includeMacOS }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self + parameters: + RealSign: ${{ parameters.RealSign }} ${{ else }}: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: @@ -96,12 +97,12 @@ extends: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - SignTypeSelection: ${{ parameters.SignTypeSelection }} Packaging.EnableSBOMSigning: false Codeql.Enabled: true jobs: - template: /azure-pipelines/build.yml@self parameters: + RealSign: ${{ parameters.RealSign }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -117,3 +118,5 @@ extends: includeMacOS: ${{ parameters.includeMacOS }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self + parameters: + RealSign: ${{ parameters.RealSign }} diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 98e5232c..821054aa 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -2,11 +2,14 @@ parameters: - name: ArchiveSymbols type: boolean default: true +- name: RealSign + displayName: Real sign? + type: boolean stages: - stage: release displayName: Publish - condition: and(succeeded(), eq(dependencies.Build.outputs['Windows.SetPipelineVariables.SignType'], 'Real')) + condition: and(succeeded(), eq('${{ parameters.RealSign }}', 'true')) jobs: - ${{ if parameters.ArchiveSymbols }}: - job: symbol_archive diff --git a/azure-pipelines/variables/SignType.ps1 b/azure-pipelines/variables/SignType.ps1 deleted file mode 100644 index 0c1a335a..00000000 --- a/azure-pipelines/variables/SignType.ps1 +++ /dev/null @@ -1,11 +0,0 @@ -if ($env:SYSTEM_COLLECTIONID -eq '011b8bdf-6d56-4f87-be0d-0092136884d9') { - if ($env:BUILD_REASON -eq 'Schedule') { - 'real' - } else { - if ($env:SIGNTYPESELECTION) { - $env:SIGNTYPESELECTION - } else { - 'test' - } - } - } diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 062a5abf..742b73d7 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -26,13 +26,13 @@ extends: variables: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - SignTypeSelection: Real BuildConfiguration: Release ValidationBuild: true jobs: - template: /azure-pipelines/build.yml@self parameters: + RealSign: true windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: false RunTests: false @@ -40,6 +40,7 @@ extends: - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: ArchiveSymbols: false + RealSign: ${{ parameters.RealSign }} - stage: insertion displayName: VS insertion From 0ed99f2668434d9f3519cd49791de64a3ba338df Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 6 Feb 2024 12:50:10 -0700 Subject: [PATCH 21/86] Bump MicroBuild to 2.0.149 --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index a2ca4d22..649d94cf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ true true - 2.0.147 + 2.0.149 From f8b892b0a524b5bfe70f0dbfd6e34d450bb0d3a4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 6 Feb 2024 18:50:57 -0700 Subject: [PATCH 22/86] Remove compliance tasks These are redundant with those injected by 1ES PT. --- azure-pipelines/build.yml | 6 ----- azure-pipelines/secure-development-tools.yml | 24 -------------------- 2 files changed, 30 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index d110df1d..03fd0ff7 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -73,12 +73,6 @@ jobs: displayName: ๐Ÿ“ข Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows artifactName: VSInsertion-Windows - # This is useful when false positives appear so we can copy some of the output into the suppressions file. - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish Guardian failures - targetPath: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions - artifactName: guardian_failures_as_suppressions - condition: failed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 1bbacefe..ad254e21 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -30,27 +30,3 @@ steps: condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) env: AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) - -- task: SdtReport@2 - displayName: ๐Ÿ“ƒ Create Security Analysis Report - inputs: - GdnExportAllTools: true - -- task: PublishSecurityAnalysisLogs@3 - displayName: ๐Ÿ“ข Publish Code Analysis Logs - inputs: - ArtifactName: CodeAnalysisLogs - ArtifactType: Container - PublishProcessedResults: true - AllTools: true - ToolLogsNotFoundAction: Standard - -- task: PostAnalysis@2 - displayName: ๐Ÿ‹๏ธโ€โ™€๏ธ Break on compliance issues - inputs: - GdnBreakAllTools: true - GdnBreakGdnToolBinSkimSeverity: Warning - GdnBreakSuppressionFiles: $(System.DefaultWorkingDirectory)/azure-pipelines/falsepositives.gdnsuppress - GdnBreakSuppressionSets: falsepositives - GdnBreakOutputSuppressionFile: $(Build.ArtifactStagingDirectory)/guardian_failures_as_suppressions/ - GdnBreakOutputSuppressionSet: falsepositives From 1ce395c70e63b9b6ec8464f8adf74828ddc67cf6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 7 Feb 2024 12:17:39 -0700 Subject: [PATCH 23/86] Bring back PoliCheck --- azure-pipelines/secure-development-tools.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index ad254e21..b0c33cf6 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -6,6 +6,13 @@ steps: - powershell: echo "##vso[build.addbuildtag]compliance" displayName: ๐Ÿท๏ธ Tag run with 'compliance' +- task: PoliCheck@2 + displayName: ๐Ÿ” Run PoliCheck + inputs: + targetType: F + targetArgument: $(System.DefaultWorkingDirectory) + optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml + - task: CopyFiles@2 displayName: ๐Ÿ“‚ Collect APIScan inputs inputs: From 9e2495e991734fed6c5d26df3eb7cc903fa234a6 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 7 Feb 2024 13:54:30 -0700 Subject: [PATCH 24/86] Drop Codeql.Enabled variable CodeQL is injected by default by the 1ES PT templates now, so we don't need this variable. --- azure-pipelines/official.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 3a506b91..68af7f75 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -64,7 +64,6 @@ extends: BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ Packaging.EnableSBOMSigning: false - Codeql.Enabled: true jobs: - template: /azure-pipelines/build.yml@self parameters: @@ -98,7 +97,6 @@ extends: BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ Packaging.EnableSBOMSigning: false - Codeql.Enabled: true jobs: - template: /azure-pipelines/build.yml@self parameters: From 38502866e53e42f2027474bb97e90f1d0fb8ff68 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 7 Feb 2024 14:30:26 -0700 Subject: [PATCH 25/86] File bugs for APIScan issues --- azure-pipelines/TSAOptions.json | 19 +++++++++++++++++++ azure-pipelines/secure-development-tools.yml | 8 ++++++++ 2 files changed, 27 insertions(+) create mode 100644 azure-pipelines/TSAOptions.json diff --git a/azure-pipelines/TSAOptions.json b/azure-pipelines/TSAOptions.json new file mode 100644 index 00000000..6dfeac4c --- /dev/null +++ b/azure-pipelines/TSAOptions.json @@ -0,0 +1,19 @@ +{ + "tsaVersion": "TsaV2", + "codebase": "NewOrUpdate", + "codebaseName": "LibraryName", + "tsaStamp": "DevDiv", + "tsaEnvironment": "PROD", + "notificationAliases": [ + "vsidemicrobuild@microsoft.com" + ], + "codebaseAdmins": [ + "REDMOND\\andarno" + ], + "instanceUrl": "https://devdiv.visualstudio.com", + "projectName": "DevDiv", + "areaPath": "DevDiv\\VS Core", + "iterationPath": "DevDiv", + "alltools": true, + "repositoryName": "Library.Template" +} diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index b0c33cf6..4ed67dc0 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -37,3 +37,11 @@ steps: condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) env: AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) + +# File bugs when APIScan finds issues +- task: TSAUpload@2 + displayName: ๐Ÿชณ TSA upload + inputs: + GdnPublishTsaOnboard: True + GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json + condition: false # enable in individual repos only AFTER updating TSAOptions.json with your own values From 863c52418aea159620ccdba978bf70cff6bfae1f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 7 Feb 2024 14:22:20 -0700 Subject: [PATCH 26/86] Move Policheck to the 1ES PT --- azure-pipelines/official.yml | 10 ++++++++++ azure-pipelines/secure-development-tools.yml | 7 ------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 68af7f75..f48535e4 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -57,6 +57,11 @@ extends: parameters: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + policheck: + enabled: ${{ parameters.EnableCompliance }} + exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml + suppression: + suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress stages: - stage: Build variables: @@ -90,6 +95,11 @@ extends: parameters: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + policheck: + enabled: ${{ parameters.EnableCompliance }} + exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml + suppression: + suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress stages: - stage: Build variables: diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 4ed67dc0..9b1aeaa0 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -6,13 +6,6 @@ steps: - powershell: echo "##vso[build.addbuildtag]compliance" displayName: ๐Ÿท๏ธ Tag run with 'compliance' -- task: PoliCheck@2 - displayName: ๐Ÿ” Run PoliCheck - inputs: - targetType: F - targetArgument: $(System.DefaultWorkingDirectory) - optionsUEPATH: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml - - task: CopyFiles@2 displayName: ๐Ÿ“‚ Collect APIScan inputs inputs: From 42d183755210479986b9cff6075cddb7f4741e62 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 7 Feb 2024 14:53:54 -0700 Subject: [PATCH 27/86] Disable Policheck on non-compliance runs --- azure-pipelines/official.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index f48535e4..e1a732bb 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -58,7 +58,7 @@ extends: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES policheck: - enabled: ${{ parameters.EnableCompliance }} + enabled: true exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml suppression: suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress @@ -95,9 +95,6 @@ extends: parameters: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES - policheck: - enabled: ${{ parameters.EnableCompliance }} - exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml suppression: suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress stages: From 3c87039a3ce8d40e48b9d5f3c3690fba33b8677a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 8 Feb 2024 11:49:42 -0700 Subject: [PATCH 28/86] Fix missing TeamEmail and TeamName in insertVS task --- azure-pipelines/vs-insertion.yml | 2 ++ azure-pipelines/vs-validation.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index c948b713..87166f29 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -50,6 +50,8 @@ extends: - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: + TeamName: $(TeamName) + TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicy: Request Perf DDRITs AutoCompletePR: true diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 742b73d7..5ebf0c71 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -70,6 +70,8 @@ extends: - task: MicroBuildInsertVsPayload@4 displayName: ๐Ÿญ Insert VS Payload inputs: + TeamName: $(TeamName) + TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) VALIDATION BUILD $(Build.BuildNumber) ($(Build.SourceBranch)) [Skip-SymbolCheck] InsertionDescription: | This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. From 5e36c8bed0cc5597783a0a71c013e0f4801064f0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 9 Feb 2024 10:06:45 -0700 Subject: [PATCH 29/86] Drop ComponentGovernance from the pipeline 1ES PT includes it already. --- azure-pipelines/microbuild.before.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 8e9c8c6a..c53ae9a6 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -1,7 +1,4 @@ steps: -- task: ComponentGovernanceComponentDetection@0 - displayName: ๐Ÿ” Component Detection - - task: notice@0 displayName: ๐Ÿ› ๏ธ Generate NOTICE file inputs: From 420207fd6383bbcea6408fdaf4f719bf8434913f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 9 Feb 2024 19:54:55 +0000 Subject: [PATCH 30/86] Recommend prettier --- .prettierrc.yaml | 0 .vscode/extensions.json | 1 + 2 files changed, 1 insertion(+) create mode 100644 .prettierrc.yaml diff --git a/.prettierrc.yaml b/.prettierrc.yaml new file mode 100644 index 00000000..e69de29b diff --git a/.vscode/extensions.json b/.vscode/extensions.json index ca3a2aa9..acaf0213 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,6 +7,7 @@ "ms-dotnettools.csharp", "k--kato.docomment", "editorconfig.editorconfig", + "esbenp.prettier-vscode", "pflannery.vscode-versionlens", "davidanson.vscode-markdownlint", "dotjoshjohnson.xml", From c70b6560f101b924b4adb85cafab7048ea3e2ee0 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 9 Feb 2024 19:55:09 +0000 Subject: [PATCH 31/86] Identify AzP-related YAML files --- .vscode/settings.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ce72437c..efbe74f4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,16 @@ "editor.formatOnSave": true, "[xml]": { "editor.wordWrap": "off" - } + }, + // Treat these files as Azure Pipelines files + "files.associations": { + "**/azure-pipelines/**/*.yml": "azure-pipelines", + "azure-pipelines.yml": "azure-pipelines" + }, + // Use Prettier as the default formatter for Azure Pipelines files. + // Needs to be explicitly configured: https://github.com/Microsoft/azure-pipelines-vscode#document-formatting + "[azure-pipelines]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": false // enable this when the conform + }, } From 823f34de13cc1f5a26a7d5186421dbe5461f2c6b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 9 Feb 2024 13:20:43 -0700 Subject: [PATCH 32/86] Add missing $ characters in pipeline --- azure-pipelines/prepare-insertion-stages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 821054aa..58218fa6 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -47,8 +47,8 @@ stages: # outputs: # - output: nuget # displayName: ๐Ÿ“ฆ Push nuget packages - # packagesToPush: '(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg' - # packageParentPath: (Pipeline.Workspace)/deployables-Windows/NuGet + # packagesToPush: '$(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg' + # packageParentPath: $(Pipeline.Workspace)/deployables-Windows/NuGet # allowPackageConflicts: true # nuGetFeedType: external # publishFeedCredentials: azure-public/vs-impl From 0f2cb2ea14fda56fbe600733c6698a694d2609c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 09:03:32 -0700 Subject: [PATCH 33/86] Bump Microsoft.NET.Test.Sdk from 17.8.0 to 17.9.0 (#251) Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.8.0 to 17.9.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Changelog](https://github.com/microsoft/vstest/blob/main/docs/releases.md) - [Commits](https://github.com/microsoft/vstest/compare/v17.8.0...v17.9.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 1438548b..3096e57c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,7 +6,7 @@ true - + From f301981ff9d11f86ad176fe99b38630452a1d0ca Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 12 Feb 2024 15:42:31 -0700 Subject: [PATCH 34/86] Publish artifacts even on failed pipelines --- azure-pipelines/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 03fd0ff7..1074264f 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -69,6 +69,7 @@ jobs: displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Windows targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Windows artifactName: ${{ artifact_name }}-Windows + condition: succeededOrFailed() - output: pipelineArtifact displayName: ๐Ÿ“ข Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows From 09f29b8638ddcd07335f629022012229788e5c6c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 12 Feb 2024 15:52:53 -0700 Subject: [PATCH 35/86] Switch job disabling style This should reduce merge conflicts going forward since the job's content doesn't entirely change just to enable it. --- azure-pipelines/prepare-insertion-stages.yml | 59 ++++++++++---------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 58218fa6..97e6117f 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -33,32 +33,33 @@ stages: SymbolsAgentPath: $(Pipeline.Workspace)/symbols-legacy azureSubscription: Symbols Upload (DevDiv) - # - job: push - # displayName: azure-public/vs-impl feed - # ${{ if parameters.ArchiveSymbols }}: - # dependsOn: symbol_archive - # pool: - # name: AzurePipelines-EO - # demands: - # - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT - # os: Linux - # templateContext: - # outputParentDirectory: $(Pipeline.Workspace) - # outputs: - # - output: nuget - # displayName: ๐Ÿ“ฆ Push nuget packages - # packagesToPush: '$(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg' - # packageParentPath: $(Pipeline.Workspace)/deployables-Windows/NuGet - # allowPackageConflicts: true - # nuGetFeedType: external - # publishFeedCredentials: azure-public/vs-impl - # steps: - # - checkout: none - # - download: current - # artifact: Variables-Windows - # displayName: ๐Ÿ”ป Download Variables-Windows artifact - # - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 - # displayName: โš™๏ธ Set pipeline variables based on artifacts - # - download: current - # artifact: deployables-Windows - # displayName: ๐Ÿ”ป Download deployables-Windows artifact + - ${{ if false }}: # Switch to true to enable, but leave the condition to avoid merge conflicts later. + - job: push + displayName: azure-public/vs-impl feed + ${{ if parameters.ArchiveSymbols }}: + dependsOn: symbol_archive + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + os: Linux + templateContext: + outputParentDirectory: $(Pipeline.Workspace) + outputs: + - output: nuget + displayName: ๐Ÿ“ฆ Push nuget packages + packagesToPush: '$(Pipeline.Workspace)/deployables-Windows/NuGet/*.nupkg' + packageParentPath: $(Pipeline.Workspace)/deployables-Windows/NuGet + allowPackageConflicts: true + nuGetFeedType: external + publishFeedCredentials: azure-public/vs-impl + steps: + - checkout: none + - download: current + artifact: Variables-Windows + displayName: ๐Ÿ”ป Download Variables-Windows artifact + - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 + displayName: โš™๏ธ Set pipeline variables based on artifacts + - download: current + artifact: deployables-Windows + displayName: ๐Ÿ”ป Download deployables-Windows artifact From 4ac532d55694d202c5bbda47a806121b33bebfae Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 14:52:59 -0700 Subject: [PATCH 36/86] Remove `EnableSBOMSigning` variable --- azure-pipelines/official.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index e1a732bb..79317477 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -68,7 +68,6 @@ extends: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - Packaging.EnableSBOMSigning: false jobs: - template: /azure-pipelines/build.yml@self parameters: @@ -103,7 +102,6 @@ extends: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true BuildConfiguration: Release NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ - Packaging.EnableSBOMSigning: false jobs: - template: /azure-pipelines/build.yml@self parameters: From 82795963e2a88b40faca81fe9248c63f64e75d36 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 15:03:30 -0700 Subject: [PATCH 37/86] Share variables across all entrypoints --- azure-pipelines.yml | 5 +---- azure-pipelines/BuildStageVariables.yml | 5 +++++ azure-pipelines/official.yml | 8 ++------ 3 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 azure-pipelines/BuildStageVariables.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b4001ab9..6415847e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,10 +24,7 @@ parameters: default: true variables: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - BuildConfiguration: Release - # codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ +- template: /azure-pipelines/BuildStageVariables.yml@self jobs: - template: azure-pipelines/build.yml diff --git a/azure-pipelines/BuildStageVariables.yml b/azure-pipelines/BuildStageVariables.yml new file mode 100644 index 00000000..2a683569 --- /dev/null +++ b/azure-pipelines/BuildStageVariables.yml @@ -0,0 +1,5 @@ +variables: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + BuildConfiguration: Release + NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ + # codecov_token: 4dc9e7e2-6b01-4932-a180-847b52b43d35 # Get a new one from https://codecov.io/ diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 79317477..ccb5a4d1 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -65,9 +65,7 @@ extends: stages: - stage: Build variables: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - BuildConfiguration: Release - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ + - template: /azure-pipelines/BuildStageVariables.yml@self jobs: - template: /azure-pipelines/build.yml@self parameters: @@ -99,9 +97,7 @@ extends: stages: - stage: Build variables: - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - BuildConfiguration: Release - NUGET_PACKAGES: $(Agent.TempDirectory)/.nuget/packages/ + - template: /azure-pipelines/BuildStageVariables.yml@self jobs: - template: /azure-pipelines/build.yml@self parameters: From 1f5f12750e115bcbeef07e4757032c2143235de1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 15:31:35 -0700 Subject: [PATCH 38/86] Fix Expand-Template.ps1 --- Expand-Template.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Expand-Template.ps1 b/Expand-Template.ps1 index 8d4c2f4b..3367d240 100755 --- a/Expand-Template.ps1 +++ b/Expand-Template.ps1 @@ -149,12 +149,15 @@ try { "(?m)^\s+- microbuild`r?`n"="" } Replace-Placeholders -Path "azure-pipelines\official.yml" -Replacements $YmlReplacements + Replace-Placeholders -Path "azure-pipelines.yml" -Replacements $YmlReplacements + + $YmlReplacements = @{} if ($CodeCovToken) { $YmlReplacements['(codecov_token: ).*(#.*)'] = "`$1$CodeCovToken" } else { $YmlReplacements['(codecov_token: ).*(#.*)'] = "#`$1`$2" } - Replace-Placeholders -Path "azure-pipelines.yml" -Replacements $YmlReplacements + Replace-Placeholders -Path "azure-pipelines/BuildStageVariables.yml" -Replacements $YmlReplacements Replace-Placeholders -Path "azure-pipelines/variables/InsertVersionsValues.ps1" -Replacements @{ 'LibraryName' = $LibraryName; From 64b2ae3bc6c384919e138cbd3fa50b884e073003 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 17:10:45 -0700 Subject: [PATCH 39/86] Clarify purpose of nuGetServiceConnection --- azure-pipelines/install-dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/install-dependencies.yml b/azure-pipelines/install-dependencies.yml index 0976e548..dc35cece 100644 --- a/azure-pipelines/install-dependencies.yml +++ b/azure-pipelines/install-dependencies.yml @@ -7,7 +7,7 @@ steps: displayName: ๐Ÿ” Authenticate NuGet feeds inputs: ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - nuGetServiceConnections: azure-public/msft_consumption + nuGetServiceConnections: azure-public/msft_consumption # Only necessary for GitHub-hosted repos forceReinstallCredentialProvider: true - powershell: | From 4475f937772658ebdb1e9b20752c97ec2b32cd9f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 17:11:33 -0700 Subject: [PATCH 40/86] Drop Ref12Analyze task It publishes to a web site that no longer operates. --- azure-pipelines/microbuild.after.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 7531cf54..d1f322c0 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -10,19 +10,6 @@ steps: $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration)/NuGet condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- task: Ref12Analyze@0 - displayName: ๐Ÿ“‘ Ref12 (Codex) Analyze - inputs: - codexoutputroot: $(Build.ArtifactStagingDirectory)\Codex - workflowArguments: | - /sourcesDirectory:$(Build.SourcesDirectory) - /codexRepoUrl:$(Build.Repository.Uri) - /repoName:$(Build.Repository.Name) - /additionalCodexArguments:-bld - /additionalCodexArguments:$(Build.ArtifactStagingDirectory)/build_logs - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['Agent.OS'], 'Windows_NT')) - continueOnError: true - - ${{ if eq(parameters.EnableCompliance, 'true') }}: - template: secure-development-tools.yml parameters: From 6dc738bcb5a88209e3b4b3eaec9e0fcaee8955d7 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 14 Feb 2024 13:55:09 -0700 Subject: [PATCH 41/86] Fix publish for scheduled pipeline runs --- azure-pipelines/build.yml | 6 +++--- azure-pipelines/official.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 1074264f..501c6269 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -51,7 +51,7 @@ jobs: signing: enabled: true zipSources: false - ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + ${{ if parameters.RealSign }}: signType: real ${{ else }}: signType: test @@ -107,7 +107,7 @@ jobs: ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: mb: - ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + ${{ if parameters.RealSign }}: signing: enabled: false # enable when building unique artifacts on this agent that must be signed signType: real @@ -137,7 +137,7 @@ jobs: ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: mb: - ${{ if or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }}: + ${{ if parameters.RealSign }}: signing: enabled: false # enable when building unique artifacts on this agent that must be signed signType: real diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index ccb5a4d1..c28137f6 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -69,7 +69,7 @@ extends: jobs: - template: /azure-pipelines/build.yml@self parameters: - RealSign: ${{ parameters.RealSign }} + RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -86,7 +86,7 @@ extends: RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: - RealSign: ${{ parameters.RealSign }} + RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} ${{ else }}: template: azure-pipelines/MicroBuild.1ES.Unofficial.yml@MicroBuildTemplate parameters: @@ -101,7 +101,7 @@ extends: jobs: - template: /azure-pipelines/build.yml@self parameters: - RealSign: ${{ parameters.RealSign }} + RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -118,4 +118,4 @@ extends: RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: - RealSign: ${{ parameters.RealSign }} + RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} From a7dbfbc8acacf7293e3c46e4b7e1466c338b5fc8 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 16 Feb 2024 10:15:56 -0700 Subject: [PATCH 42/86] Add missing elements for turnkey localization --- azure-pipelines/artifacts/LocBin.ps1 | 15 +++++++++++++++ azure-pipelines/build.yml | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 azure-pipelines/artifacts/LocBin.ps1 diff --git a/azure-pipelines/artifacts/LocBin.ps1 b/azure-pipelines/artifacts/LocBin.ps1 new file mode 100644 index 00000000..85bf5c7b --- /dev/null +++ b/azure-pipelines/artifacts/LocBin.ps1 @@ -0,0 +1,15 @@ +# Identify LCE files and the binary files they describe +$BinRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin") +$FilesToCopy = @() +$FilesToCopy += Get-ChildItem -Recurse -File -Path $BinRoot |? { $_.FullName -match '\\Localize\\' } + +Get-ChildItem -rec "$BinRoot\*.lce" -File | % { + $FilesToCopy += $_ + $FilesToCopy += $_.FullName.SubString(0, $_.FullName.Length - 4) +} + +$FilesToCopy += Get-ChildItem -rec "$BinRoot\*.lcg" -File | % { [xml](Get-Content $_) } | % { $_.lcx.name } + +@{ + "$BinRoot" = $FilesToCopy; +} diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 501c6269..5d3fbd06 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -19,6 +19,9 @@ parameters: - name: RunTests type: boolean default: true +- name: EnableLocalization + type: boolean + default: false - name: EnableCompliance type: boolean default: false @@ -58,7 +61,7 @@ jobs: sbom: enabled: true localization: - enabled: false + enabled: ${{ parameters.EnableLocalization }} ${{ if eq(variables['Build.Reason'], 'pullRequest') }}: languages: ENU,JPN outputParentDirectory: $(Build.ArtifactStagingDirectory) @@ -74,6 +77,11 @@ jobs: displayName: ๐Ÿ“ข Publish VSInsertion-Windows targetPath: $(Build.ArtifactStagingDirectory)/VSInsertion-Windows artifactName: VSInsertion-Windows + - ${{ if parameters.EnableLocalization }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish LocBin-Windows + targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows + artifactName: LocBin-Windows steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. From c7fe3ec5099b2c7648ed53f78b040d9d91adfadf Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 16 Feb 2024 11:36:08 -0700 Subject: [PATCH 43/86] Fix LocBin to not fail before a build occurs --- azure-pipelines/artifacts/LocBin.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/artifacts/LocBin.ps1 b/azure-pipelines/artifacts/LocBin.ps1 index 85bf5c7b..3b6945f7 100644 --- a/azure-pipelines/artifacts/LocBin.ps1 +++ b/azure-pipelines/artifacts/LocBin.ps1 @@ -1,5 +1,7 @@ # Identify LCE files and the binary files they describe $BinRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..\bin") +if (!(Test-Path $BinRoot)) { return } + $FilesToCopy = @() $FilesToCopy += Get-ChildItem -Recurse -File -Path $BinRoot |? { $_.FullName -match '\\Localize\\' } From 988ff3298ea7d13cd2a4631e2c041d0c53a9e608 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 16 Feb 2024 11:45:52 -0700 Subject: [PATCH 44/86] Drop `outputParentDirectory` from release pipelines Setting it to `$(Pipeline.Workspace)` was problematic because it included other files dropped by 1ES PT that credscan then failed on (go figure). And anyway, these jobs have only one output so there's no reason to specify `outputParentDirectory` as no optimization is applicable. --- azure-pipelines/prepare-insertion-stages.yml | 1 - azure-pipelines/release.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 97e6117f..6ffff531 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -44,7 +44,6 @@ stages: - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT os: Linux templateContext: - outputParentDirectory: $(Pipeline.Workspace) outputs: - output: nuget displayName: ๐Ÿ“ฆ Push nuget packages diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 8b371771..7444d7d3 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -30,7 +30,6 @@ extends: - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT os: Linux templateContext: - outputParentDirectory: $(Pipeline.Workspace) outputs: - output: nuget displayName: ๐Ÿ“ฆ Push packages to nuget.org From e5c43bf44a89dc508fc2a48d26359e8fefa23b53 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 16 Feb 2024 11:53:59 -0700 Subject: [PATCH 45/86] Fix release pipeline --- azure-pipelines/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 7444d7d3..2da6ddf0 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -14,6 +14,9 @@ resources: tags: - auto-release +variables: +- template: GlobalVariables.yml + extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: From aa19fdde1dbd4cbb0c2ee4f353563481b8f6d2ea Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 19 Feb 2024 07:35:40 -0700 Subject: [PATCH 46/86] Bump .NET SDK from 8.0.100 to 8.0.201 --- .devcontainer/Dockerfile | 2 +- global.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index b680b1db..34e56908 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # Refer to https://hub.docker.com/_/microsoft-dotnet-sdk for available versions -FROM mcr.microsoft.com/dotnet/sdk:8.0.100-jammy +FROM mcr.microsoft.com/dotnet/sdk:8.0.201-jammy # Installing mono makes `dotnet test` work without errors even for net472. # But installing it takes a long time, so it's excluded by default. diff --git a/global.json b/global.json index d24a9b70..2565f236 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "8.0.100", + "version": "8.0.201", "rollForward": "patch", "allowPrerelease": false } From 4d78e3c1af91d81e9ed7baac5f2c92ef81b859ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 07:36:25 -0700 Subject: [PATCH 47/86] Bump xunit.runner.visualstudio from 2.5.6 to 2.5.7 (#254) Bumps [xunit.runner.visualstudio](https://github.com/xunit/visualstudio.xunit) from 2.5.6 to 2.5.7. - [Release notes](https://github.com/xunit/visualstudio.xunit/releases) - [Commits](https://github.com/xunit/visualstudio.xunit/compare/2.5.6...2.5.7) --- updated-dependencies: - dependency-name: xunit.runner.visualstudio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 3096e57c..32271e50 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -7,7 +7,7 @@ - + From 4540dc53c2e406f83a3a547da0f77d31a04591a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 07:36:45 -0700 Subject: [PATCH 48/86] Bump dotnet-coverage from 17.10.2 to 17.10.3 (#252) Bumps [dotnet-coverage](https://github.com/microsoft/codecoverage) from 17.10.2 to 17.10.3. - [Commits](https://github.com/microsoft/codecoverage/commits) --- updated-dependencies: - dependency-name: dotnet-coverage dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 6eff0445..dfd00e46 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -9,7 +9,7 @@ ] }, "dotnet-coverage": { - "version": "17.10.2", + "version": "17.10.3", "commands": [ "dotnet-coverage" ] From 7dec0100751f801cd01cea01bac5f12db6ecf504 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 14:43:08 +0000 Subject: [PATCH 49/86] Bump xunit from 2.6.6 to 2.7.0 (#253) --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 32271e50..60ffd922 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -8,7 +8,7 @@ - + From 9651284db829b94d40ee6e342fcba7f9873935cb Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 12:34:45 -0700 Subject: [PATCH 50/86] Add team as reviewer for insertion PRs --- azure-pipelines/variables/InsertReviewers.ps1 | 12 ------------ azure-pipelines/vs-insertion.yml | 1 + 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 azure-pipelines/variables/InsertReviewers.ps1 diff --git a/azure-pipelines/variables/InsertReviewers.ps1 b/azure-pipelines/variables/InsertReviewers.ps1 deleted file mode 100644 index 6fd6648b..00000000 --- a/azure-pipelines/variables/InsertReviewers.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# This is a list of AzDO account names or email addresses. -# Add your team DL and/or whoever should be notified of insertion PRs. -$contacts = ,$env:BUILD_REQUESTEDFOREMAIL - -if (Test-Path "$PSScriptRoot\TeamEmail.ps1") { - $contacts += & "$PSScriptRoot\TeamEmail.ps1" -} - -$contacts = $contacts |? { $_ } -if ($contacts) { - [string]::Join(',', $contacts) -} diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 87166f29..9cb597ea 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -54,6 +54,7 @@ extends: TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicy: Request Perf DDRITs + InsertionReviewers: $(TeamEmail),$(Build.RequestedForEmail) AutoCompletePR: true AutoCompleteMergeStrategy: Squash - powershell: | From be633596de5cb11efe41e2094ec61eb7a8a7549f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 13:00:09 -0700 Subject: [PATCH 51/86] Couple APIScan and TSAUpload under the same switch --- azure-pipelines/official.yml | 2 +- azure-pipelines/secure-development-tools.yml | 39 ++++++++++---------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index c28137f6..8cdf3afb 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -39,7 +39,7 @@ parameters: - name: EnableAPIScan displayName: Include APIScan with Compliance tools type: boolean - default: true + default: false # enable in individual repos only AFTER updating TSAOptions.json with your own values resources: repositories: diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml index 9b1aeaa0..409f48d0 100644 --- a/azure-pipelines/secure-development-tools.yml +++ b/azure-pipelines/secure-development-tools.yml @@ -1,5 +1,7 @@ parameters: - EnableAPIScan: +- name: EnableAPIScan + type: boolean + default: false steps: @@ -19,22 +21,21 @@ steps: !**/osx*/** TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs -- task: APIScan@2 - displayName: ๐Ÿ” Run APIScan - inputs: - softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs - softwareName: $(SymbolsFeatureName) - softwareVersionNum: $(NBGV_MajorMinorVersion) - isLargeApp: false - toolVersion: Latest - condition: and(succeeded(), ${{ parameters.EnableAPIScan }}, ne(variables.ApiScanClientId, '')) - env: - AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) +- ${{ if parameters.EnableAPIScan }}: + - task: APIScan@2 + displayName: ๐Ÿ” Run APIScan + inputs: + softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs + softwareName: $(SymbolsFeatureName) + softwareVersionNum: $(NBGV_MajorMinorVersion) + isLargeApp: false + toolVersion: Latest + env: + AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) -# File bugs when APIScan finds issues -- task: TSAUpload@2 - displayName: ๐Ÿชณ TSA upload - inputs: - GdnPublishTsaOnboard: True - GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json - condition: false # enable in individual repos only AFTER updating TSAOptions.json with your own values + # File bugs when APIScan finds issues + - task: TSAUpload@2 + displayName: ๐Ÿชณ TSA upload + inputs: + GdnPublishTsaOnboard: True + GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json From 56f91e1ff4c4cd55b059a0842d9317b251845c5c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 13 Feb 2024 17:10:21 -0700 Subject: [PATCH 52/86] Add OptProf support to Library.Template --- Expand-Template.ps1 | 13 ++ azure-pipelines.yml | 1 + .../InsertionMetadataPackage.nuspec | 16 ++ azure-pipelines/OptProf.yml | 112 +++++++++++ azure-pipelines/OptProf_part2.yml | 89 ++++++++ azure-pipelines/ProfilingInputs.props | 5 + azure-pipelines/artifacts/VSInsertion.ps1 | 40 +++- azure-pipelines/build.yml | 190 +++++++++++------- azure-pipelines/dotnet.yml | 24 ++- azure-pipelines/microbuild.after.yml | 36 +++- azure-pipelines/microbuild.before.yml | 37 ++++ azure-pipelines/official.yml | 8 + .../variables/ProfilingInputsDropName.ps1 | 5 + azure-pipelines/vs-validation.yml | 1 + 14 files changed, 484 insertions(+), 93 deletions(-) create mode 100644 azure-pipelines/InsertionMetadataPackage.nuspec create mode 100644 azure-pipelines/OptProf.yml create mode 100644 azure-pipelines/OptProf_part2.yml create mode 100644 azure-pipelines/ProfilingInputs.props create mode 100644 azure-pipelines/variables/ProfilingInputsDropName.ps1 diff --git a/Expand-Template.ps1 b/Expand-Template.ps1 index 3367d240..da2f6b7e 100755 --- a/Expand-Template.ps1 +++ b/Expand-Template.ps1 @@ -168,6 +168,19 @@ try { 'LibraryName' = $LibraryName; } + Replace-Placeholders -Path "azure-pipelines/InsertionMetadataPackage.nuspec" -Replacements @{ + 'LibraryName' = $LibraryName; + } + Replace-Placeholders -Path "azure-pipelines/artifacts/VSInsertion.ps1" -Replacements @{ + 'LibraryName' = $LibraryName; + } + Replace-Placeholders -Path "azure-pipelines/OptProf.yml" -Replacements @{ + 'LibraryName' = $LibraryName; + } + Replace-Placeholders -Path "azure-pipelines/OptProf_part2.yml" -Replacements @{ + 'LibraryName' = $LibraryName; + } + # Self destruct git rm Expand-Template.* Apply-Template.ps1 if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6415847e..e6ceafef 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,5 +29,6 @@ variables: jobs: - template: azure-pipelines/build.yml parameters: + Is1ESPT: false includeMacOS: ${{ parameters.includeMacOS }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/InsertionMetadataPackage.nuspec b/azure-pipelines/InsertionMetadataPackage.nuspec new file mode 100644 index 00000000..f7aa3e3f --- /dev/null +++ b/azure-pipelines/InsertionMetadataPackage.nuspec @@ -0,0 +1,16 @@ + + + + LibraryName.VSInsertionMetadata + $version$ + Microsoft + Microsoft + + false + Contains metadata for insertion into VS. + ยฉ Microsoft Corporation. All rights reserved. + + + + + diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml new file mode 100644 index 00000000..6d81439a --- /dev/null +++ b/azure-pipelines/OptProf.yml @@ -0,0 +1,112 @@ +trigger: none +pr: none +schedules: +- cron: "0 3 * * Fri" # Thu @ 8 or 9 PM Mountain Time (depending on DST) + displayName: Weekly OptProf run + branches: + include: + - 'v*.*' + - main + always: true # we must keep data fresh since optimizationdata drops are purged after 30 days + +# Avoid errant CI builds: https://developercommunity.visualstudio.com/content/problem/1154409/azure-pipeline-is-triggering-due-to-events-that-ne.html +#resources: +# repositories: +# - repository: scripts +# type: git +# name: DeploymentScripts +# ref: refs/heads/test + +variables: +- template: GlobalVariables.yml +- name: PublicRelease + value: false # avoid using nice version since we're building a preliminary/unoptimized package +- name: IsOptProf + value: true + +stages: +- stage: Library + variables: + - name: OptProf + value: true + - template: BuildStageVariables.yml + jobs: + - template: build.yml + parameters: + Is1ESPT: false + RealSign: true + EnableLocalization: false + EnableCompliance: false + windowsPool: VSEngSS-MicroBuild2022-1ES + includeMacOS: false + ShouldSkipOptimize: true + IsOptProf: true + RunTests: false +- stage: QueueVSBuild + jobs: + - job: QueueOptProf + pool: VSEngSS-MicroBuild2022-1ES + variables: + InsertPayloadName: LibraryName + InsertTopicBranch: team/VS-IDE/LibraryName-OptProf-run-$(Build.BuildId) + steps: + - checkout: none # We don't need source from our own repo + clean: true + + # Pipeline YAML does not yet support checking out other repos. So we'll do it by hand. +# - checkout: scripts # We DO need source from the DeploymentScripts repo +# clean: true +# path: $(Agent.TempDirectory)/DeploymentScripts +# fetchDepth: 1 + - script: 'git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" clone https://devdiv.visualstudio.com/DevDiv/_git/DeploymentScripts --depth 1 --branch test "$(Agent.TempDirectory)/DeploymentScripts"' + displayName: Download DeploymentScripts repo + + - task: DownloadBuildArtifacts@0 + displayName: Download insertion artifacts + inputs: + artifactName: VSInsertion-Windows + downloadPath: $(Agent.TempDirectory) + - task: DownloadBuildArtifacts@0 + displayName: Download variables artifacts + inputs: + artifactName: Variables-Windows + downloadPath: $(Agent.TempDirectory) + - task: PowerShell@2 + displayName: Set pipeline variables based on artifacts + inputs: + targetType: filePath + filePath: $(Agent.TempDirectory)/Variables-Windows/_pipelines.ps1 + - task: NuGetCommand@2 + displayName: Push CoreXT packages to VS feed + inputs: + command: push + packagesToPush: $(Agent.TempDirectory)/VSInsertion-Windows/*.nupkg + publishVstsFeed: 97a41293-2972-4f48-8c0e-05493ae82010 # VS feed + allowPackageConflicts: true + - task: MicroBuildInsertVsPayload@4 + displayName: Insert VS Payload + inputs: + SkipCreatePR: true + CustomScriptExecutionCommand: src\VSSDK\NuGet\AllowUnstablePackages.ps1 + - task: benjhuser.tfs-extensions-build-tasks.trigger-build-task.TriggerBuild@3 + displayName: Trigger a new build of DD-CB-TestSignVS-devCI + inputs: + buildDefinition: DD-CB-TestSignVS-devCI + useSameBranch: false + branchToUse: $(InsertTopicBranch) + storeInEnvironmentVariable: true + queueBuildForUserThatTriggeredBuild: false + authenticationMethod: OAuth Token + password: $(System.AccessToken) + - task: PowerShell@2 + displayName: Associate InsertionOutputs artifacts with CloudBuild + inputs: + targetType: filePath + filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/WriteArtifact.ps1 + arguments: '-oldBuildID $(Build.BuildId) -newBuildID $(TriggeredBuildIds) -artifactName "InsertionOutputs" -accessToken $(System.AccessToken)' + - task: PowerShell@2 + displayName: Tag the build with LibraryName-insertion + inputs: + targetType: filePath + filePath: $(Agent.TempDirectory)/DeploymentScripts/Scripts/Insertion/TagBuild.ps1 + arguments: '-buildID $(TriggeredBuildIds) -tagName "LibraryName-insertion" -accessToken $(System.AccessToken)' diff --git a/azure-pipelines/OptProf_part2.yml b/azure-pipelines/OptProf_part2.yml new file mode 100644 index 00000000..60ac39b8 --- /dev/null +++ b/azure-pipelines/OptProf_part2.yml @@ -0,0 +1,89 @@ +trigger: none +pr: none + +resources: + pipelines: + - pipeline: VisualStudioBuildUnderTest + source: DD-CB-TestSignVS-devCI + trigger: + tags: + - LibraryName-insertion + - pipeline: DartLab + source: DartLab + branch: main + - pipeline: DartLab.OptProf + source: DartLab.OptProf + branch: main + repositories: + - repository: DartLabTemplates + type: git + name: DartLab.Templates + ref: refs/heads/main + - repository: DartLabOptProfTemplates + type: git + name: DartLab.OptProf + ref: refs/heads/main + +parameters: + +# The prefix naming of the OptimizationInputs drop +- name: optimizationDropPrefix + type: string + default: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) + +stages: +- template: \templates\stages\visual-studio\single-runsettings.yml@DartLabOptProfTemplates + parameters: + ##### Required ##### + runSettingsURI: $(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\runsettings\LibraryName.OptProf.runsettings + visualStudioBootstrapperURI: https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe + ##### Optional ##### + name: OptProfProfilingWorkflow + displayName: OptProf Profiling Workflow + optOptimizationInputsDropName: $(OptimizationInputsDropName) + previousOptimizationInputsDropName: $(PreviousOptimizationInputsDropName) + testLabPoolName: VS-Platform + ##### Step Hooks ##### + preTestMachineConfigurationStepList: + - download: VisualStudioBuildUnderTest + - task: PowerShell@2 + name: SetProductsDropName + displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName' + inputs: + filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1 + arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName' + preDeployAndRunTestsStepList: + - download: VisualStudioBuildUnderTest + prePublishOptimizationInputsDropStepList: + # Set parameter for PreviousOptimizationInputsDropName, MicroBuildCommitID, and OptimizationInputsDropName + - powershell: | + try { + $artifactName = 'InsertionOutputs' + $BuildID = $(resources.pipeline.VisualStudioBuildUnderTest.runID) + $artifact = Get-BuildArtifact -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) + $containerName = $artifact.Resource.Data -Split '/' | Select-Object -Last 1 + $fileName = Join-Path $containerName 'Metadata.json' + $jsonString = Read-BuildArtifactFile -InstanceURL 'https://dev.azure.com/devdiv' -ProjectName 'DevDiv' -BuildID $BuildID -ArtifactName $artifactName -FileName $fileName -OAuthAccessToken (ConvertTo-SecureString '$(System.AccessToken)' -AsPlainText -Force) + $json = $jsonString | ConvertFrom-Json + + Write-Host "The content of the metadata.json file was $json" + + $dropname = $json.OptimizationData + $commitID = $json.CommitID + $OptimizationInputsDropName = "${{parameters.optimizationDropPrefix}}/$($commitID)/$(Build.BuildId)/$(System.StageId)/$(System.StageAttempt)" + + Write-Host "PreviousOptimizationInputsDropName: $dropname" + Set-AzurePipelinesVariable 'PreviousOptimizationInputsDropName' $dropname + + Write-Host "MicroBuildCommitID: $commitID" + Set-AzurePipelinesVariable 'MicroBuildCommitID' $commitID + + Write-Host "OptimizationInputsDropName: $OptimizationInputsDropName" + Set-AzurePipelinesVariable 'OptimizationInputsDropName' $OptimizationInputsDropName + } + catch { + Write-Host $_ + Write-Error "Failed to set OptimizationInputsDropName pipeline variable" + throw + } + displayName: Set MicroBuildCommitID, PreviousOptimizationInputsDropName, and OptimizationInputsDropName diff --git a/azure-pipelines/ProfilingInputs.props b/azure-pipelines/ProfilingInputs.props new file mode 100644 index 00000000..fb19d604 --- /dev/null +++ b/azure-pipelines/ProfilingInputs.props @@ -0,0 +1,5 @@ + + + + + diff --git a/azure-pipelines/artifacts/VSInsertion.ps1 b/azure-pipelines/artifacts/VSInsertion.ps1 index ba6af320..4757c9c2 100644 --- a/azure-pipelines/artifacts/VSInsertion.ps1 +++ b/azure-pipelines/artifacts/VSInsertion.ps1 @@ -16,13 +16,43 @@ if (!$BuildConfiguration) { $BuildConfiguration = 'Debug' } -$PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" +$NuGetPackages = "$RepoRoot/bin/Packages/$BuildConfiguration/NuGet" -if (!(Test-Path $PackagesRoot)) { - Write-Warning "Skipping because packages haven't been built yet." +if (!(Test-Path $NuGetPackages)) { + Write-Warning "Skipping because NuGet packages haven't been built yet." return @{} } -@{ - "$PackagesRoot" = (Get-ChildItem $PackagesRoot -Recurse) +$result = @{ + "$NuGetPackages" = (Get-ChildItem $NuGetPackages -Recurse) } + +if ($env:IsOptProf) { + $CoreXTPackages = "$RepoRoot/bin/Packages/$BuildConfiguration/CoreXT" + + $ArtifactBasePath = "$RepoRoot\obj\_artifacts" + $ArtifactPath = "$ArtifactBasePath\VSInsertion" + if (-not (Test-Path $ArtifactPath)) { New-Item -ItemType Directory -Path $ArtifactPath | Out-Null } + + $profilingInputs = [xml](Get-Content -Path "$PSScriptRoot\..\ProfilingInputs.props") + $profilingInputs.Project.ItemGroup.TestStore.Include = "vstsdrop:" + (& "$PSScriptRoot\..\variables\ProfilingInputsDropName.ps1") + $profilingInputs.Save("$ArtifactPath\ProfilingInputs.props") + + $InsertionMetadataVersion = $(dotnet tool run nbgv get-version -p "$RepoRoot\src" -f json | ConvertFrom-Json).NuGetPackageVersion + if ($env:BUILD_BUILDID) { + # We must ensure unique versions for the insertion metadata package so + # it can contain information that is unique to this build. + # In particular it includes the ProfilingInputsDropName, which contains the BuildId. + # A non-unique package version here may collide with a prior run of this same commit, + # ultimately resulting in a failure of the optprof run. + $InsertionMetadataVersion += '.' + $env:BUILD_BUILDID + } + & (& "$PSScriptRoot\..\Get-NuGetTool.ps1") pack "$PSScriptRoot\..\InsertionMetadataPackage.nuspec" -OutputDirectory $CoreXTPackages -BasePath $ArtifactPath -Version $InsertionMetadataVersion | Out-Null + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + + $result["$CoreXTPackages"] = (Get-ChildItem "$CoreXTPackages\LibraryName.VSInsertionMetadata.$InsertionMetadataVersion.nupkg"); +} + +$result diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 5d3fbd06..95f3b9d4 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -1,4 +1,6 @@ parameters: +- name: Is1ESPT + type: boolean - name: RealSign type: boolean default: false @@ -16,6 +18,15 @@ parameters: vmImage: macOS-12 - name: includeMacOS type: boolean +- name: EnableOptProf + type: boolean + default: false +- name: IsOptProf + type: boolean + default: false +- name: ShouldSkipOptimize + type: boolean + default: false - name: RunTests type: boolean default: true @@ -64,6 +75,21 @@ jobs: enabled: ${{ parameters.EnableLocalization }} ${{ if eq(variables['Build.Reason'], 'pullRequest') }}: languages: ENU,JPN + optprof: + enabled: ${{ parameters.EnableOptProf }} + ProfilingInputsDropName: $(ProfilingInputsDropName) + OptimizationInputsLookupMethod: DropPrefix + DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) + ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} + AccessToken: $(System.AccessToken) + mbpresteps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - ${{ if parameters.EnableOptProf }}: + - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(azure-pipelines/variables/ProfilingInputsDropName.ps1)" + displayName: โš™ Set ProfilingInputsDropName for optprof + outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: @@ -83,10 +109,6 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows artifactName: LocBin-Windows steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - template: schedule-only-steps.yml @@ -97,94 +119,106 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.before.yml + parameters: + EnableLocalization: ${{ parameters.EnableLocalization }} + EnableOptProf: ${{ parameters.EnableOptProf }} + IsOptProf: ${{ parameters.IsOptProf }} + ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - template: dotnet.yml parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} RunTests: ${{ parameters.RunTests }} + IsOptProf: ${{ parameters.IsOptProf }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} + EnableOptProf: ${{ parameters.EnableOptProf }} + IsOptProf: ${{ parameters.IsOptProf }} - template: expand-template.yml -- job: Linux - pool: ${{ parameters.linuxPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux - artifactName: ${{ artifact_name }}-Linux - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - RunTests: ${{ parameters.RunTests }} - - script: dotnet format --verify-no-changes --no-restore - displayName: ๐Ÿ’… Verify formatted code - - template: expand-template.yml +- ${{ if not(parameters.IsOptProf) }}: + - job: Linux + pool: ${{ parameters.linuxPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux + artifactName: ${{ artifact_name }}-Linux + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + - script: dotnet format --verify-no-changes --no-restore + displayName: ๐Ÿ’… Verify formatted code + - template: expand-template.yml -- job: macOS - condition: ${{ parameters.includeMacOS }} - pool: ${{ parameters.macOSPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS - artifactName: ${{ artifact_name }}-macOS - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - RunTests: ${{ parameters.RunTests }} - - template: expand-template.yml + - job: macOS + condition: ${{ parameters.includeMacOS }} + pool: ${{ parameters.macOSPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS + artifactName: ${{ artifact_name }}-macOS + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + - template: expand-template.yml -- job: WrapUp - dependsOn: - - Windows - - Linux - - macOS - pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - parameters: - initArgs: -NoRestore - - template: publish-symbols.yml - parameters: - includeMacOS: ${{ parameters.includeMacOS }} - - ${{ if parameters.RunTests }}: - - template: publish-codecoverage.yml + - job: WrapUp + dependsOn: + - Windows + - Linux + - macOS + pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). + condition: succeededOrFailed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + parameters: + initArgs: -NoRestore + - template: publish-symbols.yml parameters: includeMacOS: ${{ parameters.includeMacOS }} + - ${{ if parameters.RunTests }}: + - template: publish-codecoverage.yml + parameters: + includeMacOS: ${{ parameters.includeMacOS }} diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 546e9083..3f4ac753 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -1,27 +1,37 @@ parameters: - RunTests: +- name: RunTests +- name: IsOptProf + type: boolean + default: false +- name: Is1ESPT + type: boolean steps: - script: dotnet build -t:build,pack --no-restore -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog" displayName: ๐Ÿ›  dotnet build -- powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults - displayName: ๐Ÿงช dotnet test - condition: and(succeeded(), ${{ parameters.RunTests }}) +- ${{ if not(parameters.IsOptProf) }}: + - powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults + displayName: ๐Ÿงช dotnet test + condition: and(succeeded(), ${{ parameters.RunTests }}) + +- ${{ if parameters.IsOptProf }}: + # We have to artifically run this script so that the extra .nupkg is produced for variables/InsertConfigValues.ps1 to notice. + - powershell: azure-pipelines\artifacts\VSInsertion.ps1 + displayName: ๐Ÿ”ง Prepare VSInsertion artifact - powershell: azure-pipelines/variables/_pipelines.ps1 failOnStderr: true displayName: โš™ Update pipeline variables based on build outputs condition: succeededOrFailed() -- ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: +- ${{ if parameters.Is1ESPT }}: - powershell: azure-pipelines/artifacts/_pipelines.ps1 -StageOnly -AvoidSymbolicLinks -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose failOnStderr: true displayName: ๐Ÿ“ข Stage artifacts condition: succeededOrFailed() - -- ${{ if ne(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: +- ${{ else }}: - powershell: azure-pipelines/artifacts/_pipelines.ps1 -ArtifactNameSuffix "-$(Agent.JobName)" -Verbose failOnStderr: true displayName: ๐Ÿ“ข Publish artifacts diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index d1f322c0..f9698e3e 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,6 +1,16 @@ parameters: - EnableCompliance: - EnableAPIScan: +- name: EnableCompliance + type: boolean + default: false +- name: EnableAPIScan + type: boolean + default: false +- name: EnableOptProf + type: boolean + default: false +- name: IsOptProf + type: boolean + default: false steps: - task: MicroBuildCodesignVerify@3 @@ -10,7 +20,27 @@ steps: $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration)/NuGet condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- ${{ if eq(parameters.EnableCompliance, 'true') }}: +- ${{ if parameters.EnableOptProf }}: + - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 + inputs: + dropServiceURI: https://devdiv.artifacts.visualstudio.com + buildNumber: $(ProfilingInputsDropName) + sourcePath: $(Build.ArtifactStagingDirectory)\OptProf\ProfilingInputs + toLowerCase: false + usePat: true + displayName: ๐Ÿ“ข Publish to Artifact Services - ProfilingInputs + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + continueOnError: true + + - ${{ if parameters.IsOptProf }}: + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/InsertionOutputs + ArtifactName: InsertionOutputs + ArtifactType: Container + displayName: ๐Ÿ“ข Publish InsertionOutputs as Azure DevOps artifacts + +- ${{ if parameters.EnableCompliance }}: - template: secure-development-tools.yml parameters: EnableAPIScan: ${{ parameters.EnableAPIScan }} diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index c53ae9a6..949b1b47 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -1,3 +1,17 @@ +parameters: +- name: EnableLocalization + type: boolean + default: false +- name: EnableOptProf + type: boolean + default: false +- name: IsOptProf + type: boolean + default: false +- name: ShouldSkipOptimize + type: boolean + default: false + steps: - task: notice@0 displayName: ๐Ÿ› ๏ธ Generate NOTICE file @@ -6,3 +20,26 @@ steps: outputformat: text condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded + +- ${{ if parameters.IsOptProf }}: + # We have to install these plugins ourselves for Optprof runs because those pipelines haven't migrated to 1ES PT yet. + - task: MicroBuildOptProfPlugin@6 + inputs: + ProfilingInputsDropName: $(ProfilingInputsDropName) + OptimizationInputsLookupMethod: DropPrefix + DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) + ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} + AccessToken: $(System.AccessToken) + displayName: ๐Ÿ”ง Install OptProf Plugin + + - task: MicroBuildSigningPlugin@4 + inputs: + signType: Real + zipSources: false + displayName: ๐Ÿ”ง Install MicroBuild Signing Plugin + + - ${{ if parameters.EnableLocalization }}: + - task: MicroBuildLocalizationPlugin@4 + inputs: + languages: $(LocLanguages) + displayName: ๐Ÿ”ง Install MicroBuild Localization Plugin diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index c28137f6..ed9a0aca 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -24,6 +24,10 @@ parameters: displayName: Real sign? type: boolean default: false +# - name: ShouldSkipOptimize # Enable this when setting EnableOptProf to true in build.yml. +# displayName: Skip OptProf optimization +# type: boolean +# default: false - name: includeMacOS displayName: Build on macOS type: boolean @@ -69,7 +73,9 @@ extends: jobs: - template: /azure-pipelines/build.yml@self parameters: + Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} + ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -101,7 +107,9 @@ extends: jobs: - template: /azure-pipelines/build.yml@self parameters: + Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} + ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES diff --git a/azure-pipelines/variables/ProfilingInputsDropName.ps1 b/azure-pipelines/variables/ProfilingInputsDropName.ps1 new file mode 100644 index 00000000..88d21f69 --- /dev/null +++ b/azure-pipelines/variables/ProfilingInputsDropName.ps1 @@ -0,0 +1,5 @@ +if ($env:SYSTEM_TEAMPROJECT) { + "ProfilingInputs/$env:SYSTEM_TEAMPROJECT/$env:BUILD_REPOSITORY_NAME/$env:BUILD_SOURCEBRANCHNAME/$env:BUILD_BUILDID" +} else { + Write-Warning "No Azure Pipelines build detected. No Azure Pipelines drop name will be computed." +} diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 5ebf0c71..195bfff3 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -32,6 +32,7 @@ extends: jobs: - template: /azure-pipelines/build.yml@self parameters: + Is1ESPT: true RealSign: true windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: false From 3dd7fab92b6aa1a7c7b9796900ce6bc525c01372 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 13:05:15 -0700 Subject: [PATCH 53/86] Disable NOTICE file generation for OptProf runs Just a little something to make optprof runs a little more reliable. --- azure-pipelines/build.yml | 1 + azure-pipelines/microbuild.before.yml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 95f3b9d4..2a69192e 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -124,6 +124,7 @@ jobs: EnableOptProf: ${{ parameters.EnableOptProf }} IsOptProf: ${{ parameters.IsOptProf }} ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} + RealSign: ${{ parameters.RealSign }} - template: dotnet.yml parameters: diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index 949b1b47..ce711a0a 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -11,15 +11,19 @@ parameters: - name: ShouldSkipOptimize type: boolean default: false +- name: RealSign + type: boolean steps: -- task: notice@0 - displayName: ๐Ÿ› ๏ธ Generate NOTICE file - inputs: - outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE - outputformat: text - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded +- ${{ if not(parameters.IsOptProf) }}: + - task: notice@0 + displayName: ๐Ÿ› ๏ธ Generate NOTICE file + inputs: + outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE + outputformat: text + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + retryCountOnTaskFailure: 5 # fails when the cloud service is overloaded + continueOnError: ${{ not(parameters.RealSign) }} # Tolerate failures when we're not building something that may ship. - ${{ if parameters.IsOptProf }}: # We have to install these plugins ourselves for Optprof runs because those pipelines haven't migrated to 1ES PT yet. From 5a329c4011165e861af9a9724f394b1ec0f4e710 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 14:00:26 -0700 Subject: [PATCH 54/86] Update ubuntu image names --- azure-pipelines/archive-sourcecode.yml | 2 +- azure-pipelines/official.yml | 4 ++-- azure-pipelines/prepare-insertion-stages.yml | 2 +- azure-pipelines/release.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index afdf0f6b..c550e2c2 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -44,7 +44,7 @@ extends: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + - ImageOverride -equals 1ESPT-Ubuntu22.04 os: Linux steps: diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 8cdf3afb..a3d64e68 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -76,7 +76,7 @@ extends: linuxPool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + - ImageOverride -equals 1ESPT-Ubuntu22.04 os: Linux macOSPool: name: Azure Pipelines @@ -108,7 +108,7 @@ extends: linuxPool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + - ImageOverride -equals 1ESPT-Ubuntu22.04 os: Linux macOSPool: name: Azure Pipelines diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index 6ffff531..da163c76 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -41,7 +41,7 @@ stages: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + - ImageOverride -equals 1ESPT-Ubuntu22.04 os: Linux templateContext: outputs: diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 2da6ddf0..0f425fb6 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -30,7 +30,7 @@ extends: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliantGPT + - ImageOverride -equals 1ESPT-Ubuntu22.04 os: Linux templateContext: outputs: From 1eba738358ff53acd1ffa70f7b11b9c102d61b97 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 15:51:50 -0700 Subject: [PATCH 55/86] Publish logs on build/test failures for mac/linux We were already doing this right on Windows. --- azure-pipelines/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 5d3fbd06..4b6aa164 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -127,6 +127,7 @@ jobs: displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux artifactName: ${{ artifact_name }}-Linux + condition: succeededOrFailed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. @@ -157,6 +158,7 @@ jobs: displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS artifactName: ${{ artifact_name }}-macOS + condition: succeededOrFailed() steps: - checkout: self fetchDepth: 0 # avoid shallow clone so nbgv can do its work. From 1d3bd2b944838856145805c77d12b19a71d3c3e1 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 14:22:55 -0700 Subject: [PATCH 56/86] OptProf pipeline fixes --- azure-pipelines/OptProf.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml index 6d81439a..dc0e4435 100644 --- a/azure-pipelines/OptProf.yml +++ b/azure-pipelines/OptProf.yml @@ -35,7 +35,6 @@ stages: parameters: Is1ESPT: false RealSign: true - EnableLocalization: false EnableCompliance: false windowsPool: VSEngSS-MicroBuild2022-1ES includeMacOS: false @@ -86,6 +85,8 @@ stages: - task: MicroBuildInsertVsPayload@4 displayName: Insert VS Payload inputs: + TeamName: $(TeamName) + TeamEmail: $(TeamEmail) SkipCreatePR: true CustomScriptExecutionCommand: src\VSSDK\NuGet\AllowUnstablePackages.ps1 - task: benjhuser.tfs-extensions-build-tasks.trigger-build-task.TriggerBuild@3 From 891d82b51df52818793084ab6f5e74053ef2648a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 20 Feb 2024 16:02:03 -0700 Subject: [PATCH 57/86] Fix `notice@0` task on non-default branches --- azure-pipelines/microbuild.before.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/microbuild.before.yml b/azure-pipelines/microbuild.before.yml index ce711a0a..05acd319 100644 --- a/azure-pipelines/microbuild.before.yml +++ b/azure-pipelines/microbuild.before.yml @@ -15,14 +15,18 @@ parameters: type: boolean steps: -- ${{ if not(parameters.IsOptProf) }}: +- ${{ if and(not(parameters.IsOptProf), ne(variables['Build.Reason'], 'PullRequest')) }}: + # notice@0 requires CG detection to run first, and non-default branches don't inject it automatically. + - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - task: ComponentGovernanceComponentDetection@0 + displayName: ๐Ÿ” Component Detection + - task: notice@0 displayName: ๐Ÿ› ๏ธ Generate NOTICE file inputs: outputfile: $(System.DefaultWorkingDirectory)/obj/NOTICE outputformat: text - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - retryCountOnTaskFailure: 5 # fails when the cloud service is overloaded + retryCountOnTaskFailure: 3 # fails when the cloud service is overloaded continueOnError: ${{ not(parameters.RealSign) }} # Tolerate failures when we're not building something that may ship. - ${{ if parameters.IsOptProf }}: From e5e41d4a503ef08a0a0f8e68232e76e5325ba7e6 Mon Sep 17 00:00:00 2001 From: Matteo Prosperi <41970398+matteo-prosperi@users.noreply.github.com> Date: Tue, 20 Feb 2024 17:02:18 -0800 Subject: [PATCH 58/86] Disable the generation of SBOMs for artifacts in unsigned builds --- azure-pipelines/official.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index a3d64e68..610900b7 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -62,6 +62,8 @@ extends: exclusionsFile: $(System.DefaultWorkingDirectory)\azure-pipelines\PoliCheckExclusions.xml suppression: suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress + sbom: + enabled: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} # Disable the generation for SBOMs for artifacts in unsigned builds since it's slow stages: - stage: Build variables: From d92acd3528bc80128b2a6b6cbc628ffa2b4f7067 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 12:43:59 -0700 Subject: [PATCH 59/86] Fix `ShouldSkipOptimize` parameter references --- azure-pipelines/official.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 3caa9f9d..72dcb4f0 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -24,7 +24,7 @@ parameters: displayName: Real sign? type: boolean default: false -# - name: ShouldSkipOptimize # Enable this when setting EnableOptProf to true in build.yml. +# - name: ShouldSkipOptimize # Enable this and references to it below when setting EnableOptProf to true in build.yml. # displayName: Skip OptProf optimization # type: boolean # default: false @@ -75,7 +75,7 @@ extends: parameters: Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} + # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES @@ -109,7 +109,7 @@ extends: parameters: Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} + # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableCompliance: ${{ parameters.EnableCompliance }} EnableAPIScan: ${{ parameters.EnableAPIScan }} windowsPool: VSEngSS-MicroBuild2022-1ES From 214c048692cc64ff58481c963ceda59d13ad029b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 13:44:49 -0700 Subject: [PATCH 60/86] Make mac/linux agents trivially excludable --- azure-pipelines.yml | 4 +- azure-pipelines/OptProf.yml | 2 +- azure-pipelines/build.yml | 145 +++++++++++++---------- azure-pipelines/official.yml | 6 +- azure-pipelines/publish-codecoverage.yml | 24 ++-- azure-pipelines/publish-symbols.yml | 59 ++++----- azure-pipelines/vs-validation.yml | 2 +- 7 files changed, 134 insertions(+), 108 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e6ceafef..79f69c22 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,7 +14,7 @@ trigger: - azure-pipelines/release.yml parameters: -- name: includeMacOS +- name: EnableMacOSBuild displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines @@ -30,5 +30,5 @@ jobs: - template: azure-pipelines/build.yml parameters: Is1ESPT: false - includeMacOS: ${{ parameters.includeMacOS }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml index dc0e4435..0f4ba24f 100644 --- a/azure-pipelines/OptProf.yml +++ b/azure-pipelines/OptProf.yml @@ -37,7 +37,7 @@ stages: RealSign: true EnableCompliance: false windowsPool: VSEngSS-MicroBuild2022-1ES - includeMacOS: false + EnableMacOSBuild: false ShouldSkipOptimize: true IsOptProf: true RunTests: false diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 215512c9..f82afad6 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -16,8 +16,12 @@ parameters: type: object default: vmImage: macOS-12 -- name: includeMacOS +- name: EnableLinuxBuild type: boolean + default: true +- name: EnableMacOSBuild + type: boolean + default: true - name: EnableOptProf type: boolean default: false @@ -39,6 +43,9 @@ parameters: - name: EnableAPIScan type: boolean default: false +- name: EnableDotNetFormatCheck + type: boolean + default: true - name: artifact_names type: object default: @@ -132,6 +139,10 @@ jobs: RunTests: ${{ parameters.RunTests }} IsOptProf: ${{ parameters.IsOptProf }} + - ${{ if and(parameters.EnableDotNetFormatCheck, not(parameters.EnableLinuxBuild)) }}: + - script: dotnet format --verify-no-changes --no-restore + displayName: ๐Ÿ’… Verify formatted code + - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: @@ -143,72 +154,76 @@ jobs: - template: expand-template.yml - ${{ if not(parameters.IsOptProf) }}: - - job: Linux - pool: ${{ parameters.linuxPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux - artifactName: ${{ artifact_name }}-Linux - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - - script: dotnet format --verify-no-changes --no-restore - displayName: ๐Ÿ’… Verify formatted code - - template: expand-template.yml + - ${{ if parameters.EnableLinuxBuild }}: + - job: Linux + pool: ${{ parameters.linuxPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-Linux + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-Linux + artifactName: ${{ artifact_name }}-Linux + condition: succeededOrFailed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + - ${{ if parameters.EnableDotNetFormatCheck }}: + - script: dotnet format --verify-no-changes --no-restore + displayName: ๐Ÿ’… Verify formatted code + - template: expand-template.yml - - job: macOS - condition: ${{ parameters.includeMacOS }} - pool: ${{ parameters.macOSPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real - outputParentDirectory: $(Build.ArtifactStagingDirectory) - outputs: - - ${{ each artifact_name in parameters.artifact_names }}: - - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: - - output: pipelineArtifact - displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS - targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS - artifactName: ${{ artifact_name }}-macOS - condition: succeededOrFailed() - steps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - template: install-dependencies.yml - - template: dotnet.yml - parameters: - Is1ESPT: ${{ parameters.Is1ESPT }} - RunTests: ${{ parameters.RunTests }} - - template: expand-template.yml + - ${{ if parameters.EnableMacOSBuild }}: + - job: macOS + pool: ${{ parameters.macOSPool }} + ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + templateContext: + mb: + ${{ if parameters.RealSign }}: + signing: + enabled: false # enable when building unique artifacts on this agent that must be signed + signType: real + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ each artifact_name in parameters.artifact_names }}: + - ${{ if or(ne(artifact_name, 'testResults'), parameters.RunTests) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish ${{ artifact_name }}-macOS + targetPath: $(Build.ArtifactStagingDirectory)/${{ artifact_name }}-macOS + artifactName: ${{ artifact_name }}-macOS + condition: succeededOrFailed() + steps: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - template: install-dependencies.yml + - template: dotnet.yml + parameters: + Is1ESPT: ${{ parameters.Is1ESPT }} + RunTests: ${{ parameters.RunTests }} + - template: expand-template.yml - job: WrapUp dependsOn: - Windows - - Linux - - macOS + - ${{ if parameters.EnableLinuxBuild }}: + - Linux + - ${{ if parameters.EnableMacOSBuild }}: + - macOS pool: ${{ parameters.windowsPool }} # Use Windows agent because PublishSymbols task requires it (https://github.com/microsoft/azure-pipelines-tasks/issues/13821). condition: succeededOrFailed() steps: @@ -220,8 +235,10 @@ jobs: initArgs: -NoRestore - template: publish-symbols.yml parameters: - includeMacOS: ${{ parameters.includeMacOS }} + EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - ${{ if parameters.RunTests }}: - template: publish-codecoverage.yml parameters: - includeMacOS: ${{ parameters.includeMacOS }} + EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 72dcb4f0..1bd088b1 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -28,7 +28,7 @@ parameters: # displayName: Skip OptProf optimization # type: boolean # default: false -- name: includeMacOS +- name: EnableMacOSBuild displayName: Build on macOS type: boolean default: false # macOS is often bogged down in Azure Pipelines @@ -88,7 +88,7 @@ extends: name: Azure Pipelines vmImage: macOS-12 os: macOS - includeMacOS: ${{ parameters.includeMacOS }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: @@ -122,7 +122,7 @@ extends: name: Azure Pipelines vmImage: macOS-12 os: macOS - includeMacOS: ${{ parameters.includeMacOS }} + EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: diff --git a/azure-pipelines/publish-codecoverage.yml b/azure-pipelines/publish-codecoverage.yml index fbb6a39a..c6247a11 100644 --- a/azure-pipelines/publish-codecoverage.yml +++ b/azure-pipelines/publish-codecoverage.yml @@ -1,20 +1,24 @@ parameters: - includeMacOS: +- name: EnableMacOSBuild + type: boolean +- name: EnableLinuxBuild + type: boolean steps: - download: current artifact: coverageResults-Windows displayName: ๐Ÿ”ป Download Windows code coverage results continueOnError: true -- download: current - artifact: coverageResults-Linux - displayName: ๐Ÿ”ป Download Linux code coverage results - continueOnError: true -- download: current - artifact: coverageResults-macOS - displayName: ๐Ÿ”ป Download macOS code coverage results - continueOnError: true - condition: and(succeeded(), ${{ parameters.includeMacOS }}) +- ${{ if parameters.EnableLinuxBuild }}: + - download: current + artifact: coverageResults-Linux + displayName: ๐Ÿ”ป Download Linux code coverage results + continueOnError: true +- ${{ if parameters.EnableMacOSBuild }}: + - download: current + artifact: coverageResults-macOS + displayName: ๐Ÿ”ป Download macOS code coverage results + continueOnError: true - powershell: azure-pipelines/Merge-CodeCoverage.ps1 -Path '$(Pipeline.Workspace)' -OutputFile coveragereport/merged.cobertura.xml -Format Cobertura -Verbose displayName: โš™ Merge coverage - task: PublishCodeCoverageResults@1 diff --git a/azure-pipelines/publish-symbols.yml b/azure-pipelines/publish-symbols.yml index 00b5a3a6..ddf82352 100644 --- a/azure-pipelines/publish-symbols.yml +++ b/azure-pipelines/publish-symbols.yml @@ -1,5 +1,8 @@ parameters: - includeMacOS: +- name: EnableMacOSBuild + type: boolean +- name: EnableLinuxBuild + type: boolean steps: - task: DownloadPipelineArtifact@2 @@ -8,19 +11,20 @@ steps: path: $(Pipeline.Workspace)/symbols/Windows displayName: ๐Ÿ”ป Download Windows symbols continueOnError: true -- task: DownloadPipelineArtifact@2 - inputs: - artifact: symbols-Linux - path: $(Pipeline.Workspace)/symbols/Linux - displayName: ๐Ÿ”ป Download Linux symbols - continueOnError: true -- task: DownloadPipelineArtifact@2 - inputs: - artifact: symbols-macOS - path: $(Pipeline.Workspace)/symbols/macOS - displayName: ๐Ÿ”ป Download macOS symbols - continueOnError: true - condition: ${{ parameters.includeMacOS }} +- ${{ if parameters.EnableLinuxBuild }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: symbols-Linux + path: $(Pipeline.Workspace)/symbols/Linux + displayName: ๐Ÿ”ป Download Linux symbols + continueOnError: true +- ${{ if parameters.EnableMacOSBuild }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: symbols-macOS + path: $(Pipeline.Workspace)/symbols/macOS + displayName: ๐Ÿ”ป Download macOS symbols + continueOnError: true - task: DownloadPipelineArtifact@2 inputs: @@ -28,19 +32,20 @@ steps: path: $(Pipeline.Workspace)/test_symbols/Windows displayName: ๐Ÿ”ป Download Windows test symbols continueOnError: true -- task: DownloadPipelineArtifact@2 - inputs: - artifact: test_symbols-Linux - path: $(Pipeline.Workspace)/test_symbols/Linux - displayName: ๐Ÿ”ป Download Linux test symbols - continueOnError: true -- task: DownloadPipelineArtifact@2 - inputs: - artifact: test_symbols-macOS - path: $(Pipeline.Workspace)/test_symbols/macOS - displayName: ๐Ÿ”ป Download macOS test symbols - continueOnError: true - condition: ${{ parameters.includeMacOS }} +- ${{ if parameters.EnableLinuxBuild }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: test_symbols-Linux + path: $(Pipeline.Workspace)/test_symbols/Linux + displayName: ๐Ÿ”ป Download Linux test symbols + continueOnError: true +- ${{ if parameters.EnableMacOSBuild }}: + - task: DownloadPipelineArtifact@2 + inputs: + artifact: test_symbols-macOS + path: $(Pipeline.Workspace)/test_symbols/macOS + displayName: ๐Ÿ”ป Download macOS test symbols + continueOnError: true - task: PublishSymbols@2 inputs: diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 195bfff3..c541be60 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -35,7 +35,7 @@ extends: Is1ESPT: true RealSign: true windowsPool: VSEngSS-MicroBuild2022-1ES - includeMacOS: false + EnableMacOSBuild: false RunTests: false - template: /azure-pipelines/prepare-insertion-stages.yml@self From c7d1dd3f26d3d24dee6471114fb7f4381bf10283 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 14:35:54 -0700 Subject: [PATCH 61/86] Clarify code comment --- azure-pipelines/official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 1bd088b1..cabae2d2 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -24,7 +24,7 @@ parameters: displayName: Real sign? type: boolean default: false -# - name: ShouldSkipOptimize # Enable this and references to it below when setting EnableOptProf to true in build.yml. +# - name: ShouldSkipOptimize # Uncomment this and references to it below when setting EnableOptProf to true in build.yml. # displayName: Skip OptProf optimization # type: boolean # default: false From f38dd307bfdaf94863b877b085f1ee24ca0d893d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 14:42:12 -0700 Subject: [PATCH 62/86] Fix assigning reviewers to insertion PRs --- azure-pipelines/vs-insertion.yml | 2 +- azure-pipelines/vs-validation.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/vs-insertion.yml b/azure-pipelines/vs-insertion.yml index 9cb597ea..8a6ffa0e 100644 --- a/azure-pipelines/vs-insertion.yml +++ b/azure-pipelines/vs-insertion.yml @@ -54,7 +54,7 @@ extends: TeamEmail: $(TeamEmail) InsertionPayloadName: $(Build.Repository.Name) $(Build.BuildNumber) InsertionBuildPolicy: Request Perf DDRITs - InsertionReviewers: $(TeamEmail),$(Build.RequestedForEmail) + InsertionReviewers: $(Build.RequestedFor) # Append `,Your team name` (without quotes) AutoCompletePR: true AutoCompleteMergeStrategy: Squash - powershell: | diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index c541be60..4d385796 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -78,7 +78,7 @@ extends: This PR is for **validation purposes only** for !$(System.PullRequest.PullRequestId). **Do not complete**. CustomScriptExecutionCommand: src/VSSDK/NuGet/AllowUnstablePackages.ps1 InsertionBuildPolicy: Request Perf DDRITs - InsertionReviewers: $(Build.RequestedForEmail) + InsertionReviewers: $(Build.RequestedFor) AutoCompletePR: false - powershell: | $insertionPRId = azure-pipelines/Get-InsertionPRId.ps1 From b21987e11f1d74129d02351bc983098706c8a554 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 15:48:10 -0700 Subject: [PATCH 63/86] fix typo --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index efbe74f4..45467326 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,6 @@ // Needs to be explicitly configured: https://github.com/Microsoft/azure-pipelines-vscode#document-formatting "[azure-pipelines]": { "editor.defaultFormatter": "esbenp.prettier-vscode", - "editor.formatOnSave": false // enable this when the conform + "editor.formatOnSave": false // enable this when they conform }, } From aa80117c6126647da771c73d013c0a1614e89bb8 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 21 Feb 2024 18:58:30 -0700 Subject: [PATCH 64/86] Avoid shallow cloning for optprof pipelines --- azure-pipelines/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index f82afad6..f3b5f2fb 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -116,6 +116,14 @@ jobs: targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows artifactName: LocBin-Windows steps: + - ${{ if not(parameters.Is1ESPT) }}: + - checkout: self + fetchDepth: 0 # avoid shallow clone so nbgv can do its work. + clean: true + - ${{ if parameters.EnableOptProf }}: + - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(azure-pipelines/variables/ProfilingInputsDropName.ps1)" + displayName: โš™ Set ProfilingInputsDropName for optprof + - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - template: schedule-only-steps.yml From 00f463153e4257899c160f2d0df97c6b97d3b569 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 22 Feb 2024 13:36:20 -0700 Subject: [PATCH 65/86] Extract APIScan into its own job --- azure-pipelines/apiscan.yml | 43 ++++++++++++++++++++ azure-pipelines/artifacts/APIScanInputs.ps1 | 24 +++++++++++ azure-pipelines/build.yml | 17 +++++--- azure-pipelines/microbuild.after.yml | 11 ----- azure-pipelines/secure-development-tools.yml | 41 ------------------- 5 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 azure-pipelines/apiscan.yml create mode 100644 azure-pipelines/artifacts/APIScanInputs.ps1 delete mode 100644 azure-pipelines/secure-development-tools.yml diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml new file mode 100644 index 00000000..2f482ca1 --- /dev/null +++ b/azure-pipelines/apiscan.yml @@ -0,0 +1,43 @@ +parameters: +- name: windowsPool + type: object + +jobs: +- job: apiscan + displayName: APIScan + dependsOn: Windows + pool: ${{ parameters.windowsPool }} + variables: + - name: SymbolsFeatureName + value: $[ dependencies.Windows.outputs['SetPipelineVariables.SymbolsFeatureName'] ] + - name: NBGV_MajorMinorVersion + value: $[ dependencies.Windows.outputs['nbgv.NBGV_MajorMinorVersion'] ] + - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline + - group: VSCloudServices-APIScan # Expected to provide ApiScanClientId, ApiScanSecret, ApiScanTenant + steps: + # We need TSAOptions.json + - checkout: self + fetchDepth: 1 + + - download: current + artifact: APIScanInputs + displayName: ๐Ÿ”ป Download APIScanInputs artifact + + - task: APIScan@2 + displayName: ๐Ÿ” Run APIScan + inputs: + softwareFolder: $(Pipeline.Workspace)/APIScanInputs + softwareName: $(SymbolsFeatureName) + softwareVersionNum: $(NBGV_MajorMinorVersion) + isLargeApp: false + toolVersion: Latest + env: + AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) + + # File bugs when APIScan finds issues + - task: TSAUpload@2 + displayName: ๐Ÿชณ TSA upload + inputs: + GdnPublishTsaOnboard: True + GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json diff --git a/azure-pipelines/artifacts/APIScanInputs.ps1 b/azure-pipelines/artifacts/APIScanInputs.ps1 new file mode 100644 index 00000000..15556413 --- /dev/null +++ b/azure-pipelines/artifacts/APIScanInputs.ps1 @@ -0,0 +1,24 @@ +$inputs = & "$PSScriptRoot/symbols.ps1" + +if (!$inputs) { return } + +# Filter out specific files that APIScan does not support. +# Specifically, APIScan doesn't support Windows ARM64 binaries, nor linux/OSX binaries. +$outputs = @{} +$forbiddenSubPaths = @( + , 'arm64' + , 'win-arm64' + , 'linux-*' + , 'osx*' +) + +$inputs.GetEnumerator() | % { + $list = $_.Value | ? { + $path = $_.Replace('\', '/') + return !($forbiddenSubPaths | ? { $path -like "*/$_/*" }) + } + $outputs[$_.Key] = $list +} + + +$outputs diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index f3b5f2fb..ba6014ff 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -62,10 +62,6 @@ jobs: - job: Windows pool: ${{ parameters.windowsPool }} timeoutInMinutes: 180 # Give plenty of time due to real signing - variables: - - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline - - group: VSCloudServices-APIScan # Expected to provide ApiScanClientId, ApiScanSecret, ApiScanTenant ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: templateContext: mb: @@ -115,6 +111,11 @@ jobs: displayName: ๐Ÿ“ข Publish LocBin-Windows targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows artifactName: LocBin-Windows + - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan) }}: + - output: pipelineArtifact + displayName: ๐Ÿ“ข Publish APIScanInputs + targetPath: $(Build.ArtifactStagingDirectory)/APIScanInputs-Windows + artifactName: APIScanInputs steps: - ${{ if not(parameters.Is1ESPT) }}: - checkout: self @@ -131,6 +132,7 @@ jobs: - script: dotnet nbgv cloud -ca displayName: โš™ Set build number + name: nbgv - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.before.yml @@ -154,8 +156,6 @@ jobs: - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - template: microbuild.after.yml parameters: - EnableCompliance: ${{ parameters.EnableCompliance }} - EnableAPIScan: ${{ parameters.EnableAPIScan }} EnableOptProf: ${{ parameters.EnableOptProf }} IsOptProf: ${{ parameters.IsOptProf }} @@ -250,3 +250,8 @@ jobs: parameters: EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} + + - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan) }}: + - template: apiscan.yml + parameters: + windowsPool: ${{ parameters.windowsPool }} diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index f9698e3e..94e83d34 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -1,10 +1,4 @@ parameters: -- name: EnableCompliance - type: boolean - default: false -- name: EnableAPIScan - type: boolean - default: false - name: EnableOptProf type: boolean default: false @@ -39,8 +33,3 @@ steps: ArtifactName: InsertionOutputs ArtifactType: Container displayName: ๐Ÿ“ข Publish InsertionOutputs as Azure DevOps artifacts - -- ${{ if parameters.EnableCompliance }}: - - template: secure-development-tools.yml - parameters: - EnableAPIScan: ${{ parameters.EnableAPIScan }} diff --git a/azure-pipelines/secure-development-tools.yml b/azure-pipelines/secure-development-tools.yml deleted file mode 100644 index 409f48d0..00000000 --- a/azure-pipelines/secure-development-tools.yml +++ /dev/null @@ -1,41 +0,0 @@ -parameters: -- name: EnableAPIScan - type: boolean - default: false - -steps: - -- powershell: echo "##vso[build.addbuildtag]compliance" - displayName: ๐Ÿท๏ธ Tag run with 'compliance' - -- task: CopyFiles@2 - displayName: ๐Ÿ“‚ Collect APIScan inputs - inputs: - SourceFolder: $(Build.ArtifactStagingDirectory)/Symbols-$(Agent.JobName) - # Exclude any patterns from the Contents (e.g. `!**/git2*`) that we have symbols for but do not need to run APIScan on. - Contents: | - ** - !**/arm64/** - !**/win-arm64/** - !**/linux-*/** - !**/osx*/** - TargetFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs - -- ${{ if parameters.EnableAPIScan }}: - - task: APIScan@2 - displayName: ๐Ÿ” Run APIScan - inputs: - softwareFolder: $(Build.ArtifactStagingDirectory)/APIScanInputs - softwareName: $(SymbolsFeatureName) - softwareVersionNum: $(NBGV_MajorMinorVersion) - isLargeApp: false - toolVersion: Latest - env: - AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) - - # File bugs when APIScan finds issues - - task: TSAUpload@2 - displayName: ๐Ÿชณ TSA upload - inputs: - GdnPublishTsaOnboard: True - GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json From 617fb13463952ed613bd17231c346651b3a861f2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 22 Feb 2024 15:26:49 -0700 Subject: [PATCH 66/86] Skip APIScan on pull requests --- azure-pipelines/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index ba6014ff..08c9e1d4 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -111,7 +111,7 @@ jobs: displayName: ๐Ÿ“ข Publish LocBin-Windows targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows artifactName: LocBin-Windows - - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan) }}: + - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }}: - output: pipelineArtifact displayName: ๐Ÿ“ข Publish APIScanInputs targetPath: $(Build.ArtifactStagingDirectory)/APIScanInputs-Windows @@ -251,7 +251,7 @@ jobs: EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan) }}: + - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }}: - template: apiscan.yml parameters: windowsPool: ${{ parameters.windowsPool }} From cdb447aac1c5fecf071265ad5c65edff6acc5e18 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 23 Feb 2024 15:05:09 -0700 Subject: [PATCH 67/86] Document pipeline parameters --- azure-pipelines/OptProf.yml | 1 - azure-pipelines/build.yml | 103 +++++++++++++++++++++++------------ azure-pipelines/official.yml | 9 +-- 3 files changed, 73 insertions(+), 40 deletions(-) diff --git a/azure-pipelines/OptProf.yml b/azure-pipelines/OptProf.yml index 0f4ba24f..d95c84c8 100644 --- a/azure-pipelines/OptProf.yml +++ b/azure-pipelines/OptProf.yml @@ -35,7 +35,6 @@ stages: parameters: Is1ESPT: false RealSign: true - EnableCompliance: false windowsPool: VSEngSS-MicroBuild2022-1ES EnableMacOSBuild: false ShouldSkipOptimize: true diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 08c9e1d4..26b731ca 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -1,62 +1,95 @@ parameters: -- name: Is1ESPT +##### The following parameters are not set by other YAML files that import this one, +##### but we use parameters because they support rich types and defaults. +##### Feel free to adjust their default value as needed. + +# Whether this repo uses OptProf to optimize the built binaries. +- name: EnableOptProf type: boolean -- name: RealSign + default: false +# Whether this repo is localized. +- name: EnableLocalization type: boolean default: false -- name: windowsPool - type: object - default: - vmImage: windows-2022 -- name: linuxPool - type: object - default: - vmImage: ubuntu-20.04 -- name: macOSPool +# Whether to run `dotnet format` as part of the build to ensure code style consistency. +# This is just one of a a few mechanisms to enforce code style consistency. +- name: EnableDotNetFormatCheck + type: boolean + default: true +# This lists the names of the artifacts that will be published *from every OS build agent*. +# Any new azure-pipelines/artifacts/*.ps1 script needs to be added to this list. +# If an artifact is only generated or collected on one OS, it should NOT be listed here, +# but should be manually added to the `outputs:` field in the appropriate OS job. +- name: artifact_names type: object default: - vmImage: macOS-12 + - build_logs + - coverageResults + - deployables + - projectAssetsJson + - symbols + - testResults + - test_symbols + - Variables +# The Enable*Build parameters turn non-Windows agents on or off. +# Their default value should be based on whether the build and tests are expected/required to pass on that platform. +# Callers (e.g. Official.yml) *may* expose these parameters at queue-time in order to turn OFF optional agents. - name: EnableLinuxBuild type: boolean default: true - name: EnableMacOSBuild type: boolean default: true -- name: EnableOptProf + +##### ๐Ÿ‘†๐Ÿผ You MAY change the defaults above. +##### ๐Ÿ‘‡๐Ÿผ You should NOT change the defaults below. + +##### The following parameters are expected to be set by other YAML files that import this one. +##### Those without defaults require explicit values to be provided by our importers. + +# Indicates whether the entrypoint file is 1ESPT compliant. Use this parameter to switch between publish tasks to fit 1ES or non-1ES needs. +- name: Is1ESPT type: boolean - default: false -- name: IsOptProf + +- name: RealSign type: boolean default: false -- name: ShouldSkipOptimize + +# Whether this particular run is an OptProf profiling run. +# This is used to skip unit tests and other non-essential work to improve reliability of the OptProf pipeline. +- name: IsOptProf type: boolean default: false + - name: RunTests type: boolean default: true -- name: EnableLocalization - type: boolean - default: false -- name: EnableCompliance - type: boolean - default: false + - name: EnableAPIScan type: boolean default: false -- name: EnableDotNetFormatCheck + +# This parameter exists to provide a workaround to get a build out even when no OptProf profiling outputs can be found. +# Entrypoint yaml files like official.yml should expose this as a queue-time setting when EnableOptProf is true in this file. +# The OptProf.yml entrypoint sets this parameter to true so that collecting profile data isn't blocked by a prior lack of profile data. +- name: ShouldSkipOptimize type: boolean - default: true -- name: artifact_names + default: false + +# The pool parameters are set to defaults that work in the azure-public AzDO account. +# They are overridden by callers for the devdiv AzDO account to use 1ES compliant pools. +- name: windowsPool + type: object + default: + vmImage: windows-2022 +- name: linuxPool type: object default: - - build_logs - - coverageResults - - deployables - - projectAssetsJson - - symbols - - testResults - - test_symbols - - Variables + vmImage: ubuntu-20.04 +- name: macOSPool + type: object + default: + vmImage: macOS-12 jobs: - job: Windows @@ -111,7 +144,7 @@ jobs: displayName: ๐Ÿ“ข Publish LocBin-Windows targetPath: $(Build.ArtifactStagingDirectory)/LocBin-Windows artifactName: LocBin-Windows - - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }}: + - ${{ if parameters.EnableAPIScan }}: - output: pipelineArtifact displayName: ๐Ÿ“ข Publish APIScanInputs targetPath: $(Build.ArtifactStagingDirectory)/APIScanInputs-Windows @@ -251,7 +284,7 @@ jobs: EnableLinuxBuild: ${{ parameters.EnableLinuxBuild }} EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} - - ${{ if and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }}: + - ${{ if parameters.EnableAPIScan }}: - template: apiscan.yml parameters: windowsPool: ${{ parameters.windowsPool }} diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index ee6dc92c..98e21d22 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -20,6 +20,9 @@ trigger: # - microbuild parameters: +# As an entrypoint pipeline yml file, all parameters here show up in the Queue Run dialog. +# If any paramaters should NOT be queue-time options, they should be removed from here +# and references to them in this file replaced with hard-coded values. - name: RealSign displayName: Real sign? type: boolean @@ -78,8 +81,7 @@ extends: Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - EnableCompliance: ${{ parameters.EnableCompliance }} - EnableAPIScan: ${{ parameters.EnableAPIScan }} + EnableAPIScan: ${{ and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }} windowsPool: VSEngSS-MicroBuild2022-1ES linuxPool: name: AzurePipelines-EO @@ -112,8 +114,7 @@ extends: Is1ESPT: true RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - EnableCompliance: ${{ parameters.EnableCompliance }} - EnableAPIScan: ${{ parameters.EnableAPIScan }} + EnableAPIScan: ${{ and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }} windowsPool: VSEngSS-MicroBuild2022-1ES linuxPool: name: AzurePipelines-EO From caf95ea5722003e7164c10ca2825d592a53368cb Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 26 Feb 2024 15:24:50 -0700 Subject: [PATCH 68/86] Remove artifact publish task from non-OptProf runs --- azure-pipelines/microbuild.after.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/azure-pipelines/microbuild.after.yml b/azure-pipelines/microbuild.after.yml index 94e83d34..34bdb452 100644 --- a/azure-pipelines/microbuild.after.yml +++ b/azure-pipelines/microbuild.after.yml @@ -14,7 +14,7 @@ steps: $(Build.SourcesDirectory)/bin/Packages/$(BuildConfiguration)/NuGet condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) -- ${{ if parameters.EnableOptProf }}: +- ${{ if parameters.IsOptProf }}: - task: ms-vscs-artifact.build-tasks.artifactDropTask-1.artifactDropTask@0 inputs: dropServiceURI: https://devdiv.artifacts.visualstudio.com @@ -26,10 +26,9 @@ steps: condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) continueOnError: true - - ${{ if parameters.IsOptProf }}: - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/InsertionOutputs - ArtifactName: InsertionOutputs - ArtifactType: Container - displayName: ๐Ÿ“ข Publish InsertionOutputs as Azure DevOps artifacts + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/InsertionOutputs + ArtifactName: InsertionOutputs + ArtifactType: Container + displayName: ๐Ÿ“ข Publish InsertionOutputs as Azure DevOps artifacts From 8d19841a37fc725ba37da73bb21d47d5cb3e37ad Mon Sep 17 00:00:00 2001 From: Vivian Lim Date: Mon, 26 Feb 2024 21:42:55 -0800 Subject: [PATCH 69/86] Extend default timeout for APIScan job I've seen it take over an hour on a handful of repos, so it seems like a longer timeout is a reasonable default to specify. --- azure-pipelines/apiscan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml index 2f482ca1..4e4c0c31 100644 --- a/azure-pipelines/apiscan.yml +++ b/azure-pipelines/apiscan.yml @@ -7,6 +7,7 @@ jobs: displayName: APIScan dependsOn: Windows pool: ${{ parameters.windowsPool }} + timeoutInMinutes: 120 variables: - name: SymbolsFeatureName value: $[ dependencies.Windows.outputs['SetPipelineVariables.SymbolsFeatureName'] ] From 3598bad8c118453bb492198f5809e58c66a17aee Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 27 Feb 2024 13:44:34 -0700 Subject: [PATCH 70/86] Collect APIScan logs as an artifact Also only file bugs when APIScan is run against main. --- azure-pipelines/apiscan.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml index 4e4c0c31..554df0eb 100644 --- a/azure-pipelines/apiscan.yml +++ b/azure-pipelines/apiscan.yml @@ -8,6 +8,12 @@ jobs: dependsOn: Windows pool: ${{ parameters.windowsPool }} timeoutInMinutes: 120 + templateContext: + outputs: + - output: pipelineArtifact + displayName: ๐Ÿ“ข collect apiscan artifact + targetPath: $(Pipeline.Workspace)/.gdn/.r/apiscan/001/Logs + artifactName: apiscan-logs variables: - name: SymbolsFeatureName value: $[ dependencies.Windows.outputs['SetPipelineVariables.SymbolsFeatureName'] ] @@ -42,3 +48,4 @@ jobs: inputs: GdnPublishTsaOnboard: True GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\azure-pipelines\TSAOptions.json + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) From aa44aabb455656e0adf9d0e640fb6979e6277a2e Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 27 Feb 2024 18:02:25 -0700 Subject: [PATCH 71/86] Preserve more apiscan logs --- azure-pipelines/apiscan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml index 554df0eb..c3141d05 100644 --- a/azure-pipelines/apiscan.yml +++ b/azure-pipelines/apiscan.yml @@ -39,6 +39,7 @@ jobs: softwareVersionNum: $(NBGV_MajorMinorVersion) isLargeApp: false toolVersion: Latest + preserveLogsFolder: true env: AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) From 36a27d828388373b91d424b196ebda381f9ef2cf Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 4 Mar 2024 14:15:17 -0700 Subject: [PATCH 72/86] Stop filtering out win-arm64 binaries from APIScan --- azure-pipelines/artifacts/APIScanInputs.ps1 | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines/artifacts/APIScanInputs.ps1 b/azure-pipelines/artifacts/APIScanInputs.ps1 index 15556413..b1550bfa 100644 --- a/azure-pipelines/artifacts/APIScanInputs.ps1 +++ b/azure-pipelines/artifacts/APIScanInputs.ps1 @@ -2,12 +2,10 @@ $inputs = & "$PSScriptRoot/symbols.ps1" if (!$inputs) { return } -# Filter out specific files that APIScan does not support. -# Specifically, APIScan doesn't support Windows ARM64 binaries, nor linux/OSX binaries. +# Filter out specific files that target OS's that are not subject to APIScan. +# Files that are subject but are not supported must be scanned and an SEL exception filed. $outputs = @{} $forbiddenSubPaths = @( - , 'arm64' - , 'win-arm64' , 'linux-*' , 'osx*' ) From 2a27d267d990d2a0a7078da1593de322bd051263 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 4 Mar 2024 14:54:06 -0700 Subject: [PATCH 73/86] Document use case for nuspec file --- azure-pipelines/InsertionMetadataPackage.nuspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/InsertionMetadataPackage.nuspec b/azure-pipelines/InsertionMetadataPackage.nuspec index f7aa3e3f..aae33f5c 100644 --- a/azure-pipelines/InsertionMetadataPackage.nuspec +++ b/azure-pipelines/InsertionMetadataPackage.nuspec @@ -1,5 +1,6 @@ - + + LibraryName.VSInsertionMetadata $version$ From 7220c37ffe0189b2e831bdb9fc7b99dd1f28b13d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 15 Mar 2024 11:56:27 -0600 Subject: [PATCH 74/86] Fix validation pipeline --- azure-pipelines/vs-validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/vs-validation.yml b/azure-pipelines/vs-validation.yml index 4d385796..ca2a7a68 100644 --- a/azure-pipelines/vs-validation.yml +++ b/azure-pipelines/vs-validation.yml @@ -41,7 +41,7 @@ extends: - template: /azure-pipelines/prepare-insertion-stages.yml@self parameters: ArchiveSymbols: false - RealSign: ${{ parameters.RealSign }} + RealSign: true - stage: insertion displayName: VS insertion From dd4956a1b8fecdd7570c49f1cbb5a77fce2d061b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Tue, 19 Mar 2024 10:15:18 -0600 Subject: [PATCH 75/86] Switch APIScan to use a managed identity --- azure-pipelines/apiscan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml index c3141d05..73af5259 100644 --- a/azure-pipelines/apiscan.yml +++ b/azure-pipelines/apiscan.yml @@ -21,7 +21,7 @@ jobs: value: $[ dependencies.Windows.outputs['nbgv.NBGV_MajorMinorVersion'] ] - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline - - group: VSCloudServices-APIScan # Expected to provide ApiScanClientId, ApiScanSecret, ApiScanTenant + - group: VSEng sponsored APIScan # Expected to provide ApiScanClientId steps: # We need TSAOptions.json - checkout: self @@ -41,7 +41,7 @@ jobs: toolVersion: Latest preserveLogsFolder: true env: - AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId);TenantId=$(ApiScanTenant);AppKey=$(ApiScanSecret) + AzureServicesAuthConnectionString: runAs=App;AppId=$(ApiScanClientId) # File bugs when APIScan finds issues - task: TSAUpload@2 From aeee57d3b34412c07507b3baab0e0388e161cac2 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Wed, 20 Mar 2024 13:38:08 -0600 Subject: [PATCH 76/86] Pipeline fixes --- azure-pipelines/apiscan.yml | 2 +- azure-pipelines/archive-sourcecode.yml | 5 +- azure-pipelines/build.yml | 61 +++----------------- azure-pipelines/dotnet.yml | 7 ++- azure-pipelines/integration-test.yml | 5 +- azure-pipelines/official.yml | 26 +++------ azure-pipelines/prepare-insertion-stages.yml | 37 ++---------- azure-pipelines/release.yml | 5 +- azure-pipelines/signstep1.yml | 4 +- azure-pipelines/signstep2.yml | 2 +- 10 files changed, 35 insertions(+), 119 deletions(-) diff --git a/azure-pipelines/apiscan.yml b/azure-pipelines/apiscan.yml index 73af5259..3b87a022 100644 --- a/azure-pipelines/apiscan.yml +++ b/azure-pipelines/apiscan.yml @@ -19,7 +19,7 @@ jobs: value: $[ dependencies.Windows.outputs['SetPipelineVariables.SymbolsFeatureName'] ] - name: NBGV_MajorMinorVersion value: $[ dependencies.Windows.outputs['nbgv.NBGV_MajorMinorVersion'] ] - - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + - ${{ if parameters.Is1ESPT }}: # https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/25351/APIScan-step-by-step-guide-to-setting-up-a-Pipeline - group: VSEng sponsored APIScan # Expected to provide ApiScanClientId steps: diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index c550e2c2..5f64c115 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -42,10 +42,7 @@ extends: jobs: - job: archive pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals 1ESPT-Ubuntu22.04 - os: Linux + vmImage: ubuntu-20.04 steps: - checkout: self diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 49c86ae2..9a78f533 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -15,7 +15,7 @@ parameters: # This is just one of a a few mechanisms to enforce code style consistency. - name: EnableDotNetFormatCheck type: boolean - default: true + default: false # false positives break the build # This lists the names of the artifacts that will be published *from every OS build agent*. # Any new azure-pipelines/artifacts/*.ps1 script needs to be added to this list. # If an artifact is only generated or collected on one OS, it should NOT be listed here, @@ -97,37 +97,8 @@ jobs: pool: ${{ parameters.windowsPool }} variables: TestFilter: "" - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ${{ if parameters.Is1ESPT }}: templateContext: - mb: - signing: - enabled: true - zipSources: false - ${{ if parameters.RealSign }}: - signType: real - ${{ else }}: - signType: test - sbom: - enabled: true - localization: - enabled: ${{ parameters.EnableLocalization }} - ${{ if eq(variables['Build.Reason'], 'pullRequest') }}: - languages: ENU,JPN - optprof: - enabled: ${{ parameters.EnableOptProf }} - ProfilingInputsDropName: $(ProfilingInputsDropName) - OptimizationInputsLookupMethod: DropPrefix - DropNamePrefix: OptimizationInputs/$(System.TeamProject)/$(Build.Repository.Name) - ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} - AccessToken: $(System.AccessToken) - mbpresteps: - - checkout: self - fetchDepth: 0 # avoid shallow clone so nbgv can do its work. - clean: true - - ${{ if parameters.EnableOptProf }}: - - powershell: Write-Host "##vso[task.setvariable variable=PROFILINGINPUTSDROPNAME]$(azure-pipelines/variables/ProfilingInputsDropName.ps1)" - displayName: โš™ Set ProfilingInputsDropName for optprof - outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: @@ -168,30 +139,21 @@ jobs: Is1ESPT: ${{ parameters.Is1ESPT }} RunTests: ${{ parameters.RunTests }} IsOptProf: ${{ parameters.IsOptProf }} + RealSign: ${{ parameters.RealSign }} - ${{ if and(parameters.EnableDotNetFormatCheck, not(parameters.EnableLinuxBuild)) }}: - script: dotnet format --verify-no-changes --no-restore displayName: ๐Ÿ’… Verify formatted code - - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: - - template: microbuild.after.yml - parameters: - EnableOptProf: ${{ parameters.EnableOptProf }} - IsOptProf: ${{ parameters.IsOptProf }} - - - template: expand-template.yml - - ${{ if not(parameters.IsOptProf) }}: - ${{ if parameters.EnableLinuxBuild }}: - job: Linux pool: ${{ parameters.linuxPool }} - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + variables: + TestFilter: "&WindowsOnly!=true" + Platform: NonWindows + ${{ if parameters.Is1ESPT }}: templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: @@ -213,7 +175,6 @@ jobs: - ${{ if parameters.EnableDotNetFormatCheck }}: - script: dotnet format --verify-no-changes --no-restore displayName: ๐Ÿ’… Verify formatted code - - template: expand-template.yml - ${{ if parameters.EnableMacOSBuild }}: - job: macOS @@ -221,13 +182,8 @@ jobs: variables: TestFilter: "&WindowsOnly!=true" Platform: NonWindows - ${{ if eq(variables['system.collectionId'], '011b8bdf-6d56-4f87-be0d-0092136884d9') }}: + ${{ if parameters.Is1ESPT }}: templateContext: - mb: - ${{ if parameters.RealSign }}: - signing: - enabled: false # enable when building unique artifacts on this agent that must be signed - signType: real outputParentDirectory: $(Build.ArtifactStagingDirectory) outputs: - ${{ each artifact_name in parameters.artifact_names }}: @@ -246,7 +202,6 @@ jobs: parameters: Is1ESPT: ${{ parameters.Is1ESPT }} RunTests: ${{ parameters.RunTests }} - - template: expand-template.yml - job: WrapUp dependsOn: diff --git a/azure-pipelines/dotnet.yml b/azure-pipelines/dotnet.yml index 6b2801b3..12531de4 100644 --- a/azure-pipelines/dotnet.yml +++ b/azure-pipelines/dotnet.yml @@ -5,19 +5,22 @@ parameters: default: false - name: Is1ESPT type: boolean +- name: RealSign + type: boolean + default: false steps: - script: dotnet build -t:build,pack --no-restore -c $(BuildConfiguration) -warnaserror /bl:"$(Build.ArtifactStagingDirectory)/build_logs/build.binlog" displayName: ๐Ÿ›  dotnet build -- ${{ if eq(variables['system.collectionId'], 'cb55739e-4afe-46a3-970f-1b49d8ee7564') }}: +- ${{ if parameters.RealSign }}: - template: ./signstep1.yml - script: dotnet pack --no-build /p:buildprojectreferences=false -c $(BuildConfiguration) /v:m /bl:"$(Build.ArtifactStagingDirectory)/build_logs/pack.binlog" displayName: ๐Ÿ“ฆ dotnet pack -- ${{ if eq(variables['system.collectionId'], 'cb55739e-4afe-46a3-970f-1b49d8ee7564') }}: +- ${{ if parameters.RealSign }}: - template: ./signstep2.yml - powershell: azure-pipelines/dotnet-test-cloud.ps1 -Configuration $(BuildConfiguration) -Agent $(Agent.JobName) -PublishResults diff --git a/azure-pipelines/integration-test.yml b/azure-pipelines/integration-test.yml index c023c631..8eb3e45e 100644 --- a/azure-pipelines/integration-test.yml +++ b/azure-pipelines/integration-test.yml @@ -1,6 +1,7 @@ jobs: # Matrix over VS 2019 and VS 2022 +# Pool options are documented at: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/migration/ed-sharedpool - job: integrationtest displayName: ๐Ÿงช Integration tests strategy: @@ -12,7 +13,9 @@ jobs: vmImage: windows-2022 currentSdk: true pool: - vmImage: $[ variables['vmImage'] ] + name: Azure-Pipelines-1ESPT-ExDShared + image: $(vmImage) + os: windows dependsOn: Windows steps: - checkout: self diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index e3fbda4c..4be6224e 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -79,18 +79,11 @@ extends: RealSign: ${{ or(parameters.RealSign, eq(variables['Build.Reason'],'Schedule')) }} # ShouldSkipOptimize: ${{ parameters.ShouldSkipOptimize }} EnableAPIScan: ${{ and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }} - windowsPool: - name: 1es-managed-pool-ss - os: windows + windowsPool: Azure-Pipelines-1ESPT-ExDShared linuxPool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals 1ESPT-Ubuntu22.04 - os: Linux - macOSPool: - name: Azure Pipelines - vmImage: macOS-12 - os: macOS + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-2004 + os: linux EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self @@ -116,14 +109,9 @@ extends: EnableAPIScan: ${{ and(parameters.EnableCompliance, parameters.EnableAPIScan, ne(variables['Build.Reason'], 'pullRequest')) }} windowsPool: Azure-Pipelines-1ESPT-ExDShared linuxPool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals 1ESPT-Ubuntu22.04 - os: Linux - macOSPool: - name: Azure Pipelines - vmImage: macOS-12 - os: macOS + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-2004 + os: linux EnableMacOSBuild: ${{ parameters.EnableMacOSBuild }} RunTests: ${{ parameters.RunTests }} - template: /azure-pipelines/prepare-insertion-stages.yml@self diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index bf30c7b3..a5717a4a 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -1,46 +1,19 @@ parameters: -- name: ArchiveSymbols +- name: RealSign type: boolean - default: true + default: false stages: - stage: release displayName: Publish - condition: and(succeeded(), eq(dependencies.Build.outputs['Windows.SetPipelineVariables.IsSigned'], 'true')) + condition: and(succeeded(), eq('${{ parameters.RealSign }}', 'true')) jobs: - - ${{ if parameters.ArchiveSymbols }}: - - job: symbol_archive - displayName: Archive symbols - pool: ubuntu-latest - steps: - - checkout: none - - download: current - artifact: Variables-Windows - displayName: ๐Ÿ”ป Download Variables-Windows artifact - - powershell: $(Pipeline.Workspace)/Variables-Windows/_pipelines.ps1 - displayName: โš™๏ธ Set pipeline variables based on artifacts - - download: current - artifact: symbols-legacy - displayName: ๐Ÿ”ป Download symbols-legacy artifact - - task: MicroBuildArchiveSymbols@4 - displayName: ๐Ÿ”ฃ Archive symbols to Symweb - inputs: - SymbolsFeatureName: $(SymbolsFeatureName) - SymbolsProject: VS - SymbolsAgentPath: $(Pipeline.Workspace)/symbols-legacy - azureSubscription: Symbols Upload (DevDiv) - - ${{ if true }}: # leave the condition to avoid merge conflicts later. - job: push - displayName: azure-public/vs-impl feed - ${{ if parameters.ArchiveSymbols }}: - dependsOn: symbol_archive + displayName: azure-public/winsdk feed pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals 1ESPT-Ubuntu22.04 - os: Linux + vmImage: ubuntu-20.04 templateContext: outputs: - output: nuget diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 178de646..d4b28a1c 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -29,10 +29,7 @@ extends: jobs: - job: release pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals 1ESPT-Ubuntu22.04 - os: Linux + vmImage: ubuntu-20.04 templateContext: outputs: - output: nuget diff --git a/azure-pipelines/signstep1.yml b/azure-pipelines/signstep1.yml index 1c4ba461..16d6e794 100644 --- a/azure-pipelines/signstep1.yml +++ b/azure-pipelines/signstep1.yml @@ -41,7 +41,7 @@ steps: SessionTimeout: 60 MaxConcurrency: 50 MaxRetryAttempts: 5 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['IsSigned'], 'true')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - task: EsrpCodeSigning@1 enabled: false # till we get it working @@ -80,4 +80,4 @@ steps: SessionTimeout: 60 MaxConcurrency: 50 MaxRetryAttempts: 5 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['IsSigned'], 'true')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/azure-pipelines/signstep2.yml b/azure-pipelines/signstep2.yml index 4649fdb5..e20468e9 100644 --- a/azure-pipelines/signstep2.yml +++ b/azure-pipelines/signstep2.yml @@ -27,4 +27,4 @@ steps: SessionTimeout: 60 MaxConcurrency: 50 MaxRetryAttempts: 5 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['IsSigned'], 'true')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) From ecfe315f3f602472ac1d2350e7d43f62b829089b Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 21 Mar 2024 15:00:56 -0600 Subject: [PATCH 77/86] Integration testing only on azure-public --- azure-pipelines/build.yml | 3 ++- azure-pipelines/integration-test.yml | 5 +---- azure-pipelines/official.yml | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/azure-pipelines/build.yml b/azure-pipelines/build.yml index 9a78f533..a7680df1 100644 --- a/azure-pipelines/build.yml +++ b/azure-pipelines/build.yml @@ -234,5 +234,6 @@ jobs: parameters: windowsPool: ${{ parameters.windowsPool }} -- ${{ if parameters.RunTests }}: + # Run the integration tests, but only on azure-public so we don't have to worry about multiple pools +- ${{ if and(parameters.RunTests, not(parameters.Is1ESPT)) }}: - template: integration-test.yml diff --git a/azure-pipelines/integration-test.yml b/azure-pipelines/integration-test.yml index 8eb3e45e..c023c631 100644 --- a/azure-pipelines/integration-test.yml +++ b/azure-pipelines/integration-test.yml @@ -1,7 +1,6 @@ jobs: # Matrix over VS 2019 and VS 2022 -# Pool options are documented at: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/migration/ed-sharedpool - job: integrationtest displayName: ๐Ÿงช Integration tests strategy: @@ -13,9 +12,7 @@ jobs: vmImage: windows-2022 currentSdk: true pool: - name: Azure-Pipelines-1ESPT-ExDShared - image: $(vmImage) - os: windows + vmImage: $[ variables['vmImage'] ] dependsOn: Windows steps: - checkout: self diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 4be6224e..21951d48 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -55,6 +55,7 @@ resources: variables: - template: GlobalVariables.yml +# Pool options are documented at: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/migration/ed-sharedpool extends: ${{ if parameters.EnableCompliance }}: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines From c9c0b23a9a3430404a1d85804f6d91ee92700ff7 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Thu, 21 Mar 2024 15:10:58 -0600 Subject: [PATCH 78/86] Fix release.yml --- azure-pipelines/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index d4b28a1c..72ce28d9 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -3,9 +3,9 @@ pr: none resources: repositories: - - repository: MicroBuildTemplate + - repository: 1esPipelines type: git - name: 1ESPipelineTemplates/MicroBuildTemplate + name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release pipelines: - pipeline: CI @@ -19,7 +19,7 @@ variables: - group: CsWin32 Publishing secrets extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES From 9d90153d40c936fe9abdaade6b573728cf92c527 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 25 Mar 2024 08:40:21 -0600 Subject: [PATCH 79/86] Fix pool for winsdk feed publishing --- azure-pipelines/prepare-insertion-stages.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/prepare-insertion-stages.yml b/azure-pipelines/prepare-insertion-stages.yml index a5717a4a..fd0e771c 100644 --- a/azure-pipelines/prepare-insertion-stages.yml +++ b/azure-pipelines/prepare-insertion-stages.yml @@ -13,7 +13,9 @@ stages: - job: push displayName: azure-public/winsdk feed pool: - vmImage: ubuntu-20.04 + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-2004 + os: linux templateContext: outputs: - output: nuget From 7afe553cd9711c29ae2c5cbe35860a9797bb1af4 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 25 Mar 2024 08:41:31 -0600 Subject: [PATCH 80/86] Fix release pipeline pool --- azure-pipelines/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 72ce28d9..b2e053c1 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -29,7 +29,9 @@ extends: jobs: - job: release pool: - vmImage: ubuntu-20.04 + name: Azure-Pipelines-1ESPT-ExDShared + image: ubuntu-2004 + os: linux templateContext: outputs: - output: nuget From 070aaac132745745c3b71c11d3fd30a1175adba5 Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 25 Mar 2024 08:42:53 -0600 Subject: [PATCH 81/86] Remove unused variable group --- azure-pipelines/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index b2e053c1..6d54f42c 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -16,7 +16,6 @@ resources: variables: - template: GlobalVariables.yml -- group: CsWin32 Publishing secrets extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines From 4590d69d95b867392307d98e17c9107dc7e9ec5c Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Mon, 25 Mar 2024 08:44:27 -0600 Subject: [PATCH 82/86] Fix `sourceAnalysisPool` --- azure-pipelines/archive-sourcecode.yml | 2 +- azure-pipelines/official.yml | 2 +- azure-pipelines/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines/archive-sourcecode.yml b/azure-pipelines/archive-sourcecode.yml index 5f64c115..e3e18986 100644 --- a/azure-pipelines/archive-sourcecode.yml +++ b/azure-pipelines/archive-sourcecode.yml @@ -35,7 +35,7 @@ extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: Azure-Pipelines-1ESPT-ExDShared stages: - stage: archive diff --git a/azure-pipelines/official.yml b/azure-pipelines/official.yml index 21951d48..e32bb224 100644 --- a/azure-pipelines/official.yml +++ b/azure-pipelines/official.yml @@ -94,7 +94,7 @@ extends: template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: Azure-Pipelines-1ESPT-ExDShared suppression: suppressionFile: $(System.DefaultWorkingDirectory)\azure-pipelines\falsepositives.gdnsuppress stages: diff --git a/azure-pipelines/release.yml b/azure-pipelines/release.yml index 6d54f42c..f1c75964 100644 --- a/azure-pipelines/release.yml +++ b/azure-pipelines/release.yml @@ -21,7 +21,7 @@ extends: template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines parameters: sdl: - sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + sourceAnalysisPool: Azure-Pipelines-1ESPT-ExDShared stages: - stage: release From 200857d791042cb73d7dec5efb81d82cd219b21f Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 May 2024 15:44:53 -0600 Subject: [PATCH 83/86] Move full SourceGenerator test to its own new class --- .../CSharpSourceGeneratorVerifier.cs | 87 +++++++++++++++++++ .../GeneratorTestBase.cs | 38 ++------ .../GeneratorTests.cs | 51 ----------- .../Microsoft.Windows.CsWin32.Tests.csproj | 2 +- .../MyReferenceAssemblies.cs | 29 +++++++ .../SourceGeneratorTests.cs | 30 +++++++ 6 files changed, 155 insertions(+), 82 deletions(-) create mode 100644 test/Microsoft.Windows.CsWin32.Tests/CSharpSourceGeneratorVerifier.cs create mode 100644 test/Microsoft.Windows.CsWin32.Tests/MyReferenceAssemblies.cs create mode 100644 test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs diff --git a/test/Microsoft.Windows.CsWin32.Tests/CSharpSourceGeneratorVerifier.cs b/test/Microsoft.Windows.CsWin32.Tests/CSharpSourceGeneratorVerifier.cs new file mode 100644 index 00000000..b36e5fc3 --- /dev/null +++ b/test/Microsoft.Windows.CsWin32.Tests/CSharpSourceGeneratorVerifier.cs @@ -0,0 +1,87 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Testing; + +internal static class CSharpSourceGeneratorVerifier +{ + internal class Test : CSharpSourceGeneratorTest + { + private readonly string testFile; + private readonly string testMethod; + + public Test([CallerFilePath] string? testFile = null, [CallerMemberName] string? testMethod = null) + { + this.testFile = testFile ?? throw new ArgumentNullException(nameof(testFile)); + this.testMethod = testMethod ?? throw new ArgumentNullException(nameof(testMethod)); + + // We don't mean to use record/playback verification. + this.TestBehaviors |= TestBehaviors.SkipGeneratedSourcesCheck; + + this.ReferenceAssemblies = MyReferenceAssemblies.NetStandard20; + this.TestState.Sources.Add(string.Empty); + this.TestState.AnalyzerConfigFiles.Add(("/.globalconfig", ConstructGlobalConfigString())); + } + + public LanguageVersion LanguageVersion { get; set; } = LanguageVersion.CSharp9; + + public string? NativeMethodsTxt { get; set; } + + [StringSyntax(StringSyntaxAttribute.Json)] + public string? NativeMethodsJson { get; set; } + + protected override IEnumerable GetSourceGenerators() + { + yield return typeof(SourceGenerator); + } + + protected override ParseOptions CreateParseOptions() + { + return ((CSharpParseOptions)base.CreateParseOptions()).WithLanguageVersion(this.LanguageVersion); + } + + protected override CompilationOptions CreateCompilationOptions() + { + var compilationOptions = (CSharpCompilationOptions)base.CreateCompilationOptions(); + return compilationOptions + .WithAllowUnsafe(true) + .WithWarningLevel(99) + .WithSpecificDiagnosticOptions(compilationOptions.SpecificDiagnosticOptions.SetItem("CS1591", ReportDiagnostic.Suppress)); + } + + protected override Task RunImplAsync(CancellationToken cancellationToken) + { + if (this.NativeMethodsTxt is not null) + { + this.TestState.AdditionalFiles.Add(("NativeMethods.txt", this.NativeMethodsTxt)); + } + + if (this.NativeMethodsJson is not null) + { + this.TestState.AdditionalFiles.Add(("NativeMethods.json", this.NativeMethodsJson)); + } + + return base.RunImplAsync(cancellationToken); + } + + private static string ConstructGlobalConfigString(bool omitDocs = false) + { + StringBuilder globalConfigBuilder = new(); + globalConfigBuilder.AppendLine("is_global = true"); + globalConfigBuilder.AppendLine(); + globalConfigBuilder.AppendLine($"build_property.CsWin32InputMetadataPaths = {JoinAssemblyMetadata("ProjectionMetadataWinmd")}"); + if (!omitDocs) + { + globalConfigBuilder.AppendLine($"build_property.CsWin32InputDocPaths = {JoinAssemblyMetadata("ProjectionDocs")}"); + } + + return globalConfigBuilder.ToString(); + + static string JoinAssemblyMetadata(string name) + { + return string.Join(";", typeof(GeneratorTests).Assembly.GetCustomAttributes().Where(metadata => metadata.Key == name).Select(metadata => metadata.Value)); + } + } + } +} diff --git a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTestBase.cs b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTestBase.cs index f05518b7..ab18d080 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTestBase.cs +++ b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTestBase.cs @@ -314,10 +314,15 @@ protected async Task CreateCompilationAsync(ReferenceAssembli // Workaround for https://github.com/dotnet/roslyn-sdk/issues/699 const string winRTPackageId = "Microsoft.Windows.SDK.Contracts"; - metadataReferences = metadataReferences.AddRange( - Directory.GetFiles(Path.Combine(Path.GetTempPath(), "test-packages", $"{winRTPackageId}.{references.Packages.Single(id => string.Equals(id.Id, winRTPackageId, StringComparison.OrdinalIgnoreCase)).Version}", "ref", "netstandard2.0"), "*.winmd").Select(p => MetadataReference.CreateFromFile(p))); + var winRTPackage = references.Packages.SingleOrDefault(id => string.Equals(id.Id, winRTPackageId, StringComparison.OrdinalIgnoreCase)); + if (winRTPackage is not null) + { + metadataReferences = metadataReferences.AddRange( + Directory.GetFiles(Path.Combine(Path.GetTempPath(), "test-packages", $"{winRTPackageId}.{winRTPackage.Version}", "ref", "netstandard2.0"), "*.winmd").Select(p => MetadataReference.CreateFromFile(p))); + } - // CONSIDER: How can I pass in the source generator itself, with AdditionalFiles, so I'm exercising that code too? + // QUESTION: How can I pass in the source generator itself, with AdditionalFiles, so I'm exercising that code too? + // ANSWER: Follow the pattern now used in SourceGeneratorTests.cs var compilation = CSharpCompilation.Create( assemblyName: "test", references: metadataReferences, @@ -365,31 +370,4 @@ private static void AssertConsistentLineEndings(SyntaxTree syntaxTree) lineCount++; } } - - protected static class MyReferenceAssemblies - { -#pragma warning disable SA1202 // Elements should be ordered by access - because field initializer depend on each other - private static readonly ImmutableArray AdditionalLegacyPackages = ImmutableArray.Create( - new PackageIdentity("Microsoft.Windows.SDK.Contracts", "10.0.22621.2")); - - private static readonly ImmutableArray AdditionalModernPackages = AdditionalLegacyPackages.AddRange(ImmutableArray.Create( - new PackageIdentity("System.Runtime.CompilerServices.Unsafe", "6.0.0"), - new PackageIdentity("System.Memory", "4.5.5"), - new PackageIdentity("Microsoft.Win32.Registry", "5.0.0"))); - - internal static readonly ReferenceAssemblies NetStandard20 = ReferenceAssemblies.NetStandard.NetStandard20.AddPackages(AdditionalModernPackages); -#pragma warning restore SA1202 // Elements should be ordered by access - - internal static class NetFramework - { - internal static readonly ReferenceAssemblies Net35 = ReferenceAssemblies.NetFramework.Net35.WindowsForms.AddPackages(AdditionalLegacyPackages); - internal static readonly ReferenceAssemblies Net472 = ReferenceAssemblies.NetFramework.Net472.WindowsForms.AddPackages(AdditionalModernPackages); - } - - internal static class Net - { - internal static readonly ReferenceAssemblies Net60 = ReferenceAssemblies.Net.Net60.AddPackages(AdditionalModernPackages); - internal static readonly ReferenceAssemblies Net70 = ReferenceAssemblies.Net.Net70.AddPackages(AdditionalModernPackages); - } - } } diff --git a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs index 248225e9..f508bff3 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs +++ b/test/Microsoft.Windows.CsWin32.Tests/GeneratorTests.cs @@ -1,8 +1,6 @@ ๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using VerifyTest = Microsoft.CodeAnalysis.CSharp.Testing.CSharpSourceGeneratorTest; - public class GeneratorTests : GeneratorTestBase { public GeneratorTests(ITestOutputHelper logger) @@ -928,36 +926,6 @@ void LogProject(string name) } } - [Fact] - public async Task UnparseableNativeMethodsJson() - { - await new VerifyTest - { - TestState = - { - ReferenceAssemblies = MyReferenceAssemblies.NetStandard20, - Sources = { string.Empty }, - AdditionalFiles = - { - ("NativeMethods.txt", "CreateFile"), - ("NativeMethods.json", @"{ ""allowMarshaling"": f }"), // the point where the user is typing "false" - }, - AnalyzerConfigFiles = - { - ("/.globalconfig", ConstructGlobalConfigString()), - }, - GeneratedSources = - { - // Nothing generated, but no exceptions thrown that would lead Roslyn to disable the source generator in the IDE either. - }, - ExpectedDiagnostics = - { - new DiagnosticResult(SourceGenerator.OptionsParsingError.Id, DiagnosticSeverity.Error), - }, - }, - }.RunAsync(); - } - [Fact] public void OpensMetadataForSharedReading() { @@ -1059,23 +1027,4 @@ public void SeekOriginEnumPreferred() QualifiedNameSyntax seekParamType = Assert.IsType(seekMethod.ParameterList.Parameters[1].Type); Assert.Equal(nameof(SeekOrigin), seekParamType.Right.Identifier.ValueText); } - - private static string ConstructGlobalConfigString(bool omitDocs = false) - { - StringBuilder globalConfigBuilder = new(); - globalConfigBuilder.AppendLine("is_global = true"); - globalConfigBuilder.AppendLine(); - globalConfigBuilder.AppendLine($"build_property.CsWin32InputMetadataPaths = {JoinAssemblyMetadata("ProjectionMetadataWinmd")}"); - if (!omitDocs) - { - globalConfigBuilder.AppendLine($"build_property.CsWin32InputDocPaths = {JoinAssemblyMetadata("ProjectionDocs")}"); - } - - return globalConfigBuilder.ToString(); - - static string JoinAssemblyMetadata(string name) - { - return string.Join(";", typeof(GeneratorTests).Assembly.GetCustomAttributes().Where(metadata => metadata.Key == name).Select(metadata => metadata.Value)); - } - } } diff --git a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj index f04b13ff..615cfee0 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj +++ b/test/Microsoft.Windows.CsWin32.Tests/Microsoft.Windows.CsWin32.Tests.csproj @@ -1,7 +1,7 @@ ๏ปฟ - net6.0 + net8.0 diff --git a/test/Microsoft.Windows.CsWin32.Tests/MyReferenceAssemblies.cs b/test/Microsoft.Windows.CsWin32.Tests/MyReferenceAssemblies.cs new file mode 100644 index 00000000..5a47c274 --- /dev/null +++ b/test/Microsoft.Windows.CsWin32.Tests/MyReferenceAssemblies.cs @@ -0,0 +1,29 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +internal static class MyReferenceAssemblies +{ +#pragma warning disable SA1202 // Elements should be ordered by access - because field initializer depend on each other + private static readonly ImmutableArray AdditionalLegacyPackages = ImmutableArray.Create( + new PackageIdentity("Microsoft.Windows.SDK.Contracts", "10.0.22621.2")); + + private static readonly ImmutableArray AdditionalModernPackages = AdditionalLegacyPackages.AddRange(ImmutableArray.Create( + new PackageIdentity("System.Runtime.CompilerServices.Unsafe", "6.0.0"), + new PackageIdentity("System.Memory", "4.5.5"), + new PackageIdentity("Microsoft.Win32.Registry", "5.0.0"))); + + internal static readonly ReferenceAssemblies NetStandard20 = ReferenceAssemblies.NetStandard.NetStandard20.AddPackages(AdditionalModernPackages); +#pragma warning restore SA1202 // Elements should be ordered by access + + internal static class NetFramework + { + internal static readonly ReferenceAssemblies Net35 = ReferenceAssemblies.NetFramework.Net35.WindowsForms.AddPackages(AdditionalLegacyPackages); + internal static readonly ReferenceAssemblies Net472 = ReferenceAssemblies.NetFramework.Net472.WindowsForms.AddPackages(AdditionalModernPackages); + } + + internal static class Net + { + internal static readonly ReferenceAssemblies Net60 = ReferenceAssemblies.Net.Net60.AddPackages(AdditionalModernPackages); + internal static readonly ReferenceAssemblies Net70 = ReferenceAssemblies.Net.Net70.AddPackages(AdditionalModernPackages); + } +} diff --git a/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs b/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs new file mode 100644 index 00000000..426f42a4 --- /dev/null +++ b/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs @@ -0,0 +1,30 @@ +๏ปฟ// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using VerifyCS = CSharpSourceGeneratorVerifier; + +public class SourceGeneratorTests +{ + [Fact] + public async Task UnparseableNativeMethodsJson() + { + await new VerifyCS.Test + { + NativeMethodsTxt = "CreateFile", +#pragma warning disable JSON001 // Invalid JSON pattern -- deliberate point of the test + NativeMethodsJson = @"{ ""allowMarshaling"": f }", +#pragma warning restore JSON001 // Invalid JSON pattern + TestState = + { + GeneratedSources = + { + // Nothing generated, but no exceptions thrown that would lead Roslyn to disable the source generator in the IDE either. + }, + ExpectedDiagnostics = + { + new DiagnosticResult(SourceGenerator.OptionsParsingError.Id, DiagnosticSeverity.Error), + }, + }, + }.RunAsync(); + } +} From 6d6fc606002603267e1a27d50c533700a6b5754a Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 May 2024 15:47:04 -0600 Subject: [PATCH 84/86] Drop the implicit `System.Memory` reference from the package Because we advise users to set `PrivateAssets=all` on the CsWin32 package itself, its transitive dependencies don't pass through to the CsWin32 user's own consumers, leading to compilation or binding redirect issues because System.Memory is missing. By dropping it from the package itself and requiring CsWin32 users to reference it directly, we put a bit more responsibility on them but solve the problem of the missing reference. To help users get this right, we report a new warning when System.Memory hasn't been referenced. Fixes #1158 --- README.md | 13 +++--- .../Microsoft.Windows.CsWin32.nuspec | 8 ---- .../SourceGenerator.cs | 13 ++++++ src/Microsoft.Windows.CsWin32/readme.txt | 17 ++++---- .../SourceGeneratorTests.cs | 42 +++++++++++++++++++ 5 files changed, 69 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index eda2cd6c..7b9e446e 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,16 @@ Install the `Microsoft.Windows.CsWin32` package: dotnet add package Microsoft.Windows.CsWin32 --prerelease ``` -**Tip**: Remove the `IncludeAssets` metadata from the package reference so that you get better code generation by allowing nuget to bring in the `System.Memory` package as a transitive dependency. +You should also install the `System.Memory` package when targeting .NET Framework 4.5+ or .NET Standard 2.0, +as that adds APIs that significantly improve much of the code generated by CsWin32: -```diff - - all -- runtime; build; native; contentfiles; analyzers; buildtransitive - +```ps1 +dotnet add package System.Memory ``` +Projects targeting .NET Core 2.1+ or .NET 5+ do *not* need to add the `System.Memory` package reference, +although it is harmless to do so. + Your project must allow unsafe code to support the generated code that will likely use pointers. This does *not* automatically make all your code *unsafe*. Use of the `unsafe` keyword is required anywhere you use pointers. diff --git a/src/Microsoft.Windows.CsWin32/Microsoft.Windows.CsWin32.nuspec b/src/Microsoft.Windows.CsWin32/Microsoft.Windows.CsWin32.nuspec index 7c73bead..f12dc5ad 100644 --- a/src/Microsoft.Windows.CsWin32/Microsoft.Windows.CsWin32.nuspec +++ b/src/Microsoft.Windows.CsWin32/Microsoft.Windows.CsWin32.nuspec @@ -24,29 +24,21 @@ - - - - - - - - diff --git a/src/Microsoft.Windows.CsWin32/SourceGenerator.cs b/src/Microsoft.Windows.CsWin32/SourceGenerator.cs index a45bd110..57b6cf4a 100644 --- a/src/Microsoft.Windows.CsWin32/SourceGenerator.cs +++ b/src/Microsoft.Windows.CsWin32/SourceGenerator.cs @@ -122,6 +122,14 @@ public class SourceGenerator : ISourceGenerator "Configuration", DiagnosticSeverity.Error, isEnabledByDefault: true); + + public static readonly DiagnosticDescriptor MissingRecommendedReference = new( + "PInvoke009", + "Missing package reference", + "Missing reference to recommended package: \"{0}\"", + "Configuration", + DiagnosticSeverity.Warning, + isEnabledByDefault: true); #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member private const string NativeMethodsTxtAdditionalFileName = "NativeMethods.txt"; @@ -185,6 +193,11 @@ public void Execute(GeneratorExecutionContext context) context.ReportDiagnostic(Diagnostic.Create(UnsafeCodeRequired, location: null)); } + if (compilation.GetTypeByMetadataName("System.Memory`1") is null) + { + context.ReportDiagnostic(Diagnostic.Create(MissingRecommendedReference, location: null, "System.Memory")); + } + Docs? docs = ParseDocs(context); SuperGenerator superGenerator = SuperGenerator.Combine(CollectMetadataPaths(context).Select(path => new Generator(path, docs, options, compilation, parseOptions))); try diff --git a/src/Microsoft.Windows.CsWin32/readme.txt b/src/Microsoft.Windows.CsWin32/readme.txt index a1b782f7..1c7684d2 100644 --- a/src/Microsoft.Windows.CsWin32/readme.txt +++ b/src/Microsoft.Windows.CsWin32/readme.txt @@ -7,19 +7,16 @@ methods and supporting types to a C# project. To get started, create a "NativeMethods.txt" file in your project directory that lists the names of Win32 APIs for which you need to have generated, one per line. -Tips ----- +You should also install the `System.Memory` package when targeting .NET Framework 4.5+ or .NET Standard 2.0, +as that adds APIs that significantly improve much of the code generated by CsWin32: -Remove the `IncludeAssets` metadata from the package reference so that you get better code generation -by allowing nuget to bring in the `System.Memory` package as a transitive dependency. - -```diff - - all -- runtime; build; native; contentfiles; analyzers; buildtransitive - +```ps1 +dotnet add package System.Memory ``` +Projects targeting .NET Core 2.1+ or .NET 5+ do *not* need to add the `System.Memory` package reference, +although it is harmless to do so. + Your project must allow unsafe code to support the generated code that will likely use pointers. Learn more from our README on GitHub: https://github.com/microsoft/CsWin32#readme diff --git a/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs b/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs index 426f42a4..a5ad4874 100644 --- a/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs +++ b/test/Microsoft.Windows.CsWin32.Tests/SourceGeneratorTests.cs @@ -27,4 +27,46 @@ public async Task UnparseableNativeMethodsJson() }, }.RunAsync(); } + + /// + /// Asserts that no warning is produced even without the required reference, when no source is being generated anyway. + /// + [Fact] + public async Task MissingSystemMemoryReference_NoGeneratedCode() + { + await new VerifyCS.Test + { + ReferenceAssemblies = ReferenceAssemblies.NetFramework.Net472.Default, + }.RunAsync(); + } + + /// + /// Asserts that a warning is produced when targeting a framework that our generated code requires the System.Memory reference for, but the reference is missing. + /// + [Fact] + public async Task MissingSystemMemoryReference_WithGeneratedCode_NetFx472() + { + await new VerifyCS.Test + { + ReferenceAssemblies = ReferenceAssemblies.NetFramework.Net472.Default, + NativeMethodsTxt = "CreateFile", + ExpectedDiagnostics = + { + new DiagnosticResult(SourceGenerator.MissingRecommendedReference.Id, DiagnosticSeverity.Warning), + }, + }.RunAsync(); + } + + /// + /// Asserts that when targeting a framework that implicitly includes the references we need, no warning is generated. + /// + [Fact] + public async Task MissingSystemMemoryReference_WithGeneratedCode_Net60() + { + await new VerifyCS.Test + { + ReferenceAssemblies = ReferenceAssemblies.Net.Net60, + NativeMethodsTxt = "CreateFile", + }.RunAsync(); + } } From eaeb452227cd84ad3765a1b7fcb48f257699f98d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 May 2024 17:25:18 -0600 Subject: [PATCH 85/86] Add NativeMethods.json to integration test --- integration-tests/nonsdk/NativeMethods.json | 3 +++ integration-tests/sdk/NativeMethods.json | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 integration-tests/nonsdk/NativeMethods.json create mode 100644 integration-tests/sdk/NativeMethods.json diff --git a/integration-tests/nonsdk/NativeMethods.json b/integration-tests/nonsdk/NativeMethods.json new file mode 100644 index 00000000..4b7b1bc0 --- /dev/null +++ b/integration-tests/nonsdk/NativeMethods.json @@ -0,0 +1,3 @@ +{ + // This file may be blank, but it exercises the analyzer's System.Text.Json and dependencies. +} \ No newline at end of file diff --git a/integration-tests/sdk/NativeMethods.json b/integration-tests/sdk/NativeMethods.json new file mode 100644 index 00000000..4b7b1bc0 --- /dev/null +++ b/integration-tests/sdk/NativeMethods.json @@ -0,0 +1,3 @@ +{ + // This file may be blank, but it exercises the analyzer's System.Text.Json and dependencies. +} \ No newline at end of file From e15541a96753ec7ac6a2c2ae97a013b8af8eab4d Mon Sep 17 00:00:00 2001 From: Andrew Arnott Date: Fri, 3 May 2024 17:22:03 -0600 Subject: [PATCH 86/86] Reference and ship the same version of System.Text.Encodings.Web Fixes #1163 --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 76d86d71..16defa7e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,7 +31,7 @@ - +