Skip to content

Commit

Permalink
Fixes to Azure Public feed usage (PowerShell#24149)
Browse files Browse the repository at this point in the history
* Fix code PATs

* Fix feed PATs

* remove gallery commit

* update feed url

* Update to new variable groups

* Fix Variable name

* Fix credential template

* Disable Signing setup in prep stage

* Capture nuget source list

* lock down the firewall

* Add creds to feed switch to allow single switch location

* Use switch from build.psm1

* Use switch template instead of commands

* update to test feed

* disable codeql in jobs where we don't compile

* disable code sign validation for prep

* move capture steps to restore phase to see if it speeds things up

* remove duplicate capture of nuget config

* update test service

* Only build windows test service on windows

* warn when no config is generated

* try to fix test service

* fix web listener refs

* try removing dotnet tool

* update feedname with user info

* update package version that is not found

* try moving failing jobs to restore phase

* allow nuget inset in either phase

* update package ref

* use the right reporoot

* Move everything to restore

* Try adding build phase

* put nuget files in the right place

* move bootstrap into yaml

* remove onebranch agent items from macos build

* switch to environment variable

* bump a couple of packages

* fix formatting

* Fix static analysis issue

* update feed url to test restoring everything

* install the AzFeed cred provider

* fix binlog issues
  • Loading branch information
TravisEz13 authored Aug 15, 2024
1 parent 8c98b5c commit 5919a24
Show file tree
Hide file tree
Showing 27 changed files with 237 additions and 346 deletions.
28 changes: 19 additions & 9 deletions .pipelines/PowerShell-Coordinated_Packages-Official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,20 @@ variables:
value: ${{ parameters.ReleaseTagVar }}
- name: SKIP_SIGNING
value: ${{ parameters.SKIP_SIGNING }}
- group: 'AzDevOpsArtifacts'
- group: 'mscodehub-feed-read-akv'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: ENABLE_MSBUILD_BINLOGS
value: ${{ parameters.ENABLE_MSBUILD_BINLOGS }}

extends:
template: v2/OneBranch.Official.CrossPlat.yml@onebranchTemplates
parameters:
customTags: 'ES365AIMigrationTooling'
featureFlags:
LinuxHostVersion:
Network: KS3
WindowsHostVersion:
Network: KS3
globalSdl:
disableLegacyManifest: true
# disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates.
Expand Down Expand Up @@ -121,15 +126,18 @@ extends:
type: windows

variables:
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/BuildJson'
- name: ob_sdl_codeSignValidation_enabled
value: false
- name: ob_sdl_codeql_compiled_enabled
value: false
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- name: ob_sdl_tsa_configFile
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_signing_setup_enabled
value: false

steps:
- checkout: self
Expand All @@ -140,6 +148,8 @@ extends:
- pwsh: |
Get-ChildItem Env:
displayName: Capture environment variables
env:
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
- template: /.pipelines/templates/SetVersionVariables.yml@self
parameters:
Expand Down
1 change: 1 addition & 0 deletions .pipelines/PowerShell-Packages-Official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ variables:
value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project
- name: LinuxContainerImage
value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- name: branchCounterKey
value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])]
Expand Down
5 changes: 2 additions & 3 deletions .pipelines/templates/checkAzureContainer.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false

displayName: Delete blob is exists
pool:
Expand Down
64 changes: 39 additions & 25 deletions .pipelines/templates/insert-nuget-config-azfeed.yml
Original file line number Diff line number Diff line change
@@ -1,39 +1,53 @@
parameters:
- name: "repoRoot"
default: $(REPOROOT)
- name: "ob_restore_phase"
type: boolean
default: true

steps:
- task: NuGetAuthenticate@1
displayName: Install Azure Artifacts Credential Provider
inputs:
forceReinstallCredentialProvider: true

- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
try {
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
Write-Verbose -Verbose "Running: Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'"
Switch-PSNugetConfig -Source Private -UserName '$(AzDevopsFeedUserNameKVPAT)' -ClearTextPAT '$(powershellPackageReadPat)'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for PSGallery modules'
condition: and(succeededOrFailed(), ne(variables['AzDevOpsFeed'], ''))
catch {
Get-Error
throw
}
displayName: 'Switch to production Azure DevOps feed for all nuget.configs'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}/src/Modules
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}

- pwsh: |
$configPath = "${env:NugetConfigDir}/nuget.config"
Import-Module ${{ parameters.repoRoot }}/build.psm1 -Force
$powerShellPublicPackages = New-NugetPackageSource -Url '$(PowerShellCore_PublicPackages)' -Name 'AzDevOpsFeed'
New-NugetConfigFile -NugetPackageSource $powerShellPublicPackages -UserName $(AzDevopsFeedUserNameKVPAT) -ClearTextPAT $(mscodehubPackageReadPat) -Destination "${env:NugetConfigDir}"
if (-not (Test-Path $configPath))
{
throw "nuget.config is not created"
Get-ChildItem ${{ parameters.repoRoot }}/nuget.config -Recurse | Foreach-Object {
Write-Verbose -Verbose "--- START $($_.fullname) ---"
get-content $_.fullname | Out-String -width 9999 -Stream | write-Verbose -Verbose
Write-Verbose -Verbose "--- END $($_.fullname) ---"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PSInternalNugetFeed'], ''))
displayName: 'Capture all nuget.config files'
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
ob_restore_phase: ${{ parameters.ob_restore_phase }}

- pwsh: |
Get-ChildItem -Path env:VSS* | Out-String -width 9999 -Stream | write-Verbose -Verbose
displayName: Capture VSS* Environment
condition: and(succeededOrFailed(), ne(variables['UseAzDevOpsFeed'], ''))
env:
NugetConfigDir: ${{ parameters.repoRoot }}
ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
ob_restore_phase: ${{ parameters.ob_restore_phase }}
18 changes: 9 additions & 9 deletions .pipelines/templates/mac.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@ jobs:
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
- name: ob_sdl_binskim_enabled
value: true
- name: ob_sdl_credscan_suppressionsfileforartifacts
value: $(Build.SourcesDirectory)/PowerShell/.config/suppress.json
- name: PowerShellRoot
value: $(Build.SourcesDirectory)

steps:
- checkout: self
clean: true
Expand All @@ -39,19 +38,19 @@ jobs:
# make the current user the owner
sudo chown $env:USER "$(Agent.TempDirectory)/PowerShell"
displayName: 'Create $(Agent.TempDirectory)/PowerShell'
- template: /.pipelines/templates/cloneToOfficialPath.yml@self
parameters:
nativePathRoot: '$(Agent.TempDirectory)'
- pwsh: |
tools/releaseBuild/macOS/PowerShellPackageVsts.ps1 -location $(PowerShellRoot) -BootStrap
Import-Module $(PowerShellRoot)/build.psm1 -Force
Start-PSBootstrap -Package
displayName: 'Bootstrap VM'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)
- pwsh: |
$env:AzDevOpsFeedPAT2 = '$(AzDevOpsFeedPAT2)'
$env:AzDevOpsFeedPAT2 = '$(powershellPackageReadPat)'
# Add -SkipReleaseChecks as a mitigation to unblock release.
# macos-10.15 does not allow creating a folder under root. Hence, moving the folder.
Expand All @@ -76,6 +75,7 @@ jobs:
displayName: 'Build'
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
- template: /.pipelines/templates/step/finalize.yml@self

- job: sign_${{ parameters.buildArchitecture }}
Expand Down
23 changes: 5 additions & 18 deletions .pipelines/templates/nupkg.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- group: 'AzDevOpsArtifacts'
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess

steps:
Expand Down Expand Up @@ -89,23 +90,9 @@ jobs:
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- pwsh: |
$repoRoot = "$(PowerShellRoot)"
Write-Verbose -Verbose "repoRoot: $repoRoot"
$configPath = "$repoRoot/nuget.config"
Import-Module "$repoRoot/build.psm1" -Force
New-NugetConfigFile -NugetFeedUrl $(PowerShellCore_PublicPackages) -UserName $(AzDevOpsFeedUserName) -ClearTextPAT $(AzDevOpsFeedPAT2) -FeedName AzDevOpsFeed -Destination "$(PowerShellRoot)"
if(-not (Test-Path $configPath))
{
throw "nuget.config is not created"
}
Get-Content $configPath | Write-Verbose -Verbose
displayName: 'Add nuget.config for Azure DevOps feed for packages'
condition: and(succeededOrFailed(), ne(variables['PowerShellCore_PublicPackages'], ''))
env:
ob_restore_phase: true # This ensures this done in restore phase to workaround signing issue
- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(PowerShellRoot)

- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
Expand Down
5 changes: 2 additions & 3 deletions .pipelines/templates/release-MakeBlobPublic.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false

steps:
- checkout: self
Expand Down
3 changes: 2 additions & 1 deletion .pipelines/templates/release-validate-sdk.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ jobs:
type: ${{ parameters.jobtype }}

variables:
- group: AzDevOpsArtifacts
- group: mscodehub-feed-read-general
- group: mscodehub-feed-read-akv
- group: DotNetPrivateBuildAccess
- name: ob_outputDirectory
value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
Expand Down
31 changes: 29 additions & 2 deletions .pipelines/templates/testartifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true

- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true

- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Expand Down Expand Up @@ -58,6 +65,13 @@ jobs:
BuildTestPackage -runtime win-arm64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task
- job: build_testartifacts_nonwin
variables:
Expand All @@ -75,15 +89,22 @@ jobs:
steps:
- checkout: self
clean: true
env:
ob_restore_phase: true

- template: /.pipelines/templates/insert-nuget-config-azfeed.yml@self
parameters:
repoRoot: $(Build.SourcesDirectory)
repoRoot: $(Build.SourcesDirectory)/PowerShell
ob_restore_phase: true

- pwsh: |
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Start-PSBootstrap
displayName: Bootstrap
env:
__DOTNET_RUNTIME_FEED_KEY: $(RUNTIME_SOURCEFEED_KEY)
ob_restore_phase: true
- pwsh: |
New-Item -Path '$(ob_outputDirectory)' -ItemType Directory -Force
Import-Module $(Build.SourcesDirectory)/PowerShell/build.psm1
Expand Down Expand Up @@ -113,3 +134,9 @@ jobs:
BuildTestPackage -runtime linux-musl-x64
displayName: Build test package and upload
retryCountOnTaskFailure: 1
env:
ob_restore_phase: true
- pwsh: |
Write-Host "This doesn't do anything but make the build phase run."
displayName: Dummy build task
5 changes: 2 additions & 3 deletions .pipelines/templates/uploadToAzure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,8 @@ jobs:
value: $(Build.SourcesDirectory)\PowerShell\.config\tsaoptions.json
- name: ob_sdl_credscan_suppressionsFile
value: $(Build.SourcesDirectory)\PowerShell\.config\suppress.json
- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/master') }}:
- name: ob_sdl_codeql_compiled_enabled
value: true
- name: ob_sdl_codeql_compiled_enabled
value: false

steps:
- checkout: self
Expand Down
2 changes: 1 addition & 1 deletion .vsts-ci/templates/ci-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ jobs:
displayName: Set Build Name for Non-PR
condition: ne(variables['Build.Reason'], 'PullRequest')

- ${{ if ne(variables['AzDevOpsFeed'], '') }}:
- ${{ if ne(variables['UseAzDevOpsFeed'], '') }}:
- template: /tools/releaseBuild/azureDevOps/templates/insert-nuget-config-azfeed.yml

- pwsh: |
Expand Down
Loading

0 comments on commit 5919a24

Please sign in to comment.