Remove reliance of scripts on access to storage account - use offline…

… copies of files (dotnet#4761)
CodeBlanch · Jun 26, 2024 · a832d9c · a832d9c
1 parent cb11e39
commit a832d9c
Show file tree

Hide file tree

Showing 3 changed files with 96 additions and 15 deletions.
diff --git a/eng/release/Scripts/FixupManifestToLocal.ps1 b/eng/release/Scripts/FixupManifestToLocal.ps1
@@ -0,0 +1,79 @@
+param(
+  [Parameter(Mandatory=$true)][string] $ManifestPath,
+  [Parameter(Mandatory=$true)][string] $StagingPath,
+  [switch] $help,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+function Write-Help() {
+    Write-Host "Publish packages specified in a manifest. This should not be used for large manifests."
+    Write-Host "Common settings:"
+    Write-Host "  -ManifestPath <value>     Path to a publishing manifest where the NuGet packages to publish can be found."
+    Write-Host "  -StagingPath <value>      Directory containing the staged assets from blob storage."
+    Write-Host ""
+}
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
+    Write-Help
+    exit 1
+}
+
+if ($null -ne $properties) {
+    Write-Error "Unexpected extra parameters: $properties."
+    exit 1
+}
+
+if (!(Test-Path $ManifestPath))
+{
+    Write-Error "Error: unable to find manifest at '$ManifestPath'."
+    exit 1
+}
+
+$manifestSize = $(Get-ChildItem $ManifestPath).length / 1kb
+
+# Limit size. For large manifests
+if ($manifestSize -gt 500)
+{
+    Write-Error "Error: Manifest $ManifestPath too large."
+    exit 1
+}
+
+$manifestJson = Get-Content -Raw -Path $ManifestPath | ConvertFrom-Json
+
+foreach ($nugetPack in $manifestJson.NugetAssets)
+{
+    $packagePath = Join-Path $StagingPath $nugetPack.PublishRelativePath
+    if (!(Test-Path $packagePath))
+    {
+        Write-Error "Error: unable to find package at '$packagePath'."
+        continue
+    }
+    Add-Member -InputObject $nugetPack -MemberType NoteProperty -Name LocalPath -Value $packagePath
+}
+
+$toolHashToLocalPath = @{}
+
+foreach ($tool in $manifestJson.ToolBundleAssets)
+{
+    $toolPath = Join-Path $StagingPath $tool.PublishRelativePath
+    if (!(Test-Path $toolPath))
+    {
+        Write-Error "Error: unable to find package at '$toolPath'."
+        continue
+    }
+    Add-Member -InputObject $tool -MemberType NoteProperty -Name LocalPath -Value $toolPath
+    $toolHashToLocalPath.Add($tool.Sha512, $toolPath)
+}
+
+foreach ($asset in $manifestJson.PublishInstructions)
+{
+    $localAssetPath = $toolHashToLocalPath[$asset.Sha512]
+    Add-Member -InputObject $asset -MemberType NoteProperty -Name RemotePath -Value $asset.FilePath
+    $asset.FilePath = $localAssetPath
+}
+
+Copy-Item $ManifestPath "$ManifestPath.bak"
+$manifestJson | ConvertTo-Json -Depth 10 | Set-Content -Path $ManifestPath
diff --git a/eng/release/Scripts/GenerateGithubRelease.ps1 b/eng/release/Scripts/GenerateGithubRelease.ps1
@@ -3,7 +3,7 @@ param(
   [Parameter(Mandatory=$false)][string] $ReleaseNotes,
   [Parameter(Mandatory=$true)][string] $GhOrganization,
   [Parameter(Mandatory=$true)][string] $GhRepository,
-  [Parameter(Mandatory=$false)][string] $GhCliLink = "https://github.com/cli/cli/releases/download/v1.2.0/gh_1.2.0_windows_amd64.zip",
+  [Parameter(Mandatory=$false)][string] $GhCliLink = "https://github.com/cli/cli/releases/download/v2.52.0/gh_2.52.0_windows_amd64.zip",
   [Parameter(Mandatory=$true)][string] $TagName,
   [bool] $DraftRelease = $false,
   [switch] $help,
@@ -50,7 +50,7 @@ function Get-DownloadLinksAndChecksums($manifest)
     $linkTable += "</details>`n`n"
 
     $filePublishData = @{}
-    $manifest.PublishInstructions | %{ $filePublishData.Add($_.FilePath, $_) }
+    $manifest.PublishInstructions | %{ $filePublishData.Add($_.Sha512, $_) }
 
     $sortedTools = $manifest.ToolBundleAssets | Sort-Object -Property @{ Expression = "Rid" }, @{ Expression = "ToolName" }
 
@@ -65,11 +65,11 @@ function Get-DownloadLinksAndChecksums($manifest)
 
     foreach ($toolBundle in $sortedTools)
     {
-        $hash = $filePublishData[$toolBundle.PublishedPath].Sha512
+        $hash = $toolBundle.Sha512
         $name = $toolBundle.ToolName
         $rid = $toolBundle.Rid
 
-        $link = "https://download.visualstudio.microsoft.com/download/pr/" + $filePublishData[$toolBundle.PublishedPath].PublishUrlSubPath
+        $link = "https://download.visualstudio.microsoft.com/download/pr/" + $filePublishData[$hash].PublishUrlSubPath
         $linkTable += "| $name | $rid | [Download]($link) |`n";
 
         $checksumCsv += "`"$name`",`"$rid`",`"$link`",`"$hash`"`n"
@@ -93,7 +93,7 @@ function Post-GithubRelease($manifest, [string]$releaseBody, [string]$checksumCs
         Expand-Archive -Path $zipPath -DestinationPath $extractionPath
         $progressPreference = 'Continue'
     }
-    catch 
+    catch
     {
         Write-Error "Unable to get GitHub CLI for release"
         exit 1

diff --git a/eng/release/Scripts/PublishToNuget.ps1 b/eng/release/Scripts/PublishToNuget.ps1
@@ -1,6 +1,5 @@
 param(
   [Parameter(Mandatory=$true)][string] $ManifestPath,
-  [Parameter(Mandatory=$true)][string] $StagingPath,
   [Parameter(Mandatory=$true)][string] $FeedEndpoint,
   [Parameter(Mandatory=$true)][string] $FeedPat,
   [switch] $help,
@@ -10,7 +9,6 @@ function Write-Help() {
     Write-Host "Publish packages specified in a manifest. This should not be used for large manifests."
     Write-Host "Common settings:"
     Write-Host "  -ManifestPath <value>      Path to a publishing manifest where the NuGet packages to publish can be found."
-    Write-Host "  -StagingPath <value>       Path where the assets in the manifests are laid out."
     Write-Host "  -FeedEndpoint <value>      NuGet feed to publish the packages to."
     Write-Host "  -FeedPat <value>           PAT to use in the publish process."
     Write-Host ""
@@ -31,14 +29,14 @@ if ($null -ne $properties) {
 
 if (!(Test-Path $ManifestPath))
 {
-    Write-Error "Error: unable to find maifest at $ManifestPath."
+    Write-Error "Error: unable to find manifest at '$ManifestPath'."
     exit 1
 }
 
 $manifestSize = $(Get-ChildItem $ManifestPath).length / 1kb
 
 # Limit size. For large manifests
-if ($manifestSize -gt 500) 
+if ($manifestSize -gt 500)
 {
     Write-Error "Error: Manifest $ManifestPath too large."
     exit 1
@@ -49,15 +47,19 @@ $manifestJson = Get-Content -Raw -Path $ManifestPath | ConvertFrom-Json
 $failedToPublish = 0
 foreach ($nugetPack in $manifestJson.NugetAssets)
 {
-    $packagePath = Join-Path $StagingPath $nugetPack.PublishRelativePath
+    if (!($nugetPack.PSobject.Properties.Name.Contains("LocalPath")))
+    {
+        Write-Error "Error: unable to find LocalPath for '$nugetPack'. Ensure local manifest translation happened."
+        exit 1
+
+        continue
+    }
+
     try
     {
-        Write-Host "Downloading: $nugetPack."
-        $progressPreference = 'silentlyContinue'
-        Invoke-WebRequest -Uri $nugetPack.PublishedPath -OutFile (New-Item -Path $packagePath -Force)
-        $progressPreference = 'Continue'
+        $packagePath = $nugetPack.LocalPath;
 
-        if ($nugetPack.PSobject.Properties.Name.Contains("Sha512")-and $(Get-FileHash -Algorithm sha512 $packagePath).Hash -ne $nugetPack.Sha512) {
+        if ($nugetPack.PSobject.Properties.Name.Contains("Sha512") -and $(Get-FileHash -Algorithm sha512 $packagePath).Hash -ne $nugetPack.Sha512) {
             Write-Host "Sha512 verification failed for $($nugetPack.PublishRelativePath)."
             $failedToPublish++
             continue