From d39826aa6461b639f8a1121c4d95c9f48ea68e11 Mon Sep 17 00:00:00 2001 From: Medeni Baykal <433724+Haplois@users.noreply.github.com> Date: Thu, 13 Jan 2022 21:36:50 +0100 Subject: [PATCH] Fixed manifest publishing (#3246) - Fixed a small issue in `Invoke-Exe` - Fixed a trailing backslash issue in manifest publishing --- scripts/build.ps1 | 67 ++++++------ scripts/build/GenerateManifest.proj | 10 +- scripts/common.lib.ps1 | 152 ++++++++++++++++++++++++++-- 3 files changed, 182 insertions(+), 47 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 56a5885e9d..9a6d4f470b 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -74,8 +74,8 @@ if([string]::IsNullOrWhiteSpace($Version)) # Build configuration # Write-Verbose "Setup build configuration." -$TPB_Solution = "TestPlatform.sln" $TPB_TestAssets = Join-Path $env:TP_ROOT_DIR "test\TestAssets\" +$TPB_Solution = Join-Path $env:TP_ROOT_DIR "TestPlatform.sln" $TPB_TestAssets_Solution = Join-Path $TPB_TestAssets "TestAssets.sln" $TPB_TestAssets_CILAssets = Join-Path $TPB_TestAssets "CILProject\CILProject.proj" $TPB_TargetFramework45 = "net45" @@ -126,12 +126,11 @@ function Invoke-Build $dotnetExe = Get-DotNetPath Write-Log ".. .. Build: Source: $TPB_Solution" - Invoke-Exe $dotnetExe "build $TPB_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:TestPlatform.binlog" - Write-Host -ForegroundColor Yellow $LASTEXITCODE + Invoke-Exe $dotnetExe -Arguments "build $TPB_Solution --configuration $TPB_Configuration -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:TestPlatform.binlog" Write-Log ".. .. Build: Complete." Write-Log ".. .. Build: Source: $TPB_TestAssets_CILAssets" - Invoke-Exe $dotnetExe "build $TPB_TestAssets_CILAssets --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$($env:TP_ROOT_DIR)\CILAssets.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_CILAssets --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$($env:TP_ROOT_DIR)\CILAssets.binlog""" Write-Log ".. .. Build: Complete." Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}" } @@ -145,11 +144,15 @@ function Invoke-TestAssetsBuild $nugetConfig = Join-Path $TPB_TestAssets "NuGet.config" Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution" - Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" - Invoke-Exe -IgnoreExitCode 1 $nugetExe "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" - Invoke-Exe $dotnetExe "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$($env:TP_ROOT_DIR)\TestAssets.binlog""" - Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- remove NuGet source" - Invoke-Exe -IgnoreExitCode 1 $nugetExe "sources remove -Name ""locally-built-testplatform-packages"" -ConfigFile ""$nugetConfig""" + try { + Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" + Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" + Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$($env:TP_ROOT_DIR)\TestAssets.binlog""" + } + finally { + Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- remove NuGet source" + Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources remove -Name ""locally-built-testplatform-packages"" -ConfigFile ""$nugetConfig""" + } Write-Log ".. .. Build: Complete." Write-Log "Invoke-TestAssetsBuild: Complete. {$(Get-ElapsedTime($timer))}" } @@ -187,7 +190,6 @@ function Publish-Package { $timer = Start-Timer Write-Log "Publish-Package: Started." - $dotnetExe = Get-DotNetPath $fullCLRPackage451Dir = Get-FullCLRPackageDirectory $fullCLRPackage45Dir = Get-FullCLRPackageDirectory45 $uap100PackageDir = $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkUap100"); @@ -567,9 +569,8 @@ function Publish-Package function Publish-Tests { - if($TPB_PublishTests) + if($TPB_PublishTests) { - $dotnetExe = Get-DotNetPath Write-Log "Publish-Tests: Started." # Adding only Perf project for now @@ -595,12 +596,14 @@ function Publish-Tests function Publish-PackageInternal($packagename, $framework, $output) { - Invoke-Exe $dotnetExe "publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" + $dotnetExe = Get-DotNetPath + Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) { - Invoke-Exe $dotnetExe "publish $packagename --configuration $TPB_Configuration --framework $framework --runtime $runtime --self-contained $selfcontained --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" + $dotnetExe = Get-DotNetPath + Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --runtime $runtime --self-contained $selfcontained --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) @@ -650,8 +653,8 @@ function Create-VsixPackage if($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackageDirectory $extensionsPackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } - - # Copy Microsoft.VisualStudio.Core to Extensions + + # Copy Microsoft.VisualStudio.Core to Extensions $codeCoverageCorePackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.coverage.core\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkNS20" Copy-Item $codeCoverageCorePackagesDir\Microsoft.VisualStudio.Coverage.Core.dll $extensionsPackageDir -Force @@ -731,7 +734,7 @@ function Create-VsixPackage Update-VsixVersion $vsixProjectDir # Build vsix project to get TestPlatform.vsix - Invoke-Exe $msbuildPath """$vsixProjectDir\TestPlatform.csproj"" -p:Configuration=$Configuration" + Invoke-Exe $msbuildPath -Arguments """$vsixProjectDir\TestPlatform.csproj"" -p:Configuration=$Configuration" } else { @@ -831,8 +834,7 @@ function Create-NugetPackages $uap10Nuget = $testhostUapPackageDir } - Write-Verbose "$nugetExe pack $stagingDir\$file -OutputDirectory $packageOutputDir -Version $TPB_Version -Properties Version=$TPB_Version $additionalArgs" - Invoke-Exe $nugetExe "pack $stagingDir\$file -OutputDirectory $packageOutputDir -Version $TPB_Version -Properties Version=$TPB_Version;JsonNetVersion=$JsonNetVersion;Runtime=$TPB_TargetRuntime;NetCoreTargetFramework=$TPB_TargetFrameworkCore20;FakesPackageDir=$FakesPackageDir;NetStandard10Framework=$TPB_TargetFrameworkNS10;NetStandard13Framework=$TPB_TargetFrameworkNS13;NetStandard20Framework=$TPB_TargetFrameworkNS20;Uap10Framework=$uap10Nuget;BranchName=$TPB_BRANCH;CommitId=$TPB_COMMIT $additionalArgs" + Invoke-Exe $nugetExe -Arguments "pack $stagingDir\$file -OutputDirectory $packageOutputDir -Version $TPB_Version -Properties Version=$TPB_Version;JsonNetVersion=$JsonNetVersion;Runtime=$TPB_TargetRuntime;NetCoreTargetFramework=$TPB_TargetFrameworkCore20;FakesPackageDir=$FakesPackageDir;NetStandard10Framework=$TPB_TargetFrameworkNS10;NetStandard13Framework=$TPB_TargetFrameworkNS13;NetStandard20Framework=$TPB_TargetFrameworkNS20;Uap10Framework=$uap10Nuget;BranchName=$TPB_BRANCH;CommitId=$TPB_COMMIT $additionalArgs" } # Verifies that expected number of files gets shipped in nuget packages. @@ -911,7 +913,7 @@ function Update-LocalizedResources } $localizationProject = Join-Path $env:TP_PACKAGE_PROJ_DIR "Localize\Localize.proj" - Invoke-Exe $dotnetExe "msbuild $localizationProject -m -nologo -v:minimal -t:Localize -p:LocalizeResources=true -nodeReuse:False" + Invoke-Exe $dotnetExe -Arguments "msbuild $localizationProject -m -nologo -v:minimal -t:Localize -p:LocalizeResources=true -nodeReuse:False" Write-Log ".. Update-LocalizedResources: Complete. {$(Get-ElapsedTime($timer))}" } @@ -960,6 +962,7 @@ function Locate-MSBuildPath Write-Verbose "found msbuild : '$($msbuildPath -join "','")'" $msBuild = $msBuildPath | Select-Object -First 1 + Write-Verbose "msbuildPath is : '$($msbuildPath -join "','")'" if ($null -eq $msBuild -or 0 -eq $msBuild.Count) { throw "MSBuild not found." @@ -981,13 +984,13 @@ function Locate-VsInstallPath Write-Verbose "VSInstallation requirements : $requiredPackageIds" Try - { + { if ($TPB_CIBuild) { - $vsInstallPath = Invoke-Exe $vswhere "-version ""(15.0"" -products * -requires $requiredPackageIds -property installationPath" + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" } else { # Allow using pre release versions of VS for dev builds - $vsInstallPath = Invoke-Exe $vswhere "-version ""(15.0"" -prerelease -products * -requires $requiredPackageIds -property installationPath" + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" } } Catch [System.Management.Automation.MethodInvocationException] @@ -1010,8 +1013,6 @@ function Locate-VsInstallPath function Update-VsixVersion($vsixProjectDir) { Write-Log "Update-VsixVersion: Started." - - $packageDir = Get-FullCLRPackageDirectory $vsixVersion = $Version # Build number comes in the form 20170111-01(yyyymmdd-buildNoOfThatDay) @@ -1021,7 +1022,7 @@ function Update-VsixVersion($vsixProjectDir) $vsixVersion = "$vsixVersion.$($vsixVersionSuffix[0])$($vsixVersionSuffix[1])" } - $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | % {$_.ToString().Replace("`$version`$", "$vsixVersion") } + $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object {$_.ToString().Replace("`$version`$", "$vsixVersion") } Set-Content -path "$vsixProjectDir\source.extension.vsixmanifest" -value $manifestContentWithVersion Write-Log "Update-VsixVersion: Completed." @@ -1035,7 +1036,7 @@ function Generate-Manifest ($PackageFolder) $generateManifestPath = Join-Path $env:TP_ROOT_DIR "scripts\build\GenerateManifest.proj" $msbuildPath = Locate-MSBuildPath - Invoke-Exe $msbuildPath "$generateManifestPath /t:PublishToBuildAssetRegistry /p:PackagesToPublishPattern=$PackageFolder\*.nupkg /p:BUILD_BUILDNUMBER=$BuildNumber /p:PackagesPath=""$PackageFolder"" /p:Configuration=$TPB_Configuration /bl:""$env:TP_OUT_DIR\log\$Configuration\manifest-generation-$packagesFolderName.binlog""" + Invoke-Exe $msbuildPath -Arguments "$generateManifestPath /t:PublishToBuildAssetRegistry /p:PackagesToPublishPattern=$PackageFolder\*.nupkg /p:BUILD_BUILDNUMBER=$BuildNumber /p:PackagesPath=""$PackageFolder"" /p:Configuration=$TPB_Configuration /bl:""$env:TP_OUT_DIR\log\$Configuration\manifest-generation-$packagesFolderName.binlog""" Write-Log "Generate-Manifest ($packagesFolderName): Completed." } @@ -1063,7 +1064,7 @@ function Build-SpecificProjects } } - if( $ProjectsToBuild -eq $null){ + if( $null -eq $ProjectsToBuild){ Write-Error "No csproj name match for given pattern: $ProjectNamePatterns" } @@ -1071,9 +1072,9 @@ function Build-SpecificProjects foreach($ProjectToBuild in $ProjectsToBuild) { Write-Log "Building Project $ProjectToBuild" # Restore and Build - $output = Invoke-Exe $dotnetPath "restore $ProjectToBuild" + $output = Invoke-Exe $dotnetPath -Arguments "restore $ProjectToBuild" PrintAndExit-OnError $output - $output = Invoke-Exe $dotnetPath "build $ProjectToBuild" + $output = Invoke-Exe $dotnetPath -Arguments "build $ProjectToBuild" PrintAndExit-OnError $output if (-Not ($ProjectToBuild.FullName -contains "$($env:TP_ROOT_DIR)$([IO.Path]::DirectorySeparatorChar)src")) { @@ -1090,7 +1091,7 @@ function Build-SpecificProjects Write-Log "Copying artifacts from $fromDir to $toDir" Get-ChildItem $fromDir | ForEach-Object { if(-not ($_.PSIsContainer)) { - copy $_.FullName $toDir + Copy-Item $_.FullName $toDir } } } @@ -1142,7 +1143,6 @@ if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { { Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir } - Copy-PackageIntoStaticDirectory } @@ -1151,7 +1151,6 @@ if ($Force -or $Steps -contains "PrepareAcceptanceTests") { Invoke-TestAssetsBuild Publish-Tests } - if ($Script:ScriptFailed) { Write-Log "Build failed. {$(Get-ElapsedTime($timer))}" -Level "Error" @@ -1159,4 +1158,4 @@ if ($Script:ScriptFailed) { } else { Write-Log "Build succeeded. {$(Get-ElapsedTime($timer))}" Exit 0 -} +} \ No newline at end of file diff --git a/scripts/build/GenerateManifest.proj b/scripts/build/GenerateManifest.proj index 1f62797d7b..6f80d79fb7 100644 --- a/scripts/build/GenerateManifest.proj +++ b/scripts/build/GenerateManifest.proj @@ -13,10 +13,12 @@ $(MicrosoftDotNetBuildTasksFeedFilePath)tools/net472/ $(MicrosoftDotNetBuildTasksFeedFilePath)tools/netcoreapp3.1/ - $(ArtifactsDir)\$(Configuration)\packages\ - $(PackagesPath)\ - $(PackagesPath)manifest\manifest.xml - $(PackagesPath)*.nupkg + $(PackagesPath) + $(ArtifactsDir)\$(Configuration)\packages\ + $(PackagesPathWithTrailingSlash)\ + + $(PackagesPathWithTrailingSlash)manifest\manifest.xml + $(PackagesPathWithTrailingSlash)*.nupkg https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 36a479c134..d5fbbfcd5a 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -136,12 +136,12 @@ function Install-DotNetCli Get-ChildItem "Env:\dotnet_*" "`n`n---- x64 dotnet" - Invoke-Exe "$env:DOTNET_ROOT\dotnet.exe" "--info" + Invoke-Exe "$env:DOTNET_ROOT\dotnet.exe" -Arguments "--info" "`n`n---- x86 dotnet" # avoid erroring out because we don't have the sdk for x86 that global.json requires try { - Invoke-Exe "${env:DOTNET_ROOT(x86)}\dotnet.exe" "--info" 2> $null + (Invoke-Exe "${env:DOTNET_ROOT(x86)}\dotnet.exe" -Arguments "--info") 2> $null } catch {} Write-Log "Install-DotNetCli: Complete. {$(Get-ElapsedTime($timer))}" } @@ -161,9 +161,7 @@ function Restore-Package $timer = Start-Timer Write-Log "Restore-Package: Start restoring packages to $env:TP_PACKAGES_DIR." $dotnetExe = Get-DotNetPath - - Write-Log ".. .. Restore-Package: Source: $env:TP_ROOT_DIR\src\package\external\external.csproj" - Invoke-Exe $dotnetExe "restore $env:TP_ROOT_DIR\src\package\external\external.csproj --packages $env:TP_PACKAGES_DIR -v:minimal -warnaserror -p:Version=$TPB_Version -bl:""$env:TP_OUT_DIR\log\$Configuration\external.binlog""" + Invoke-Exe $dotnetExe -Arguments "restore $env:TP_ROOT_DIR\src\package\external\external.csproj --packages $env:TP_PACKAGES_DIR -v:minimal -warnaserror -p:Version=$TPB_Version -bl:""$env:TP_OUT_DIR\log\$Configuration\external.binlog""" Write-Log ".. .. Restore-Package: Complete." Write-Log "Restore-Package: Complete. {$(Get-ElapsedTime($timer))}" } @@ -221,11 +219,147 @@ function Invoke-Exe { [Parameter(Mandatory)] [string] $Command, [string] $Arguments, - [int[]] $IgnoreExitCode + [int[]] $IgnoreExitCode, + [switch] $CaptureOutput ) - Write-Verbose "Invoking: $Command $Arguments" - & $Command ($Arguments -split ' ') - if ($IgnoreExitCode -notcontains $LASTEXITCODE) { + Write-Verbose "Invoking: > $Command $Arguments" + Write-Verbose " > Ignored exit-codes: 0, $($IgnoreExitCode -join ', ')" + + $workingDirectory = [System.IO.Path]::GetDirectoryName($Command) + $process = Start-InlineProcess -Path $Command -Arguments $Arguments -WorkingDirectory $workingDirectory -SuppressOutput:$CaptureOutput.IsPresent + $exitCode = $process.ExitCode + + Write-Verbose "Done. Exit code: $exitCode" + + if ($exitCode -ne 0 -and ($IgnoreExitCode -notcontains $exitCode)) { + if($CaptureOutput) + { + $process.StdErr + } Set-ScriptFailedOnError -Command $Command -Arguments $Arguments } + + if($CaptureOutput) + { + $process.StdOut + } +} + +Add-Type -TypeDefinition @' +using System; +using System.Text; +using System.Collections.Generic; + +public class ProcessOutputter { + private readonly ConsoleColor _color; + private readonly List _output; + private int nullCount = 0; + + public ProcessOutputter (ConsoleColor color, bool suppressOutput = false) { + _color = color; + _output = new List(); + + OutputHandler = (s, e) => { + AppendLine(e.Data); + + if (!suppressOutput) { + var fg = Console.ForegroundColor; + try + { + Console.ForegroundColor = _color; + Console.WriteLine(e.Data); + } + finally + { + Console.ForegroundColor = fg; + } + } + }; + } + + public ProcessOutputter () { + _output = new List(); + OutputHandler = (s, e) => AppendLine(e.Data); + } + + public System.Diagnostics.DataReceivedEventHandler OutputHandler { get; private set; } + public IEnumerable Output { get { return _output; } } + + private void AppendLine(string line) { + if (string.IsNullOrEmpty(line)) { + nullCount++; + return; + } + + while (nullCount > 0) { + --nullCount; + _output.Add(string.Empty); + } + + _output.Add(line); + } +} +'@ + +function Start-InlineProcess { + param ( + [string] + $Path, + + [string] + $WorkingDirectory, + + [string] + $Arguments, + + [switch] + $Elevate, + + [switch] + $SuppressOutput + ) + + $processInfo = [System.Diagnostics.ProcessStartInfo]::new() + $processInfo.FileName = $Path + $processInfo.Arguments = $Arguments + $processInfo.WorkingDirectory = $WorkingDirectory + + $processInfo.RedirectStandardError = $true + $processInfo.RedirectStandardOutput = $true + $processInfo.UseShellExecute = $false + + if ($Elevate) { + $processInfo.Verb = "runas" + } + + $outputHandler = [ProcessOutputter]::new("White", $SuppressOutput.IsPresent) + $errorHandler = [ProcessOutputter]::new("Red", $SuppressOutput.IsPresent) + $outputDataReceived = $outputHandler.OutputHandler + $errorDataReceivedEvent = $errorHandler.OutputHandler + + $process = New-Object System.Diagnostics.Process + $process.EnableRaisingEvents = $true + $process.add_OutputDataReceived($outputDataReceived) + $process.add_ErrorDataReceived($errorDataReceivedEvent) + + try { + $process.StartInfo = $processInfo + if (-not $process.Start()) { + return $null + } + $process.BeginOutputReadLine() + $process.BeginErrorReadLine() + $process.WaitForExit() + + return @{ + ExitCode = $process.ExitCode + StdOut = $outputHandler.Output + StdErr = $errorHandler.Output + } + } + finally { + $process.remove_OutputDataReceived($outputDataReceived) + $process.remove_ErrorDataReceived($errorDataReceived) + $process.Dispose() + } } \ No newline at end of file