diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..986aba0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,5 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 498fe0f..53b24c7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -151,14 +151,14 @@ Additional references: (See [Contributing to documentation related to PowerShell](#contributing-to-documentation-related-to-powershell) for more info.) * If your change adds a new source file, ensure the appropriate copyright and license headers is on top. It is standard practice to have both a copyright and license notice for each source file. - * For `.h`, `.cpp`, and `.cs` files use: + * For `.h`, `.cpp`, `.cs`, and `.rc` files use: - // Copyright (c) Microsoft Corporation. All rights reserved. + // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. * For `.ps1` and `.psm1` files use: - # Copyright (c) Microsoft Corporation. All rights reserved. + # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. * If your change adds a new module manifest (.psd1 file), ensure that: @@ -166,7 +166,7 @@ Additional references: ```powershell Author = "PowerShell" Company = "Microsoft Corporation" - Copyright = "Copyright (c) Microsoft Corporation. All rights reserved." + Copyright = "Copyright (c) Microsoft Corporation." ``` ### Pull Request - Work in Progress diff --git a/.github/SECURITY.md b/.github/SECURITY.md new file mode 100644 index 0000000..f941d30 --- /dev/null +++ b/.github/SECURITY.md @@ -0,0 +1,41 @@ + + +## Security + +Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin) and [PowerShell](https://github.com/PowerShell). + +If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. + +## Reporting Security Issues + +**Please do not report security vulnerabilities through public GitHub issues.** + +Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). + +If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). + +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). + +Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: + + * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) + * Full paths of source file(s) related to the manifestation of the issue + * The location of the affected source code (tag/branch/commit or direct URL) + * Any special configuration required to reproduce the issue + * Step-by-step instructions to reproduce the issue + * Proof-of-concept or exploit code (if possible) + * Impact of the issue, including how an attacker might exploit the issue + +This information will help us triage your report more quickly. + +If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. + +## Preferred Languages + +We prefer all communications to be in English. + +## Policy + +Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). + + diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..89754d2 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,62 @@ +name: "CodeQL" + +on: + push: + branches: [master] + pull_request: + # The branches below must be a subset of the branches above + branches: [master] + +defaults: + run: + shell: pwsh + +permissions: + contents: read + +jobs: + analyze: + permissions: + actions: read # for github/codeql-action/init to get workflow details + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/analyze to upload SARIF results + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + # Override automatic language detection by changing the below list + # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] + language: ['cpp'] + # Learn more... + # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection + + steps: + - name: Checkout repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: '0' + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@2d790406f505036ef40ecba973cc774a50395aac # v3.25.13 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + - run: | + Get-ChildItem -Path env: + name: Capture Environment + + - run: | + Import-Module ./build.psm1 + Start-PSBootstrap + Start-BuildNativeUnixBinaries + name: Build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@2d790406f505036ef40ecba973cc774a50395aac # v3.25.13 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e06cb4c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +dotnet-install.sh +dotnet-uninstall-pkgs.sh +src/powershell-unix diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 886f748..bfb3233 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -9,12 +9,12 @@ variables: resources: - repo: self clean: true -phases: -- phase: Build - - queue: - name: Hosted Ubuntu 1604 - parallel: 2 +jobs: +- job: Build + pool: + vmImage: ubuntu-latest + displayName: Linux + strategy: matrix: Linux ARM Native: buildName: ubuntu.16.04-arm diff --git a/.vsts-ci/mac.yml b/.vsts-ci/mac.yml index af762aa..d5c130e 100644 --- a/.vsts-ci/mac.yml +++ b/.vsts-ci/mac.yml @@ -9,11 +9,13 @@ variables: resources: - repo: self clean: true -phases: -- phase: Build - queue: - name: Hosted macOS +jobs: +- job: Build + + pool: + vmImage: macOs-latest + steps: - powershell: | tools/releaseBuild/PowershellNative.ps1 -Arch osx -Configuration Release -RepoRoot $(Build.SourcesDirectory) -TargetLocation "$(System.ArtifactsDirectory)/Packages" -Verbose diff --git a/.vsts-ci/misc-analysis.yml b/.vsts-ci/misc-analysis.yml index d8a63fb..ec0b46e 100644 --- a/.vsts-ci/misc-analysis.yml +++ b/.vsts-ci/misc-analysis.yml @@ -14,8 +14,19 @@ pr: - release* resources: -- repo: self - clean: true + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance jobs: -- template: templates/credscan.yml \ No newline at end of file +- job: Compliance_Job + pool: + vmImage: windows-latest + steps: + - checkout: self + clean: true + - checkout: ComplianceRepo + clean: true + - template: ci-compliance.yml@ComplianceRepo diff --git a/.vsts-ci/templates/credscan.yml b/.vsts-ci/templates/credscan.yml deleted file mode 100644 index 3aa26a3..0000000 --- a/.vsts-ci/templates/credscan.yml +++ /dev/null @@ -1,31 +0,0 @@ -parameters: - pool: 'Hosted VS2017' - jobName: 'credscan' - displayName: Secret Scan - -jobs: -- job: ${{ parameters.jobName }} - pool: - name: ${{ parameters.pool }} - - displayName: ${{ parameters.displayName }} - - steps: - - powershell: Write-Host "##vso[build.updatebuildnumber]$env:BUILD_SOURCEBRANCHNAME-$env:BUILD_SOURCEVERSION-$((get-date).ToString("yyyyMMddhhmmss"))" - displayName: Set Build Name for Non-PR - condition: ne(variables['Build.Reason'], 'PullRequest') - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Scan for secrets' - inputs: - debugMode: false - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Secret Scan Logs to Build Artifacts' - continueOnError: true - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@1 - displayName: 'Check for failures' - inputs: - CredScan: true - ToolLogsNotFoundAction: Error \ No newline at end of file diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index 5d34caa..1ccfab5 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -9,12 +9,16 @@ variables: resources: - repo: self clean: true -phases: -- phase: Build - queue: - name: Hosted Windows Container - parallel: 4 +jobs: +- job: Build + + pool: + name: PS-PowerShell-x64 + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + + strategy: matrix: Windows x86: buildName: x86 @@ -27,21 +31,9 @@ phases: steps: - powershell: | - choco install cmake.install --installargs 'ADD_CMAKE_TO_PATH=System' - displayName: Install cmake - condition: succeeded() - - powershell: | - choco install windows-sdk-10.1 - displayName: Install Windows SDK 10.1 - condition: succeeded() - - powershell: | - Invoke-WebRequest "https://aka.ms/vs/15/release/vs_BuildTools.exe" -OutFile vs_BuildTools.exe -UseBasicParsing - Start-Process -FilePath 'vs_BuildTools.exe' -ArgumentList '--quiet', '--norestart', '--locale en-US', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM', '--add Microsoft.VisualStudio.Component.VC.Tools.ARM64', '--includeRecommended', '--add Microsoft.VisualStudio.Workload.VCTools', '--add Microsoft.VisualStudio.Component.Windows10SDK.16299.Desktop.arm', '--add Microsoft.VisualStudio.Component.VC.ATL.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATLMFC.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.ATL.ARM64.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.ARM.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.ARM64.Spectre', '--add Microsoft.VisualStudio.Component.VC.Runtimes.x86.x64.Spectre' -Wait - Remove-Item .\vs_BuildTools.exe - Remove-Item -Force -Recurse 'C:\Program Files (x86)\Microsoft Visual Studio\Installer' $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' Write-Host "##vso[task.prependpath]$vsPath" - displayName: Install Visual Studio 2017 + displayName: Set Visual Studio Path condition: succeeded() - powershell: | $cmakeBinPath = "$env:ProgramFiles\CMake\bin\" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index dddfa22..686e5e7 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,8 +1,10 @@ -# Code of Conduct +# Microsoft Open Source Code of Conduct -This project has adopted the [Microsoft Open Source Code of Conduct][conduct-code]. -For more information see the [Code of Conduct FAQ][conduct-FAQ] or contact [opencode@microsoft.com][conduct-email] with any additional questions or comments. +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -[conduct-code]: http://opensource.microsoft.com/codeofconduct/ -[conduct-FAQ]: http://opensource.microsoft.com/codeofconduct/faq/ -[conduct-email]: mailto:opencode@microsoft.com +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns +- Employees can reach out at [aka.ms/opensource/moderation-support](https://aka.ms/opensource/moderation-support) diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..b2f52a2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 930ea36..9121018 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,11 @@ char *GetSomeString() The CoreFX team has an excellent guide for [UNIX Interop][]. [UNIX Interop]: https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/interop-guidelines.md#unix-shims + +## Code of Conduct + +Please see our [Code of Conduct](CODE_OF_CONDUCT.md) before participating in this project. + +## Security Policy + +For any security issues, please see our [Security Policy](.github/SECURITY.md). diff --git a/assets/Powershell_black_64.png b/assets/Powershell_black_64.png new file mode 100644 index 0000000..53bbbee Binary files /dev/null and b/assets/Powershell_black_64.png differ diff --git a/build.psm1 b/build.psm1 index 5319c0d..9c4ec58 100644 --- a/build.psm1 +++ b/build.psm1 @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. +# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # On Unix paths is separated by colon @@ -136,13 +136,15 @@ function Get-EnvironmentInformation $environment += @{'IsUbuntu16' = $Environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '16.04'} $environment += @{'IsUbuntu17' = $Environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '17.10'} $environment += @{'IsUbuntu18' = $Environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '18.04'} + $environment += @{'IsUbuntu22' = $Environment.IsUbuntu -and $LinuxInfo.VERSION_ID -match '22.04'} $environment += @{'IsCentOS' = $LinuxInfo.ID -match 'centos' -and $LinuxInfo.VERSION_ID -match '7'} $environment += @{'IsFedora' = $LinuxInfo.ID -match 'fedora' -and $LinuxInfo.VERSION_ID -ge 24} + $environment += @{'IsRedHat' = $LinuxInfo.ID -match 'rhel'} $environment += @{'IsOpenSUSE' = $LinuxInfo.ID -match 'opensuse'} $environment += @{'IsSLES' = $LinuxInfo.ID -match 'sles'} $environment += @{'IsOpenSUSE13' = $Environmenst.IsOpenSUSE -and $LinuxInfo.VERSION_ID -match '13'} $environment += @{'IsOpenSUSE42.1' = $Environment.IsOpenSUSE -and $LinuxInfo.VERSION_ID -match '42.1'} - $environment += @{'IsRedHatFamily' = $Environment.IsCentOS -or $Environment.IsFedora} + $environment += @{'IsRedHatFamily' = $Environment.IsCentOS -or $Environment.IsFedora -or $Environment.IsRedHat} $environment += @{'IsSUSEFamily' = $Environment.IsSLES -or $Environment.IsOpenSUSE} $environment += @{'IsAlpine' = $LinuxInfo.ID -match 'alpine'} @@ -462,6 +464,11 @@ function Start-BuildNativeUnixBinaries { Start-NativeExecution { cmake -DCMAKE_TOOLCHAIN_FILE="./arm64.toolchain.cmake" . } Start-NativeExecution { make -j } } + elseif ($IsMacOS) { + Start-NativeExecution { cmake -DCMAKE_TOOLCHAIN_FILE="./macos.toolchain.cmake" . } + Start-NativeExecution { make -j } + Start-NativeExecution { ctest --verbose } + } else { Start-NativeExecution { cmake -DCMAKE_BUILD_TYPE=Debug . } Start-NativeExecution { make -j } @@ -524,16 +531,7 @@ function Start-BuildPowerShellNativePackage [Parameter(Mandatory = $true)] [ValidateScript({Test-Path $_ -PathType Leaf})] - [string] $macOSZipPath, - - [Parameter(Mandatory = $true)] - [ValidateScript({Test-Path $_ -PathType Leaf})] - [string] $psrpZipPath, - - [Parameter(Mandatory = $true)] - [string] $NuGetOutputPath, - - [switch] $SkipCleanup = $false + [string] $macOSZipPath ) if(-not (Test-Path $PackageRoot)) @@ -554,7 +552,6 @@ function Start-BuildPowerShellNativePackage $BinFolderLinuxARM64 = Join-Path $tempExtractionPath "LinuxARM64" $BinFolderLinuxAlpine = Join-Path $tempExtractionPath "LinuxAlpine" $BinFolderMacOS = Join-Path $tempExtractionPath "MacOS" - $BinFolderPSRP = Join-Path $tempExtractionPath "PSRP" Expand-Archive -Path $WindowsX64ZipPath -DestinationPath $BinFolderX64 -Force Expand-Archive -Path $WindowsX86ZipPath -DestinationPath $BinFolderX86 -Force @@ -565,11 +562,10 @@ function Start-BuildPowerShellNativePackage Expand-Archive -Path $LinuxARMZipPath -DestinationPath $BinFolderLinuxARM -Force Expand-Archive -Path $LinuxARM64ZipPath -DestinationPath $BinFolderLinuxARM64 -Force Expand-Archive -Path $macOSZipPath -DestinationPath $BinFolderMacOS -Force - Expand-Archive -Path $psrpZipPath -DestinationPath $BinFolderPSRP -Force PlaceWindowsNativeBinaries -PackageRoot $PackageRoot -BinFolderX64 $BinFolderX64 -BinFolderX86 $BinFolderX86 -BinFolderARM $BinFolderARM -BinFolderARM64 $BinFolderARM64 - PlaceUnixBinaries -PackageRoot $PackageRoot -BinFolderLinux $BinFolderLinux -BinFolderLinuxARM $BinFolderLinuxARM -BinFolderLinuxARM64 $BinFolderLinuxARM64 -BinFolderOSX $BinFolderMacOS -BinFolderPSRP $BinFolderPSRP -BinFolderLinuxAlpine $BinFolderLinuxAlpine + PlaceUnixBinaries -PackageRoot $PackageRoot -BinFolderLinux $BinFolderLinux -BinFolderLinuxARM $BinFolderLinuxARM -BinFolderLinuxARM64 $BinFolderLinuxARM64 -BinFolderOSX $BinFolderMacOS -BinFolderLinuxAlpine $BinFolderLinuxAlpine $Nuspec = @' @@ -579,40 +575,53 @@ function Start-BuildPowerShellNativePackage {0} Microsoft Microsoft,PowerShell - true + false Native binaries for PowerShell Core - https://github.com/PowerShell/PowerShell - https://github.com/PowerShell/PowerShell/blob/master/assets/Powershell_black_64.png?raw=true - MIT - PowerShell - en-US - © Microsoft Corporation. All rights reserved. - - - - + https://github.com/PowerShell/PowerShell-Native + {1} + MIT + PowerShell + en-US + © Microsoft Corporation. All rights reserved. + + + + '@ - $Nuspec -f $Version | Out-File -FilePath (Join-Path $PackageRoot -ChildPath 'Microsoft.PowerShell.Native.nuspec') -Force + $iconFileName = "Powershell_black_64.png" + $iconPath = Join-Path $PSScriptRoot -ChildPath "assets\$iconFileName" -Resolve - if(-not (Test-Path $NuGetOutputPath)) - { - $null = New-Item $NuGetOutputPath -Force -Verbose -ItemType Directory - } + Copy-Item $iconPath (Join-Path $PackageRoot -ChildPath $iconFileName) -Verbose + + $Nuspec -f $Version, $iconFileName | Out-File -FilePath (Join-Path $PackageRoot -ChildPath 'Microsoft.PowerShell.Native.nuspec') -Force +} + +<# +.SYNOPSIS + Pack the new nuget package from af folder. +#> +function New-NugetPackage { + param ( + [Parameter(Mandatory)] + [string] $PackageRoot, + + [Parameter(Mandatory)] + [string] $NuGetOutputPath + ) try { + if (-not (Test-Path $NuGetOutputPath)) { + $null = New-Item $NuGetOutputPath -Force -Verbose -ItemType Directory + } + Push-Location $PackageRoot nuget.exe pack . -OutputDirectory $NuGetOutputPath } finally { Pop-Location } - - if(-not $SkipCleanup -and (Test-Path $tempExtractionPath)) - { - Remove-Item $tempExtractionPath -Recurse -Force -ErrorAction SilentlyContinue - } } <# @@ -644,11 +653,7 @@ function PlaceUnixBinaries [Parameter(Mandatory = $true)] [ValidateScript({Test-Path $_ -PathType Container})] - $BinFolderOSX, - - [Parameter(Mandatory = $true)] - [ValidateScript({Test-Path $_ -PathType Container})] - $BinFolderPSRP + $BinFolderOSX ) $RuntimePathLinux = New-Item -ItemType Directory -Path (Join-Path $PackageRoot -ChildPath 'runtimes/linux-x64/native') -Force @@ -662,12 +667,6 @@ function PlaceUnixBinaries Copy-Item "$BinFolderLinuxARM64\*" -Destination $RuntimePathLinuxARM64 -Verbose Copy-Item "$BinFolderLinuxAlpine\*" -Destination $RuntimePathLinuxAlpine -Verbose Copy-Item "$BinFolderOSX\*" -Destination $RuntimePathOSX -Verbose - - ## LinuxARM is not supported by PSRP - Get-ChildItem -Recurse $BinFolderPSRP/*.dylib | ForEach-Object { Copy-Item $_.FullName -Destination $RuntimePathOSX -Verbose } - Get-ChildItem -Recurse $BinFolderPSRP/*.so | ForEach-Object { Copy-Item $_.FullName -Destination $RuntimePathLinux -Verbose } - - Copy-Item $BinFolderPSRP/version.txt -Destination "$PackageRoot/PSRP_version.txt" -Verbose } <# @@ -769,6 +768,7 @@ function Start-PSBuild { # If this parameter is not provided it will get determined automatically. [ValidateSet("win7-x64", "win7-x86", + "osx-arm64", "osx-x64", "linux-x64", "linux-arm", @@ -981,7 +981,7 @@ Fix steps: Start-NativeExecution { & "~/.rcedit/rcedit-x64.exe" $pwshPath --set-icon "$PSScriptRoot\assets\Powershell_black.ico" ` --set-file-version $fileVersion --set-product-version $ReleaseVersion --set-version-string "ProductName" "PowerShell Core 6" ` - --set-version-string "LegalCopyright" "(C) Microsoft Corporation. All Rights Reserved." ` + --set-version-string "LegalCopyright" "(c) Microsoft Corporation." ` --application-manifest "$PSScriptRoot\assets\pwsh.manifest" } | Write-Verbose } @@ -1082,6 +1082,7 @@ function New-PSOptions { [ValidateSet("", "win7-x86", "win7-x64", + "osx-arm64", "osx-x64", "linux-x64", "linux-arm", @@ -1134,7 +1135,12 @@ function New-PSOptions { if ($Environment.IsLinux) { $Runtime = "linux-x64" } elseif ($Environment.IsMacOS) { - $Runtime = "osx-x64" + if ($PSVersionTable.OS.Contains('ARM64')) { + $Runtime = "osx-arm64" + } + else { + $Runtime = "osx-x64" + } } else { $RID = dotnet --info | ForEach-Object { if ($_ -match "RID") { @@ -1887,7 +1893,7 @@ function Install-Dotnet { } function Get-RedHatPackageManager { - if ($Environment.IsCentOS) { + if ($Environment.IsCentOS -or $Environment.IsRedHat) { "yum install -y -q" } elseif ($Environment.IsFedora) { "dnf install -y -q" @@ -1974,13 +1980,13 @@ function Start-PSBootstrap { } } elseif ($Environment.IsRedHatFamily) { # Build tools - $Deps += "which", "curl", "gcc-c++", "cmake", "make" + $Deps += "which", "curl", "wget" # .NET Core required runtime libraries - $Deps += "libicu", "libunwind" + $Deps += "libicu", "openssl-libs" # Packaging tools - if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel' } + if ($Package) { $Deps += "ruby-devel", "rpm-build", "groff", 'libffi-devel', "gcc-c++" } $PackageManager = Get-RedHatPackageManager @@ -2048,7 +2054,7 @@ function Start-PSBootstrap { # Install patched version of curl Start-NativeExecution { brew install curl --with-openssl --with-gssapi } -IgnoreExitcode } elseif ($Environment.IsAlpine) { - $Deps += "build-base", "gcc", "abuild", "binutils", "git", "python", "bash", "cmake" + $Deps += "build-base", "gcc", "abuild", "binutils", "git", "python3", "bash", "cmake" # Install dependencies Start-NativeExecution { apk update } @@ -2136,63 +2142,19 @@ function Start-PSBootstrap { $cmakePresent = precheck 'cmake' $null $sdkPresent = Test-Win10SDK - # Install chocolatey - $chocolateyPath = "$env:AllUsersProfile\chocolatey\bin" - - if(precheck 'choco' $null) { - Write-Log "Chocolatey is already installed. Skipping installation." - } - elseif(($cmakePresent -eq $false) -or ($sdkPresent -eq $false)) { - Write-Log "Chocolatey not present. Installing chocolatey." - if ($Force -or "Install chocolatey via https://chocolatey.org/install.ps1") { - Invoke-Expression ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) - if (-not ($machinePath.ToLower().Contains($chocolateyPath.ToLower()))) { - Write-Log "Adding $chocolateyPath to Path environment variable" - $env:Path += ";$chocolateyPath" - $newMachineEnvironmentPath += ";$chocolateyPath" - } else { - Write-Log "$chocolateyPath already present in Path environment variable" - } - } else { - Write-Error "Chocolatey is required to install missing dependencies. Please install it from https://chocolatey.org/ manually. Alternatively, install cmake and Windows 10 SDK." - return - } - } else { - Write-Log "Skipping installation of chocolatey, cause both cmake and Win 10 SDK are present." - } - # Install cmake - $cmakePath = "${env:ProgramFiles}\CMake\bin" - if($cmakePresent -and !($force.IsPresent)) { - Write-Log "Cmake is already installed. Skipping installation." + #$cmakePath = "${env:ProgramFiles}\CMake\bin" + if($cmakePresent) { + Write-Log "Cmake is already installed." } else { - Write-Log "Cmake not present or -Force used. Installing cmake." - Start-NativeExecution { choco install cmake -y --version 3.10.0 } - if (-not ($machinePath.ToLower().Contains($cmakePath.ToLower()))) { - Write-Log "Adding $cmakePath to Path environment variable" - $env:Path += ";$cmakePath" - $newMachineEnvironmentPath = "$cmakePath;$newMachineEnvironmentPath" - } else { - Write-Log "$cmakePath already present in Path environment variable" - } + throw "Cmake not present." } - # Install Windows 10 SDK - $packageName = "windows-sdk-10.0" - - if (-not $sdkPresent) { - Write-Log "Windows 10 SDK not present. Installing $packageName." - Start-NativeExecution { choco install windows-sdk-10.0 -y } - } else { - Write-Log "Windows 10 SDK present. Skipping installation." + if ($sdkPresent) { + Write-Log "Windows 10 SDK is already installed." } - - # Update path machine environment variable - if ($newMachineEnvironmentPath -ne $machinePath) { - Write-Log "Updating Path machine environment variable" - if ($Force -or $PSCmdlet.ShouldProcess("Update Path machine environment variable to $newMachineEnvironmentPath")) { - [Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE') - } + else { + throw "Windows 10 SDK not present." } } } @@ -2535,6 +2497,7 @@ function Start-CrossGen { [Parameter(Mandatory=$true)] [ValidateSet("win7-x86", "win7-x64", + "osx-arm64", "osx-x64", "linux-x64", "linux-arm", @@ -2605,6 +2568,8 @@ function Start-CrossGen { throw "crossgen is not available for 'linux-arm64'" } elseif ($Environment.IsLinux) { "linux-x64" + } elseif ($Runtime -eq "osx-arm64") { + "osx-arm64" } elseif ($Environment.IsMacOS) { "osx-x64" } diff --git a/src/PowerShell.Core.Instrumentation/PowerShell.Core.Instrumentation.man b/src/PowerShell.Core.Instrumentation/PowerShell.Core.Instrumentation.man index 350323f..8019539 100644 --- a/src/PowerShell.Core.Instrumentation/PowerShell.Core.Instrumentation.man +++ b/src/PowerShell.Core.Instrumentation/PowerShell.Core.Instrumentation.man @@ -1,4 +1,4 @@ - + - - - - diff --git a/tools/releaseBuild/signing.xml b/tools/releaseBuild/signing.xml deleted file mode 100644 index 426ed7e..0000000 --- a/tools/releaseBuild/signing.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/tools/releaseBuild/updateSigning.ps1 b/tools/releaseBuild/updateSigning.ps1 deleted file mode 100644 index 3241915..0000000 --- a/tools/releaseBuild/updateSigning.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -param( - [string] $SigningXmlPath = (Join-Path -Path $PSScriptRoot -ChildPath 'signing.xml') -) -# Script for use in VSTS to update signing.xml - -# Parse the signing xml -$signingXml = [xml](Get-Content $signingXmlPath) - -# Get any variables to updating 'signType' in the XML -# Define a varabile named `SignType' in VSTS to updating that signing type -# Example: $env:AuthenticodeSignType='newvalue' -# will cause all files with the 'Authenticode' signtype to be updated with the 'newvalue' signtype -$signTypes = @{} -Get-ChildItem -Path env:/*SignType | ForEach-Object -Process { - $signType = $_.Name.ToUpperInvariant().Replace('SIGNTYPE','') - Write-Host "Found SigningType $signType with value $($_.value)" - $signTypes[$signType] = $_.Value -} - -# examine each job in the xml -$signingXml.SignConfigXML.job | ForEach-Object -Process { - # examine each file in the job - $_.file | ForEach-Object -Process { - # if the sign type is one of the variables we found, update it to the new value - $signType = $_.SignType.ToUpperInvariant() - if($signTypes.ContainsKey($signType)) - { - $newSignType = $signTypes[$signType] - Write-Host "Updating $($_.src) to $newSignType" - $_.signType = $newSignType - } - } -} - -$signingXml.Save($signingXmlPath) diff --git a/tools/releaseBuild/vstsBuild.ps1 b/tools/releaseBuild/vstsBuild.ps1 index 23a0ea3..2b81d45 100644 --- a/tools/releaseBuild/vstsBuild.ps1 +++ b/tools/releaseBuild/vstsBuild.ps1 @@ -1,4 +1,4 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. +# Copyright (c) Microsoft Corporation. # Licensed under the MIT License. [cmdletbinding(DefaultParameterSetName = 'Build')] diff --git a/tools/releaseBuild/yaml/compliance.yml b/tools/releaseBuild/yaml/compliance.yml deleted file mode 100644 index 354eeb9..0000000 --- a/tools/releaseBuild/yaml/compliance.yml +++ /dev/null @@ -1,57 +0,0 @@ -steps: - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 - displayName: 'Run BinSkim ' - inputs: - InputType: Basic - AnalyzeTarget: '$(System.ArtifactsDirectory)\uncompressed\*.dll;$(System.ArtifactsDirectory)\uncompressed\*.exe' - AnalyzeSymPath: 'SRV*http://symweb' - AnalyzeVerbose: true - AnalyzeHashes: true - AnalyzeStatistics: true - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run MpCmdRun.exe' - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@1 - displayName: 'Run APIScan' - inputs: - softwareFolder: '$(System.ArtifactsDirectory)\uncompressed' - softwareName: PowerShell - softwareVersionNum: '$(NugetPackageVersion)' - isLargeApp: false - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Run CredScan' - inputs: - debugMode: false - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 - displayName: 'Run PoliCheck' - inputs: - targetType: F - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs' - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 - displayName: 'TSA upload to Codebase: PowerShellNative Stamp: Azure' - inputs: - codebase: NewOrUpdate - tsaStamp: Azure - codeBaseName: PowerShellNative - notificationAlias: '4e50053a.microsoft.com@amer.teams.ms' - codeBaseAdmins: 'adityap;tplunk;dongbow;slee' - instanceUrlAzure: MSAZURE - projectNameMSAZURE: One - areaPath: 'One\MGMT\Azure Automation\PowerShell\PowerShell Core' - iterationPath: 'One\_Default' - uploadFortifySCA: false - uploadFxCop: false - uploadModernCop: false - uploadPREfast: false - uploadRoslyn: false - uploadTSLint: false - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' diff --git a/tools/releaseBuild/yaml/nuget.yml b/tools/releaseBuild/yaml/nuget.yml index bf106f3..a45ec90 100644 --- a/tools/releaseBuild/yaml/nuget.yml +++ b/tools/releaseBuild/yaml/nuget.yml @@ -13,15 +13,6 @@ steps: targetPath: "$(System.ArtifactsDirectory)/signed" displayName: Download signed windows native packages -- powershell: | - $null = New-Item $(PackageRoot) -ItemType Directory -Force -Verbose - if(-not (Test-Path '$(System.ArtifactsDirectory)/release' )) - { - New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/release' -Force - } - Invoke-WebRequest -Uri '$(PSRPBlobUrl)' -OutFile $(System.ArtifactsDirectory)/release/psrp.zip -Verbose - displayName: 'Download PSRP package' - - powershell: 'Get-ChildItem $(System.ArtifactsDirectory)/release' displayName: 'Capture downloaded zips' @@ -43,8 +34,6 @@ steps: Get-ChildItem -Recurse $extractedRoot -File displayName: 'Extract All Zips' -- template: compliance.yml - - template: upload.yml parameters: fileName: 'x64_arm64-symbols.zip' @@ -67,32 +56,7 @@ steps: inputs: versionSpec: 5.3.1 -- powershell: | - $Branch = $env:BUILD_SOURCEBRANCH - $branchOnly = $Branch -replace '^refs/heads/'; - $branchOnly = $branchOnly -replace '[_\-]' - - $packageVersion = if($env:PACKAGE_VERSION -eq 'fromBranch' -or !$env:PACKAGE_VERSION) - { - if($Branch -match '^.*(release[-/])') - { - Write-verbose "release branch:" -verbose - $Branch -replace '^.*(release[-/]v)' - } - else - { - "0.0.0" - } - } - else { - "0.0.0" - } - - $vstsCommandString = "vso[task.setvariable variable=NugetPackageVersion]$packageVersion" - Write-Verbose -Message "setting $Variable to $packageVersion" -Verbose - Write-Host -Object "##$vstsCommandString" - - displayName: Set NuGetPackageVersion +- template: setVersion.yml - powershell: | Import-Module $(Build.SourcesDirectory)\build.psm1 @@ -107,42 +71,49 @@ steps: $LinuxARM64ZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-arm64-symbols.zip' $LinuxAlpineZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-musl-x64-symbols.zip' $macOSZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'osx-symbols.zip' - $psrpZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'psrp.zip' - Start-BuildPowerShellNativePackage -PackageRoot $PackageRoot -Version $(NugetPackageVersion) -WindowsX64ZipPath $WindowsX64ZipPath -WindowsX86ZipPath $WindowsX86ZipPath -WindowsARMZipPath $WindowsARMZipPath -WindowsARM64ZipPath $WindowsARM64ZipPath -LinuxZipPath $LinuxZipPath -LinuxARMZipPath $LinuxARMZipPath -LinuxARM64ZipPath $LinuxARM64ZipPath -LinuxAlpineZipPath $LinuxAlpineZipPath -macOSZipPath $macOSZipPath -psrpZipPath $psrpZipPath -NuGetOutputPath $(NuGetPackagePath) + Start-BuildPowerShellNativePackage -PackageRoot $PackageRoot -Version $(PackageVersion) -WindowsX64ZipPath $WindowsX64ZipPath -WindowsX86ZipPath $WindowsX86ZipPath -WindowsARMZipPath $WindowsARMZipPath -WindowsARM64ZipPath $WindowsARM64ZipPath -LinuxZipPath $LinuxZipPath -LinuxARMZipPath $LinuxARMZipPath -LinuxARM64ZipPath $LinuxARM64ZipPath -LinuxAlpineZipPath $LinuxAlpineZipPath -macOSZipPath $macOSZipPath - displayName: 'Build NuGet package' + displayName: 'Collate NuGet package structure' -- powershell: 'Get-Childitem $(NuGetPackagePath)' - displayName: 'Capture nuget package' - -- powershell: | - ## Get the nuget file paths - $files = (Get-ChildItem $(NuGetPackagePath)\*.nupkg).FullName - - if($files.Count -lt 1) { throw "No packages available to publish" } - - & $(Build.SourcesDirectory)\tools\releaseBuild\generatePackgeSigning.ps1 -Path "$(NuGetPackagePath)\NugetSigning.xml" -NuPkgFiles $files +- ${{ if ne(variables['SkipSigning'], 'True') }}: + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)\NugetPackage' + Build_Repository_Uri: https://github.com/PowerShell/PowerShell-Native + PackageName: Microsoft.PowerShell.Native + PackageVersion: $(PackageVersion) - displayName: 'Generate signing file' +- pwsh: | + Import-Module $(Build.SourcesDirectory)\build.psm1 + $PackageRoot = "$(System.ArtifactsDirectory)\NugetPackage" + New-NugetPackage -PackageRoot $PackageRoot -NuGetOutputPath $(NuGetPackagePath) + displayName: Build NuGet package -- powershell: 'Get-Content $(NuGetPackagePath)\NugetSigning.xml' - displayName: 'Capture package signing file' +- powershell: 'Get-Childitem $(NuGetPackagePath)' + displayName: 'Capture nuget package' -- task: PkgESCodeSign@10 - displayName: 'Package ES - CodeSign $(NuGetPackagePath)\NugetSigning.xml' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(NuGetPackagePath)\NugetSigning.xml' - inPathRoot: ' $(NuGetPackagePath)' - outPathRoot: '$(System.ArtifactsDirectory)\signed' - condition: ne(variables['SKIP_SIGNING'], 'True') +- template: EsrpSign.yml@ComplianceRepo + parameters: + # the folder which contains the binaries to sign + buildOutputPath: $(NuGetPackagePath) + # the location to put the signed output + signOutputPath: '$(System.ArtifactsDirectory)\signed' + # the certificate ID to use + certificateId: "CP-401405" + # The file pattern to use + # If not using minimatch: comma separated, with * supported + # If using minimatch: newline separated, with !, **, and * supported. + # See link in the useMinimatch comments. + pattern: '*.nupkg' + # decides if the task should use minimatch for the pattern matching. + # https://github.com/isaacs/minimatch#features + useMinimatch: false - powershell: | Copy-Item $(NuGetPackagePath)\*.nupkg -DestinationPath '$(System.ArtifactsDirectory)\signed' -Force -Verbose displayName: Copy unsigned nuget packge as signing is disabled - condition: eq(variables['SKIP_SIGNING'], 'True') + condition: eq(variables['SkipSigning'], 'True') - powershell: | Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 -Force @@ -160,5 +131,5 @@ steps: - template: uploadArtifact.yml parameters: artifactPath: '$(System.ArtifactsDirectory)\signed' - artifactFilter: 'Microsoft.PowerShell.Native.$(NugetPackageVersion).nupkg' + artifactFilter: 'Microsoft.PowerShell.Native.$(PackageVersion).nupkg' artifactName: 'finalResults' diff --git a/tools/releaseBuild/yaml/releaseBuild.yml b/tools/releaseBuild/yaml/releaseBuild.yml index 2330cf2..16e0161 100644 --- a/tools/releaseBuild/yaml/releaseBuild.yml +++ b/tools/releaseBuild/yaml/releaseBuild.yml @@ -1,8 +1,23 @@ +trigger: none + variables: - AuthenticodeSignType: '400' - BuildConfiguration: 'Release' - NuGetPackagePath: '$(System.ArtifactsDirectory)/NuGetRelease' - PackageRoot: '$(System.ArtifactsDirectory)/Packages' + - name: BuildConfiguration + value: 'Release' + - name: NuGetPackagePath + value: '$(System.ArtifactsDirectory)/NuGetRelease' + - name: PackageRoot + value: '$(System.ArtifactsDirectory)/Packages' + - group: ApiScan + - group: ApiScanMeta + - group: SymbolPublish + - group: Azure Blob variable group + +resources: + repositories: + - repository: ComplianceRepo + type: github + endpoint: ComplianceGHRepo + name: PowerShell/compliance stages: - stage: Build @@ -11,10 +26,9 @@ stages: jobs: - job: BuildWin pool: - name: PowerShell + name: PowerShell1ES demands: - - DotNetFramework - - Agent.Image + - ImageOverride -equals PSMMS2019-PS-Native-Secure displayName: Windows strategy: matrix: @@ -27,12 +41,37 @@ stages: x64ARM64: ARCHITECTURE: x64_arm64 steps: - - template: windows.yml + - template: windows-build.yml + + - job: SignWin + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + + displayName: Sign Windows + variables: + - group: ESRP + dependsOn: BuildWin + strategy: + matrix: + x64: + ARCHITECTURE: x64 + x86: + ARCHITECTURE: x86 + x64ARM: + ARCHITECTURE: x64_arm + x64ARM64: + ARCHITECTURE: x64_arm64 + steps: + - template: windows-sign.yml - job: BuildLinux displayName: Linux pool: - vmImage: ubuntu-16.04 + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure strategy: matrix: UbuntuArm: @@ -48,25 +87,146 @@ stages: - job: BuildMac displayName: Build Native Binaries on macOS - pool: Hosted Mac Internal + pool: + vmImage: macOS-Latest steps: - template: mac.yml - job: BuildNuGetPkg displayName: Build NuGet Package dependsOn: - - BuildWin + - SignWin - BuildLinux - BuildMac pool: - name: PowerShell + name: PowerShell1ES demands: - - DotNetFramework - - Agent.Image + - ImageOverride -equals PSMMS2019-PS-Native-Secure + variables: + - group: ESRP steps: - template: nuget.yml +- stage: PublishSymbol + displayName: Publish pwrshplugin symbols + dependsOn: Build + jobs: + - job: Symbols_Job + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + steps: + - download: current + artifact: release + + - pwsh: | + Write-Verbose -Verbose "Enumerating $(Pipeline.Workspace)/release" + $downloadedArtifacts = Get-ChildItem -Recurse "$(Pipeline.Workspace)/release" + $downloadedArtifacts + + $expandedRoot = New-Item -Path "$(Pipeline.Workspace)/expanded" -ItemType Directory -Verbose + $symbolsRoot = New-Item -Path "$(Pipeline.Workspace)/symbols" -ItemType Directory -Verbose + + $downloadedArtifacts | Where-Object { $_.Name -like 'x*-symbols.zip'} | ForEach-Object { + $destFolder = New-Item -Path "$expandedRoot/$($_.BaseName)/" -ItemType Directory -Verbose + Expand-Archive -Path $_.FullName -DestinationPath $destFolder -Force + + Get-ChildItem -Path $destFolder -Recurse -Filter '*.pdb' | ForEach-Object { + $runtimeFolder = New-Item -Path "$symbolsRoot/$($destFolder.BaseName)/" -ItemType Directory -Verbose + Copy-Item -Path $_.FullName -Destination $runtimeFolder -Verbose + } + } + + Write-Verbose -Verbose "Enumerating $symbolsRoot" + Get-ChildItem -Path $symbolsRoot -Recurse + + $vstsCommandString = "vso[task.setvariable variable=SymbolsPath]$symbolsRoot" + Write-Verbose -Message "$vstsCommandString" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Expand and capture symbols folders + + - task: PublishSymbols@2 + inputs: + symbolsFolder: '$(SymbolsPath)' + searchPattern: '**/*.pdb' + indexSources: false + publishSymbols: true + symbolServerType: teamServices + detailedLog: true + +- stage: compliance + displayName: Compliance + dependsOn: Build + jobs: + - job: Compliance_Job + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + steps: + - checkout: self + clean: true + - checkout: ComplianceRepo + clean: true + + - download: current + artifact: release + + - download: current + artifact: signed + + - template: setVersion.yml + + - pwsh: | + Get-ChildItem -Path 'ENV:' + displayName: Capture environment + + - powershell: 'Get-ChildItem $(Pipeline.Workspace)/release' + displayName: 'Capture downloaded zips' + + - powershell: | + $extractedRoot = New-Item $(Pipeline.Workspace)/uncompressed -ItemType Directory -Force -Verbose + Get-ChildItem $(Pipeline.Workspace)/release/*.zip | ForEach-Object { + $baseName = $_.BaseName + if($baseName -match 'x64_arm') { + Write-Verbose "Skipping expanding file $_.Name" -Verbose + } + else { + $folderPath = Join-Path $extractedRoot $baseName + Expand-Archive $_.FullName -DestinationPath $folderPath -Force + } + } + Write-Host "Extracted files:" + Get-ChildItem -Recurse $extractedRoot -File + displayName: 'Extract All Zips' + + - pwsh: | + Write-Verbose -Verbose "$(PackageVersion)" + displayName: 'Write the package version' + + - template: assembly-module-compliance.yml@ComplianceRepo + parameters: + # binskim + AnalyzeTarget: '$(Pipeline.Workspace)/uncompressed/*.dll' + AnalyzeSymPath: 'SRV*' + # component-governance + sourceScanPath: '$(Build.SourcesDirectory)/powershell-native' + # credscan + suppressionsFile: '' + # TermCheck + optionsRulesDBPath: '' + optionsFTPath: '' + # tsa-upload + codeBaseName: 'PowerShellNative' + # selections + softwareName: 'PowerShellNative' + softwareNameFolder: '$(Pipeline.Workspace)/uncompressed' + softwareVersion: '$(PackageVersion)' + connectionString: RunAs=App;AppId=$(APIScanClient);TenantId=$(APIScanTenant);AppKey=$(APIScanSecret) + APIScan: true # set to false when not using Windows APIs. + - template: publish.yml parameters: stageName: AzArtifactsFeed diff --git a/tools/releaseBuild/yaml/setVersion.yml b/tools/releaseBuild/yaml/setVersion.yml new file mode 100644 index 0000000..fb755da --- /dev/null +++ b/tools/releaseBuild/yaml/setVersion.yml @@ -0,0 +1,32 @@ +steps: +- pwsh: | + $Branch = $env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + Write-Verbose -Verbose "Branch == $Branch" + Write-Verbose -Verbose "BranchOnly == $branchOnly" + + $packageVersion = if($env:PACKAGE_VERSION -eq 'fromBranch' -or !$env:PACKAGE_VERSION) + { + if($Branch -match '^.*(release[-/])') + { + Write-verbose "release branch:" -verbose + $Branch -replace '^.*(release[-/]v)' + } + else + { + Write-Verbose -Verbose "Branch is not a release branch. Defaulting to 99.99.99" + "99.99.99" + } + } + else { + Write-Verbose -Verbose "PACKAGE_VERSION is not equal to 'fromBranch'" + "99.99.99" + } + + $vstsCommandString = "vso[task.setvariable variable=PackageVersion]$packageVersion" + Write-Verbose -Message "setting $Variable to $packageVersion" -Verbose + Write-Host -Object "##$vstsCommandString" + + displayName: Set version variable diff --git a/tools/releaseBuild/yaml/upload.yml b/tools/releaseBuild/yaml/upload.yml index 256bde8..af7100a 100644 --- a/tools/releaseBuild/yaml/upload.yml +++ b/tools/releaseBuild/yaml/upload.yml @@ -3,12 +3,44 @@ parameters: sourcePath: '' steps: -- task: AzureFileCopy@2 + +- powershell: | + [System.Net.ServicePointManager]::SecurityProtocol = + [System.Net.ServicePointManager]::SecurityProtocol -bor + [System.Security.Authentication.SslProtocols]::Tls12 -bor + [System.Security.Authentication.SslProtocols]::Tls11 + + Get-PackageProvider -Name NuGet -ForceBootstrap + Install-Module PowerShellGet -RequiredVersion 2.2.5 -SkipPublisherCheck -Force + displayName: Update PowerShell Get + +- powershell: | + $modules = 'Az.Accounts', 'Az.Storage', 'Az.Resources' + foreach($module in $modules) { + if(!(get-module $module -listavailable)) { + Write-Verbose "installing $module..." -verbose + Install-Module $module -force -AllowClobber + } else { + #Write-Verbose "updating $module..." -verbose + #Update-Module $module -verbose + } + } + displayName: Install PowerShell modules + +- powershell: | + $azureRMModule = Get-Module -Name AzureRM -ListAvailable -ErrorAction SilentlyContinue + if ($azureRMModule) { + Uninstall-AzureRm + } + displayName: Uninstall Uninstall-AzureRm + continueOnError: true + +- task: AzureFileCopy@4 displayName: 'Upload ${{ parameters.fileName }} to azure blob' inputs: SourcePath: '${{ parameters.sourcePath }}' - azureSubscription: 'MGMT-Powershell-CICDInfra (94cf12ad-4fe9-490b-b281-0a260198a4e0)' + azureSubscription: '$(AzureFileCopySubscription)' Destination: AzureBlob - storage: pscoretestdata + storage: '$(StorageAccount)' ContainerName: 'PowerShell-Native-Symbols' BlobPrefix: '$(Build.SourceBranchName)' diff --git a/tools/releaseBuild/yaml/windows-build.yml b/tools/releaseBuild/yaml/windows-build.yml new file mode 100644 index 0000000..f1ed011 --- /dev/null +++ b/tools/releaseBuild/yaml/windows-build.yml @@ -0,0 +1,16 @@ +steps: + - powershell: | + $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' + Write-Host "##vso[task.prependpath]$vsPath" + displayName: Set Visual Studio Path + condition: succeeded() + - powershell: | + $cmakeBinPath = "$env:ProgramFiles\CMake\bin\" + if(Test-Path $cmakeBinPath) { $env:Path = "$cmakeBinPath;$env:PATH" } else { throw "CMake not installed under $cmakeBinPath" } + $(Build.SourcesDirectory)\tools\releaseBuild\PowerShellNative.ps1 -RepoRoot $(Build.SourcesDirectory) -TargetLocation "$(System.ArtifactsDirectory)\Packages" -Arch $(ARCHITECTURE) -Configuration Release -Symbols + displayName: Start build - $(ARCHITECTURE) + condition: succeeded() + - powershell: | + Write-Host "##vso[artifact.upload containerfolder=release;artifactname=release]$(System.ArtifactsDirectory)\Packages\$(ARCHITECTURE)-symbols.zip" + displayName: Upload artifacts + condition: succeeded() diff --git a/tools/releaseBuild/yaml/windows-sign.yml b/tools/releaseBuild/yaml/windows-sign.yml new file mode 100644 index 0000000..05acec2 --- /dev/null +++ b/tools/releaseBuild/yaml/windows-sign.yml @@ -0,0 +1,58 @@ +steps: +- task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: release + +- powershell: | + $zipFilePath = '$(System.ArtifactsDirectory)\release\$(ARCHITECTURE)-symbols.zip' + Get-ChildItem $zipFilePath -Verbose + + Expand-Archive $zipFilePath -Destination '$(System.ArtifactsDirectory)\Expanded' -Force + + $vstsCommandString = "vso[task.setvariable variable=Symbols]$(System.ArtifactsDirectory)\Expanded" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Expand artifact $(ARCHITECTURE)-symbols.zip + +- powershell: | + $vstsCommandString = "vso[task.setvariable variable=SignedOutput]$(System.ArtifactsDirectory)\Signed" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: Define signedOutput variable + +- template: EsrpSign.yml@ComplianceRepo + parameters: + # the folder which contains the binaries to sign + buildOutputPath: $(Symbols) + # the location to put the signed output + signOutputPath: '$(SignedOutput)' + # the certificate ID to use + certificateId: "CP-230012" + # The file pattern to use + # If not using minimatch: comma separated, with * supported + # If using minimatch: newline separated, with !, **, and * supported. + # See link in the useMinimatch comments. + pattern: '*.dll' + # decides if the task should use minimatch for the pattern matching. + # https://github.com/isaacs/minimatch#features + useMinimatch: false + +- powershell: | + Compress-Archive -Path '$(SignedOutput)\*' -DestinationPath '$(SignedOutput)\win-$(ARCHITECTURE).zip' + displayName: Compress signed binaries + condition: eq(variables['SkipSigning'], 'false') + +- powershell: | + Get-ChildItem -Path '$(Symbols)\*' -Recurse | Copy-Item -Destination '$(SignedOutput)' -Force -Verbose + displayName: Copy unsigned binaries as signing is skipped + condition: eq(variables['SkipSigning'], 'True') + +- template: uploadArtifact.yml + parameters: + artifactPath: '$(SignedOutput)' + artifactFilter: 'win-*.zip' + artifactName: 'signed' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + displayName: 'Run MpCmdRun.exe' diff --git a/tools/releaseBuild/yaml/windows.yml b/tools/releaseBuild/yaml/windows.yml deleted file mode 100644 index da55144..0000000 --- a/tools/releaseBuild/yaml/windows.yml +++ /dev/null @@ -1,94 +0,0 @@ -steps: -- task: PkgESSetupBuild@10 - displayName: 'Initialize build' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - useDfs: false - productName: PowerShellCore - branchVersion: true - disableWorkspace: true - disableBuildTools: true - disableNugetPack: true - condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) - -- powershell: | - docker container prune --force - docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | %{docker container rm --force --volumes $_} - docker image ls --format '{{ json .}}' | ConvertFrom-Json |% { - if($_.tag -eq '') - { - $formatString = 'yyyy-MM-dd HH:mm:ss zz00' - $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) - $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) - if($createdAt -lt (Get-Date).adddays(-1)) - { - docker image rm $_.ID - } - } - } - - Write-verbose "--docker info---" -verbose - docker info - Write-verbose "--docker image ls---" -verbose - docker image ls - Write-verbose "--docker container ls --all---" -verbose - docker container ls --all - Write-verbose "--git branch ---" -verbose - git branch - exit 0 - displayName: 'Docker cleanup and get environment' - continueOnError: true - -- task: PowerShell@2 - displayName: 'Build Windows Universal' - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/tools/releaseBuild/vstsBuild.ps1 - arguments: '-Name win-$(ARCHITECTURE)-symbols -Verbose' - -- task: PowerShell@2 - displayName: 'Update Signing Xml' - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)/tools/releaseBuild/updateSigning.ps1 - -- powershell: | - $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(ARCHITECTURE)}" - Write-Host "sending " + $vstsCommandString - Write-Host "##$vstsCommandString" - displayName: 'Get Symbols Path' - -- task: PkgESCodeSign@10 - displayName: 'CodeSign $(ARCHITECTURE)' - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' - inPathRoot: '$(Symbols)' - outPathRoot: '$(Symbols)\Signed' - condition: ne(variables['SKIP_SIGNING'], 'True') - -- powershell: | - Compress-Archive -Path '$(Symbols)\Signed\*' -DestinationPath '$(Symbols)\Signed\win-$(ARCHITECTURE).zip' - displayName: Compress signed binaries - condition: ne(variables['SKIP_SIGNING'], 'True') - -- powershell: | - Get-ChildItem -Path '$(Symbols)\*' -Recurse | Copy-Item -Destination '$(Symbols)\Signed' -Force -Verbose - displayName: Copy unsigned binaries as signing is skipped - condition: eq(variables['SKIP_SIGNING'], 'True') - -- template: uploadArtifact.yml - parameters: - artifactPath: '$(Symbols)\Signed' - artifactFilter: 'win-*.zip' - artifactName: 'signed' - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'Run MpCmdRun.exe' - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection' - inputs: - sourceScanPath: '$(Build.SourcesDirectory)'