diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cb65b0dd2749..09bbe2f736b9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,7 +23,7 @@ /sdk/eventhub/ @annatisch @yunhaoling @YijunXieMS # PRLabel: %Storage -/sdk/storage/ @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft +/sdk/storage/ @amishra-dev @zezha-msft @annatisch @rakshith91 @xiafu-msft @kasobol-msft /sdk/applicationinsights/ @alexeldeib diff --git a/NOTICE.txt b/NOTICE.txt index 29fd1c19d257..e7ab85b06abd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,7 +1,17 @@ -This file list any third-party libraries or other resources that may be -distributed under licenses different than the Azure SDK for Python software. +NOTICES AND INFORMATION +Do Not Translate or Localize -In the event that we accidentally failed to list a required notice, please -bring it to our attention by opening an issue. +This software incorporates material from third parties. Microsoft makes certain +open source code available at https://3rdpartysource.microsoft.com, or you may +send a check or money order for US $5.00, including the product name, the open +source component name, and version number, to: -The attached notices are provided for information only. \ No newline at end of file +Source Code Compliance Team +Microsoft Corporation +One Microsoft Way +Redmond, WA 98052 +USA + +Notwithstanding any other terms, you may reverse engineer this software to the +extent required to debug changes to any libraries licensed under the GNU Lesser +General Public License. diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 9e281d04eb66..90e496d2e83d 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -154,6 +154,9 @@ if ($ProvisionerApplicationId) { $subscriptionArgs = if ($SubscriptionId) { @{SubscriptionId = $SubscriptionId} } + else { + @{} + } $provisionerAccount = Retry { Connect-AzAccount -Force:$Force -Tenant $TenantId -Credential $provisionerCredential -ServicePrincipal -Environment $Environment @subscriptionArgs @@ -161,7 +164,11 @@ if ($ProvisionerApplicationId) { $exitActions += { Write-Verbose "Logging out of service principal '$($provisionerAccount.Context.Account)'" - $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + + # Only attempt to disconnect if the -WhatIf flag was not set. Otherwise, this call is not necessary and will fail. + if ($PSCmdlet.ShouldProcess($ProvisionerApplicationId)) { + $null = Disconnect-AzAccount -AzureContext $provisionerAccount.Context + } } } @@ -176,7 +183,6 @@ if ($TestApplicationId -and !$TestApplicationOid) { } } - # If the ServiceDirectory is an absolute path use the last directory name # (e.g. D:\foo\bar\ -> bar) $serviceName = if (Split-Path -IsAbsolute $ServiceDirectory) { @@ -229,6 +235,13 @@ if ($resourceGroup.ProvisioningState -eq 'Succeeded') { # New-AzResourceGroup would've written an error and stopped the pipeline by default anyway. Write-Verbose "Successfully created resource group '$($resourceGroup.ResourceGroupName)'" } +elseif (($resourceGroup -eq $null) -and (-not $PSCmdlet.ShouldProcess($resourceGroupName))) { + # If the -WhatIf flag was passed, there will be no resource group created. Fake it. + $resourceGroup = [PSCustomObject]@{ + ResourceGroupName = $resourceGroupName + Location = $Location + } +} # Populate the template parameters and merge any additional specified. $templateParameters = @{ @@ -299,6 +312,9 @@ foreach ($templateFile in $templateFiles) { "$($serviceDirectoryPrefix)RESOURCE_GROUP" = $resourceGroup.ResourceGroupName; "$($serviceDirectoryPrefix)LOCATION" = $resourceGroup.Location; "$($serviceDirectoryPrefix)ENVIRONMENT" = $context.Environment.Name; + "$($serviceDirectoryPrefix)AZURE_AUTHORITY_HOST" = $context.Environment.ActiveDirectoryAuthority; + "$($serviceDirectoryPrefix)RESOURCE_MANAGER_URL" = $context.Environment.ResourceManagerUrl; + "$($serviceDirectoryPrefix)SERVICE_MANAGEMENT_URL" = $context.Environment.ServiceManagementUrl; } foreach ($key in $deployment.Outputs.Keys) { @@ -331,7 +347,7 @@ foreach ($templateFile in $templateFiles) { } else { - + if (!$CI) { # Write an extra new line to isolate the environment variables for easy reading. Log "Persist the following environment variables based on your detected shell ($shell):`n" @@ -340,7 +356,7 @@ foreach ($templateFile in $templateFiles) { foreach ($key in $deploymentOutputs.Keys) { $value = $deploymentOutputs[$key] - + if ($CI) { # Treat all ARM template output variables as secrets since "SecureString" variables do not set values. # In order to mask secrets but set environment variables for any given ARM template, we set variables twice as shown below. diff --git a/eng/common/scripts/ChangeLog-Operations.ps1 b/eng/common/scripts/ChangeLog-Operations.ps1 new file mode 100644 index 000000000000..8397e1d0a05a --- /dev/null +++ b/eng/common/scripts/ChangeLog-Operations.ps1 @@ -0,0 +1,122 @@ +# Common Changelog Operations + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)" + +# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG +function Get-ChangeLogEntries { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation + ) + + $changeLogEntries = @{} + if (!(Test-Path $ChangeLogLocation)) { + Write-Error "ChangeLog[${ChangeLogLocation}] does not exist" + return $null + } + + try { + $contents = Get-Content $ChangeLogLocation + # walk the document, finding where the version specifiers are and creating lists + $changeLogEntry = $null + foreach ($line in $contents) { + if ($line -match $RELEASE_TITLE_REGEX) { + $changeLogEntry = [pscustomobject]@{ + ReleaseVersion = $matches["version"] + ReleaseStatus = $matches["releaseStatus"] + ReleaseTitle = $line + ReleaseContent = @() # Release content without the version title + } + $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry + } + else { + if ($changeLogEntry) { + $changeLogEntry.ReleaseContent += $line + } + } + } + } + catch { + Write-Host "Error parsing $ChangeLogLocation." + Write-Host $_.Exception.Message + } + return $changeLogEntries +} + +# Returns single changeLogEntry object containing the ChangeLog for a particular version +function Get-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation + + if ($changeLogEntries -and $changeLogEntries.ContainsKey($VersionString)) { + return $changeLogEntries[$VersionString] + } + return $null +} + +#Returns the changelog for a particular version as string +function Get-ChangeLogEntryAsString { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + return ChangeLogEntryAsString $changeLogEntry +} + +function ChangeLogEntryAsString($changeLogEntry) { + if (!$changeLogEntry) { + return "[Missing change log entry]" + } + [string]$releaseTitle = $changeLogEntry.ReleaseTitle + [string]$releaseContent = $changeLogEntry.ReleaseContent -Join [Environment]::NewLine + return $releaseTitle, $releaseContent -Join [Environment]::NewLine +} + +function Confirm-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString, + [boolean]$ForRelease = $false + ) + + $changeLogEntry = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + + if (!$changeLogEntry) { + Write-Error "ChangeLog[${ChangeLogLocation}] does not have an entry for version ${VersionString}." + return $false + } + + Write-Host "Found the following change log entry for version '${VersionString}' in [${ChangeLogLocation}]." + Write-Host "-----" + Write-Host (ChangeLogEntryAsString $changeLogEntry) + Write-Host "-----" + + if ([System.String]::IsNullOrEmpty($changeLogEntry.ReleaseStatus)) { + Write-Error "Entry does not have a correct release status. Please ensure the status is set to a date '(yyyy-MM-dd)' or '(Unreleased)' if not yet released." + return $false + } + + if ($ForRelease -eq $True) { + if ($changeLogEntry.ReleaseStatus -eq "(Unreleased)") { + Write-Error "Entry has no release date set. Please ensure to set a release date with format 'yyyy-MM-dd'." + return $false + } + + if ([System.String]::IsNullOrWhiteSpace($changeLogEntry.ReleaseContent)) { + Write-Error "Entry has no content. Please ensure to provide some content of what changed in this version." + return $false + } + } + return $true +} \ No newline at end of file diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1 new file mode 100644 index 000000000000..cba89157a879 --- /dev/null +++ b/eng/common/scripts/Package-Properties.ps1 @@ -0,0 +1,182 @@ +# Helper functions for retireving useful information from azure-sdk-for-* repo +# Example Use : Import-Module .\eng\common\scripts\modules +class PackageProps +{ + [string]$pkgName + [string]$pkgVersion + [string]$pkgDirectoryPath + [string]$pkgServiceName + [string]$pkgReadMePath + [string]$pkgChangeLogPath + [string]$pkgGroup + + PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName) + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName) + } + + PackageProps([string]$pkgName,[string]$pkgVersion,[string]$pkgDirectoryPath,[string]$pkgServiceName,[string]$pkgGroup="") + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName, $pkgGroup) + } + + hidden [void]Initialize( + [string]$pkgName, + [string]$pkgVersion, + [string]$pkgDirectoryPath, + [string]$pkgServiceName + ) + { + $this.pkgName = $pkgName + $this.pkgVersion = $pkgVersion + $this.pkgDirectoryPath = $pkgDirectoryPath + $this.pkgServiceName = $pkgServiceName + + if (Test-Path (Join-Path $pkgDirectoryPath "README.md")) + { + $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md" + } + else + { + $this.pkgReadMePath = $null + } + + if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md")) + { + $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md" + } + else + { + $this.pkgChangeLogPath = $null + } + } + + hidden [void]Initialize( + [string]$pkgName, + [string]$pkgVersion, + [string]$pkgDirectoryPath, + [string]$pkgServiceName, + [string]$pkgGroup + ) + { + $this.Initialize($pkgName, $pkgVersion, $pkgDirectoryPath, $pkgServiceName) + $this.pkgGroup = $pkgGroup + } +} + +# Takes package name and service Name +# Returns important properties of the package as related to the language repo +# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +# Note: python is required for parsing python package properties. +function Get-PkgProperties +{ + Param + ( + [Parameter(Mandatory=$true)] + [string]$PackageName, + [Parameter(Mandatory=$true)] + [string]$ServiceName + ) + + $pkgDirectoryName = $null + $pkgDirectoryPath = $null + $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName + if (!(Test-Path $serviceDirectoryPath)) + { + Write-Error "Service Directory $ServiceName does not exist" + exit 1 + } + + $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory + + foreach ($directory in $directoriesPresent) + { + $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name + if ($ExtractPkgProps) + { + $pkgProps = &$ExtractPkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName + } + else + { + Write-Error "The function '${ExtractPkgProps}' was not found." + } + + if ($pkgProps -ne $null) + { + return $pkgProps + } + } + Write-Error "Failed to retrive Properties for $PackageName" +} + +# Takes ServiceName and Repo Root Directory +# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified +# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +function Get-AllPkgProperties ([string]$ServiceName=$null) +{ + $pkgPropsResult = @() + + if ([string]::IsNullOrEmpty($ServiceName)) + { + $searchDir = Join-Path $RepoRoot "sdk" + foreach ($dir in (Get-ChildItem $searchDir -Directory)) + { + $serviceDir = Join-Path $searchDir $dir.Name + + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -pkgPropsResult $pkgPropsResult + } + } + } + } + else + { + $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -pkgPropsResult $pkgPropsResult + } + } + } + + return $pkgPropsResult +} + +function Operate-OnPackages ($activePkgList, $serviceName, [Array]$pkgPropsResult) +{ + foreach ($pkg in $activePkgList) + { + $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName + $pkgPropsResult += $pkgProps + } + return $pkgPropsResult +} + +function Get-PkgListFromYml ($ciYmlPath) +{ + $ProgressPreference = "SilentlyContinue" + Register-PSRepository -Default -ErrorAction:SilentlyContinue + Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser + $ciYmlContent = Get-Content $ciYmlPath -Raw + $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered + if ($ciYmlObj.Contains("stages")) + { + $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"] + } + elseif ($ciYmlObj.Contains("extends")) + { + $artifactsInCI = $ciYmlObj["extends"]["parameters"]["Artifacts"] + } + if ($artifactsInCI -eq $null) + { + Write-Error "Failed to retrive package names in ci $ciYmlPath" + } + return $artifactsInCI +} \ No newline at end of file diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 40b99e632de7..af7c9401e644 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -406,7 +406,7 @@ function IsPythonPackageVersionPublished($pkgId, $pkgVersion) { # Retrieves the list of all tags that exist on the target repository function GetExistingTags($apiUrl) { try { - return (Invoke-WebRequest -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } + return (Invoke-RestMethod -Method "GET" -Uri "$apiUrl/git/refs/tags" -MaximumRetryCount 3 -RetryIntervalSec 10) | % { $_.ref.Replace("refs/tags/", "") } } catch { Write-Host $_ diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 new file mode 100644 index 000000000000..f1677b21425f --- /dev/null +++ b/eng/common/scripts/common.ps1 @@ -0,0 +1,27 @@ +$global:RepoRoot = Resolve-Path "${PSScriptRoot}..\..\..\.." +$global:EngDir = Join-Path $global:RepoRoot "eng" +$global:EngCommonDir = Join-Path $global:EngDir "common" +$global:EngCommonScriptsDir = Join-Path $global:EngCommonDir "scripts" +$global:EngScriptsDir = Join-Path $global:EngDir "scripts" + +# Import required scripts +. (Join-Path $global:EngCommonScriptsDir SemVer.ps1) +. (Join-Path $global:EngCommonScriptsDir Changelog-Operations.ps1) +. (Join-Path $global:EngCommonScriptsDir Package-Properties.ps1) + +# Setting expected from common languages settings +$global:Language = "Unknown" +$global:PackageRepository = "Unknown" +$global:packagePattern = "Unknown" +$global:MetadataUri = "Unknown" + +# Import common language settings +$EngScriptsLanguageSettings = Join-path $global:EngScriptsDir "Language-Settings.ps1" +if (Test-Path $EngScriptsLanguageSettings) { + . $EngScriptsLanguageSettings +} + +# Transformed Functions +$GetPackageInfoFromRepoFn = "Get-${Language}-PackageInfoFromRepo" +$GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile" +$PublishGithubIODocsFn = "Publish-${Language}-GithubIODocs" \ No newline at end of file diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml index 2c8d11b79b88..629bee369cff 100644 --- a/eng/pipelines/aggregate-reports.yml +++ b/eng/pipelines/aggregate-reports.yml @@ -1,10 +1,6 @@ trigger: none pr: none -variables: - Skip.MyPy: true - Skip.Pylint: true - Skip.ApiStubGen: true jobs: - job: 'ValidateDependencies' @@ -19,7 +15,7 @@ jobs: parameters: Directory: "" - - template: ./templates/steps/analyze.yml + - template: /eng/pipelines/templates/steps/analyze_dependency.yml - task: AzureFileCopy@2 displayName: 'Upload dependency report' diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml index 54fcbf2df067..b1e145929e08 100644 --- a/eng/pipelines/templates/jobs/archetype-sdk-client.yml +++ b/eng/pipelines/templates/jobs/archetype-sdk-client.yml @@ -7,6 +7,7 @@ parameters: ToxEnvParallel: '--tenvparallel' InjectedPackages: '' BuildDocs: true + SkipPythonVersion: '' TestMatrix: Linux_Python27: OSVmImage: 'ubuntu-18.04' @@ -92,8 +93,10 @@ jobs: matrix: ${{ each matrixEntry in parameters.TestMatrix }}: ${{ if or(eq(matrixEntry.value.RunForPR, 'true'), ne(variables['Build.Reason'], 'PullRequest')) }}: - ${{ matrixEntry.key }}: - ${{ insert }}: ${{ matrixEntry.value }} + # Skip python version if any specific service has opted out + ${{ if ne(parameters.SkipPythonVersion, matrixEntry.value.PythonVersion) }}: + ${{ matrixEntry.key }}: + ${{ insert }}: ${{ matrixEntry.value }} pool: vmImage: '$(OSVmImage)' diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 2cda41e8f1f3..5781942a3cec 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -20,6 +20,9 @@ parameters: - name: TargetDocRepoName type: string default: azure-docs-sdk-python +- name: SkipPythonVersion + type: string + default: '' stages: - stage: Build @@ -30,6 +33,7 @@ stages: ToxEnvParallel: ${{parameters.ToxEnvParallel}} BuildDocs: ${{parameters.BuildDocs}} InjectedPackages: ${{parameters.InjectedPackages}} + SkipPythonVersion: ${{parameters.SkipPythonVersion}} # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch. - ${{if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal'))}}: @@ -41,4 +45,4 @@ stages: ArtifactName: packages DocArtifact: documentation TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} \ No newline at end of file + TargetDocRepoName: ${{parameters.TargetDocRepoName}} \ No newline at end of file diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index eae12578bda1..2e1aea91f7f3 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -5,31 +5,7 @@ parameters: AdditionalTestArgs: '' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - inputs: - versionSpec: '$(PythonVersion)' - - - task: DownloadPipelineArtifact@0 - inputs: - artifactName: 'artifacts' - targetPath: $(Build.ArtifactStagingDirectory) - - - script: | - pip install -r eng/ci_tools.txt - ward scan -d $(Build.SourcesDirectory) -c $(Build.SourcesDirectory)/eng/.docsettings.yml - displayName: 'Verify Readmes' - - - pwsh: | - mkdir "$(Build.ArtifactStagingDirectory)/reports" - Copy-Item -Path "$(Build.SourcesDirectory)/eng/common/InterdependencyGraph.html" -Destination "$(Build.ArtifactStagingDirectory)/reports/InterdependencyGraph.html" - displayName: 'Populate Reports Staging Folder' - - - task: PythonScript@0 - displayName: 'Analyze dependencies' - inputs: - scriptPath: 'scripts/analyze_deps.py' - arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/reports/dependencies.html" --dump "$(Build.ArtifactStagingDirectory)/reports/data.js"' + - template: /eng/pipelines/templates/steps/analyze_dependency.yml - task: PythonScript@0 displayName: 'Verify Change Log' @@ -97,9 +73,22 @@ steps: BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} + - task: DownloadPipelineArtifact@0 + condition: ne(variables['Skip.ApiStubGen'],'true') + inputs: + artifactName: 'artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + - template: ../steps/run_apistub.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} BuildTargetingString: ${{ parameters.BuildTargetingString }} TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: ${{parameters.AdditionalTestArgs}} + + - template: ../steps/run_bandit.yml + parameters: + ServiceDirectory: ${{ parameters.ServiceDirectory }} + BuildTargetingString: ${{ parameters.BuildTargetingString }} + TestMarkArgument: ${{ parameters.TestMarkArgument }} + AdditionalTestArgs: ${{parameters.AdditionalTestArgs}} diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml new file mode 100644 index 000000000000..77ab69df6dd3 --- /dev/null +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -0,0 +1,21 @@ +steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + inputs: + versionSpec: '$(PythonVersion)' + + - script: | + pip install -r eng/ci_tools.txt + ward scan -d $(Build.SourcesDirectory) -c $(Build.SourcesDirectory)/eng/.docsettings.yml + displayName: 'Verify Readmes' + + - pwsh: | + mkdir "$(Build.ArtifactStagingDirectory)/reports" + Copy-Item -Path "$(Build.SourcesDirectory)/eng/common/InterdependencyGraph.html" -Destination "$(Build.ArtifactStagingDirectory)/reports/InterdependencyGraph.html" + displayName: 'Populate Reports Staging Folder' + + - task: PythonScript@0 + displayName: 'Analyze dependencies' + inputs: + scriptPath: 'scripts/analyze_deps.py' + arguments: '--verbose --out "$(Build.ArtifactStagingDirectory)/reports/dependencies.html" --dump "$(Build.ArtifactStagingDirectory)/reports/data.js"' \ No newline at end of file diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index a8691c39d809..5ad5273b64bc 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -5,17 +5,6 @@ parameters: AdditionalTestArgs: '' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.7' - condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) - inputs: - versionSpec: '3.7' - - - script: | - pip install -r eng/ci_tools.txt - displayName: 'Prep Environment' - condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) - - task: PythonScript@0 displayName: 'Run Api Stub Generation' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) @@ -28,4 +17,4 @@ steps: --service="${{ parameters.ServiceDirectory }}" --toxenv="apistub" --disablecov - --omit-management \ No newline at end of file + --filter-type="Omit_management" \ No newline at end of file diff --git a/eng/pipelines/templates/steps/run_bandit.yml b/eng/pipelines/templates/steps/run_bandit.yml new file mode 100644 index 000000000000..02e6c5e8cb1d --- /dev/null +++ b/eng/pipelines/templates/steps/run_bandit.yml @@ -0,0 +1,20 @@ +parameters: + BuildTargetingString: 'azure-*' + ServiceDirectory: '' + TestMarkArgument: '' + EnvVars: {} + +steps: + - task: PythonScript@0 + displayName: 'Run Bandit' + inputs: + scriptPath: 'scripts/devops_tasks/setup_execute_tests.py' + arguments: >- + "${{ parameters.BuildTargetingString }}" + --mark_arg="${{ parameters.TestMarkArgument }}" + --service="${{ parameters.ServiceDirectory }}" + --toxenv="bandit" + --disablecov + --filter-type="Omit_management" + env: ${{ parameters.EnvVars }} + condition: and(succeededOrFailed(), ne(variables['Skip.Bandit'],'true')) \ No newline at end of file diff --git a/eng/pipelines/templates/steps/run_pylint.yml b/eng/pipelines/templates/steps/run_pylint.yml index b4b15d966d51..1386c14c00a4 100644 --- a/eng/pipelines/templates/steps/run_pylint.yml +++ b/eng/pipelines/templates/steps/run_pylint.yml @@ -9,8 +9,7 @@ steps: displayName: 'Use Python 3.7' inputs: versionSpec: '3.7' - condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true')) - + condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true')) - script: | pip install -r eng/ci_tools.txt @@ -27,6 +26,6 @@ steps: --service="${{ parameters.ServiceDirectory }}" --toxenv="lint" --disablecov - --omit-management + --filter-type="Omit_management" env: ${{ parameters.EnvVars }} condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true')) \ No newline at end of file diff --git a/eng/test_tools.txt b/eng/test_tools.txt index 566e3d544c0c..4db3e30c49e9 100644 --- a/eng/test_tools.txt +++ b/eng/test_tools.txt @@ -7,6 +7,7 @@ pytest-custom-exit-code==0.3.0 pytest-xdist==1.32.0 # we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed coverage==4.5.4 +bandit==1.6.2 # locking packages defined as deps from azure-sdk-tools or azure-devtools pytoml==0.1.21 diff --git a/eng/tox/run_bandit.py b/eng/tox/run_bandit.py new file mode 100644 index 000000000000..1275bc3efb87 --- /dev/null +++ b/eng/tox/run_bandit.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# This script is used to execute bandit within a tox environment. Depending on which package is being executed against, +# a failure may be suppressed. + +from subprocess import check_call, CalledProcessError +import argparse +import os +import logging +import sys + + +logging.getLogger().setLevel(logging.INFO) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run bandit against target folder.") + + parser.add_argument( + "-t", + "--target", + dest="target_package", + help="The target package directory on disk. The target module passed to bandit will be /azure.", + required=True, + ) + + args = parser.parse_args() + + package_name = os.path.basename(os.path.abspath(args.target_package)) + try: + check_call( + [ + sys.executable, + "-m", + "bandit", + "-r", + os.path.join(args.target_package, "azure"), + "-ll", + ] + ) + except CalledProcessError as e: + logging.error("{} exited with error {}".format(package_name, e.returncode)) + exit(1) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index c279af3e0426..9b38156d47da 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -235,4 +235,16 @@ commands = # install API stub generator {envbindir}/python -m pip install "git+https://github.com/azure/azure-sdk-tools.git#subdirectory=packages/python-packages/api-stub-generator&egg=api-stub-generator" {envbindir}/python -m pip freeze - {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} \ No newline at end of file + {envbindir}/python {toxinidir}/../../../eng/tox/run_apistubgen.py -t {toxinidir} -w {envtmpdir} + + +[testenv:bandit] +skipsdist = false +skip_install = false +usedevelop = false +changedir = {envtmpdir} +deps = + {[base]deps} +commands = + {envbindir}/python -m pip freeze + {envbindir}/python {toxinidir}/../../../eng/tox/run_bandit.py -t {toxinidir} \ No newline at end of file diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 1c41506e1f1f..932e9b801d05 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -71,6 +71,7 @@ lambda_filter_azure_pkg = lambda x: x.startswith("azure") and "-nspkg" not in x omit_mgmt = lambda x: "mgmt" not in x or os.path.basename(x) in MANAGEMENT_PACKAGES_FILTER_EXCLUSIONS + # dict of filter type and filter function omit_funct_dict = { "Build": omit_build, diff --git a/scripts/devops_tasks/setup_execute_tests.py b/scripts/devops_tasks/setup_execute_tests.py index e928f33fdd6b..a099f98fc122 100644 --- a/scripts/devops_tasks/setup_execute_tests.py +++ b/scripts/devops_tasks/setup_execute_tests.py @@ -276,13 +276,14 @@ def execute_global_install_and_test( ) parser.add_argument( - "--omit-management", - dest="omit_management", - default=False, - action="store_true", - help="Flag that indicates to omit any management packages except any management packages that should not be filtered. for e.g azure-mgmt-core", + "--filter-type", + dest="filter_type", + default='Build', + help="Filter type to identify eligible packages. for e.g. packages filtered in Build can pass filter type as Build,", + choices=['Build', "Docs", "Regression", "Omit_management"] ) + args = parser.parse_args() # We need to support both CI builds of everything and individual service @@ -293,10 +294,7 @@ def execute_global_install_and_test( else: target_dir = root_dir - if args.omit_management: - targeted_packages = process_glob_string(args.glob_string, target_dir, "", "Omit_management") - else: - targeted_packages = process_glob_string(args.glob_string, target_dir) + targeted_packages = process_glob_string(args.glob_string, target_dir, "", args.filter_type) extended_pytest_args = [] if len(targeted_packages) == 0: diff --git a/sdk/cosmos/azure-cosmos/tox.ini b/sdk/cosmos/azure-cosmos/tox.ini index 4de20e13f2b5..5c1604df1cf3 100644 --- a/sdk/cosmos/azure-cosmos/tox.ini +++ b/sdk/cosmos/azure-cosmos/tox.ini @@ -7,7 +7,8 @@ envlist = py27, py34 [base] -deps = pytest, -rdev_requirements.txt +deps = pytest + -rdev_requirements.txt [pytest] python_files=test/**.py diff --git a/sdk/identity/azure-identity/azure/identity/_credentials/__init__.py b/sdk/identity/azure-identity/azure/identity/_credentials/__init__.py index baf64e6d5102..78e3e51e4cb5 100644 --- a/sdk/identity/azure-identity/azure/identity/_credentials/__init__.py +++ b/sdk/identity/azure-identity/azure/identity/_credentials/__init__.py @@ -14,7 +14,7 @@ from .azure_cli import AzureCliCredential from .device_code import DeviceCodeCredential from .user_password import UsernamePasswordCredential -from .vscode_credential import VSCodeCredential +from .vscode import VSCodeCredential __all__ = [ diff --git a/sdk/identity/azure-identity/azure/identity/_credentials/default.py b/sdk/identity/azure-identity/azure/identity/_credentials/default.py index a6354cd59da5..a2cadd674ca2 100644 --- a/sdk/identity/azure-identity/azure/identity/_credentials/default.py +++ b/sdk/identity/azure-identity/azure/identity/_credentials/default.py @@ -13,7 +13,7 @@ from .managed_identity import ManagedIdentityCredential from .shared_cache import SharedTokenCacheCredential from .azure_cli import AzureCliCredential -from .vscode_credential import VSCodeCredential +from .vscode import VSCodeCredential try: diff --git a/sdk/identity/azure-identity/azure/identity/_credentials/vscode_credential.py b/sdk/identity/azure-identity/azure/identity/_credentials/vscode.py similarity index 100% rename from sdk/identity/azure-identity/azure/identity/_credentials/vscode_credential.py rename to sdk/identity/azure-identity/azure/identity/_credentials/vscode.py diff --git a/sdk/identity/azure-identity/azure/identity/_internal/__init__.py b/sdk/identity/azure-identity/azure/identity/_internal/__init__.py index 60d776175e4f..236972381529 100644 --- a/sdk/identity/azure-identity/azure/identity/_internal/__init__.py +++ b/sdk/identity/azure-identity/azure/identity/_internal/__init__.py @@ -37,7 +37,7 @@ def get_default_authority(): from .certificate_credential_base import CertificateCredentialBase from .client_secret_credential_base import ClientSecretCredentialBase from .decorators import wrap_exceptions -from .msal_credentials import InteractiveCredential +from .interactive import InteractiveCredential def _scopes_to_resource(*scopes): diff --git a/sdk/identity/azure-identity/azure/identity/_internal/interactive.py b/sdk/identity/azure-identity/azure/identity/_internal/interactive.py new file mode 100644 index 000000000000..4e226bc0c357 --- /dev/null +++ b/sdk/identity/azure-identity/azure/identity/_internal/interactive.py @@ -0,0 +1,197 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ +"""Base class for credentials using MSAL for interactive user authentication""" + +import abc +import base64 +import json +import logging +import time +from typing import TYPE_CHECKING + +import msal +from six.moves.urllib_parse import urlparse +from azure.core.credentials import AccessToken +from azure.core.exceptions import ClientAuthenticationError + +from .msal_credentials import MsalCredential +from .._auth_record import AuthenticationRecord +from .._constants import KnownAuthorities +from .._exceptions import AuthenticationRequiredError, CredentialUnavailableError +from .._internal import wrap_exceptions + +if TYPE_CHECKING: + # pylint:disable=ungrouped-imports,unused-import + from typing import Any, Optional + +_LOGGER = logging.getLogger(__name__) + +_DEFAULT_AUTHENTICATE_SCOPES = { + "https://" + KnownAuthorities.AZURE_CHINA: ("https://management.core.chinacloudapi.cn//.default",), + "https://" + KnownAuthorities.AZURE_GERMANY: ("https://management.core.cloudapi.de//.default",), + "https://" + KnownAuthorities.AZURE_GOVERNMENT: ("https://management.core.usgovcloudapi.net//.default",), + "https://" + KnownAuthorities.AZURE_PUBLIC_CLOUD: ("https://management.core.windows.net//.default",), +} + + +def _decode_client_info(raw): + """Taken from msal.oauth2cli.oidc""" + + raw += "=" * (-len(raw) % 4) + raw = str(raw) # On Python 2.7, argument of urlsafe_b64decode must be str, not unicode. + return base64.urlsafe_b64decode(raw).decode("utf-8") + + +def _build_auth_record(response): + """Build an AuthenticationRecord from the result of an MSAL ClientApplication token request""" + + try: + id_token = response["id_token_claims"] + + if "client_info" in response: + client_info = json.loads(_decode_client_info(response["client_info"])) + home_account_id = "{uid}.{utid}".format(**client_info) + else: + # MSAL uses the subject claim as home_account_id when the STS doesn't provide client_info + home_account_id = id_token["sub"] + + return AuthenticationRecord( + authority=urlparse(id_token["iss"]).netloc, # "iss" is the URL of the issuing tenant + client_id=id_token["aud"], + home_account_id=home_account_id, + tenant_id=id_token["tid"], # tenant which issued the token, not necessarily user's home tenant + username=id_token["preferred_username"], + ) + except (KeyError, ValueError): + # surprising: msal.ClientApplication always requests an id token, whose shape shouldn't change + return None + + +class InteractiveCredential(MsalCredential): + def __init__(self, **kwargs): + self._disable_automatic_authentication = kwargs.pop("disable_automatic_authentication", False) + self._auth_record = kwargs.pop("authentication_record", None) # type: Optional[AuthenticationRecord] + if self._auth_record: + kwargs.pop("client_id", None) # authentication_record overrides client_id argument + tenant_id = kwargs.pop("tenant_id", None) or self._auth_record.tenant_id + super(InteractiveCredential, self).__init__( + client_id=self._auth_record.client_id, + authority=self._auth_record.authority, + tenant_id=tenant_id, + **kwargs + ) + else: + super(InteractiveCredential, self).__init__(**kwargs) + + def get_token(self, *scopes, **kwargs): + # type: (*str, **Any) -> AccessToken + """Request an access token for `scopes`. + + .. note:: This method is called by Azure SDK clients. It isn't intended for use in application code. + + :param str scopes: desired scopes for the access token. This method requires at least one scope. + :rtype: :class:`azure.core.credentials.AccessToken` + :raises CredentialUnavailableError: the credential is unable to attempt authentication because it lacks + required data, state, or platform support + :raises ~azure.core.exceptions.ClientAuthenticationError: authentication failed. The error's ``message`` + attribute gives a reason. + :raises AuthenticationRequiredError: user interaction is necessary to acquire a token, and the credential is + configured not to begin this automatically. Call :func:`authenticate` to begin interactive authentication. + """ + if not scopes: + message = "'get_token' requires at least one scope" + _LOGGER.warning("%s.get_token failed: %s", self.__class__.__name__, message) + raise ValueError(message) + + allow_prompt = kwargs.pop("_allow_prompt", not self._disable_automatic_authentication) + try: + token = self._acquire_token_silent(*scopes, **kwargs) + _LOGGER.info("%s.get_token succeeded", self.__class__.__name__) + return token + except Exception as ex: # pylint:disable=broad-except + if not (isinstance(ex, AuthenticationRequiredError) and allow_prompt): + _LOGGER.warning( + "%s.get_token failed: %s", + self.__class__.__name__, + ex, + exc_info=_LOGGER.isEnabledFor(logging.DEBUG), + ) + raise + + # silent authentication failed -> authenticate interactively + now = int(time.time()) + + try: + result = self._request_token(*scopes, **kwargs) + if "access_token" not in result: + message = "Authentication failed: {}".format(result.get("error_description") or result.get("error")) + raise ClientAuthenticationError(message=message) + + # this may be the first authentication, or the user may have authenticated a different identity + self._auth_record = _build_auth_record(result) + except Exception as ex: # pylint:disable=broad-except + _LOGGER.warning( + "%s.get_token failed: %s", self.__class__.__name__, ex, exc_info=_LOGGER.isEnabledFor(logging.DEBUG), + ) + raise + + _LOGGER.info("%s.get_token succeeded", self.__class__.__name__) + return AccessToken(result["access_token"], now + int(result["expires_in"])) + + def authenticate(self, **kwargs): + # type: (**Any) -> AuthenticationRecord + """Interactively authenticate a user. + + :keyword Iterable[str] scopes: scopes to request during authentication, such as those provided by + :func:`AuthenticationRequiredError.scopes`. If provided, successful authentication will cache an access token + for these scopes. + :rtype: ~azure.identity.AuthenticationRecord + :raises ~azure.core.exceptions.ClientAuthenticationError: authentication failed. The error's ``message`` + attribute gives a reason. + """ + + scopes = kwargs.pop("scopes", None) + if not scopes: + if self._authority not in _DEFAULT_AUTHENTICATE_SCOPES: + # the credential is configured to use a cloud whose ARM scope we can't determine + raise CredentialUnavailableError( + message="Authenticating in this environment requires a value for the 'scopes' keyword argument." + ) + + scopes = _DEFAULT_AUTHENTICATE_SCOPES[self._authority] + + _ = self.get_token(*scopes, _allow_prompt=True, **kwargs) + return self._auth_record # type: ignore + + @wrap_exceptions + def _acquire_token_silent(self, *scopes, **kwargs): + # type: (*str, **Any) -> AccessToken + result = None + if self._auth_record: + app = self._get_app() + for account in app.get_accounts(username=self._auth_record.username): + if account.get("home_account_id") != self._auth_record.home_account_id: + continue + + now = int(time.time()) + result = app.acquire_token_silent_with_error(list(scopes), account=account, **kwargs) + if result and "access_token" in result and "expires_in" in result: + return AccessToken(result["access_token"], now + int(result["expires_in"])) + + # if we get this far, result is either None or the content of an AAD error response + if result: + details = result.get("error_description") or result.get("error") + raise AuthenticationRequiredError(scopes, error_details=details) + raise AuthenticationRequiredError(scopes) + + def _get_app(self): + # type: () -> msal.PublicClientApplication + if not self._msal_app: + self._msal_app = self._create_app(msal.PublicClientApplication) + return self._msal_app + + @abc.abstractmethod + def _request_token(self, *scopes, **kwargs): + pass diff --git a/sdk/identity/azure-identity/azure/identity/_internal/linux_vscode_adapter.py b/sdk/identity/azure-identity/azure/identity/_internal/linux_vscode_adapter.py index d31b202678df..e25dfc7bcb90 100644 --- a/sdk/identity/azure-identity/azure/identity/_internal/linux_vscode_adapter.py +++ b/sdk/identity/azure-identity/azure/identity/_internal/linux_vscode_adapter.py @@ -4,9 +4,12 @@ # ------------------------------------ import os import json +import logging import ctypes as ct from .._constants import VSCODE_CREDENTIALS_SECTION +_LOGGER = logging.getLogger(__name__) + def _c_str(string): return ct.c_char_p(string.encode("utf-8")) @@ -96,5 +99,8 @@ def get_credentials(): environment_name = _get_user_settings() credentials = _get_refresh_token(VSCODE_CREDENTIALS_SECTION, environment_name) return credentials - except Exception: # pylint: disable=broad-except + except Exception as ex: # pylint: disable=broad-except + _LOGGER.debug( + 'Exception retrieving VS Code credentials: "%s"', ex, exc_info=_LOGGER.isEnabledFor(logging.DEBUG) + ) return None diff --git a/sdk/identity/azure-identity/azure/identity/_internal/macos_vscode_adapter.py b/sdk/identity/azure-identity/azure/identity/_internal/macos_vscode_adapter.py index 003f1daca549..4db3a38ae423 100644 --- a/sdk/identity/azure-identity/azure/identity/_internal/macos_vscode_adapter.py +++ b/sdk/identity/azure-identity/azure/identity/_internal/macos_vscode_adapter.py @@ -4,9 +4,12 @@ # ------------------------------------ import os import json +import logging from msal_extensions.osx import Keychain, KeychainError from .._constants import VSCODE_CREDENTIALS_SECTION +_LOGGER = logging.getLogger(__name__) + def _get_user_settings_path(): app_data_folder = os.environ["USER"] @@ -37,5 +40,8 @@ def get_credentials(): environment_name = _get_user_settings() credentials = _get_refresh_token(VSCODE_CREDENTIALS_SECTION, environment_name) return credentials - except Exception: # pylint: disable=broad-except + except Exception as ex: # pylint: disable=broad-except + _LOGGER.debug( + 'Exception retrieving VS Code credentials: "%s"', ex, exc_info=_LOGGER.isEnabledFor(logging.DEBUG) + ) return None diff --git a/sdk/identity/azure-identity/azure/identity/_internal/msal_credentials.py b/sdk/identity/azure-identity/azure/identity/_internal/msal_credentials.py index ac2966c93e1d..fd5034acd4bb 100644 --- a/sdk/identity/azure-identity/azure/identity/_internal/msal_credentials.py +++ b/sdk/identity/azure-identity/azure/identity/_internal/msal_credentials.py @@ -3,22 +3,13 @@ # Licensed under the MIT License. # ------------------------------------ import abc -import base64 -import json -import logging -import time import msal -from six.moves.urllib_parse import urlparse from azure.core.credentials import AccessToken -from azure.core.exceptions import ClientAuthenticationError from .msal_client import MsalClient from .persistent_cache import load_user_cache -from .._constants import KnownAuthorities -from .._exceptions import AuthenticationRequiredError, CredentialUnavailableError -from .._internal import get_default_authority, normalize_authority, wrap_exceptions -from .._auth_record import AuthenticationRecord +from .._internal import get_default_authority, normalize_authority try: ABC = abc.ABC @@ -34,48 +25,6 @@ # pylint:disable=ungrouped-imports,unused-import from typing import Any, Mapping, Optional, Type, Union -_LOGGER = logging.getLogger(__name__) - -_DEFAULT_AUTHENTICATE_SCOPES = { - "https://" + KnownAuthorities.AZURE_CHINA: ("https://management.core.chinacloudapi.cn//.default",), - "https://" + KnownAuthorities.AZURE_GERMANY: ("https://management.core.cloudapi.de//.default",), - "https://" + KnownAuthorities.AZURE_GOVERNMENT: ("https://management.core.usgovcloudapi.net//.default",), - "https://" + KnownAuthorities.AZURE_PUBLIC_CLOUD: ("https://management.core.windows.net//.default",), -} - - -def _decode_client_info(raw): - """Taken from msal.oauth2cli.oidc""" - - raw += "=" * (-len(raw) % 4) - raw = str(raw) # On Python 2.7, argument of urlsafe_b64decode must be str, not unicode. - return base64.urlsafe_b64decode(raw).decode("utf-8") - - -def _build_auth_record(response): - """Build an AuthenticationRecord from the result of an MSAL ClientApplication token request""" - - try: - id_token = response["id_token_claims"] - - if "client_info" in response: - client_info = json.loads(_decode_client_info(response["client_info"])) - home_account_id = "{uid}.{utid}".format(**client_info) - else: - # MSAL uses the subject claim as home_account_id when the STS doesn't provide client_info - home_account_id = id_token["sub"] - - return AuthenticationRecord( - authority=urlparse(id_token["iss"]).netloc, # "iss" is the URL of the issuing tenant - client_id=id_token["aud"], - home_account_id=home_account_id, - tenant_id=id_token["tid"], # tenant which issued the token, not necessarily user's home tenant - username=id_token["preferred_username"], - ) - except (KeyError, ValueError): - # surprising: msal.ClientApplication always requests an id token, whose shape shouldn't change - return None - class MsalCredential(ABC): """Base class for credentials wrapping MSAL applications""" @@ -123,132 +72,3 @@ def _create_app(self, cls): ) return app - - -class InteractiveCredential(MsalCredential): - def __init__(self, **kwargs): - self._disable_automatic_authentication = kwargs.pop("disable_automatic_authentication", False) - self._auth_record = kwargs.pop("authentication_record", None) # type: Optional[AuthenticationRecord] - if self._auth_record: - kwargs.pop("client_id", None) # authentication_record overrides client_id argument - tenant_id = kwargs.pop("tenant_id", None) or self._auth_record.tenant_id - super(InteractiveCredential, self).__init__( - client_id=self._auth_record.client_id, - authority=self._auth_record.authority, - tenant_id=tenant_id, - **kwargs - ) - else: - super(InteractiveCredential, self).__init__(**kwargs) - - def get_token(self, *scopes, **kwargs): - # type: (*str, **Any) -> AccessToken - """Request an access token for `scopes`. - - .. note:: This method is called by Azure SDK clients. It isn't intended for use in application code. - - :param str scopes: desired scopes for the access token. This method requires at least one scope. - :rtype: :class:`azure.core.credentials.AccessToken` - :raises CredentialUnavailableError: the credential is unable to attempt authentication because it lacks - required data, state, or platform support - :raises ~azure.core.exceptions.ClientAuthenticationError: authentication failed. The error's ``message`` - attribute gives a reason. - :raises AuthenticationRequiredError: user interaction is necessary to acquire a token, and the credential is - configured not to begin this automatically. Call :func:`authenticate` to begin interactive authentication. - """ - if not scopes: - message = "'get_token' requires at least one scope" - _LOGGER.warning("%s.get_token failed: %s", self.__class__.__name__, message) - raise ValueError(message) - - allow_prompt = kwargs.pop("_allow_prompt", not self._disable_automatic_authentication) - try: - token = self._acquire_token_silent(*scopes, **kwargs) - _LOGGER.info("%s.get_token succeeded", self.__class__.__name__) - return token - except Exception as ex: # pylint:disable=broad-except - if not (isinstance(ex, AuthenticationRequiredError) and allow_prompt): - _LOGGER.warning( - "%s.get_token failed: %s", - self.__class__.__name__, - ex, - exc_info=_LOGGER.isEnabledFor(logging.DEBUG), - ) - raise - - # silent authentication failed -> authenticate interactively - now = int(time.time()) - - try: - result = self._request_token(*scopes, **kwargs) - if "access_token" not in result: - message = "Authentication failed: {}".format(result.get("error_description") or result.get("error")) - raise ClientAuthenticationError(message=message) - - # this may be the first authentication, or the user may have authenticated a different identity - self._auth_record = _build_auth_record(result) - except Exception as ex: # pylint:disable=broad-except - _LOGGER.warning( - "%s.get_token failed: %s", self.__class__.__name__, ex, exc_info=_LOGGER.isEnabledFor(logging.DEBUG), - ) - raise - - _LOGGER.info("%s.get_token succeeded", self.__class__.__name__) - return AccessToken(result["access_token"], now + int(result["expires_in"])) - - def authenticate(self, **kwargs): - # type: (**Any) -> AuthenticationRecord - """Interactively authenticate a user. - - :keyword Iterable[str] scopes: scopes to request during authentication, such as those provided by - :func:`AuthenticationRequiredError.scopes`. If provided, successful authentication will cache an access token - for these scopes. - :rtype: ~azure.identity.AuthenticationRecord - :raises ~azure.core.exceptions.ClientAuthenticationError: authentication failed. The error's ``message`` - attribute gives a reason. - """ - - scopes = kwargs.pop("scopes", None) - if not scopes: - if self._authority not in _DEFAULT_AUTHENTICATE_SCOPES: - # the credential is configured to use a cloud whose ARM scope we can't determine - raise CredentialUnavailableError( - message="Authenticating in this environment requires a value for the 'scopes' keyword argument." - ) - - scopes = _DEFAULT_AUTHENTICATE_SCOPES[self._authority] - - _ = self.get_token(*scopes, _allow_prompt=True, **kwargs) - return self._auth_record # type: ignore - - @wrap_exceptions - def _acquire_token_silent(self, *scopes, **kwargs): - # type: (*str, **Any) -> AccessToken - result = None - if self._auth_record: - app = self._get_app() - for account in app.get_accounts(username=self._auth_record.username): - if account.get("home_account_id") != self._auth_record.home_account_id: - continue - - now = int(time.time()) - result = app.acquire_token_silent_with_error(list(scopes), account=account, **kwargs) - if result and "access_token" in result and "expires_in" in result: - return AccessToken(result["access_token"], now + int(result["expires_in"])) - - # if we get this far, result is either None or the content of an AAD error response - if result: - details = result.get("error_description") or result.get("error") - raise AuthenticationRequiredError(scopes, error_details=details) - raise AuthenticationRequiredError(scopes) - - @abc.abstractmethod - def _request_token(self, *scopes, **kwargs): - # type: (*str, **Any) -> dict - """Request an access token via a non-silent MSAL token acquisition method, returning that method's result""" - - def _get_app(self): - # type: () -> msal.PublicClientApplication - if not self._msal_app: - self._msal_app = self._create_app(msal.PublicClientApplication) - return self._msal_app diff --git a/sdk/identity/azure-identity/azure/identity/_internal/win_vscode_adapter.py b/sdk/identity/azure-identity/azure/identity/_internal/win_vscode_adapter.py index 1a0f4c86a05b..a3fb9a2ec5b9 100644 --- a/sdk/identity/azure-identity/azure/identity/_internal/win_vscode_adapter.py +++ b/sdk/identity/azure-identity/azure/identity/_internal/win_vscode_adapter.py @@ -4,6 +4,7 @@ # ------------------------------------ import os import json +import logging import ctypes as ct from .._constants import VSCODE_CREDENTIALS_SECTION @@ -12,6 +13,7 @@ except (IOError, ValueError): pass +_LOGGER = logging.getLogger(__name__) SUPPORTED_CREDKEYS = set(("Type", "TargetName", "Persist", "UserName", "Comment", "CredentialBlob")) @@ -49,8 +51,7 @@ def _read_credential(service_name, account_name): if _advapi.CredReadW(target, 1, 0, ct.byref(cred_ptr)): cred_blob = cred_ptr.contents.CredentialBlob cred_blob_size = cred_ptr.contents.CredentialBlobSize - password_as_list = [int.from_bytes(cred_blob[pos : pos + 1], "little") for pos in range(0, cred_blob_size)] - cred = "".join(map(chr, password_as_list)) + cred = "".join(map(chr, cred_blob[:cred_blob_size])) _advapi.CredFree(cred_ptr) return cred return None @@ -81,5 +82,8 @@ def get_credentials(): environment_name = _get_user_settings() credentials = _get_refresh_token(VSCODE_CREDENTIALS_SECTION, environment_name) return credentials - except Exception: # pylint: disable=broad-except + except Exception as ex: # pylint: disable=broad-except + _LOGGER.debug( + 'Exception retrieving VS Code credentials: "%s"', ex, exc_info=_LOGGER.isEnabledFor(logging.DEBUG) + ) return None diff --git a/sdk/identity/azure-identity/azure/identity/aio/_credentials/__init__.py b/sdk/identity/azure-identity/azure/identity/aio/_credentials/__init__.py index d9146708694d..c5553286c582 100644 --- a/sdk/identity/azure-identity/azure/identity/aio/_credentials/__init__.py +++ b/sdk/identity/azure-identity/azure/identity/aio/_credentials/__init__.py @@ -11,7 +11,7 @@ from .client_secret import ClientSecretCredential from .shared_cache import SharedTokenCacheCredential from .azure_cli import AzureCliCredential -from .vscode_credential import VSCodeCredential +from .vscode import VSCodeCredential __all__ = [ diff --git a/sdk/identity/azure-identity/azure/identity/aio/_credentials/default.py b/sdk/identity/azure-identity/azure/identity/aio/_credentials/default.py index 2c9d37f68e50..8cbe49b30cae 100644 --- a/sdk/identity/azure-identity/azure/identity/aio/_credentials/default.py +++ b/sdk/identity/azure-identity/azure/identity/aio/_credentials/default.py @@ -13,7 +13,7 @@ from .environment import EnvironmentCredential from .managed_identity import ManagedIdentityCredential from .shared_cache import SharedTokenCacheCredential -from .vscode_credential import VSCodeCredential +from .vscode import VSCodeCredential if TYPE_CHECKING: from typing import Any diff --git a/sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode_credential.py b/sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode.py similarity index 98% rename from sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode_credential.py rename to sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode.py index c27247aedce3..f49d10cbd152 100644 --- a/sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode_credential.py +++ b/sdk/identity/azure-identity/azure/identity/aio/_credentials/vscode.py @@ -9,7 +9,7 @@ from ..._constants import AZURE_VSCODE_CLIENT_ID from .._internal.aad_client import AadClient from .._internal.decorators import log_get_token_async -from ..._credentials.vscode_credential import get_credentials +from ..._credentials.vscode import get_credentials if TYPE_CHECKING: # pylint:disable=unused-import,ungrouped-imports diff --git a/sdk/identity/azure-identity/tests/test_default.py b/sdk/identity/azure-identity/tests/test_default.py index 88c477dc593a..9191462da2a4 100644 --- a/sdk/identity/azure-identity/tests/test_default.py +++ b/sdk/identity/azure-identity/tests/test_default.py @@ -10,11 +10,11 @@ DefaultAzureCredential, InteractiveBrowserCredential, SharedTokenCacheCredential, + VSCodeCredential, ) from azure.identity._constants import EnvironmentVariables from azure.identity._credentials.azure_cli import AzureCliCredential from azure.identity._credentials.managed_identity import ManagedIdentityCredential -from azure.identity._credentials.vscode_credential import VSCodeCredential import pytest from six.moves.urllib_parse import urlparse diff --git a/sdk/identity/azure-identity/tests/test_default_async.py b/sdk/identity/azure-identity/tests/test_default_async.py index afd89b3f0b8a..a385a5b2a579 100644 --- a/sdk/identity/azure-identity/tests/test_default_async.py +++ b/sdk/identity/azure-identity/tests/test_default_async.py @@ -2,17 +2,19 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # ------------------------------------ -import asyncio import os from unittest.mock import Mock, patch from urllib.parse import urlparse from azure.core.credentials import AccessToken from azure.identity import CredentialUnavailableError -from azure.identity.aio import DefaultAzureCredential, SharedTokenCacheCredential -from azure.identity.aio._credentials.azure_cli import AzureCliCredential -from azure.identity.aio._credentials.managed_identity import ManagedIdentityCredential -from azure.identity.aio._credentials.vscode_credential import VSCodeCredential +from azure.identity.aio import ( + AzureCliCredential, + DefaultAzureCredential, + ManagedIdentityCredential, + SharedTokenCacheCredential, + VSCodeCredential, +) from azure.identity._constants import EnvironmentVariables import pytest diff --git a/sdk/identity/azure-identity/tests/test_interactive_credential.py b/sdk/identity/azure-identity/tests/test_interactive_credential.py index 8bfeaac041a4..645e74f21bd0 100644 --- a/sdk/identity/azure-identity/tests/test_interactive_credential.py +++ b/sdk/identity/azure-identity/tests/test_interactive_credential.py @@ -9,7 +9,7 @@ KnownAuthorities, CredentialUnavailableError, ) -from azure.identity._internal.msal_credentials import InteractiveCredential +from azure.identity._internal import InteractiveCredential from msal import TokenCache import pytest diff --git a/sdk/identity/azure-identity/tests/test_vscode_credential.py b/sdk/identity/azure-identity/tests/test_vscode_credential.py index ed5a0f5af235..ef43604017ae 100644 --- a/sdk/identity/azure-identity/tests/test_vscode_credential.py +++ b/sdk/identity/azure-identity/tests/test_vscode_credential.py @@ -9,7 +9,7 @@ from azure.core.pipeline.policies import SansIOHTTPPolicy from azure.identity._constants import EnvironmentVariables from azure.identity._internal.user_agent import USER_AGENT -from azure.identity._credentials.vscode_credential import get_credentials +from azure.identity._credentials.vscode import get_credentials import pytest from six.moves.urllib_parse import urlparse diff --git a/sdk/servicebus/azure-servicebus/samples/async_samples/sample_code_servicebus_async.py b/sdk/servicebus/azure-servicebus/samples/async_samples/sample_code_servicebus_async.py index b4ded73c2eba..e4b9fa76d014 100644 --- a/sdk/servicebus/azure-servicebus/samples/async_samples/sample_code_servicebus_async.py +++ b/sdk/servicebus/azure-servicebus/samples/async_samples/sample_code_servicebus_async.py @@ -22,7 +22,7 @@ async def process_message(message): - print(message) + print(str(message)) def example_create_servicebus_client_async(): @@ -206,14 +206,14 @@ async def example_send_and_receive_async(): async with servicebus_receiver: messages = await servicebus_receiver.peek_messages() for message in messages: - print(message) + print(str(message)) # [END peek_messages_async] # [START receive_async] async with servicebus_receiver: messages = await servicebus_receiver.receive_messages(max_wait_time=5) for message in messages: - print(message) + print(str(message)) await message.complete() # [END receive_async] @@ -240,7 +240,7 @@ async def example_receive_deferred_async(): messages = await servicebus_receiver.receive_messages(max_wait_time=5) for message in messages: deferred_sequenced_numbers.append(message.sequence_number) - print(message) + print(str(message)) await message.defer() received_deferred_msg = await servicebus_receiver.receive_deferred_messages( diff --git a/sdk/servicebus/azure-servicebus/samples/sync_samples/sample_code_servicebus.py b/sdk/servicebus/azure-servicebus/samples/sync_samples/sample_code_servicebus.py index ffa3a68f7f78..1e6393b8f288 100644 --- a/sdk/servicebus/azure-servicebus/samples/sync_samples/sample_code_servicebus.py +++ b/sdk/servicebus/azure-servicebus/samples/sync_samples/sample_code_servicebus.py @@ -18,7 +18,7 @@ def process_message(message): - print(message) + print(str(message)) def example_create_servicebus_client_sync(): @@ -213,7 +213,7 @@ def example_send_and_receive_sync(): with servicebus_receiver: messages = servicebus_receiver.peek_messages() for message in messages: - print(message) + print(str(message)) # [END peek_messages_sync] # [START auto_lock_renew_message_sync] @@ -232,7 +232,7 @@ def example_send_and_receive_sync(): with servicebus_receiver: messages = servicebus_receiver.receive_messages(max_wait_time=5) for message in messages: - print(message) + print(str(message)) message.complete() # [END receive_sync] @@ -269,7 +269,7 @@ def example_receive_deferred_sync(): messages = servicebus_receiver.receive_messages(max_wait_time=5) for message in messages: deferred_sequenced_numbers.append(message.sequence_number) - print(message) + print(str(message)) message.defer() received_deferred_msg = servicebus_receiver.receive_deferred_messages( diff --git a/sdk/servicebus/ci.yml b/sdk/servicebus/ci.yml index 61c369a0270c..29f5e55ae4ef 100644 --- a/sdk/servicebus/ci.yml +++ b/sdk/servicebus/ci.yml @@ -33,3 +33,5 @@ extends: safeName: azuremgmtservicebus - name: azure_servicebus safeName: azureservicebus + SkipPythonVersion: 'pypy3' + diff --git a/sdk/template/azure-template/CHANGELOG.md b/sdk/template/azure-template/CHANGELOG.md index 39cbf1f691c6..c85f69aeddd9 100644 --- a/sdk/template/azure-template/CHANGELOG.md +++ b/sdk/template/azure-template/CHANGELOG.md @@ -1,5 +1,8 @@ # Release History +## 0.0.12 (Unreleased) +- Test a successful Release + ## 0.0.7 (2020-07-02) - Test a successful Release diff --git a/sdk/template/azure-template/azure/template/_version.py b/sdk/template/azure-template/azure/template/_version.py index 61dc99b21913..8fb7d48af3e3 100644 --- a/sdk/template/azure-template/azure/template/_version.py +++ b/sdk/template/azure-template/azure/template/_version.py @@ -1,2 +1,2 @@ # matches SEMVER -VERSION = "0.0.7" \ No newline at end of file +VERSION = "0.0.12" \ No newline at end of file diff --git a/sdk/template/azure-template/sdk_packaging.toml b/sdk/template/azure-template/sdk_packaging.toml index 0d4f9aa5f7d1..901bc8ccbfa6 100644 --- a/sdk/template/azure-template/sdk_packaging.toml +++ b/sdk/template/azure-template/sdk_packaging.toml @@ -1,9 +1,2 @@ [packaging] auto_update = false -package_name = "azure-template" -package_pprint_name = "Template Package" -is_stable = false -is_arm = false - -# Package owners should uncomment and set this doc id. -# package_doc_id = "" diff --git a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md index f08fefb11aa5..82260740944b 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md +++ b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md @@ -1,8 +1,12 @@ # Release History -## 1.0.1 (Unreleased) +## 5.0.1 (Unreleased) +## 5.0.0 (2020-07-27) + +- Re-release of GA version 1.0.0 with an updated version + ## 1.0.0 (2020-06-09) - First stable release of the azure-ai-textanalytics package. Targets the service's v3.0 API. diff --git a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py index f61b5ba71d0d..715b122ebe53 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py +++ b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py @@ -4,4 +4,4 @@ # Licensed under the MIT License. # ------------------------------------ -VERSION = "1.0.1" +VERSION = "5.0.1"