Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into fix_autorest_links * 'master' of https://github.com/Azure/azure-sdk-for-python: (44 commits) Added credscan steps for python (Azure#16136) [Communication] Move core dependency to setup.py from dev_requirements in communication management package (Azure#16077) Increment package version after release of azure_security_attestation (Azure#16218) T2 compute 2021 01 19 (Azure#16246) Update error message in tools repo (Azure#16245) Add LanguageDisplayName variable to LanguageSettings file (Azure#16239) Fix Eventgrid sample (Azure#16217) Adding 2020-09-01-hybrid profile (Azure#14642) Increment version for storage releases (Azure#16152) Increment package version after release of azure_storage_file_datalake (Azure#16157) make the globbing of the packages work in dev_setup from any invocation directory (Azure#16209) [Queues] Generated Queues using newest autorest version (Azure#16148) T2 confluent 2021 01 15 (Azure#16221) we are using a pool not a specific vmImage (Azure#16213) Naming Feedback - part 2 (Azure#16210) Naming feedback - part 1 (Azure#16208) Raise on bad credential (Azure#16206) Updated release date (Azure#16205) Required params must be positional (Azure#16194) Replace EventgridSharedAccessSignatureCredential with AzureSasCredential (Azure#16147) ...
iscai-msft · Jan 20, 2021 · 33e10ae · 33e10ae
2 parents e86f557 + 4e7bbb5
commit 33e10ae
Show file tree

Hide file tree

Showing 428 changed files with 57,176 additions and 14,238 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -20,7 +20,7 @@
 /sdk/identity/                                                       @chlowell @mccoyp @schaabs
 
 # PRLabel: %Event Hubs
-/sdk/eventhub/                                                       @annatisch @yunhaoling @KieranBrantnerMagee
+/sdk/eventhub/                                                       @annatisch @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %Storage
 /sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @xiafu-msft @tasherif-msft @kasobol-msft
@@ -66,7 +66,7 @@
 /sdk/datadatamigration/                                              @vchske
 
 # PRLabel: %Event Grid
-/sdk/eventgrid/                                                      @lmazuel @rakshith91 @KieranBrantnerMagee
+/sdk/eventgrid/                                                      @lmazuel @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %HDInsight
 /sdk/hdinsight/                                                      @idear1203
@@ -90,7 +90,7 @@
 /sql/sql/                                                            @jaredmoo
 
 # PRLabel: %Service Bus
-/sdk/servicebus/                                                     @annatisch @yunhaoling @KieranBrantnerMagee @rakshith91
+/sdk/servicebus/                                                     @annatisch @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %Synapse
 /sdk/synapse/                                                        @aim-for-better @idear1203
@@ -110,6 +110,9 @@
 # PRLabel: %Tables
 /sdk/tables/                                                         @seankane-msft
 
+# PRLabel: %Media
+/sdk/media/                                                          @naiteeks @bennage @giakas
+
 # Smoke Tests
 /common/smoketest/                                                   @lmazuel @chlowell @annatisch @rakshith91 @shurd @southpolesteve
 

diff --git a/common/smoketest/requirements-release.txt b/common/smoketest/requirements-release.txt
@@ -0,0 +1,8 @@
+azure-core>=0.0.0b1
+azure-identity>=0.0.0b1
+azure-cosmos>=4.0.0b5
+azure-eventhub>=0.0.0b1
+azure-keyvault-certificates>=0.0.0b1
+azure-keyvault-keys>=0.0.0b1
+azure-keyvault-secrets>=0.0.0b1
+azure-storage-blob>=0.0.0b1
diff --git a/common/smoketest/requirements.txt b/common/smoketest/requirements.txt
@@ -5,4 +5,4 @@ azure-eventhub
 azure-keyvault-certificates
 azure-keyvault-keys
 azure-keyvault-secrets
-azure-storage-blob
+azure-storage-blob
diff --git a/common/smoketest/smoke-test.yml b/common/smoketest/smoke-test.yml
@@ -1,123 +1,4 @@
-variables:
-  InstallAsyncRequirements: true
-
 jobs:
-  - job:
-    strategy:
-      matrix:
-        Python_27_Linux (AzureCloud):
-          PythonVersion: '2.7'
-          InstallAsyncRequirements: false
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Linux (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureCloud Canary):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-          Location: 'eastus2euap'
-        Python_37_Windows (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Windows (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Mac (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Mac (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureUSGovernment):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_37_Windows (AzureUSGovernment):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_38_Linux (AzureChinaCloud):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-        Python_37_Windows (AzureChinaCloud):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-
-    pool:
-      vmImage: $(OSVmImage)
-
-    variables:
-      Location: ''
-      azureCloudArmParameters: "@{ storageEndpointSuffix = 'core.windows.net'; azureCloud = 'AzureCloud'; }"
-      azureUSGovernmentArmParameters: "@{ storageEndpointSuffix = 'core.usgovcloudapi.net'; azureCloud = 'AzureUSGovernment'; }"
-      azureChinaCloudArmParameters: "@{ storageEndpointSuffix = 'core.chinacloudapi.cn'; azureCloud = 'AzureChinaCloud'; }"
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: "Use Python $(PythonVersion)"
-        inputs:
-          versionSpec: $(PythonVersion)
-
-      - script: |
-          python -m pip install pip==20.0.2
-          pip --version
-        displayName: pip --version
-
-      - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple
-        displayName: Install requirements from dev feed without dependencies
-
-      - script: pip install -r ./common/smoketest/requirements_async.txt
-        displayName: "Install requirements_async.txt"
-        condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true'))
-
-      - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt
-        displayName: Create dependency list from installed dev packages
-
-      - script: pip install -r ./common/smoketest/requirements_dependencies.txt
-        displayName: Install dev package dependencies from PyPI
-
-      - script: pip freeze
-        displayName: Show installed packages (pip freeze)
-
-      - template: /eng/common/TestResources/deploy-test-resources.yml
-        parameters:
-          ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
-          Location: $(Location)
-          ArmTemplateParameters: $(ArmTemplateParameters)
-
-      - script: python ./common/smoketest/program.py
-        displayName: Run Smoke Test
-
-      - template: /eng/common/TestResources/remove-test-resources.yml
-        parameters:
-          ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
-
+  - template: /eng/pipelines/templates/jobs/smoke-test.yml
+    parameters:
+      Daily: true
diff --git a/doc/dev/mgmt/tests.md b/doc/dev/mgmt/tests.md
@@ -35,13 +35,13 @@ sdk/storage/azure-mgmt-storage> pytest
 
 You can provide directories or individual files as positional arguments to specify particular tests to run rather than running the entire test suite. For example:
 ```Shell
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/tests/test_mgmt_storage.py
+sdk/storage/azure-mgmt-storage> pytest
+sdk/storage/azure-mgmt-storage> pytest tests/test_mgmt_storage.py
 ```
 
 If you have print statements in your tests for debugging you can add the `-s` flag to send those print statements to standard output:
 ```Shell
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/ -s
+sdk/storage/azure-mgmt-storage> pytest -s
 ```
 
 ## Getting Azure credentials
@@ -106,6 +106,7 @@ def get_credentials(**kwargs):
 ```python
 def get_azure_core_credentials(**kwargs):
     from azure.identity import ClientSecretCredential
+    import os
     return ClientSecretCredential(
         client_id = os.environ['AZURE_CLIENT_ID'],
         client_secret = os.environ['AZURE_CLIENT_SECRET'],

diff --git a/doc/dev/tests.md b/doc/dev/tests.md
@@ -84,6 +84,12 @@ azure-sdk-for-python\sdk\my-directory\my-library> pytest
 azure-sdk-for-python\sdk\my-directory\my-library> pytest <test_file.py>
 ```
 
+If your tests are broken up into multiple folders for organization, you can run specific folders:
+```cmd
+azure-sdk-for-python\sdk\my-directory\my-library> pytest .\tests\async_tests\
+azure-sdk-for-python\sdk\my-directory\my-library> pytest .\tests\async_tests\<test_file.py>
+```
+
 In addition you can provide keywords to run specific tests within the suite or within a specific file
 ```cmd
 azure-sdk-for-python\sdk\my-directory\my-library> pytest -k <keyword>
@@ -293,4 +299,4 @@ For more information, refer to the [advanced tests notes][advanced_tests_notes]
 [azure_devtools]: https://pypi.org/project/azure-devtools/
 [engsys_wiki]: https://dev.azure.com/azure-sdk/internal/_wiki/wikis/internal.wiki/48/Create-a-new-Live-Test-pipeline?anchor=test-resources.json
 [mgmt_settings_fake]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/mgmt_settings_fake.py
-[packaging]: https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/packaging.md
+[packaging]: https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/packaging.md
diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml
@@ -8,6 +8,7 @@ omitted_paths:
   - doc/*
   - sdk/**/samples/*
   - sdk/identity/azure-identity/tests/*
+  - sdk/**/tests/perfstress_tests/*
 
 language: python
 root_check_enabled: True
@@ -90,6 +91,7 @@ known_content_issues:
   - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
+  - ['sdk/servicebus/azure-servicebus/tests/perf_tests/README.md', '#4554']
   - ['sdk/servicefabric/azure-servicefabric/README.md', '#4554']
   - ['sdk/storage/azure-storage-nspkg/README.md', '#4554']
   - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']

diff --git a/eng/CredScanSuppression.json b/eng/CredScanSuppression.json
@@ -0,0 +1,4 @@
+{
+    "tool": "Credential Scanner",
+    "suppressions": []
+}
diff --git a/eng/common/docgeneration/Generate-DocIndex.ps1 b/eng/common/docgeneration/Generate-DocIndex.ps1
@@ -188,5 +188,7 @@ if ($GetGithubIoDocIndexFn -and (Test-Path "function:$GetGithubIoDocIndexFn"))
 }
 else
 {
-    LogWarning "The function 'GetGithubIoDocIndexFn' was not found."
+    LogWarning "The function for 'GetGithubIoDocIndexFn' was not found.`
+    Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+    See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
 }
diff --git a/eng/common/scripts/Create-APIReview.ps1 b/eng/common/scripts/Create-APIReview.ps1
@@ -59,7 +59,9 @@ if ($FindArtifactForApiReviewFn -and (Test-Path "Function:$FindArtifactForApiRev
 }
 else
 {
-    Write-Host "Function 'FindArtifactForApiReviewFn' is not found"
+    Write-Host "The function for 'FindArtifactForApiReviewFn' was not found.`
+    Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+    See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
     exit(1)
 }
 

diff --git a/eng/common/scripts/Package-Properties.ps1 b/eng/common/scripts/Package-Properties.ps1
@@ -99,7 +99,9 @@ function Get-PkgProperties
         }
         else
         {
-            LogError "The function '$GetPackageInfoFromRepoFn' was not found."
+            LogError "The function for '$GetPackageInfoFromRepoFn' was not found.`
+            Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+            See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
         }
 
         if ($pkgProps -ne $null)

diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1
@@ -99,7 +99,9 @@ function RetrievePackages($artifactLocation) {
   }
   else
   {
-    LogError "The function '$GetPackageInfoFromPackageFileFn' was not found."
+    LogError "The function for '$GetPackageInfoFromPackageFileFn' was not found.`
+    Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+    See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
   }
 }
 

diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1
@@ -238,6 +238,8 @@ if ($PublishGithubIODocsFn -and (Test-Path "Function:$PublishGithubIODocsFn"))
 }
 else
 {
-    LogWarning "The function '$PublishGithubIODocsFn' was not found."
+    LogWarning "The function for '$PublishGithubIODocsFn' was not found.`
+    Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+    See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
 }
 
diff --git a/eng/common/scripts/update-docs-ci.ps1 b/eng/common/scripts/update-docs-ci.ps1
@@ -62,7 +62,9 @@ foreach ($config in $targets) {
     }
     else
     {
-      LogWarning "The function '$UpdateDocCIFn' was not found."
+      LogWarning "The function for '$UpdateDocCIFn' was not found.`
+      Make sure it is present in eng/scripts/Language-Settings.ps1 and referenced in eng/common/scripts/common.ps1.`
+      See https://github.com/Azure/azure-sdk-tools/blob/master/doc/common/common_engsys.md#code-structure"
     }
   }
 }
diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -17,6 +17,23 @@ jobs:
     vmImage: 'windows-2019'
 
   steps:
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+    displayName: 'Run CredScan'
+    inputs:
+      suppressionsFile: 'eng\CredScanSuppression.json'
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+    displayName: 'Post Analysis'
+    inputs:
+      GdnBreakAllTools: false
+      GdnBreakGdnToolCredScan: true
+      GdnBreakGdnToolCredScanSeverity: Error
+      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
+      GdnBreakBaselines: baseline     
+    continueOnError: true
+  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+    displayName: 'Publish Security Analysis Logs'
+    continueOnError: true
+    condition: succeededOrFailed()
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
 
   - task: AzureFileCopy@2

diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml
@@ -233,7 +233,7 @@ jobs:
       - 'Build'
 
     pool:
-      vmImage: ${{ parameters.LinuxPool }}
+      name: ${{ parameters.LinuxPool }}
 
     steps:
     - template: ../steps/test_regression.yml