Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into new_polling

* 'master' of https://github.com/Azure/azure-sdk-for-python: (250 commits)
  Converting to pure unit tests (Azure#16499)
  Fix replace keys for nested json (Azure#16557)
  update release date (Azure#16543)
  Feature/communication chat preview3 new model (Azure#16561)
  [text analytics] Exposed the length property in classes impacted by `string_index_type` (Azure#16538)
  [textanalytics] add :keyword: to docstrings for TextDocumentInput and DetectLanguageInput (Azure#16542)
  [SB] combine conn str parser logic in base handler and _common (Azure#16464)
  [text analytics] add analyze readme bullet point (Azure#16552)
  resolve issues with failing search documents on mindependency checks (Azure#16553)
  [text analytics] analyze changes (Azure#16418)
  [Core] Added enum meta class (Azure#16316)
  Adding missing supported fields to invoice sample (Azure#16547)
  Update EventHub ci.yml to skip unsupported pypy3 (Azure#16545)
  patching the publishing of artifacts only on successful run (Azure#16539)
  need to bump msrest up one (Azure#16544)
  [EventHub&ServiceBUs] Update readme pointing to uamqp installation guidance (added alpine installation) (Azure#16515)
  add dotenv as a dep for azure-sdk-tools (Azure#16532)
  skip prebuilt from_url tests for now (Azure#16534)
  Latest/Minimum filter retrieved packages by pyVersion compatibility (Azure#16510)
  CertificateCredential accepts certs as bytes (Azure#16410)
  ...

.coveragerc

@@ -6,6 +6,11 @@ omit =
 	*/test*
 	env*
 
+[paths]
+source =
+	sdk/
+	**/sdk
+
 [report]
 exclude_lines =
 	pragma: no cover

.github/CODEOWNERS

@@ -20,7 +20,7 @@
 /sdk/identity/                                                       @chlowell @mccoyp @schaabs
 
 # PRLabel: %Event Hubs
-/sdk/eventhub/                                                       @annatisch @yunhaoling @KieranBrantnerMagee
+/sdk/eventhub/                                                       @annatisch @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %Storage
 /sdk/storage/                                                        @amishra-dev @zezha-msft @annatisch @xiafu-msft @tasherif-msft @kasobol-msft
@@ -36,7 +36,7 @@
 /sdk/cognitiveservices/azure-cognitiveservices-vision-customvision/  @areddish
 
 # PRLabel: %Communication
-/sdk/communication/                                                  @RezaJooyandeh @turalf @ankitarorabit @memontic-ms @Azure/azure-sdk-communication-code-reviewers
+/sdk/communication/                                                  @turalf @ankitarorabit @memontic-ms @Azure/azure-sdk-communication-code-reviewers
 
 # PRLabel: %KeyVault
 /sdk/keyvault/                                                       @schaabs @chlowell @mccoyp
@@ -45,7 +45,7 @@
 /sdk/loganalytics/                                                   @alexeldeib
 
 # PRLabel: %Monitor - Exporter
-/sdk/monitor/opentelemetry-exporter-azuremonitor                     @rakshith91 @lmazuel @hectorhdzg @lzchen
+/sdk/monitor/azure-opentelemetry-exporter-azuremonitor               @rakshith91 @lmazuel @hectorhdzg @lzchen
 
 # PRLabel: %Consumption
 /sdk/consumption/                                                    @sandeepnl
@@ -66,7 +66,7 @@
 /sdk/datadatamigration/                                              @vchske
 
 # PRLabel: %Event Grid
-/sdk/eventgrid/                                                      @lmazuel @rakshith91 @KieranBrantnerMagee
+/sdk/eventgrid/                                                      @lmazuel @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %HDInsight
 /sdk/hdinsight/                                                      @idear1203
@@ -90,7 +90,7 @@
 /sql/sql/                                                            @jaredmoo
 
 # PRLabel: %Service Bus
-/sdk/servicebus/                                                     @annatisch @yunhaoling @KieranBrantnerMagee @rakshith91
+/sdk/servicebus/                                                     @annatisch @yunhaoling @swathipil @rakshith91
 
 # PRLabel: %Synapse
 /sdk/synapse/                                                        @aim-for-better @idear1203
@@ -110,11 +110,14 @@
 # PRLabel: %Tables
 /sdk/tables/                                                         @seankane-msft
 
+# PRLabel: %Media
+/sdk/media/                                                          @naiteeks @bennage @giakas
+
 # Smoke Tests
 /common/smoketest/                                                   @lmazuel @chlowell @annatisch @rakshith91 @shurd @southpolesteve
 
 # Management Plane
-/**/*mgmt*/     @msyyc @jsntcy @changlong-liu
+/**/*mgmt*/     @00Kai0 @msyyc @jsntcy @changlong-liu
 
 
 ###########

.gitignore

@@ -33,6 +33,7 @@ build/
 
 # Test results
 TestResults/
+ENV_DIR/
 
 # tox generated artifacts
 test-junit-*.xml

build_package.py

@@ -9,6 +9,7 @@
 import argparse
 import os
 import glob
+import sys
 from subprocess import check_call
 
 
@@ -19,8 +20,8 @@ def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
     absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
 
     absdirpath = os.path.abspath(absdirs[0])
-    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
-    check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
+    check_call([sys.executable, 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
 
 
 if __name__ == '__main__':

ci_template.yml

@@ -0,0 +1,33 @@
+# DO NOT EDIT THIS FILE
+# This file is generated automatically and any changes will be lost.
+
+trigger:
+  branches:
+    include:
+    - master
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+pr:
+  branches:
+    include:
+    - master
+    - feature/*
+    - hotfix/*
+    - release/*
+    - restapi*
+  paths:
+    include:
+    - sdk/MyService/
+
+extends:
+  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
+  parameters:
+    ServiceDirectory: MyService
+    Artifacts:
+    - name: azure_mgmt_MyService
+      safeName: azuremgmtMyService

common/smoketest/requirements-release.txt

@@ -0,0 +1,8 @@
+azure-core>=0.0.0b1
+azure-identity>=0.0.0b1
+azure-cosmos>=4.0.0b5
+azure-eventhub>=0.0.0b1
+azure-keyvault-certificates>=0.0.0b1
+azure-keyvault-keys>=0.0.0b1
+azure-keyvault-secrets>=0.0.0b1
+azure-storage-blob>=0.0.0b1

common/smoketest/requirements.txt

@@ -5,4 +5,4 @@ azure-eventhub
 azure-keyvault-certificates
 azure-keyvault-keys
 azure-keyvault-secrets
-azure-storage-blob
+azure-storage-blob

common/smoketest/smoke-test.yml

@@ -1,123 +1,4 @@
-variables:
-  InstallAsyncRequirements: true
-
 jobs:
-  - job:
-    strategy:
-      matrix:
-        Python_27_Linux (AzureCloud):
-          PythonVersion: '2.7'
-          InstallAsyncRequirements: false
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Linux (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureCloud Canary):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-          Location: 'eastus2euap'
-        Python_37_Windows (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Windows (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_37_Mac (AzureCloud):
-          PythonVersion: '3.7'
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Mac (AzureCloud):
-          PythonVersion: '3.8'
-          OSVmImage: macOS-10.15
-          SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
-          ArmTemplateParameters: $(azureCloudArmParameters)
-        Python_38_Linux (AzureUSGovernment):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_37_Windows (AzureUSGovernment):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-gov-test-resources)
-          ArmTemplateParameters: $(azureUSGovernmentArmParameters)
-        Python_38_Linux (AzureChinaCloud):
-          PythonVersion: '3.8'
-          OSVmImage: ubuntu-18.04
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-        Python_37_Windows (AzureChinaCloud):
-          PythonVersion: '3.7'
-          OSVmImage: windows-2019
-          SubscriptionConfiguration: $(sub-config-cn-test-resources)
-          Location: 'chinanorth'
-          ArmTemplateParameters: $(azureChinaCloudArmParameters)
-
-    pool:
-      vmImage: $(OSVmImage)
-
-    variables:
-      Location: ''
-      azureCloudArmParameters: "@{ storageEndpointSuffix = 'core.windows.net'; azureCloud = 'AzureCloud'; }"
-      azureUSGovernmentArmParameters: "@{ storageEndpointSuffix = 'core.usgovcloudapi.net'; azureCloud = 'AzureUSGovernment'; }"
-      azureChinaCloudArmParameters: "@{ storageEndpointSuffix = 'core.chinacloudapi.cn'; azureCloud = 'AzureChinaCloud'; }"
-
-    steps:
-      - task: UsePythonVersion@0
-        displayName: "Use Python $(PythonVersion)"
-        inputs:
-          versionSpec: $(PythonVersion)
-
-      - script: |
-          python -m pip install pip==20.0.2
-          pip --version
-        displayName: pip --version
-
-      - script: pip install -r ./common/smoketest/requirements.txt --pre --no-deps --index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple
-        displayName: Install requirements from dev feed without dependencies
-
-      - script: pip install -r ./common/smoketest/requirements_async.txt
-        displayName: "Install requirements_async.txt"
-        condition: and(succeeded(), eq(variables['InstallAsyncRequirements'], 'true'))
-
-      - script: python ./common/smoketest/dependencies.py -r ./common/smoketest/requirements.txt | tee ./common/smoketest/requirements_dependencies.txt
-        displayName: Create dependency list from installed dev packages
-
-      - script: pip install -r ./common/smoketest/requirements_dependencies.txt
-        displayName: Install dev package dependencies from PyPI
-
-      - script: pip freeze
-        displayName: Show installed packages (pip freeze)
-
-      - template: /eng/common/TestResources/deploy-test-resources.yml
-        parameters:
-          ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
-          Location: $(Location)
-          ArmTemplateParameters: $(ArmTemplateParameters)
-
-      - script: python ./common/smoketest/program.py
-        displayName: Run Smoke Test
-
-      - template: /eng/common/TestResources/remove-test-resources.yml
-        parameters:
-          ServiceDirectory: '$(Build.SourcesDirectory)/common/smoketest/'
-          SubscriptionConfiguration: $(SubscriptionConfiguration)
-
+  - template: /eng/pipelines/templates/jobs/smoke-test.yml
+    parameters:
+      Daily: true

doc/dev/mgmt/swagger/multi_api/readme.md

@@ -8,7 +8,7 @@ This is the AutoRest configuration file for Network.
 
 ## Getting Started
 
-To build the SDK for Network, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/installing-autorest.md) and in this folder, run:
+To build the SDK for Network, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/install/readme.md) and in this folder, run:
 
 > `autorest`
 

doc/dev/mgmt/swagger/single_api/readme.md

@@ -8,7 +8,7 @@ This is the AutoRest configuration file for Cdn.
 
 ---
 ## Getting Started
-To build the SDK for Cdn, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/installing-autorest.md) and in this folder, run:
+To build the SDK for Cdn, simply [Install AutoRest](https://github.com/Azure/autorest/blob/master/docs/install/readme.md) and in this folder, run:
 
 > `autorest`
 

doc/dev/mgmt/tests.md

@@ -35,13 +35,13 @@ sdk/storage/azure-mgmt-storage> pytest
 
 You can provide directories or individual files as positional arguments to specify particular tests to run rather than running the entire test suite. For example:
 ```Shell
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/tests/test_mgmt_storage.py
+sdk/storage/azure-mgmt-storage> pytest
+sdk/storage/azure-mgmt-storage> pytest tests/test_mgmt_storage.py
 ```
 
 If you have print statements in your tests for debugging you can add the `-s` flag to send those print statements to standard output:
 ```Shell
-sdk/storage/azure-mgmt-storage> pytest sdk/storage/azure-mgmt-storage/ -s
+sdk/storage/azure-mgmt-storage> pytest -s
 ```
 
 ## Getting Azure credentials
@@ -106,6 +106,7 @@ def get_credentials(**kwargs):
 ```python
 def get_azure_core_credentials(**kwargs):
     from azure.identity import ClientSecretCredential
+    import os
     return ClientSecretCredential(
         client_id = os.environ['AZURE_CLIENT_ID'],
         client_secret = os.environ['AZURE_CLIENT_SECRET'],
@@ -309,11 +310,11 @@ class ExampleSqlServerTestCase(AzureMgmtTestCase):
 ```
 
 <!-- LINKS -->
-[arm_apis]: https://docs.microsoft.com/en-us/rest/api/resources/
+[arm_apis]: https://docs.microsoft.com/rest/api/resources/
 [azure_devtools]: https://pypi.org/project/azure-devtools/
 [azure_portal]: https://portal.azure.com/
 [decorators]: https://www.python.org/dev/peps/pep-0318/
-[dev_setup]: ../dev_setup.md
+[dev_setup]: https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/dev_setup.md
 [devtools_testutils]: https://github.com/Azure/azure-sdk-for-python/tree/master/tools/azure-sdk-tools/devtools_testutils
 [mgmt_settings_fake]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/mgmt_settings_fake.py
 [pytest]: https://docs.pytest.org/en/latest/

doc/dev/tests.md

@@ -5,7 +5,7 @@ In this document we will provide the introduction to the testing framework by:
 - [Integrating with pytest](#integrate-with-the-pytest-test-framework)
 - [Using Tox](#tox)
 - [The `devtools_testutils` package](#devtools_testutils-package)
-- [Writing New Tests](#Writing-new-tests)
+- [Writing New Tests](#writing-new-tests)
 - [Define our credentials and settings](#define-credentials)
 - [Create live test resources](#create-live-test-resources)
 - [Write our test](#writing-your-test)
@@ -84,6 +84,12 @@ azure-sdk-for-python\sdk\my-directory\my-library> pytest
 azure-sdk-for-python\sdk\my-directory\my-library> pytest <test_file.py>
 ```
 
+If your tests are broken up into multiple folders for organization, you can run specific folders:
+```cmd
+azure-sdk-for-python\sdk\my-directory\my-library> pytest .\tests\async_tests\
+azure-sdk-for-python\sdk\my-directory\my-library> pytest .\tests\async_tests\<test_file.py>
+```
+
 In addition you can provide keywords to run specific tests within the suite or within a specific file
 ```cmd
 azure-sdk-for-python\sdk\my-directory\my-library> pytest -k <keyword>
@@ -289,8 +295,8 @@ For more information, refer to the [advanced tests notes][advanced_tests_notes]
 
 
 <!-- Links -->
-[advanced_tests_notes]: ./tests-advanced.md
+[advanced_tests_notes]: https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/tests-advanced.md
 [azure_devtools]: https://pypi.org/project/azure-devtools/
 [engsys_wiki]: https://dev.azure.com/azure-sdk/internal/_wiki/wikis/internal.wiki/48/Create-a-new-Live-Test-pipeline?anchor=test-resources.json
 [mgmt_settings_fake]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/mgmt_settings_fake.py
-[packaging]: ./packaging.md
+[packaging]: https://github.com/Azure/azure-sdk-for-python/blob/master/doc/dev/packaging.md

eng/.docsettings.yml

@@ -8,6 +8,7 @@ omitted_paths:
   - doc/*
   - sdk/**/samples/*
   - sdk/identity/azure-identity/tests/*
+  - sdk/**/tests/perfstress_tests/*
 
 language: python
 root_check_enabled: True
@@ -90,6 +91,7 @@ known_content_issues:
   - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/README.md', '#4554']
   - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
+  - ['sdk/servicebus/azure-servicebus/tests/perf_tests/README.md', '#4554']
   - ['sdk/servicefabric/azure-servicefabric/README.md', '#4554']
   - ['sdk/storage/azure-storage-nspkg/README.md', '#4554']
   - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']
@@ -100,8 +102,8 @@ known_content_issues:
   - ['sdk/storage/azure-storage-queue/swagger/README.md', '#4554']
   - ['sdk/storage/README.md', '#4554']
   - ['sdk/textanalytics/azure-ai-textanalytics/samples/README.md', '#4554']
-  - ['sdk/monitor/microsoft-opentelemetry-exporter-azuremonitor/swagger/README.md', '#4554']
-  - ['sdk/monitor/microsoft-opentelemetry-exporter-azuremonitor/README.md', '#4554']
+  - ['sdk/monitor/azure-opentelemetry-exporter-azuremonitor/swagger/README.md', '#4554']
+  - ['sdk/monitor/azure-opentelemetry-exporter-azuremonitor/README.md', '#4554']
   - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
   - ['sdk/media/azure-media-nspkg/README.md', '#4554']