Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into ta-v5.1.0b6-analyze

* 'master' of https://github.com/Azure/azure-sdk-for-python: (24 commits)
  [text analytics] PII updates for v5.1.0b6 (Azure#17038)
  Fix bug where imported matrix parameter duplicates are not overrided (Azure#17126)
  Add NULL to readme (Azure#17076)
  Sas batching (Azure#17133)
  dropping py3.5 (Azure#17127)
  [EventHubs] 5.3.1 update changelog (Azure#17132)
  [text analytics] assertions (Azure#17098)
  add recordings (Azure#17125)
  [core] add HttpRequest and HttpResponse reprs (Azure#16972)
  [text analytics] add actual normalized_text tests (Azure#17123)
  update samples to use actual role names (Azure#17124)
  Sync eng/common directory with azure-sdk-tools for PR 1448 (Azure#17085)
  Enable APIView status check (Azure#17107)
  Fix PackageName typo (Azure#17109)
  Move SetTestPipelineVersion.ps1 to eng/common (Azure#17103)
  Fix paths for non-windows agents (Azure#17096)
  [Communication] - Identity - Update README (Azure#17091)
  Rename CertificateCredential's certificate_bytes -> certificate_data (Azure#17090)
  fix shared reqs (Azure#17095)
  [translation] initial library (Azure#16837)
  ...

eng/common/pipelines/templates/steps/create-apireview.yml

@@ -19,4 +19,3 @@ steps:
         workingDirectory: $(Pipeline.Workspace)
       displayName: Create API Review for ${{ artifact.name}}
       condition: and(succeededOrFailed(), ne(variables['Skip.CreateApiReview'], 'true') , ne(variables['Build.Reason'],'PullRequest'), eq(variables['System.TeamProject'], 'internal'))
-      continueOnError: true

eng/common/scripts/Create-APIReview.ps1

@@ -41,11 +41,12 @@ function Submit-APIReview($packagename, $filePath, $uri, $apiKey, $apiLabel)
     try
     {
         $Response = Invoke-WebRequest -Method 'POST' -Uri $uri -Body $multipartContent -Headers $headers
-        Write-Host "API Review: $($Response)"
+        Write-Host "API Review URL: $($Response.Content)"
         $StatusCode = $Response.StatusCode
     }
     catch
     {
+        Write-Host "Exception details: $($_.Exception.Response)"
         $StatusCode = $_.Exception.Response.StatusCode
     }
 
@@ -101,6 +102,9 @@ foreach ($pkgName in $responses.Keys)
         {
             $pkgInfo = Get-Content $pkgPropPath | ConvertFrom-Json
             $version = [AzureEngSemanticVersion]::ParseVersionString($pkgInfo.Version)
+            Write-Host "Package name: $($PackageName)"
+            Write-Host "Version: $($version)"
+            Write-Host "SDK Type: $($pkgInfo.SdkType)"
             if ($version.IsPrerelease)
             {
                 Write-Host "Package version is not GA. Ignoring API view approval status"
@@ -110,12 +114,16 @@ foreach ($pkgName in $responses.Keys)
                 $FoundFailure = $True
                 if ($respCode -eq '201')
                 {
-                    Write-Error "Automatic API Review approval is pending for package $($PackageName)"
+                    Write-Host "Package version $($version) is GA and automatic API Review is not yet approved for package $($PackageName)."
+                    Write-Host "Build and release is not allowed for GA package without API review approval."
+                    Write-Host "You will need to queue another build to proceed further after API review is approved"
+                    Write-Host "You can check http://aka.ms/azsdk/engsys/apireview/faq for more details on API Approval."
                 }
                 else
                 {
-                    Write-Error "Failed to create API Review for package $($PackageName)"
-                }                
+                    Write-Host "Failed to create API Review for package $($PackageName). Please reach out to Azure SDK engineering systems on teams channel and share this build details."
+                }
+                exit 1
             }
             else
             {
@@ -124,8 +132,3 @@ foreach ($pkgName in $responses.Keys)
         }
     }
 }
-if ($FoundFailure)
-{
-    Write-Error "Automatic API review is not yet approved for package $($PackageName)"
-    exit 1
-}

eng/common/scripts/SetTestPipelineVersion.ps1

@@ -0,0 +1,33 @@
+# Sets a valid version for a package using the buildID
+
+param (
+  [Parameter(mandatory = $true)]
+  $BuildID,
+  [Parameter(mandatory = $true)]
+  $PackageName,
+  [Parameter(mandatory = $true)]
+  $ServiceDirectory
+)
+
+. common.ps1
+
+$latestTags = git tag -l "${PackageName}_*"
+$semVars = @()
+
+Foreach ($tags in $latestTags)
+{
+  $semVars += $tags.Replace("${PackageName}_", "")
+}
+
+$semVarsSorted = [AzureEngSemanticVersion]::SortVersionStrings($semVars)
+LogDebug "Last Published Version $($semVarsSorted[0])"
+
+$newVersion = [AzureEngSemanticVersion]::new($semVarsSorted[0])
+$newVersion.PrereleaseLabel = $newVersion.DefaultPrereleaseLabel
+$newVersion.PrereleaseNumber = $BuildID
+
+LogDebug "Version to publish [ $($newVersion.ToString()) ]"
+
+SetPackageVersion -PackageName $PackagName `
+  -Version $newVersion `
+  -ServiceDirectory $ServiceDirectory `

eng/common/scripts/job-matrix/job-matrix-functions.ps1

@@ -32,7 +32,7 @@ function GenerateMatrix(
     [Array]$filters = @(),
     [Array]$nonSparseParameters = @()
 ) {
-    $orderedMatrix, $importedMatrix = ProcessImport $config.orderedMatrix $selectFromMatrixType
+    $orderedMatrix, $importedMatrix, $importedDisplayNamesLookup = ProcessImport $config.orderedMatrix $selectFromMatrixType
     if ($selectFromMatrixType -eq "sparse") {
         [Array]$matrix = GenerateSparseMatrix $orderedMatrix $config.displayNamesLookup $nonSparseParameters
     } elseif ($selectFromMatrixType -eq "all") {
@@ -44,7 +44,7 @@ function GenerateMatrix(
     # Combine with imported after matrix generation, since a sparse selection should result in a full combination of the
     # top level and imported sparse matrices (as opposed to a sparse selection of both matrices).
     if ($importedMatrix) {
-        [Array]$matrix = CombineMatrices $matrix $importedMatrix
+        [Array]$matrix = CombineMatrices $matrix $importedMatrix $importedDisplayNamesLookup
     }
 
     if ($config.exclude) {
@@ -199,19 +199,19 @@ function ProcessIncludes([MatrixConfig]$config, [Array]$matrix)
 function ProcessImport([System.Collections.Specialized.OrderedDictionary]$matrix, [String]$selection)
 {
     if (!$matrix -or !$matrix.Contains($IMPORT_KEYWORD)) {
-        return $matrix
+        return $matrix, @(), @{}
     }
 
     $importPath = $matrix[$IMPORT_KEYWORD]
     $matrix.Remove($IMPORT_KEYWORD)
 
-    $matrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
-    $importedMatrix = GenerateMatrix $matrixConfig $selection
+    $importedMatrixConfig = GetMatrixConfigFromJson (Get-Content $importPath)
+    $importedMatrix = GenerateMatrix $importedMatrixConfig $selection
 
-    return $matrix, $importedMatrix
+    return $matrix, $importedMatrix, $importedMatrixConfig.displayNamesLookup
 }
 
-function CombineMatrices([Array]$matrix1, [Array]$matrix2)
+function CombineMatrices([Array]$matrix1, [Array]$matrix2, [Hashtable]$displayNamesLookup = @{})
 {
     $combined = @()
     if (!$matrix1) {
@@ -223,21 +223,22 @@ function CombineMatrices([Array]$matrix1, [Array]$matrix2)
 
     foreach ($entry1 in $matrix1) {
         foreach ($entry2 in $matrix2) {
+            $entry2name = @()
             $newEntry = @{
                 name = $entry1.name
                 parameters = CloneOrderedDictionary $entry1.parameters
             }
             foreach($param in $entry2.parameters.GetEnumerator()) {
-                if (!$newEntry.Contains($param.Name)) {
+                if (!$newEntry.parameters.Contains($param.Name)) {
                     $newEntry.parameters[$param.Name] = $param.Value
+                    $entry2name += CreateDisplayName $param.Value $displayNamesLookup
                 } else {
                     Write-Warning "Skipping duplicate parameter `"$($param.Name)`" when combining matrix."
                 }
             }
 
             # The maximum allowed matrix name length is 100 characters
-            $entry2.name = $entry2.name.TrimStart("job_")
-            $newEntry.name = $newEntry.name, $entry2.name -join "_"
+            $newEntry.name = @($newEntry.name, ($entry2name -join "_")) -join "_"
             if ($newEntry.name.Length -gt 100) {
                 $newEntry.name = $newEntry.name[0..99] -join ""
             }
@@ -305,7 +306,7 @@ function GenerateSparseMatrix(
 
     if ($nonSparse) {
         [Array]$allOfMatrix = GenerateFullMatrix $nonSparse $displayNamesLookup
-        return CombineMatrices $allOfMatrix $sparseMatrix
+        return CombineMatrices $allOfMatrix $sparseMatrix $displayNamesLookup
     }
 
     return $sparseMatrix

eng/common/scripts/job-matrix/tests/job-matrix-functions.modification.tests.ps1

@@ -216,4 +216,25 @@ Describe "Platform Matrix Import" -Tag "import" {
         $matrix.Length | Should -Be 7
         CompareMatrices $matrix $expected
     }
+
+    It "Should generate a sparse matrix with an imported a sparse matrix" {
+        $matrixJson = @'
+{
+    "matrix": {
+        "$IMPORT": "./test-import-matrix.json",
+        "Foo": [ "fooOverride1", "fooOverride2" ],
+    }
+}
+'@
+
+        $importConfig = GetMatrixConfigFromJson $matrixJson
+        $matrix = GenerateMatrix $importConfig "sparse"
+
+        $matrix[0].parameters["Foo"] | Should -Be "fooOverride1"
+        $matrix[0].name | Should -Be "fooOverride1_bar1"
+        $matrix[3].parameters["Foo"] | Should -Be "fooOverride2"
+        $matrix[3].name | Should -Be "fooOverride2_bar1"
+        $matrix[5].parameters["Foo"] | Should -Be "fooOverride2"
+        $matrix[5].name | Should -Be "fooOverride2_importedBaz"
+    }
 }

eng/scripts/Language-Settings.ps1

@@ -8,18 +8,18 @@ $BlobStorageUrl = "https://azuresdkdocs.blob.core.windows.net/%24web?restype=con
 function Get-AllPackageInfoFromRepo ($serviceDirectory)
 {
   $allPackageProps = @()
-  $searchPath = "sdk/*/*/setup.py"
+  $searchPath = Join-Path sdk * * setup.py
   if ($serviceDirectory)
   {
-    $searchPath = "sdk/${serviceDirectory}/*/setup.py"
+    $searchPath = Join-Path sdk ${serviceDirectory} * setup.py
   }
 
   $allPkgPropLines = $null
   try
   {
     Push-Location $RepoRoot
     pip install packaging==20.4 -q -I
-    $allPkgPropLines = python "eng\scripts\get_package_properties.py" -s $searchPath
+    $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath
   }
   catch
   {

sdk/communication/azure-communication-identity/README.md

@@ -2,7 +2,7 @@
 
 # Azure Communication Identity Package client library for Python
 
-Azure Communication Identity client package is intended to be used to setup the basics for opening a way to use Azure Communication Service offerings. This package helps to create identities user tokens to be used by other client packages such as chat, calling, sms. 
+Azure Communication Identity client package is intended to be used to setup the basics for opening a way to use Azure Communication Service offerings. This package helps to create identity user tokens to be used by other client packages such as chat, calling, sms.
 
 [Source code](https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/communication/azure-communication-identity) | [Package (Pypi)](https://pypi.org/project/azure-communication-identity/) | [API reference documentation](https://github.com/Azure/azure-sdk-for-python/blob/master/sdk/communication/azure-communication-identity) | [Product documentation](https://docs.microsoft.com/azure/communication-services/quickstarts/access-tokens?pivots=programming-language-python)
 
@@ -51,21 +51,12 @@ The following section provides several code snippets covering some of the most c
 
 ### Creating a new user
 
-Use the `create_user` method to create a new user. 
+Use the `create_user` method to create a new user.
 ```python
 user = identity_client.create_user()
 print("User created with id:" + user.identifier)
 ```
 
-Alternatively, use the `create_user_and_token` method to create a new user and issue a token for it.\
-For this option, a list of `CommunicationTokenScope` must be defined (see "Issuing an access token" for more information)
-
-```python
-user, tokenresponse = identity_client.create_user_and_token(scopes=[CommunicationTokenScope.CHAT])
-print("User id:" + user.identifier)
-print("Token issued with value: " + tokenresponse.token)
-```
-
 ### Issuing or Refreshing an access token for a user
 
 Use the `get_token` method to issue or refresh a scoped access token for the user. \
@@ -77,6 +68,14 @@ Pass in the user object as a parameter, and a list of `CommunicationTokenScope`.
 tokenresponse = identity_client.get_token(user, scopes=[CommunicationTokenScope.CHAT])
 print("Token issued with value: " + tokenresponse.token)
 ```
+### Creating a user and a token in a single request
+For convenience, use `create_user_and_token` to create a new user and issue a token with one function call. This translates into a single web request as opposed to creating a user first and then issuing a token.
+
+```python
+user, tokenresponse = identity_client.create_user_and_token(scopes=[CommunicationTokenScope.CHAT])
+print("User id:" + user.identifier)
+print("Token issued with value: " + tokenresponse.token)
+```
 
 ### Revoking a user's access tokens
 

sdk/core/azure-core/CHANGELOG.md

@@ -1,11 +1,15 @@
 # Release History
 
-## 1.12.0 (Unreleased)
+## 1.12.0 (2021-03-08)
 
 ### Features
 
 - Added `azure.core.messaging.CloudEvent` model that follows the cloud event spec.
-- Added `azure.core.serialization.NULL` sentinel value 
+- Added `azure.core.serialization.NULL` sentinel value
+
+### Bug Fixes
+
+- Improve `repr`s for `HttpRequest` and `HttpResponse`s  #16972
 
 ## 1.11.0 (2021-02-08)
 

sdk/core/azure-core/README.md

@@ -179,6 +179,21 @@ class MyCustomEnum(with_metaclass(CaseInsensitiveEnumMeta, str, Enum)):
     BAR = 'bar'
 ```
 
+#### Null Sentinel Value
+
+A falsy sentinel object which is supposed to be used to specify attributes
+with no data. This gets serialized to `null` on the wire.
+
+```python
+from azure.core.serialization import NULL
+
+assert bool(NULL) is False
+
+foo = Foo(
+    attr=NULL
+)
+```
+
 ## Contributing
 This project welcomes contributions and suggestions. Most contributions require
 you to agree to a Contributor License Agreement (CLA) declaring that you have
@@ -198,4 +213,4 @@ or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any
 additional questions or comments.
 
 <!-- LINKS -->
-[package]: https://pypi.org/project/azure-core/
+[package]: https://pypi.org/project/azure-core/

sdk/core/azure-core/azure/core/pipeline/transport/_base.py

@@ -234,7 +234,9 @@ def __init__(self, method, url, headers=None, files=None, data=None):
         self.multipart_mixed_info = None  # type: Optional[Tuple]
 
     def __repr__(self):
-        return "<HttpRequest [%s]>" % (self.method)
+        return "<HttpRequest [{}], url: '{}'>".format(
+            self.method, self.url
+        )
 
     def __deepcopy__(self, memo=None):
         try:
@@ -592,6 +594,15 @@ def raise_for_status(self):
         if self.status_code >= 400:
             raise HttpResponseError(response=self)
 
+    def __repr__(self):
+        # there doesn't have to be a content type
+        content_type_str = (
+            ", Content-Type: {}".format(self.content_type) if self.content_type else ""
+        )
+        return "<{}: {} {}{}>".format(
+            type(self).__name__, self.status_code, self.reason, content_type_str
+        )
+
 
 class HttpResponse(_HttpResponseBase):  # pylint: disable=abstract-method
     def stream_download(self, pipeline):

sdk/core/azure-core/tests/async_tests/test_universal_http_async.py

@@ -98,6 +98,8 @@ def __init__(self, body_bytes, headers=None):
             self._body = body_bytes
             self._headers = headers
             self._cache = {}
+            self.status = 200
+            self.reason = "OK"
 
     req_response = MockAiohttpClientResponse(body_bytes, headers)
 
@@ -120,3 +122,12 @@ async def test_aiohttp_response_text():
             {'Content-Type': 'text/plain'}
         )
         assert res.text(encoding) == '56', "Encoding {} didn't work".format(encoding)
+
+def test_repr():
+    res = _create_aiohttp_response(
+        b'\xef\xbb\xbf56',
+        {}
+    )
+    res.content_type = "text/plain"
+
+    assert repr(res) == "<AioHttpTransportResponse: 200 OK, Content-Type: text/plain>"

sdk/core/azure-core/tests/test_pipeline.py

@@ -285,7 +285,7 @@ def test_request_url_with_params(self):
         request.format_parameters({"g": "h"})
 
         self.assertIn(request.url, ["a/b/c?g=h&t=y", "a/b/c?t=y&g=h"])
-    
+
     def test_request_url_with_params_as_list(self):
 
         request = HttpRequest("GET", "/")
@@ -300,7 +300,7 @@ def test_request_url_with_params_with_none_in_list(self):
         request.url = "a/b/c?t=y"
         with pytest.raises(ValueError):
             request.format_parameters({"g": ["h",None]})
-    
+
     def test_request_url_with_params_with_none(self):
 
         request = HttpRequest("GET", "/")
@@ -328,6 +328,10 @@ def test_request_text(self):
         # We want a direct string
         assert request.data == "foo"
 
+    def test_repr(self):
+        request = HttpRequest("GET", "hello.com")
+        assert repr(request) == "<HttpRequest [GET], url: 'hello.com'>"
+
 
 if __name__ == "__main__":
     unittest.main()