Merge branch 'revert_error_logging' of https://github.com/iscai-msft/…

…azure-sdk-for-python into revert_error_logging * 'revert_error_logging' of https://github.com/iscai-msft/azure-sdk-for-python: Sync eng/common directory with azure-sdk-tools for PR 2327 (Azure#22021) Sync eng/common directory with azure-sdk-tools for PR 2327 (Azure#22017) pin mistune, dep of m2r, after breaking change release (Azure#22018) Update eng sys codeowners [AutoRelease] t2-loadtestservice-2021-11-26-84683 (Azure#21895) update release-helper (Azure#22012) skip python ci for data-plane (Azure#22011) CognitiveServices Face (Azure#21986) [stress, eventhub] Update dependency version and image/namespace. Remove redundant files. (Azure#21989) [translation] remove sample tests dependency on static storage acc (Azure#21992) Update packaging tool only (Azure#21973) Increment package version after release of azure-core (Azure#21991) Sync eng/common directory with azure-sdk-tools for PR 2380 (Azure#21993) pin crypto in depends check (Azure#21987) Restore deploymentOutputs variable in scope for post deployment script (Azure#21983)
iscai-msft · Dec 6, 2021 · d5091f4 · d5091f4
2 parents c628208 + 855160c
commit d5091f4
Show file tree

Hide file tree

Showing 72 changed files with 3,614 additions and 308 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -107,7 +107,7 @@
 /sdk/synapse/                                                        @wonner @idear1203 @xiangyan99
 
 # PRLabel: %EngSys
-/sdk/template/                                                       @Azure/azure-sdk-eng
+/sdk/template/                                                       @scbedd @weshaggard @benbp
 
 # PRLabel: %Cognitive - Text Analytics
 /sdk/textanalytics/                                                  @kristapratico @iscai-msft @abhahn
@@ -143,9 +143,9 @@
 ###########
 # Eng Sys
 ###########
-/eng/                                                                @weshaggard @scbedd @danieljurek
-/**/tests.yml                                                        @benbp
-/**/ci.yml                                                           @scbedd
+/eng/                                                                @scbedd @weshaggard @benbp
+/**/tests.yml                                                        @scbedd @benbp
+/**/ci.yml                                                           @scbedd @benbp
 
 # Add owners for notifications for specific pipelines
 /eng/pipelines/templates/jobs/tests-nightly-python.yml               @lmazuel @mccoyp

diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1
@@ -242,6 +242,8 @@ function SetDeploymentOutputs([string]$serviceName, [object]$azContext, [object]
             $key = $null
         }
     }
+
+    return $deploymentOutputs
 }
 
 # Support actions to invoke on exit.
@@ -669,7 +671,7 @@ try {
             Write-Verbose "Successfully deployed template '$($templateFile.jsonFilePath)' to resource group '$($resourceGroup.ResourceGroupName)'"
         }
 
-        SetDeploymentOutputs $serviceName $context $deployment $templateFile
+        $deploymentOutputs = SetDeploymentOutputs $serviceName $context $deployment $templateFile
 
         $postDeploymentScript = $templateFile.originalFilePath | Split-Path | Join-Path -ChildPath 'test-resources-post.ps1'
         if (Test-Path $postDeploymentScript) {

diff --git a/eng/common/pipelines/templates/steps/update-docsms-metadata.yml b/eng/common/pipelines/templates/steps/update-docsms-metadata.yml
@@ -26,7 +26,12 @@ parameters:
     type: object
     default:
       - '**'
-
+  - name: PackageSourceOverride
+    type: string
+    default: ''
+  - name: DocValidationImageId
+    type: string
+    default: ''
 steps:
 - template: /eng/common/pipelines/templates/steps/enable-long-path-support.yml
 
@@ -73,15 +78,23 @@ steps:
     parameters:
       WorkingDirectory: $(DocRepoLocation)
       DefaultBranchVariableName: TargetBranchName
-
+# Pull and build the docker image.
+- ${{ if ne(parameters.DocValidationImageId, '') }}:
+  - template: /eng/common/pipelines/templates/steps/docker-pull-image.yml
+    parameters:
+      ContainerRegistryClientId: $(azuresdkimages-cr-clientid)
+      ContainerRegistryClientSecret: $(azuresdkimages-cr-clientsecret)
+      ImageId: '${{ parameters.DocValidationImageId }}'
 - pwsh: |
     $packageInfoJson = '${{ convertToJson(parameters.PackageInfoLocations) }}'.Trim('"').Replace("\\", "/")
     $packageInfoLocations = ConvertFrom-Json $packageInfoJson
     ${{ parameters.ScriptDirectory }}/Update-DocsMsMetadata.ps1 `
       -PackageInfoJsonLocations $packageInfoLocations `
       -DocRepoLocation "$(DocRepoLocation)" `
       -Language '${{parameters.Language}}' `
-      -RepoId '${{ parameters.RepoId }}'
+      -RepoId '${{ parameters.RepoId }}' `
+      -DocValidationImageId '${{ parameters.DocValidationImageId }}' `
+      -PackageSourceOverride '${{ parameters.PackageSourceOverride }}'
   displayName: Apply Documentation Updates
 
 - template: /eng/common/pipelines/templates/steps/git-push-changes.yml

diff --git a/eng/common/scripts/Update-DocsMsMetadata.ps1 b/eng/common/scripts/Update-DocsMsMetadata.ps1
@@ -28,6 +28,15 @@ Programming language to supply to metadata
 .PARAMETER RepoId
 GitHub repository ID of the SDK. Typically of the form: 'Azure/azure-sdk-for-js'
 
+.PARAMETER DocValidationImageId
+The docker image id in format of '$containerRegistry/$imageName:$tag'
+e.g. azuresdkimages.azurecr.io/jsrefautocr:latest
+
+.PARAMETER PackageSourceOverride
+Optional parameter to supply a different package source (useful for daily dev
+docs generation from pacakges which are not published to the default feed). This
+variable is meant to be used in the domain-specific business logic in
+&$ValidateDocsMsPackagesFn
 #>
 
 param(
@@ -41,7 +50,13 @@ param(
   [string]$Language,
 
   [Parameter(Mandatory = $true)]
-  [string]$RepoId
+  [string]$RepoId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$DocValidationImageId,
+
+  [Parameter(Mandatory = $false)]
+  [string]$PackageSourceOverride
 )
 
 . (Join-Path $PSScriptRoot common.ps1)
@@ -104,21 +119,19 @@ ms.technology: azure
 ms.devlang: $Language
 ms.service: $service
 ---
-
 "@
 
   return "$header`n$ReadmeContent"
 }
 
-function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) { 
+function GetPackageInfoJson ($packageInfoJsonLocation) {
   if (!(Test-Path $packageInfoJsonLocation)) {
     LogWarning "Package metadata not found for $packageInfoJsonLocation"
     return
   }
 
   $packageInfoJson = Get-Content $packageInfoJsonLocation -Raw
   $packageInfo = ConvertFrom-Json $packageInfoJson
-  $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version)
   if ($packageInfo.DevVersion) {
     # If the package is of a dev version there may be language-specific needs to 
     # specify the appropriate version. For example, in the case of JS, the dev 
@@ -131,6 +144,11 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) {
       $packageInfo.Version = $packageInfo.DevVersion
     }
   }
+  return $packageInfo
+}
+
+function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation, $packageInfo) { 
+  $originalVersion = [AzureEngSemanticVersion]::ParseVersionString($packageInfo.Version)
 
   $packageMetadataArray = (Get-CSVMetadata).Where({ $_.Package -eq $packageInfo.Name -and $_.GroupId -eq $packageInfo.Group -and $_.Hide -ne 'true' -and $_.New -eq 'true' })
   if ($packageMetadataArray.Count -eq 0) { 
@@ -176,7 +194,18 @@ function UpdateDocsMsMetadataForPackage($packageInfoJsonLocation) {
     -Value $packageInfoJson
 }
 
-foreach ($packageInfo in $PackageInfoJsonLocations) {
+foreach ($packageInfoLocation in $PackageInfoJsonLocations) {
   Write-Host "Updating metadata for package: $packageInfo"
-  UpdateDocsMsMetadataForPackage $packageInfo
+
+  # Convert package metadata json file to metadata json property.
+  $packageInfo = GetPackageInfoJson $packageInfoLocation
+  # Add validation step for daily update and release
+  if ($ValidateDocsMsPackagesFn -and (Test-Path "Function:$ValidateDocsMsPackagesFn")) {
+    &$ValidateDocsMsPackagesFn -PackageInfo $packageInfo -PackageSourceOverride $PackageSourceOverride -DocValidationImageId $DocValidationImageId
+    if ($LASTEXITCODE) {
+      LogError "The package failed Doc.Ms validation. Check https://aka.ms/azsdk/docs/docker for more details on how to diagnose this issue."
+      exit $LASTEXITCODE
+    }
+  }
+  UpdateDocsMsMetadataForPackage $packageInfoLocation $packageInfo
 }
diff --git a/eng/common/scripts/Verify-Readme.ps1 b/eng/common/scripts/Verify-Readme.ps1
@@ -0,0 +1,15 @@
+# Wrapper Script for Readme Verification
+[CmdletBinding()]
+param (
+  [string]$DocWardenVersion = "0.7.1",
+
+  [Parameter(Mandatory = $true)]
+  [string]$ScanPath,
+
+  [Parameter(Mandatory = $true)]
+  [string]$SettingsPath
+)
+
+pip install setuptools wheel --quiet
+pip install doc-warden==$DocWardenVersion --quiet
+ward scan -d $ScanPath -c $SettingsPath
diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1
@@ -45,3 +45,4 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS
 $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
 $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
 $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"
+$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini
@@ -152,6 +152,7 @@ deps =
   sphinx==3.5.4
   sphinx_rtd_theme==0.5.0
   recommonmark==0.6.0
+  mistune<2.0.0
   m2r==0.2.1
 commands =
   {envbindir}/python {toxinidir}/../../../eng/tox/prep_sphinx_env.py -d {distdir} -t {toxinidir}
@@ -173,6 +174,7 @@ changedir = {toxinidir}
 deps =
   {[packaging]pkgs}
   requests
+  cryptography<4
 commands =
     {envbindir}/python {toxinidir}/../../../eng/tox/sanitize_setup.py -t {toxinidir}
     {envbindir}/python {toxinidir}/../../../eng/tox/create_package_and_install.py \

diff --git a/scripts/release_helper/common.py b/scripts/release_helper/common.py
@@ -1,3 +1,4 @@
+from datetime import date, datetime
 from typing import Set, List, Dict
 import os
 from utils import IssuePackage, REQUEST_REPO, AUTO_ASSIGN_LABEL, AUTO_PARSE_LABEL, get_origin_link_and_tag
@@ -6,6 +7,7 @@
 import time
 from github import Github
 from github.Repository import Repository
+import subprocess as sp
 
 _LOG = logging.getLogger(__name__)
 
@@ -15,6 +17,9 @@
 # 'github assignee': 'token'
 _ASSIGNEE_TOKEN = {'msyyc': os.getenv('PYTHON_MSYYC_TOKEN')}
 
+_SWAGGER_URL = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification'
+_SWAGGER_PULL = 'https://github.com/Azure/azure-rest-api-specs/pull'
+
 
 _SWAGGER_URL = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification'
 _SWAGGER_PULL = 'https://github.com/Azure/azure-rest-api-specs/pull'
@@ -75,7 +80,9 @@ def get_readme_from_pr_link(self, link: str) -> str:
         if len(readme_link) > 1:
             multi_link = ', '.join(readme_link)
             pr = f"{_SWAGGER_PULL}/{pr_number}"
-            self.comment(f'Hi, @{self.assignee}, by parsing {pr}, there are multi service link: {multi_link}. Please decide which one is the right.')
+            self.comment(
+                f'Hi, @{self.assignee}, by parsing {pr}, there are multi service link: {multi_link}. Please decide which one is the right.')
+
             self.bot.append('multi readme link!')
             raise Exception(f'multi link in "{pr}"')
 
@@ -196,10 +203,23 @@ def auto_assign(self) -> None:
             self.update_issue_instance()
         self.add_label(AUTO_ASSIGN_LABEL)
 
+    def bot_advice(self):
+        latest_comments = ''
+        comments = [(comment.updated_at.timestamp(), comment.user.login) for comment in
+                    self.issue_package.issue.get_comments()]
+        comments.sort()
+        if comments:
+            latest_comments = comments[-1][1]
+        if self.issue_package.issue.comments == 0:
+            self.bot = 'new issue ! <br>'
+        elif latest_comments not in self.language_owner:
+            self.bot = 'new comment.  <br>'
+
     def run(self) -> None:
         # common part(don't change the order)
         self.auto_assign()  # necessary flow
         self.auto_parse()  # necessary flow
+        self.bot_advice()
 
 
 class Common:
@@ -213,16 +233,53 @@ def __init__(self, issues: List[IssuePackage], assignee_token: Dict[str, str], l
         self.issues_package = issues
         self.assignee_candidates = set(assignee_token.keys())
         self.language_owner = language_owner
+        # arguments add to language.md
+        self.file_out_name = 'common.md'
+        self.target_release_date = ''
+        self.date_from_target = ''
+        self.package_name = ''
+
         for assignee in assignee_token:
             self.request_repo_dict[assignee] = Github(assignee_token[assignee]).get_repo(REQUEST_REPO)
 
+    def output_md(self, items):
+        with open(self.file_out_name, 'w') as file_out:
+            file_out.write(
+                '| issue | author | package | assignee | bot advice | created date of issue | target release date | date from target |\n')
+            file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n')
+            file_out.writelines([self.output_python(item) for item in items])
+
+    def output_python(self, item):
+        create_date = str(date.fromtimestamp(item.issue_package.issue.created_at.timestamp()).strftime('%m-%d'))
+
+        return '| [#{}]({}) | {} | {} | {} | {} | {} | {} | {} |\n'.format(
+            item.issue_package.issue.html_url.split('/')[-1],
+            item.issue_package.issue.html_url,
+            item.issue_package.issue.user.login,
+            self.package_name,
+            item.issue_package.issue.assignee.login,
+            item.bot,
+            create_date,
+            self.target_release_date,
+            self.date_from_target
+        )
+
+    @staticmethod
+    def push_md_to_storage():
+        cmd_list = ['git add .', 'git commit -m \"update excel\"', 'git push -f origin HEAD']
+        [sp.check_call(cmd, shell=True) for cmd in cmd_list]
+
     def run(self):
+        items = []
         for item in self.issues_package:
             issue = IssueProcess(item, self.request_repo_dict, self.assignee_candidates, self.language_owner)
             try:
                 issue.run()
+                items.append(issue)
             except Exception as e:
                 _LOG.error(f'Error happened during handling issue {item.issue.number}: {e}')
+        self.output_md(items)
+
 
 
 def common_process(issues: List[IssuePackage]):

diff --git a/scripts/release_helper/go.py b/scripts/release_helper/go.py
@@ -14,7 +14,9 @@ class IssueProcessGo(IssueProcess):
 
 
 class Go(Common):
-    pass
+    def __init__(self, issues, assignee_token, language_owner):
+        super(Go, self).__init__(issues, assignee_token, language_owner)
+        self.file_out_name = 'release_go_status.md'
 
 
 def go_process(issues: List[Any]):

diff --git a/scripts/release_helper/java.py b/scripts/release_helper/java.py
@@ -33,7 +33,10 @@ def auto_parse(self) -> None:
 
 
 class Java(Common):
-    pass
+    def __init__(self, issues, assignee_token, language_owner):
+        super(Java, self).__init__(issues, assignee_token, language_owner)
+        self.file_out_name = 'release_java_status.md'
+
 
 
 def java_process(issues: List[Any]):

diff --git a/scripts/release_helper/js.py b/scripts/release_helper/js.py
@@ -14,7 +14,9 @@ class IssueProcessJs(IssueProcess):
 
 
 class Js(Common):
-    pass
+    def __init__(self, issues, assignee_token, language_owner):
+        super(Js, self).__init__(issues, assignee_token, language_owner)
+        self.file_out_name = 'release_js_status.md'
 
 
 def js_process(issues: List[Any]):

diff --git a/scripts/release_helper/main.py b/scripts/release_helper/main.py
@@ -5,7 +5,8 @@
 from go import go_process
 from java import java_process
 from js import js_process
-from common import common_process
+from common import common_process, Common
+
 
 import os
 from typing import List
@@ -54,6 +55,7 @@ def main():
     for language in languages:
         language_issues = select_language_issues(issues, language)
         languages[language](language_issues)
+    Common.push_md_to_storage()
 
 
 if __name__ == '__main__':

diff --git a/scripts/release_helper/release_helper.yml b/scripts/release_helper/release_helper.yml
@@ -36,8 +36,8 @@ jobs:
         git config --global user.name "ReleaseHelper"
 
         # clone(REPO: https://github.com/Azure/azure-sdk-for-python.git, USR_NAME: Azure, USR_TOKEN: xxxxxxxxxxxxx)
-        # mkdir file-storage
-        # git clone ${FILE_REPO:0:8}$(USR_NAME):$(Yuchao-GitToken)@${FILE_REPO:8} $(pwd)/file-storage
+        mkdir file-storage
+        git clone ${FILE_REPO:0:8}$(USR_NAME):$(Yuchao-GitToken)@${FILE_REPO:8} $(pwd)/file-storage
 
         # import env variable
         export TOKEN=$(Yuchao-GitToken)
@@ -55,8 +55,8 @@ jobs:
         pip install -r $script_path/requirement.txt
 
         # checkout the target branch
-        # cd file-storage
-        # git checkout release-helper
+        cd file-storage
+        git checkout release-helper
 
         # run
         python $script_path/main.py