From d6f0f53003a3d89daa60444ab10accbcfd4a630b Mon Sep 17 00:00:00 2001
From: ZelinWang <zelinwang@microsoft.com>
Date: Sun, 17 Apr 2022 09:51:27 +0800
Subject: [PATCH] {CI} enable pytest group worker in module test and full test
 (#22047)

* update

* Update automation_full_test.py

* Update automation_full_test.py

* Update automation_full_test.py

* Update automation_full_test.py

* update
---
 .../templates/automation_test.yml             | 14 +++-
 scripts/ci/automation_full_test.py            | 71 +++++++++++++++++++
 2 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 scripts/ci/automation_full_test.py

diff --git a/.azure-pipelines/templates/automation_test.yml b/.azure-pipelines/templates/automation_test.yml
index 1cd74099d9c..443c0f224aa 100644
--- a/.azure-pipelines/templates/automation_test.yml
+++ b/.azure-pipelines/templates/automation_test.yml
@@ -32,12 +32,20 @@ steps:
       echo fullTest: ${{ parameters.fullTest }}
       echo module: ${{ parameters.module }}
       echo Build.Reason: $(Build.Reason)
+      serial_modules="appservice botservice cloud network azure-cli-core azure-cli-telemetry"
 
       # Test specific module
       module="${{ parameters.module }}"
       if [[ -n $module ]]; then
         echo "Running test for module '$module'"
-        azdev test --no-exitfirst --verbose --series $module
+        # Determine whether the module belongs to serial_modules
+        if [[ "$serial_modules" =~ "$module" ]]; then
+          echo "Running in series"
+          azdev test --no-exitfirst --verbose --series $module --pytest-args "--durations=0"
+        else
+          echo "Running in parallels"
+          azdev test --no-exitfirst --verbose $module --pytest-args "--durations=0"
+        fi
         exit 0
       fi
 
@@ -45,10 +53,10 @@ steps:
         echo "Running incremental test"
         # If CI is set to shallow fetch, target branch should be expilictly fetched.
         git fetch origin --depth=1 $(System.PullRequest.TargetBranch)
-        azdev test --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }} --verbose --series
+        azdev test --no-exitfirst --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) --cli-ci --profile ${{ parameters.profile }} --verbose --series --pytest-args "--durations=0"
       else
         echo "Running full test"
-        azdev test --no-exitfirst --profile ${{ parameters.profile }} --verbose --series
+        python scripts/ci/automation_full_test.py "${{ parameters.profile }}" "$serial_modules"
       fi
     displayName: "azdev test"
     env:
diff --git a/scripts/ci/automation_full_test.py b/scripts/ci/automation_full_test.py
new file mode 100644
index 00000000000..a0323525fb5
--- /dev/null
+++ b/scripts/ci/automation_full_test.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import logging
+import subprocess
+import sys
+from azdev.utilities import get_path_table
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+logger.addHandler(ch)
+profile = sys.argv[1]
+serial_modules = sys.argv[2].split()
+
+
+class AutomaticScheduling(object):
+
+    def __init__(self):
+        self.modules = {}
+        self.serial_modules = serial_modules
+        self.profile = profile
+
+    def get_all_modules(self):
+        result = get_path_table()
+        # only get modules and core, ignore extensions
+        self.modules = {**result['mod'], **result['core']}
+
+    def run_modules(self):
+        # divide all modules into parallel or serial execution
+        error_flag = False
+        serial_tests = []
+        parallel_tests = []
+        for k, v in self.modules.items():
+            if k in self.serial_modules:
+                serial_tests.append(k)
+            else:
+                parallel_tests.append(k)
+        if serial_tests:
+            cmd = ['azdev', 'test', '--no-exitfirst', '--verbose', '--series'] + \
+                  serial_tests + ['--profile', f'{profile}', '--pytest-args', '"--durations=0"']
+            logger.info(cmd)
+            try:
+                subprocess.run(cmd)
+            except subprocess.CalledProcessError:
+                error_flag = True
+        if parallel_tests:
+            cmd = ['azdev', 'test', '--no-exitfirst', '--verbose'] + \
+                  parallel_tests + ['--profile', f'{profile}', '--pytest-args', '"--durations=0"']
+            logger.info(cmd)
+            try:
+                subprocess.run(cmd)
+            except subprocess.CalledProcessError:
+                error_flag = True
+        return error_flag
+
+
+def main():
+    logger.info("Start automation full test ...\n")
+    autoschduling = AutomaticScheduling()
+    autoschduling.get_all_modules()
+    sys.exit(1) if autoschduling.run_modules() else sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()