[CI] Move 1: Copy job SourceStaticPy from Travis to ADO (1/6) (Azure#…

…1133) * Move job SourceStatic to ADO - rename as CodeStyleCheck * Fix bug caused by indent * Fix wrong-import-order in applicationinsights.commands * Fix wrong-import-order in azext_dns. - Disable lint on azext_dns * Fix W1202:logging-format-interpolation in azext_find * Fix wrong-import-order in azext_interactive * Recover scripts/ci/test_static.sh. Use scripts/ci/test_static.py instead * Fix encoding bug in Windows scripts/ci/test_static.py * Switch to scripts/ci/test_static.py * Fix logging-format-interpolation in src/mesh/azext_mesh/custom.py * Fix:Using deprecated method warn() (deprecated-method) in src/webapp/azext_webapp/tunnel.py * Fix logging-not-lazy in src/resource-graph/azext_resourcegraph/custom.py * Fix line-too-long in src/aks-preview/azext_aks_preview/custom.py * Fix unused-argument in src/aks-preview/azext_aks_preview/custom.py * Fix too-many-statements in src/aks-preview/azext_aks_preview/custom.py * Fix too-many-branches, too-many-locals in src/aks-preview/azext_aks_preview/custom.py * Fix logging-format-interpolation in src/aks-preview/azext_aks_preview/custom.py * Fix too-many-lines in src/aks-preview/azext_aks_preview/custom.py * Upgrade pylint to 2.3.0 along with azdev * Pre-install wheel * Pre-install wheel==0.30.0 * Pre-install requests * Fix mistake import reorder * Disable try-except-raise in src/dms-preview/azext_dms/custom.py * Disable arguments-differ in src/spring-cloud/azext_spring_cloud/azure_storage_file/sharedaccesssignature.py * Remove no-else-return, no-else-raise refector suggestion * Downgrade pylint to 1.9.5 since try-catch-rase checkers incompatible between 1.* and 2.* * Fix: E2057 in src/aks-preview/azext_aks_preview/custom.py * Fix error in scripts/ci/source_code_static_analysis.py * typo * Remove deprecated extension: DNS * Remove codeowner for deprecated extension: DNS * Remove flake8 exclusion configuration for deprecated extension: DNS * Fix PR Azure#802 bump SDK version without updating tests
jddarby · Dec 17, 2019 · 32dc96d · 32dc96d
1 parent 703054e
commit 32dc96d
Show file tree

Hide file tree

Showing 23 changed files with 576 additions and 308 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -35,20 +35,19 @@ jobs:
       TSLint: false
       ToolLogsNotFoundAction: 'Standard'
 
-# - job: SourceStatic
-#   displayName: "Source Code Static Check"
-#   pool:
-#     vmImage: 'ubuntu-16.04'
-#   steps:
-#   - task: UsePythonVersion@0
-#     displayName: 'Use Python 3.7'
-#     inputs:
-#       versionSpec: 3.7
-#   - task: Bash@3
-#     displayName: "Source Code Static Check"
-#     inputs:
-#       targetType: 'filePath'
-#       filePath: scripts/ci/test_static.sh
+- job: StaticAnalysis
+  displayName: "Static Analysis"
+  pool:
+    vmImage: 'ubuntu-16.04'
+  steps:
+    - task: UsePythonVersion@0
+      displayName: 'Use Python 3.6'
+      inputs:
+        versionSpec: 3.6
+    - bash: pip install wheel==0.30.0 pylint==1.9.5 flake8 requests
+      displayName: 'Install wheel, pylint, flake8, requests'
+    - bash: python scripts/ci/source_code_static_analysis.py
+      displayName: "Static Analysis"
 
 - job: IndexVerify
   displayName: "Verify Extensions Index"

diff --git a/pylintrc b/pylintrc
@@ -4,10 +4,30 @@ reports=no
 
 [MESSAGES CONTROL]
 # For all codes, run 'pylint --list-msgs' or go to 'https://pylint.readthedocs.io/en/latest/reference_guide/features.html'
-# locally-disabled: Warning locally suppressed using disable-msg
-# cyclic-import: because of https://github.com/PyCQA/pylint/issues/850
-# too-many-arguments: Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
-disable=missing-docstring,locally-disabled,fixme,cyclic-import,too-many-arguments,invalid-name,duplicate-code
+#
+# locally-disabled:
+#       Warning locally suppressed using disable-msg
+#
+# cyclic-import:
+#       because of https://github.com/PyCQA/pylint/issues/850
+#
+# too-many-arguments:
+#       Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods.
+#
+# too-many-lines:
+#       Due to unreasonable structure and history debut, files get expanding very large
+#
+# useless-object-inheritance:
+#       Due to some classes inherits from __builtin__.object(), can be safely removed from bases in python3
+#
+disable=missing-docstring,
+        locally-disabled,
+        fixme,
+        cyclic-import,
+        too-many-arguments,
+        invalid-name,
+        duplicate-code,
+        useless-object-inheritance
 
 [TYPECHECK]
 # For Azure CLI extensions, we ignore some import errors as they'll be available in the environment of the CLI

diff --git a/scripts/ci/source_code_static_analysis.py b/scripts/ci/source_code_static_analysis.py
@@ -0,0 +1,154 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import os
+import sys
+import multiprocessing
+
+from pylint import lint
+from flake8.main import application
+
+from util import get_repo_root
+
+# import argparse
+# from verify_codeowners import main as _verify_codeowners
+# from verify_license import main as _verify_license
+
+
+SDK_FILE_MARKER = r"# Code generated by Microsoft (R) AutoRest Code Generator."
+
+
+# build list of sdk files
+def _get_sdk_module_list(root_dir):
+    dir_paths = [os.path.join(root_dir, path) for path in os.listdir(root_dir)]
+    sdk_files = []
+
+    for path in dir_paths:
+        if os.path.isdir(path):
+            sdk_files.extend(_get_sdk_module_list(path))
+        elif _is_sdk_file(path):
+            sdk_files.append(os.path.dirname(path))
+
+    return sdk_files
+
+
+# check if the current file is a python sdk file
+def _is_sdk_file(file_path):
+    # don't bother opening non-python files. e.g pyc files.
+    if not file_path.endswith(".py"):
+        return False
+    with open(file_path, encoding='utf-8') as f:
+        for line in f:
+            if SDK_FILE_MARKER in line:
+                return True
+    return False
+
+
+# build list of modules that start with prefix "azext_" in the src
+def _get_azext_module_paths(root_dir):
+    src_root = os.path.join(root_dir, "src")
+    src_paths = [os.path.join(src_root, path) for path in os.listdir(src_root)]
+    ext_modules = []
+    for ext_root in src_paths:
+        if os.path.isdir(ext_root):
+            paths = [path for path in os.listdir(ext_root)]
+            ext_modules.extend([os.path.join(ext_root, path) for path in paths if path.startswith("azext_")])
+    return ext_modules
+
+
+# build list of python ci scripts
+def _get_ci_py_file_paths(directory):
+    return [os.path.join(directory, path) for path in os.listdir(directory) if path.endswith(".py")]
+
+
+def _run_pylint(module_paths, ignored_modules=None, rcfile=None, cpu_count=1):
+    pylint_opts = []
+
+    if ignored_modules:
+        pylint_opts.append("--ignore={}".format(ignored_modules))
+    if rcfile:
+        pylint_opts.append("--rcfile={}".format(rcfile))
+
+    pylint_opts.append("--jobs={}".format(cpu_count))
+    pylint_opts.extend(module_paths)
+
+    try:
+        lint.Run(pylint_opts)
+    except SystemExit as se:
+        # 0:  everything is fine
+        # 1:  Fatal message issued
+        # 2:  Error message issued
+        # 4:  Warning message issued
+        # 8:  Refactor message issued
+        # 16: Convention message issued
+        # 32: Usage error
+        if se.code not in [0, 8, 16, 24]:
+            sys.exit(se.code)
+
+
+def _run_flake8(module_paths, config_file=None):
+    flake8_opts = ["--statistics"]
+
+    if config_file:
+        flake8_opts.append("--append-config={}".format(config_file))
+
+    flake8_opts.extend(module_paths)
+
+    app = application.Application()
+    app.run(flake8_opts)
+    try:
+        app.exit()
+    except SystemExit:
+        pass
+
+    if app.result_count > 0 or app.catastrophic_failure:
+        sys.exit(app.result_count or 1)
+
+
+def main():
+    cpu_count = multiprocessing.cpu_count()
+
+    root_dir = get_repo_root()
+    sdk_modules = _get_sdk_module_list(root_dir)
+    sdk_modules.append("vendored_sdks")
+    module_paths = _get_azext_module_paths(root_dir)
+
+    # scripts_dir = os.path.join(root_dir, "scripts")
+    # ci_files = _get_ci_py_file_paths(os.path.join(scripts_dir, "ci"))
+
+    rc_file = os.path.join(root_dir, "pylintrc")
+    # config_file = os.path.join(root_dir, "flake8")
+
+    print("\nRunning pylint on extensions...")
+    _run_pylint(module_paths, ",".join(sdk_modules), rc_file, cpu_count)
+    print("Pylint OK.\n")
+
+    # print("Running flake8 on extensions...")
+    # _run_flake8(module_paths, config_file)
+    # print("Flake8 OK.\n")
+    #
+    # print("Running pylint on CI scripts...")
+    # _run_pylint(ci_files, rcfile=rcfile, cpu_count=cpu_count)
+    # print("Pylint OK.\n")
+    #
+    # print("Running flake8 on CI scripts...")
+    # _run_flake8(ci_files, config_file=config_file)
+    # print("Pylint OK.\n")
+    #
+    # print("Other Static checks...")
+    #
+    # _verify_codeowners()
+    #
+    # parser = argparse.ArgumentParser()
+    # parser.add_argument('excluded_paths', nargs='*')
+    # _verify_license(parser.parse_args())
+
+    print("All static checks successful!")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/ci/test_static.py b/scripts/ci/test_static.py
@@ -41,7 +41,7 @@ def _is_sdk_file(file_path):
     # don't bother opening non-python files. e.g pyc files.
     if not file_path.endswith(".py"):
         return False
-    with open(file_path) as f:
+    with open(file_path, encoding='utf-8') as f:
         for line in f:
             if SDK_FILE_MARKER in line:
                 return True

diff --git a/scripts/ci/test_static.sh b/scripts/ci/test_static.sh
@@ -24,4 +24,4 @@ flake8 --append-config=./.flake8 ./scripts/ci/*.py
 
 # Other static checks
 python ./scripts/ci/verify_codeowners.py
-python ./scripts/ci/verify_license.py
+python ./scripts/ci/verify_license.py