From faf989414408cfd8fa4f71341cb611c175c78fb3 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 9 Jul 2020 14:09:31 +0000 Subject: [PATCH] Bug 1650834 - Make tools/lint code black compatible r=ahal Differential Revision: https://phabricator.services.mozilla.com/D82401 --- tools/lint/android/lints.py | 275 ++++++++++++--------- tools/lint/black.yml | 3 +- tools/lint/clang-format/__init__.py | 39 +-- tools/lint/clippy/__init__.py | 41 +-- tools/lint/cpp/mingw-capitalization.py | 4 +- tools/lint/eslint/__init__.py | 69 +++--- tools/lint/eslint/setup_helper.py | 80 +++--- tools/lint/file-perm/__init__.py | 15 +- tools/lint/file-whitespace/__init__.py | 30 +-- tools/lint/hooks.py | 22 +- tools/lint/hooks_clang_format.py | 6 +- tools/lint/hooks_js_format.py | 6 +- tools/lint/libpref/__init__.py | 65 ++--- tools/lint/license/__init__.py | 63 +++-- tools/lint/mach_commands.py | 100 +++++--- tools/lint/perfdocs/__init__.py | 6 +- tools/lint/perfdocs/framework_gatherers.py | 55 ++--- tools/lint/perfdocs/gatherer.py | 46 ++-- tools/lint/perfdocs/generator.py | 149 +++++------ tools/lint/perfdocs/logger.py | 27 +- tools/lint/perfdocs/perfdocs.py | 10 +- tools/lint/perfdocs/utils.py | 26 +- tools/lint/perfdocs/verifier.py | 103 ++++---- tools/lint/python/check_compat.py | 38 +-- tools/lint/python/compat.py | 32 +-- tools/lint/python/flake8.py | 92 ++++--- tools/lint/python/l10n_lint.py | 74 +++--- tools/lint/python/pylint.py | 30 ++- tools/lint/rst/__init__.py | 41 +-- tools/lint/rust/__init__.py | 14 +- tools/lint/shell/__init__.py | 60 ++--- tools/lint/spell/__init__.py | 75 +++--- tools/lint/test/conftest.py | 42 ++-- tools/lint/test/test_clang_format.py | 8 +- tools/lint/test/test_clippy.py | 8 +- tools/lint/test/test_codespell.py | 6 +- tools/lint/test/test_eslint.py | 12 +- tools/lint/test/test_file_license.py | 4 +- tools/lint/test/test_file_perm.py | 4 +- tools/lint/test/test_file_whitespace.py | 4 +- tools/lint/test/test_flake8.py | 46 ++-- tools/lint/test/test_lintpref.py | 8 +- tools/lint/test/test_pylint.py | 14 +- tools/lint/test/test_rst.py | 8 +- tools/lint/test/test_rustfmt.py | 4 +- tools/lint/test/test_shellcheck.py | 8 +- tools/lint/test/test_yaml.py | 8 +- tools/lint/wpt/wpt.py | 31 ++- tools/lint/yamllint_/__init__.py | 61 +++-- 49 files changed, 1085 insertions(+), 887 deletions(-) diff --git a/tools/lint/android/lints.py b/tools/lint/android/lints.py index fd04ab5f49f78..f280651382dcb 100644 --- a/tools/lint/android/lints.py +++ b/tools/lint/android/lints.py @@ -14,9 +14,7 @@ import xml.etree.ElementTree as ET -from mozpack.files import ( - FileFinder, -) +from mozpack.files import FileFinder import mozpack.path as mozpath from mozlint import result @@ -28,30 +26,42 @@ def setup(root, **setupargs): - if setupargs.get('substs', {}).get('MOZ_BUILD_APP') != 'mobile/android': + if setupargs.get("substs", {}).get("MOZ_BUILD_APP") != "mobile/android": return 1 - if 'topobjdir' not in setupargs: - print('Skipping {}: a configured Android build is required!'.format(setupargs['name'])) + if "topobjdir" not in setupargs: + print( + "Skipping {}: a configured Android build is required!".format( + setupargs["name"] + ) + ) return 1 return 0 def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose=True): - sys.path.insert(0, os.path.join(topsrcdir, 'mobile', 'android')) + sys.path.insert(0, os.path.join(topsrcdir, "mobile", "android")) from gradle import gradle_lock with gradle_lock(topobjdir, max_wait_seconds=GRADLE_LOCK_MAX_WAIT_SECONDS): # The android-lint parameter can be used by gradle tasks to run special # logic when they are run for a lint using # project.hasProperty('android-lint') - cmd_args = [sys.executable, os.path.join(topsrcdir, 'mach'), - 'gradle', '--verbose', '-Pandroid-lint', '--'] + \ - tasks + \ - extra_args - - cmd = ' '.join(six.moves.shlex_quote(arg) for arg in cmd_args) + cmd_args = ( + [ + sys.executable, + os.path.join(topsrcdir, "mach"), + "gradle", + "--verbose", + "-Pandroid-lint", + "--", + ] + + tasks + + extra_args + ) + + cmd = " ".join(six.moves.shlex_quote(arg) for arg in cmd_args) log.debug(cmd) # Gradle and mozprocess do not get along well, so we use subprocess @@ -75,40 +85,45 @@ def gradle(log, topsrcdir=None, topobjdir=None, tasks=[], extra_args=[], verbose def api_lint(config, **lintargs): - topsrcdir = lintargs['root'] - topobjdir = lintargs['topobjdir'] + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] - gradle(lintargs['log'], topsrcdir=topsrcdir, topobjdir=topobjdir, - tasks=lintargs['substs']['GRADLE_ANDROID_API_LINT_TASKS'], - extra_args=lintargs.get('extra_args') or []) + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) - folder = lintargs['substs']['GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER'] + folder = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_APILINT_FOLDER"] results = [] - with open(os.path.join(topobjdir, folder, 'apilint-result.json')) as f: + with open(os.path.join(topobjdir, folder, "apilint-result.json")) as f: issues = json.load(f) - for rule in ('compat_failures', 'failures'): + for rule in ("compat_failures", "failures"): for r in issues[rule]: err = { - 'rule': r['rule'] if rule == 'failures' else 'compat_failures', - 'path': mozpath.relpath(r['file'], topsrcdir), - 'lineno': int(r['line']), - 'column': int(r.get('column') or 0), - 'message': r['msg'], + "rule": r["rule"] if rule == "failures" else "compat_failures", + "path": mozpath.relpath(r["file"], topsrcdir), + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": r["msg"], } results.append(result.from_config(config, **err)) - for r in issues['api_changes']: + for r in issues["api_changes"]: err = { - 'rule': 'api_changes', - 'path': mozpath.relpath(r['file'], topsrcdir), - 'lineno': int(r['line']), - 'column': int(r.get('column') or 0), - 'message': 'Unexpected api change. Please run ./gradlew {} for more ' - 'information'.format( - ' '.join(lintargs['substs']['GRADLE_ANDROID_API_LINT_TASKS'])), + "rule": "api_changes", + "path": mozpath.relpath(r["file"], topsrcdir), + "lineno": int(r["line"]), + "column": int(r.get("column") or 0), + "message": "Unexpected api change. Please run ./gradlew {} for more " + "information".format( + " ".join(lintargs["substs"]["GRADLE_ANDROID_API_LINT_TASKS"]) + ), } results.append(result.from_config(config, **err)) @@ -116,14 +131,18 @@ def api_lint(config, **lintargs): def javadoc(config, **lintargs): - topsrcdir = lintargs['root'] - topobjdir = lintargs['topobjdir'] + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] - gradle(lintargs['log'], topsrcdir=topsrcdir, topobjdir=topobjdir, - tasks=lintargs['substs']['GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS'], - extra_args=lintargs.get('extra_args') or []) + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) - output_files = lintargs['substs']['GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES'] + output_files = lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_DOCS_OUTPUT_FILES"] results = [] @@ -133,42 +152,49 @@ def javadoc(config, **lintargs): issues = json.load(f) for issue in issues: - issue['path'] = issue['path'].replace(lintargs['root'], '') + issue["path"] = issue["path"].replace(lintargs["root"], "") # We want warnings to be errors for linting purposes. - issue['level'] = 'error' + issue["level"] = "error" results.append(result.from_config(config, **issue)) return results def lint(config, **lintargs): - topsrcdir = lintargs['root'] - topobjdir = lintargs['topobjdir'] + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] - gradle(lintargs['log'], topsrcdir=topsrcdir, topobjdir=topobjdir, - tasks=lintargs['substs']['GRADLE_ANDROID_LINT_TASKS'], - extra_args=lintargs.get('extra_args') or []) + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_LINT_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) # It's surprising that this is the App variant name, but this is "withoutGeckoBinariesDebug" # right now and the GeckoView variant name is "withGeckoBinariesDebug". This will be addressed # as we unify variants. path = os.path.join( - lintargs['topobjdir'], - 'gradle/build/mobile/android/geckoview/reports', - 'lint-results-{}.xml'.format(lintargs['substs']['GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME'])) - tree = ET.parse(open(path, 'rt')) + lintargs["topobjdir"], + "gradle/build/mobile/android/geckoview/reports", + "lint-results-{}.xml".format( + lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"] + ), + ) + tree = ET.parse(open(path, "rt")) root = tree.getroot() results = [] - for issue in root.findall('issue'): + for issue in root.findall("issue"): location = issue[0] err = { - 'level': issue.get('severity').lower(), - 'rule': issue.get('id'), - 'message': issue.get('message'), - 'path': location.get('file').replace(lintargs['root'], ''), - 'lineno': int(location.get('line') or 0), + "level": issue.get("severity").lower(), + "rule": issue.get("id"), + "message": issue.get("message"), + "path": location.get("file").replace(lintargs["root"], ""), + "lineno": int(location.get("line") or 0), } results.append(result.from_config(config, **err)) @@ -176,40 +202,46 @@ def lint(config, **lintargs): def _parse_checkstyle_output(config, topsrcdir=None, report_path=None): - tree = ET.parse(open(report_path, 'rt')) + tree = ET.parse(open(report_path, "rt")) root = tree.getroot() - for file in root.findall('file'): - sourcepath = file.get('name').replace(topsrcdir + '/', '') + for file in root.findall("file"): + sourcepath = file.get("name").replace(topsrcdir + "/", "") - for error in file.findall('error'): + for error in file.findall("error"): # Like . # NOQA: E501 err = { - 'level': 'error', - 'rule': error.get('source'), - 'message': error.get('message'), - 'path': sourcepath, - 'lineno': int(error.get('line') or 0), - 'column': int(error.get('column') or 0), + "level": "error", + "rule": error.get("source"), + "message": error.get("message"), + "path": sourcepath, + "lineno": int(error.get("line") or 0), + "column": int(error.get("column") or 0), } yield result.from_config(config, **err) def checkstyle(config, **lintargs): - topsrcdir = lintargs['root'] - topobjdir = lintargs['topobjdir'] + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] - gradle(lintargs['log'], topsrcdir=topsrcdir, topobjdir=topobjdir, - tasks=lintargs['substs']['GRADLE_ANDROID_CHECKSTYLE_TASKS'], - extra_args=lintargs.get('extra_args') or []) + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) results = [] - for relative_path in lintargs['substs']['GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES']: - report_path = os.path.join(lintargs['topobjdir'], relative_path) - results.extend(_parse_checkstyle_output(config, - topsrcdir=lintargs['root'], - report_path=report_path)) + for relative_path in lintargs["substs"]["GRADLE_ANDROID_CHECKSTYLE_OUTPUT_FILES"]: + report_path = os.path.join(lintargs["topobjdir"], relative_path) + results.extend( + _parse_checkstyle_output( + config, topsrcdir=lintargs["root"], report_path=report_path + ) + ) return results @@ -217,77 +249,96 @@ def checkstyle(config, **lintargs): def _parse_android_test_results(config, topsrcdir=None, report_dir=None): # A brute force way to turn a Java FQN into a path on disk. Assumes Java # and Kotlin sources are in mobile/android for performance and simplicity. - sourcepath_finder = FileFinder(os.path.join(topsrcdir, 'mobile', 'android')) + sourcepath_finder = FileFinder(os.path.join(topsrcdir, "mobile", "android")) finder = FileFinder(report_dir) - reports = list(finder.find('TEST-*.xml')) + reports = list(finder.find("TEST-*.xml")) if not reports: - raise RuntimeError('No reports found under {}'.format(report_dir)) + raise RuntimeError("No reports found under {}".format(report_dir)) for report, _ in reports: - tree = ET.parse(open(os.path.join(finder.base, report), 'rt')) + tree = ET.parse(open(os.path.join(finder.base, report), "rt")) root = tree.getroot() - class_name = root.get('name') # Like 'org.mozilla.gecko.permissions.TestPermissions'. - path = '**/' + class_name.replace('.', '/') + '.*' # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501 + class_name = root.get( + "name" + ) # Like 'org.mozilla.gecko.permissions.TestPermissions'. + path = ( + "**/" + class_name.replace(".", "/") + ".*" + ) # Like '**/org/mozilla/gecko/permissions/TestPermissions.*'. # NOQA: E501 - for testcase in root.findall('testcase'): - function_name = testcase.get('name') + for testcase in root.findall("testcase"): + function_name = testcase.get("name") # Schema cribbed from http://llg.cubic.org/docs/junit/. - for unexpected in itertools.chain(testcase.findall('error'), - testcase.findall('failure')): + for unexpected in itertools.chain( + testcase.findall("error"), testcase.findall("failure") + ): sourcepaths = list(sourcepath_finder.find(path)) if not sourcepaths: - raise RuntimeError('No sourcepath found for class {class_name}' - .format(class_name=class_name)) + raise RuntimeError( + "No sourcepath found for class {class_name}".format( + class_name=class_name + ) + ) for sourcepath, _ in sourcepaths: lineno = 0 - message = unexpected.get('message') + message = unexpected.get("message") # Turn '... at org.mozilla.gecko.permissions.TestPermissions.testMultipleRequestsAreQueuedAndDispatchedSequentially(TestPermissions.java:118)' into 118. # NOQA: E501 - pattern = r'at {class_name}\.{function_name}\(.*:(\d+)\)' - pattern = pattern.format(class_name=class_name, function_name=function_name) + pattern = r"at {class_name}\.{function_name}\(.*:(\d+)\)" + pattern = pattern.format( + class_name=class_name, function_name=function_name + ) match = re.search(pattern, message) if match: lineno = int(match.group(1)) else: - msg = 'No source line found for {class_name}.{function_name}'.format( - class_name=class_name, function_name=function_name) + msg = "No source line found for {class_name}.{function_name}".format( + class_name=class_name, function_name=function_name + ) raise RuntimeError(msg) err = { - 'level': 'error', - 'rule': unexpected.get('type'), - 'message': message, - 'path': os.path.join('mobile', 'android', sourcepath), - 'lineno': lineno, + "level": "error", + "rule": unexpected.get("type"), + "message": message, + "path": os.path.join("mobile", "android", sourcepath), + "lineno": lineno, } yield result.from_config(config, **err) def test(config, **lintargs): - topsrcdir = lintargs['root'] - topobjdir = lintargs['topobjdir'] + topsrcdir = lintargs["root"] + topobjdir = lintargs["topobjdir"] - gradle(lintargs['log'], topsrcdir=topsrcdir, topobjdir=topobjdir, - tasks=lintargs['substs']['GRADLE_ANDROID_TEST_TASKS'], - extra_args=lintargs.get('extra_args') or []) + gradle( + lintargs["log"], + topsrcdir=topsrcdir, + topobjdir=topobjdir, + tasks=lintargs["substs"]["GRADLE_ANDROID_TEST_TASKS"], + extra_args=lintargs.get("extra_args") or [], + ) results = [] def capitalize(s): # Can't use str.capitalize because it lower cases trailing letters. - return (s[0].upper() + s[1:]) if s else '' + return (s[0].upper() + s[1:]) if s else "" - pairs = [('geckoview', lintargs['substs']['GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME'])] + pairs = [("geckoview", lintargs["substs"]["GRADLE_ANDROID_GECKOVIEW_VARIANT_NAME"])] for project, variant in pairs: report_dir = os.path.join( - lintargs['topobjdir'], - 'gradle/build/mobile/android/{}/test-results/test{}UnitTest' - .format(project, capitalize(variant))) - results.extend(_parse_android_test_results(config, - topsrcdir=lintargs['root'], - report_dir=report_dir)) + lintargs["topobjdir"], + "gradle/build/mobile/android/{}/test-results/test{}UnitTest".format( + project, capitalize(variant) + ), + ) + results.extend( + _parse_android_test_results( + config, topsrcdir=lintargs["root"], report_dir=report_dir + ) + ) return results diff --git a/tools/lint/black.yml b/tools/lint/black.yml index c33d4f2d14c1e..48c07abb478b9 100644 --- a/tools/lint/black.yml +++ b/tools/lint/black.yml @@ -6,8 +6,7 @@ black: - taskcluster/docker/funsize-update-generator - testing/condprofile/condprof - tools/crashreporter/system-symbols - - tools/lint/python/black.py - - tools/lint/test/test_black.py + - tools/lint/ - tools/tryselect/selectors/scriptworker.py exclude: - layout/style/ServoCSSPropList.mako.py diff --git a/tools/lint/clang-format/__init__.py b/tools/lint/clang-format/__init__.py index d5daa920c4af4..18b133bb0161c 100644 --- a/tools/lint/clang-format/__init__.py +++ b/tools/lint/clang-format/__init__.py @@ -28,8 +28,11 @@ def parse_issues(config, output, paths, log): for line in output: match = diff_line.match(line) file, line_no, col, diff, diff2 = match.groups() - log.debug("file={} line={} col={} diff={} diff2={}".format( - file, line_no, col, diff, diff2)) + log.debug( + "file={} line={} col={} diff={} diff2={}".format( + file, line_no, col, diff, diff2 + ) + ) d = diff + "\n" + diff2 res = { "path": file, @@ -78,34 +81,34 @@ def get_clang_format_binary(): clang_tools_path = os.path.join(get_state_dir(), "clang-tools") bin_path = os.path.join(clang_tools_path, "clang-tidy", "bin") - return os.path.join(bin_path, "clang-format" + substs.get('HOST_BIN_SUFFIX', '')) + return os.path.join(bin_path, "clang-format" + substs.get("HOST_BIN_SUFFIX", "")) def is_ignored_path(ignored_dir_re, topsrcdir, f): # Remove up to topsrcdir in pathname and match - if f.startswith(topsrcdir + '/'): - match_f = f[len(topsrcdir + '/'):] + if f.startswith(topsrcdir + "/"): + match_f = f[len(topsrcdir + "/") :] else: match_f = f return re.match(ignored_dir_re, match_f) def remove_ignored_path(paths, topsrcdir, log): - path_to_third_party = os.path.join(topsrcdir, '.clang-format-ignore') + path_to_third_party = os.path.join(topsrcdir, ".clang-format-ignore") ignored_dir = [] - with open(path_to_third_party, 'r') as fh: + with open(path_to_third_party, "r") as fh: for line in fh: # In case it starts with a space line = line.strip() # Remove comments and empty lines - if line.startswith('#') or len(line) == 0: + if line.startswith("#") or len(line) == 0: continue # The regexp is to make sure we are managing relative paths ignored_dir.append(r"^[\./]*" + line.rstrip()) # Generates the list of regexp - ignored_dir_re = '(%s)' % '|'.join(ignored_dir) + ignored_dir_re = "(%s)" % "|".join(ignored_dir) path_list = [] for f in paths: @@ -119,13 +122,13 @@ def remove_ignored_path(paths, topsrcdir, log): def lint(paths, config, fix=None, **lintargs): - log = lintargs['log'] - paths = list(expand_exclusions(paths, config, lintargs['root'])) + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) # We ignored some specific files for a bunch of reasons. # Not using excluding to avoid duplication - if lintargs.get('use_filters', True): - paths = remove_ignored_path(paths, lintargs['root'], log) + if lintargs.get("use_filters", True): + paths = remove_ignored_path(paths, lintargs["root"], log) # An empty path array can occur when the user passes in `-n`. If we don't # return early in this case, rustfmt will attempt to read stdin and hang. @@ -146,18 +149,20 @@ def lint(paths, config, fix=None, **lintargs): else: cmd_args.append("--dry-run") base_command = cmd_args + paths - log.debug("Command: {}".format(' '.join(cmd_args))) + log.debug("Command: {}".format(" ".join(cmd_args))) output = run_process(config, base_command) output_list = [] if len(output) % 3 != 0: - raise Exception("clang-format output should be a multiple of 3. Output: %s" % output) + raise Exception( + "clang-format output should be a multiple of 3. Output: %s" % output + ) for i in range(0, len(output), 3): # Merge the element 3 by 3 (clang-format output) line = output[i] - line += ";" + output[i+1] - line += ";" + output[i+2] + line += ";" + output[i + 1] + line += ";" + output[i + 2] output_list.append(line) if fix: diff --git a/tools/lint/clippy/__init__.py b/tools/lint/clippy/__init__.py index c6497fe028df6..f38b3697ee289 100644 --- a/tools/lint/clippy/__init__.py +++ b/tools/lint/clippy/__init__.py @@ -44,13 +44,16 @@ def parse_issues(log, config, issues, path, onlyIn): try: detail = json.loads(six.ensure_text(issue)) if "message" in detail: - p = detail['target']['src_path'] + p = detail["target"]["src_path"] detail = detail["message"] if "level" in detail: - if ((detail["level"] == "error" or detail["level"] == "failure-note") - and not detail["code"]): - log.debug("Error outside of clippy." - "This means that the build failed. Therefore, skipping this") + if ( + detail["level"] == "error" or detail["level"] == "failure-note" + ) and not detail["code"]: + log.debug( + "Error outside of clippy." + "This means that the build failed. Therefore, skipping this" + ) log.debug("File = {} / Detail = {}".format(p, detail)) continue # We are in a clippy warning @@ -61,13 +64,17 @@ def parse_issues(log, config, issues, path, onlyIn): # [], 'code': None, 'level': 'warning', 'message': # '5 warnings emitted', 'spans': []} # if this is the case, skip it - log.debug("Skipping the summary line {} for file {}".format(detail, p)) + log.debug( + "Skipping the summary line {} for file {}".format(detail, p) + ) continue l = detail["spans"][0] if onlyIn and onlyIn not in p: # Case when we have a .rs in the include list in the yaml file - log.debug("{} is not part of the list of files '{}'".format(p, onlyIn)) + log.debug( + "{} is not part of the list of files '{}'".format(p, onlyIn) + ) continue res = { "path": p, @@ -119,13 +126,9 @@ def get_clippy_version(log, binary): # --version failed, clippy isn't installed. return False - log.debug( - "Found version: {}".format( - output - ) - ) + log.debug("Found version: {}".format(output)) - version = re.findall(r'(\d+-\d+-\d+)', output)[0].replace("-", ".") + version = re.findall(r"(\d+-\d+-\d+)", output)[0].replace("-", ".") version = StrictVersion(version) return version @@ -160,11 +163,11 @@ def lint(paths, config, fix=None, **lintargs): if not cargo: print(CARGO_NOT_FOUND) - if 'MOZ_AUTOMATION' in os.environ: + if "MOZ_AUTOMATION" in os.environ: return 1 return [] - min_version_str = config.get('min_clippy_version') + min_version_str = config.get("min_clippy_version") min_version = StrictVersion(min_version_str) actual_version = get_clippy_version(log, cargo) log.debug( @@ -183,13 +186,13 @@ def lint(paths, config, fix=None, **lintargs): cmd_args_common = ["--manifest-path"] cmd_args_clippy = [ cargo, - 'clippy', - '--message-format=json', + "clippy", + "--message-format=json", ] lock_files_to_delete = [] for p in paths: - lock_file = os.path.join(p, 'Cargo.lock') + lock_file = os.path.join(p, "Cargo.lock") if not os.path.exists(lock_file): lock_files_to_delete.append(lock_file) @@ -218,7 +221,7 @@ def lint(paths, config, fix=None, **lintargs): # Make sure that we don't display that either onlyIn = p - cargo_files = get_ancestors_by_name('Cargo.toml', p, lintargs['root']) + cargo_files = get_ancestors_by_name("Cargo.toml", p, lintargs["root"]) p = cargo_files[0] log.debug("Path translated to = {}".format(p)) diff --git a/tools/lint/cpp/mingw-capitalization.py b/tools/lint/cpp/mingw-capitalization.py index fef02e63a2fe3..b5a4b07c6ae7b 100644 --- a/tools/lint/cpp/mingw-capitalization.py +++ b/tools/lint/cpp/mingw-capitalization.py @@ -8,7 +8,7 @@ from mozlint.types import LineType here = os.path.abspath(os.path.dirname(__file__)) -HEADERS_FILE = os.path.join(here, 'mingw-headers.txt') +HEADERS_FILE = os.path.join(here, "mingw-headers.txt") # generated by cd mingw-w64/mingw-w64-headers && # find . -name "*.h" | xargs -I bob -- basename bob | sort | uniq) @@ -16,7 +16,7 @@ class MinGWCapitalization(LineType): def __init__(self, *args, **kwargs): super(MinGWCapitalization, self).__init__(*args, **kwargs) - with open(HEADERS_FILE, 'r') as fh: + with open(HEADERS_FILE, "r") as fh: self.headers = fh.read().strip().splitlines() self.regex = re.compile("^#include\s*<(" + "|".join(self.headers) + ")>") diff --git a/tools/lint/eslint/__init__.py b/tools/lint/eslint/__init__.py index c32c53af130a5..cf1b29e64a530 100644 --- a/tools/lint/eslint/__init__.py +++ b/tools/lint/eslint/__init__.py @@ -47,8 +47,8 @@ def setup(root, **lintargs): def lint(paths, config, binary=None, fix=None, setup=None, **lintargs): """Run eslint.""" - log = lintargs['log'] - setup_helper.set_project_root(lintargs['root']) + log = lintargs["log"] + setup_helper.set_project_root(lintargs["root"]) module_path = setup_helper.get_project_root() # Valid binaries are: @@ -63,35 +63,44 @@ def lint(paths, config, binary=None, fix=None, setup=None, **lintargs): print(ESLINT_NOT_FOUND_MESSAGE) return 1 - extra_args = lintargs.get('extra_args') or [] + extra_args = lintargs.get("extra_args") or [] exclude_args = [] - for path in config.get('exclude', []): - exclude_args.extend(['--ignore-pattern', os.path.relpath(path, lintargs['root'])]) - - cmd_args = [binary, - os.path.join(module_path, "node_modules", "eslint", "bin", "eslint.js"), - # This keeps ext as a single argument. - '--ext', '[{}]'.format(','.join(config['extensions'])), - '--format', 'json', - '--no-error-on-unmatched-pattern', - ] + extra_args + exclude_args + paths - log.debug("Command: {}".format(' '.join(cmd_args))) + for path in config.get("exclude", []): + exclude_args.extend( + ["--ignore-pattern", os.path.relpath(path, lintargs["root"])] + ) + + cmd_args = ( + [ + binary, + os.path.join(module_path, "node_modules", "eslint", "bin", "eslint.js"), + # This keeps ext as a single argument. + "--ext", + "[{}]".format(",".join(config["extensions"])), + "--format", + "json", + "--no-error-on-unmatched-pattern", + ] + + extra_args + + exclude_args + + paths + ) + log.debug("Command: {}".format(" ".join(cmd_args))) # eslint requires that --fix be set before the --ext argument. if fix: - cmd_args.insert(2, '--fix') + cmd_args.insert(2, "--fix") shell = False - if os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64'): + if os.environ.get("MSYSTEM") in ("MINGW32", "MINGW64"): # The eslint binary needs to be run from a shell with msys shell = True - encoding = 'utf-8' + encoding = "utf-8" orig = signal.signal(signal.SIGINT, signal.SIG_IGN) - proc = subprocess.Popen(cmd_args, - shell=shell, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = subprocess.Popen( + cmd_args, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) signal.signal(signal.SIGINT, orig) try: @@ -118,16 +127,18 @@ def lint(paths, config, binary=None, fix=None, setup=None, **lintargs): results = [] for obj in jsonresult: - errors = obj['messages'] + errors = obj["messages"] for err in errors: - err.update({ - 'hint': err.get('fix'), - 'level': 'error' if err['severity'] == 2 else 'warning', - 'lineno': err.get('line') or 0, - 'path': obj['filePath'], - 'rule': err.get('ruleId'), - }) + err.update( + { + "hint": err.get("fix"), + "level": "error" if err["severity"] == 2 else "warning", + "lineno": err.get("line") or 0, + "path": obj["filePath"], + "rule": err.get("ruleId"), + } + ) results.append(result.from_config(config, **err)) return results diff --git a/tools/lint/eslint/setup_helper.py b/tools/lint/eslint/setup_helper.py index dd5b280188fde..09671f94bda61 100644 --- a/tools/lint/eslint/setup_helper.py +++ b/tools/lint/eslint/setup_helper.py @@ -13,8 +13,12 @@ from distutils.version import LooseVersion from filecmp import dircmp -from mozbuild.nodeutil import (find_node_executable, find_npm_executable, - NPM_MIN_VERSION, NODE_MIN_VERSION) +from mozbuild.nodeutil import ( + find_node_executable, + find_npm_executable, + NPM_MIN_VERSION, + NODE_MIN_VERSION, +) from mozbuild.util import ensure_subprocess_env from mozfile.mozfile import remove as mozfileremove @@ -70,11 +74,16 @@ def eslint_setup(should_clobber=False): guide you through an interactive wizard helping you configure eslint for optimal use on Mozilla projects. """ - package_setup(get_project_root(), 'eslint', should_clobber=should_clobber) + package_setup(get_project_root(), "eslint", should_clobber=should_clobber) -def package_setup(package_root, package_name, should_update=False, should_clobber=False, - no_optional=False): +def package_setup( + package_root, + package_name, + should_update=False, + should_clobber=False, + no_optional=False, +): """Ensure `package_name` at `package_root` is installed. When `should_update` is true, clobber, install, and produce a new @@ -101,8 +110,8 @@ def package_setup(package_root, package_name, should_update=False, should_clobbe if should_clobber: node_modules_path = os.path.join(project_root, "node_modules") print("Clobbering %s..." % node_modules_path) - if sys.platform.startswith('win') and have_winrm(): - process = subprocess.Popen(['winrm', '-rf', node_modules_path]) + if sys.platform.startswith("win") and have_winrm(): + process = subprocess.Popen(["winrm", "-rf", node_modules_path]) process.wait() else: mozfileremove(node_modules_path) @@ -118,7 +127,7 @@ def package_setup(package_root, package_name, should_update=False, should_clobbe extra_parameters = ["--loglevel=error"] if no_optional: - extra_parameters.append('--no-optional') + extra_parameters.append("--no-optional") package_lock_json_path = os.path.join(get_project_root(), "package-lock.json") @@ -140,13 +149,15 @@ def package_setup(package_root, package_name, should_update=False, should_clobbe cmd.extend(extra_parameters) - print("Installing %s for mach using \"%s\"..." % (package_name, " ".join(cmd))) + print('Installing %s for mach using "%s"...' % (package_name, " ".join(cmd))) result = call_process(package_name, cmd) if not result: return 1 - bin_path = os.path.join(get_project_root(), "node_modules", ".bin", package_name) + bin_path = os.path.join( + get_project_root(), "node_modules", ".bin", package_name + ) print("\n%s installed successfully!" % package_name) print("\nNOTE: Your local %s binary is at %s\n" % (package_name, bin_path)) @@ -184,15 +195,17 @@ def expected_eslint_modules(): # Also read the in-tree ESLint plugin mozilla information, to ensure the # dependencies are up to date. - mozilla_json_path = os.path.join(get_eslint_module_path(), - "eslint-plugin-mozilla", "package.json") + mozilla_json_path = os.path.join( + get_eslint_module_path(), "eslint-plugin-mozilla", "package.json" + ) with open(mozilla_json_path, "r", encoding="utf-8") as f: expected_modules.update(json.load(f)["dependencies"]) # Also read the in-tree ESLint plugin spidermonkey information, to ensure the # dependencies are up to date. - mozilla_json_path = os.path.join(get_eslint_module_path(), - "eslint-plugin-spidermonkey-js", "package.json") + mozilla_json_path = os.path.join( + get_eslint_module_path(), "eslint-plugin-spidermonkey-js", "package.json" + ) with open(mozilla_json_path, "r", encoding="utf-8") as f: expected_modules.update(json.load(f)["dependencies"]) @@ -204,7 +217,7 @@ def check_file_diffs(dcmp): # Diff files only looks at files that are different. Not for files # that are only present on one side. This should be generally OK as # new files will need to be added in the index.js for the package. - if dcmp.diff_files and dcmp.diff_files != ['package.json']: + if dcmp.diff_files and dcmp.diff_files != ["package.json"]: return True result = False @@ -216,8 +229,10 @@ def check_file_diffs(dcmp): return result - dcmp = dircmp(os.path.join(node_modules_path, name), - os.path.join(get_eslint_module_path(), name)) + dcmp = dircmp( + os.path.join(node_modules_path, name), + os.path.join(get_eslint_module_path(), name), + ) return check_file_diffs(dcmp) @@ -299,18 +314,21 @@ def get_possible_node_paths_win(): if platform.system() != "Windows": return [] - return list({ - "%s\\nodejs" % os.environ.get("SystemDrive"), - os.path.join(os.environ.get("ProgramFiles"), "nodejs"), - os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"), - os.path.join(os.environ.get("PROGRAMFILES"), "nodejs") - }) + return list( + { + "%s\\nodejs" % os.environ.get("SystemDrive"), + os.path.join(os.environ.get("ProgramFiles"), "nodejs"), + os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"), + os.path.join(os.environ.get("PROGRAMFILES"), "nodejs"), + } + ) def get_version(path): try: - version_str = subprocess.check_output([path, "--version"], stderr=subprocess.STDOUT, - universal_newlines=True) + version_str = subprocess.check_output( + [path, "--version"], stderr=subprocess.STDOUT, universal_newlines=True + ) return version_str except (subprocess.CalledProcessError, OSError): return None @@ -332,8 +350,8 @@ def set_project_root(root=None): file_found = False folder = os.getcwd() - while (folder): - if os.path.exists(os.path.join(folder, 'mach')): + while folder: + if os.path.exists(os.path.join(folder, "mach")): file_found = True break else: @@ -382,9 +400,9 @@ def check_node_executables_valid(): def have_winrm(): # `winrm -h` should print 'winrm version ...' and exit 1 try: - p = subprocess.Popen(['winrm.exe', '-h'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - return p.wait() == 1 and p.stdout.read().startswith('winrm') + p = subprocess.Popen( + ["winrm.exe", "-h"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + return p.wait() == 1 and p.stdout.read().startswith("winrm") except Exception: return False diff --git a/tools/lint/file-perm/__init__.py b/tools/lint/file-perm/__init__.py index 3828c0072493e..effa078a57d49 100644 --- a/tools/lint/file-perm/__init__.py +++ b/tools/lint/file-perm/__init__.py @@ -13,15 +13,15 @@ def lint(paths, config, fix=None, **lintargs): - if platform.system() == 'Windows': + if platform.system() == "Windows": # Windows doesn't have permissions in files # Exit now return results - files = list(expand_exclusions(paths, config, lintargs['root'])) + files = list(expand_exclusions(paths, config, lintargs["root"])) for f in files: if os.access(f, os.X_OK): - with open(f, 'r+') as content: + with open(f, "r+") as content: # Some source files have +x permissions line = content.readline() if line.startswith("#!"): @@ -34,9 +34,10 @@ def lint(paths, config, fix=None, **lintargs): os.chmod(f, 0o644) continue - res = {'path': f, - 'message': "Execution permissions on a source file", - 'level': 'error' - } + res = { + "path": f, + "message": "Execution permissions on a source file", + "level": "error", + } results.append(result.from_config(config, **res)) return results diff --git a/tools/lint/file-whitespace/__init__.py b/tools/lint/file-whitespace/__init__.py index af8e3a86c1716..1611ea50e0ad0 100644 --- a/tools/lint/file-whitespace/__init__.py +++ b/tools/lint/file-whitespace/__init__.py @@ -9,10 +9,10 @@ def lint(paths, config, fix=None, **lintargs): - files = list(expand_exclusions(paths, config, lintargs['root'])) + files = list(expand_exclusions(paths, config, lintargs["root"])) for f in files: - with open(f, 'rb') as open_file: + with open(f, "rb") as open_file: hasFix = False content_to_write = [] for i, line in enumerate(open_file): @@ -23,18 +23,19 @@ def lint(paths, config, fix=None, **lintargs): content_to_write.append(line.rstrip() + b"\n") hasFix = True else: - res = {'path': f, - 'message': "Trailing whitespace", - 'level': 'error', - 'lineno': i + 1, - } + res = { + "path": f, + "message": "Trailing whitespace", + "level": "error", + "lineno": i + 1, + } results.append(result.from_config(config, **res)) else: if fix: content_to_write.append(line) if hasFix: # Only update the file when we found a change to make - with open(f, 'wb') as open_file_to_write: + with open(f, "wb") as open_file_to_write: open_file_to_write.write(b"".join(content_to_write)) # We are still using the same fp, let's return to the first @@ -47,14 +48,15 @@ def lint(paths, config, fix=None, **lintargs): if b"\r\n" in content: if fix: # replace \r\n by \n - content = content.replace(b'\r\n', b'\n') - with open(f, 'wb') as open_file_to_write: + content = content.replace(b"\r\n", b"\n") + with open(f, "wb") as open_file_to_write: open_file_to_write.write(content) else: - res = {'path': f, - 'message': "Windows line return", - 'level': 'error' - } + res = { + "path": f, + "message": "Windows line return", + "level": "error", + } results.append(result.from_config(config, **res)) return results diff --git a/tools/lint/hooks.py b/tools/lint/hooks.py index 254b232380358..61dc17ed345fb 100755 --- a/tools/lint/hooks.py +++ b/tools/lint/hooks.py @@ -27,35 +27,35 @@ def run_process(cmd): def run_mozlint(hooktype, args): # --quiet prevents warnings on eslint, it will be ignored by other linters - python = find_executable('python3') + python = find_executable("python3") if not python: print("error: Python 3 not detected on your system! Please install it.") sys.exit(1) - cmd = [python, os.path.join(topsrcdir, 'mach'), 'lint', '--quiet'] + cmd = [python, os.path.join(topsrcdir, "mach"), "lint", "--quiet"] - if 'commit' in hooktype: + if "commit" in hooktype: # don't prevent commits, just display the lint results - run_process(cmd + ['--workdir=staged']) + run_process(cmd + ["--workdir=staged"]) return False - elif 'push' in hooktype: - return run_process(cmd + ['--outgoing'] + args) + elif "push" in hooktype: + return run_process(cmd + ["--outgoing"] + args) print("warning: '{}' is not a valid mozlint hooktype".format(hooktype)) return False def hg(ui, repo, **kwargs): - hooktype = kwargs['hooktype'] - return run_mozlint(hooktype, kwargs.get('pats', [])) + hooktype = kwargs["hooktype"] + return run_mozlint(hooktype, kwargs.get("pats", [])) def git(): hooktype = os.path.basename(__file__) - if hooktype == 'hooks.py': - hooktype = 'pre-push' + if hooktype == "hooks.py": + hooktype = "pre-push" return run_mozlint(hooktype, []) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(git()) diff --git a/tools/lint/hooks_clang_format.py b/tools/lint/hooks_clang_format.py index 70878a63574c1..a239eebe4c5ae 100755 --- a/tools/lint/hooks_clang_format.py +++ b/tools/lint/hooks_clang_format.py @@ -11,7 +11,11 @@ here = os.path.dirname(os.path.realpath(__file__)) topsrcdir = os.path.join(here, os.pardir, os.pardir) -EXTRA_PATHS = ("python/mozversioncontrol", "python/mozbuild", "testing/mozbase/mozfile",) +EXTRA_PATHS = ( + "python/mozversioncontrol", + "python/mozbuild", + "testing/mozbase/mozfile", +) sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS] from mozversioncontrol import get_repository_object, InvalidRepoPath diff --git a/tools/lint/hooks_js_format.py b/tools/lint/hooks_js_format.py index 2f2ecdbcb61b8..7af4ea9243ef3 100755 --- a/tools/lint/hooks_js_format.py +++ b/tools/lint/hooks_js_format.py @@ -11,7 +11,11 @@ here = os.path.dirname(os.path.realpath(__file__)) topsrcdir = os.path.join(here, os.pardir, os.pardir) -EXTRA_PATHS = ("python/mozversioncontrol", "python/mozbuild", "testing/mozbase/mozfile",) +EXTRA_PATHS = ( + "python/mozversioncontrol", + "python/mozbuild", + "testing/mozbase/mozfile", +) sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS] from mozversioncontrol import get_repository_object, InvalidRepoPath diff --git a/tools/lint/libpref/__init__.py b/tools/lint/libpref/__init__.py index a6e6c33ec3fe6..66551c6fbbd7e 100644 --- a/tools/lint/libpref/__init__.py +++ b/tools/lint/libpref/__init__.py @@ -16,14 +16,14 @@ # If for any reason a pref needs to appear in both files, add it to this set. IGNORE_PREFS = { - 'devtools.console.stdout.chrome', # Uses the 'sticky' attribute. - 'devtools.console.stdout.content', # Uses the 'sticky' attribute. - 'fission.autostart', # Uses the 'locked' attribute. - 'browser.dom.window.dump.enabled', # Uses the 'sticky' attribute. - 'apz.fling_curve_function_y2', # This pref is a part of a series. - 'dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled', # NOQA: E501; Uses the 'locked' attribute. + "devtools.console.stdout.chrome", # Uses the 'sticky' attribute. + "devtools.console.stdout.content", # Uses the 'sticky' attribute. + "fission.autostart", # Uses the 'locked' attribute. + "browser.dom.window.dump.enabled", # Uses the 'sticky' attribute. + "apz.fling_curve_function_y2", # This pref is a part of a series. + "dom.postMessage.sharedArrayBuffer.bypassCOOP_COEP.insecure.enabled", # NOQA: E501; Uses the 'locked' attribute. } -PATTERN = re.compile(r'\s*pref\(\s*\"(?P.+)\"\s*,\s*(?P.+)\)\s*;.*') +PATTERN = re.compile(r"\s*pref\(\s*\"(?P.+)\"\s*,\s*(?P.+)\)\s*;.*") def get_names(pref_list_filename): @@ -32,17 +32,16 @@ def get_names(pref_list_filename): # pattern does not happen in 'name', so it's fine to ignore these. # We also want to evaluate all branches of #ifdefs for pref names, so we # ignore anything else preprocessor related. - file = open(pref_list_filename).read().replace('@', '') + file = open(pref_list_filename).read().replace("@", "") try: pref_list = yaml.safe_load(file) except (IOError, ValueError) as e: - print('{}: error:\n {}' - .format(pref_list_filename, e), file=sys.stderr) + print("{}: error:\n {}".format(pref_list_filename, e), file=sys.stderr) sys.exit(1) for pref in pref_list: - if pref['name'] not in IGNORE_PREFS: - pref_names[pref['name']] = pref['value'] + if pref["name"] not in IGNORE_PREFS: + pref_names[pref["name"]] = pref["value"] return pref_names @@ -53,21 +52,23 @@ def check_against(path, pref_names): errors = [] prefs = read_prefs(path) for pref in prefs: - if pref['name'] in pref_names: - errors.extend(check_value_for_pref(pref, pref_names[pref['name']], path)) + if pref["name"] in pref_names: + errors.extend(check_value_for_pref(pref, pref_names[pref["name"]], path)) return errors def check_value_for_pref(some_pref, some_value, path): errors = [] - if some_pref['value'] == some_value: - errors.append({ - 'path': path, - 'message': some_pref['raw'], - 'lineno': some_pref['line'], - 'hint': 'Remove the duplicate pref or add it to IGNORE_PREFS.', - 'level': 'error', - }) + if some_pref["value"] == some_value: + errors.append( + { + "path": path, + "message": some_pref["raw"], + "lineno": some_pref["line"], + "hint": "Remove the duplicate pref or add it to IGNORE_PREFS.", + "level": "error", + } + ) return errors @@ -79,17 +80,19 @@ def read_prefs(path): for lineno, line in enumerate(source, start=1): match = PATTERN.match(line) if match: - prefs.append({ - 'name': match.group('pref'), - 'value': evaluate_pref(match.group('val')), - 'line': lineno, - 'raw': line - }) + prefs.append( + { + "name": match.group("pref"), + "value": evaluate_pref(match.group("val")), + "line": lineno, + "raw": line, + } + ) return prefs def evaluate_pref(value): - bools = {'true': True, 'false': False} + bools = {"true": True, "false": False} if value in bools: return bools[value] elif value.isdigit(): @@ -100,8 +103,8 @@ def evaluate_pref(value): def checkdupes(paths, config, **kwargs): results = [] errors = [] - pref_names = get_names(config['support-files'][0]) - files = list(expand_exclusions(paths, config, kwargs['root'])) + pref_names = get_names(config["support-files"][0]) + files = list(expand_exclusions(paths, config, kwargs["root"])) for file in files: errors.extend(check_against(file, pref_names)) for error in errors: diff --git a/tools/lint/license/__init__.py b/tools/lint/license/__init__.py index 2292558a7c096..35b224d3aa772 100644 --- a/tools/lint/license/__init__.py +++ b/tools/lint/license/__init__.py @@ -13,19 +13,17 @@ # Official source: https://www.mozilla.org/en-US/MPL/headers/ TEMPLATES = { - "mpl2_license": - """ + "mpl2_license": """ This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. """.strip().splitlines(), - "public_domain_license": - """ + "public_domain_license": """ Any copyright is dedicated to the public domain. http://creativecommons.org/publicdomain/zero/1.0/ """.strip().splitlines(), } -license_list = os.path.join(here, 'valid-licenses.txt') +license_list = os.path.join(here, "valid-licenses.txt") def load_valid_license(): @@ -35,7 +33,7 @@ def load_valid_license(): with open(license_list) as f: l = f.readlines() # Remove the empty lines - return list(filter(bool, [x.replace('\n', '') for x in l])) + return list(filter(bool, [x.replace("\n", "") for x in l])) def is_valid_license(licenses, filename): @@ -43,7 +41,7 @@ def is_valid_license(licenses, filename): From a given file, check if we can find the license patterns in the X first lines of the file """ - with open(filename, 'r', errors='replace') as myfile: + with open(filename, "r", errors="replace") as myfile: contents = myfile.read() # Empty files don't need a license. if not contents: @@ -60,7 +58,7 @@ def add_header(filename, header): Add the header to the top of the file """ header.append("\n") - with open(filename, 'r+') as f: + with open(filename, "r+") as f: # lines in list format lines = f.readlines() @@ -85,9 +83,17 @@ def is_test(f): if "lint/test/" in f: # For the unit tests return False - return ("/test" in f or "/gtest" in f or "/crashtest" in f or "/mochitest" in f - or "/reftest" in f or "/imptest" in f or "/androidTest" in f - or "/jit-test/" in f or "jsapi-tests/" in f) + return ( + "/test" in f + or "/gtest" in f + or "/crashtest" in f + or "/mochitest" in f + or "/reftest" in f + or "/imptest" in f + or "/androidTest" in f + or "/jit-test/" in f + or "jsapi-tests/" in f + ) def fix_me(filename): @@ -97,14 +103,26 @@ def fix_me(filename): _, ext = os.path.splitext(filename) license = [] - license_template = TEMPLATES['mpl2_license'] + license_template = TEMPLATES["mpl2_license"] test = False if is_test(filename): - license_template = TEMPLATES['public_domain_license'] + license_template = TEMPLATES["public_domain_license"] test = True - if ext in ['.cpp', '.c', '.cc', '.h', '.m', '.mm', '.rs', '.js', '.jsm', '.jsx', '.css']: + if ext in [ + ".cpp", + ".c", + ".cc", + ".h", + ".m", + ".mm", + ".rs", + ".js", + ".jsm", + ".jsx", + ".css", + ]: for i, l in enumerate(license_template): start = " " end = "" @@ -119,13 +137,13 @@ def fix_me(filename): add_header(filename, license) return - if ext in ['.py', '.ftl', '.properties'] or filename.endswith(".inc.xul"): + if ext in [".py", ".ftl", ".properties"] or filename.endswith(".inc.xul"): for l in license_template: license.append("# " + l.strip() + "\n") add_header(filename, license) return - if ext in ['.xml', '.xul', '.html', '.xhtml', '.dtd', '.svg']: + if ext in [".xml", ".xul", ".html", ".xhtml", ".dtd", ".svg"]: for i, l in enumerate(license_template): start = " - " end = "" @@ -136,7 +154,7 @@ def fix_me(filename): # Last line, we end by --> end = " -->" license.append(start + l.strip() + end) - if ext != '.svg' or end == "": + if ext != ".svg" or end == "": # When dealing with an svg, we should not have a space between # the license and the content license.append("\n") @@ -145,7 +163,7 @@ def fix_me(filename): def lint(paths, config, fix=None, **lintargs): - files = list(expand_exclusions(paths, config, lintargs['root'])) + files = list(expand_exclusions(paths, config, lintargs["root"])) licenses = load_valid_license() @@ -154,10 +172,11 @@ def lint(paths, config, fix=None, **lintargs): # For now, do not do anything with test (too many) continue if not is_valid_license(licenses, f): - res = {'path': f, - 'message': "No matching license strings found in tools/lint/license/valid-licenses.txt", # noqa - 'level': 'error' - } + res = { + "path": f, + "message": "No matching license strings found in tools/lint/license/valid-licenses.txt", # noqa + "level": "error", + } results.append(result.from_config(config, **res)) if fix: fix_me(f) diff --git a/tools/lint/mach_commands.py b/tools/lint/mach_commands.py index 1f878d3d68b29..b278fa8a6e24b 100644 --- a/tools/lint/mach_commands.py +++ b/tools/lint/mach_commands.py @@ -21,23 +21,24 @@ here = os.path.abspath(os.path.dirname(__file__)) EXCLUSION_FILES = [ - os.path.join('tools', 'rewriting', 'Generated.txt'), - os.path.join('tools', 'rewriting', 'ThirdPartyPaths.txt'), + os.path.join("tools", "rewriting", "Generated.txt"), + os.path.join("tools", "rewriting", "ThirdPartyPaths.txt"), ] EXCLUSION_FILES_OPTIONAL = [] -thunderbird_excludes = os.path.join('comm', 'tools', 'lint', 'GlobalExclude.txt') +thunderbird_excludes = os.path.join("comm", "tools", "lint", "GlobalExclude.txt") if os.path.exists(thunderbird_excludes): EXCLUSION_FILES_OPTIONAL.append(thunderbird_excludes) GLOBAL_EXCLUDES = [ - 'node_modules', - 'tools/lint/test/files', + "node_modules", + "tools/lint/test/files", ] def setup_argument_parser(): from mozlint import cli + return cli.MozlintParser() @@ -46,11 +47,16 @@ def get_global_excludes(topsrcdir): excludes = GLOBAL_EXCLUDES[:] # exclude top level paths that look like objdirs - excludes.extend([name for name in os.listdir(topsrcdir) - if name.startswith('obj') and os.path.isdir(name)]) + excludes.extend( + [ + name + for name in os.listdir(topsrcdir) + if name.startswith("obj") and os.path.isdir(name) + ] + ) for path in EXCLUSION_FILES + EXCLUSION_FILES_OPTIONAL: - with open(os.path.join(topsrcdir, path), 'r') as fh: + with open(os.path.join(topsrcdir, path), "r") as fh: excludes.extend([f.strip() for f in fh.readlines()]) return excludes @@ -58,11 +64,12 @@ def get_global_excludes(topsrcdir): @CommandProvider class MachCommands(MachCommandBase): - @Command( - 'lint', category='devenv', - description='Run linters.', - parser=setup_argument_parser) + "lint", + category="devenv", + description="Run linters.", + parser=setup_argument_parser, + ) def lint(self, *runargs, **lintargs): """Run linters.""" self._activate_virtualenv() @@ -70,35 +77,58 @@ def lint(self, *runargs, **lintargs): try: buildargs = {} - buildargs['substs'] = copy.deepcopy(dict(self.substs)) - buildargs['defines'] = copy.deepcopy(dict(self.defines)) - buildargs['topobjdir'] = self.topobjdir + buildargs["substs"] = copy.deepcopy(dict(self.substs)) + buildargs["defines"] = copy.deepcopy(dict(self.defines)) + buildargs["topobjdir"] = self.topobjdir lintargs.update(buildargs) except BuildEnvironmentNotFoundException: pass - lintargs.setdefault('root', self.topsrcdir) - lintargs['exclude'] = get_global_excludes(lintargs['root']) - lintargs['config_paths'].insert(0, here) + lintargs.setdefault("root", self.topsrcdir) + lintargs["exclude"] = get_global_excludes(lintargs["root"]) + lintargs["config_paths"].insert(0, here) for path in EXCLUSION_FILES: parser.GLOBAL_SUPPORT_FILES.append(os.path.join(self.topsrcdir, path)) return cli.run(*runargs, **lintargs) - @Command('eslint', category='devenv', - description='Run eslint or help configure eslint for optimal development.') - @CommandArgument('paths', default=None, nargs='*', - help="Paths to file or directories to lint, like " - "'browser/' Defaults to the " - "current directory if not given.") - @CommandArgument('-s', '--setup', default=False, action='store_true', - help='Configure eslint for optimal development.') - @CommandArgument('-b', '--binary', default=None, - help='Path to eslint binary.') - @CommandArgument('--fix', default=False, action='store_true', - help='Request that eslint automatically fix errors, where possible.') - @CommandArgument('extra_args', nargs=argparse.REMAINDER, - help='Extra args that will be forwarded to eslint.') + @Command( + "eslint", + category="devenv", + description="Run eslint or help configure eslint for optimal development.", + ) + @CommandArgument( + "paths", + default=None, + nargs="*", + help="Paths to file or directories to lint, like " + "'browser/' Defaults to the " + "current directory if not given.", + ) + @CommandArgument( + "-s", + "--setup", + default=False, + action="store_true", + help="Configure eslint for optimal development.", + ) + @CommandArgument("-b", "--binary", default=None, help="Path to eslint binary.") + @CommandArgument( + "--fix", + default=False, + action="store_true", + help="Request that eslint automatically fix errors, where possible.", + ) + @CommandArgument( + "extra_args", + nargs=argparse.REMAINDER, + help="Extra args that will be forwarded to eslint.", + ) def eslint(self, paths, extra_args=[], **kwargs): - self._mach_context.commands.dispatch('lint', self._mach_context, - linters=['eslint'], paths=paths, - argv=extra_args, **kwargs) + self._mach_context.commands.dispatch( + "lint", + self._mach_context, + linters=["eslint"], + paths=paths, + argv=extra_args, + **kwargs + ) diff --git a/tools/lint/perfdocs/__init__.py b/tools/lint/perfdocs/__init__.py index 8298e83f40481..fc4ca2bdd67ee 100644 --- a/tools/lint/perfdocs/__init__.py +++ b/tools/lint/perfdocs/__init__.py @@ -7,7 +7,7 @@ from mozlint.util import pip here = os.path.abspath(os.path.dirname(__file__)) -PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, 'requirements.txt') +PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, "requirements.txt") def setup(root, **lintargs): @@ -17,6 +17,4 @@ def setup(root, **lintargs): def lint(paths, config, logger, fix=False, **lintargs): - return perfdocs.run_perfdocs( - config, logger=logger, paths=paths, generate=fix - ) + return perfdocs.run_perfdocs(config, logger=logger, paths=paths, generate=fix) diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py index e8555f51107fd..0841d61774eab 100644 --- a/tools/lint/perfdocs/framework_gatherers.py +++ b/tools/lint/perfdocs/framework_gatherers.py @@ -9,48 +9,46 @@ from perfdocs.utils import read_yaml from manifestparser import TestManifest -''' +""" This file is for framework specific gatherers since manifests might be parsed differently in each of them. The gatherers must implement the FrameworkGatherer class. -''' +""" class FrameworkGatherer(object): - ''' + """ Abstract class for framework gatherers. - ''' + """ def __init__(self, yaml_path, workspace_dir): - ''' + """ Generic initialization for a framework gatherer. - ''' + """ self.workspace_dir = workspace_dir self._yaml_path = yaml_path self._suite_list = {} self._test_list = {} - self._manifest_path = '' + self._manifest_path = "" self._manifest = None def get_manifest_path(self): - ''' + """ Returns the path to the manifest based on the manifest entry in the frameworks YAML configuration file. :return str: Path to the manifest. - ''' + """ if self._manifest_path: return self._manifest_path yaml_content = read_yaml(self._yaml_path) - self._manifest_path = os.path.join( - self.workspace_dir, yaml_content["manifest"] - ) + self._manifest_path = os.path.join(self.workspace_dir, yaml_content["manifest"]) return self._manifest_path def get_suite_list(self): - ''' + """ Each framework gatherer must return a dictionary with the following structure. Note that the test names must be relative paths so that issues can be correctly issued @@ -62,32 +60,29 @@ def get_suite_list(self): 'testing/raptor/test2' ] } - ''' + """ raise NotImplementedError def _build_section_with_header(self, title, content, header_type=None): - ''' + """ Adds a section to the documentation with the title as the type mentioned and paragraph as content mentioned. :param title: title of the section :param content: content of section paragraph :param documentation: documentation object to add section to :param type: type of the title heading - ''' - heading_map = { - 'H4': '-', - 'H5': '^' - } - return [title, heading_map.get(type, '^') * len(title), content, ''] + """ + heading_map = {"H4": "-", "H5": "^"} + return [title, heading_map.get(type, "^") * len(title), content, ""] class RaptorGatherer(FrameworkGatherer): - ''' + """ Gatherer for the Raptor framework. - ''' + """ def get_suite_list(self): - ''' + """ Returns a dictionary containing a mapping from suites to the tests they contain. @@ -97,7 +92,7 @@ def get_suite_list(self): 'testing/raptor/test2' ] } - ''' + """ if self._suite_list: return self._suite_list @@ -115,7 +110,7 @@ def get_suite_list(self): self._suite_list[s] = [] # Get the individual test - fpath = re.sub(".*testing", "testing", test['manifest']) + fpath = re.sub(".*testing", "testing", test["manifest"]) if fpath not in self._suite_list[s]: self._suite_list[s].append(fpath) @@ -123,12 +118,12 @@ def get_suite_list(self): return self._suite_list def _get_subtests_from_ini(self, manifest_path): - ''' + """ Returns a list of (sub)tests from an ini file containing the test definitions. :param str manifest_path: path to the ini file :return list: the list of the tests - ''' + """ test_manifest = TestManifest([manifest_path], strict=False) test_list = test_manifest.active_tests(exists=False, disabled=False) subtest_list = {subtest["name"]: subtest["manifest"] for subtest in test_list} @@ -136,7 +131,7 @@ def _get_subtests_from_ini(self, manifest_path): return subtest_list def get_test_list(self): - ''' + """ Returns a dictionary containing the tests in every suite ini file. :return dict: A dictionary with the following structure: { @@ -145,7 +140,7 @@ def get_test_list(self): 'raptor_test2' ] } - ''' + """ if self._test_list: return self._test_list diff --git a/tools/lint/perfdocs/gatherer.py b/tools/lint/perfdocs/gatherer.py index eab98416abf63..4c23c9f041398 100644 --- a/tools/lint/perfdocs/gatherer.py +++ b/tools/lint/perfdocs/gatherer.py @@ -18,18 +18,18 @@ class Gatherer(object): - ''' + """ Gatherer produces the tree of the perfdoc's entries found and can obtain manifest-based test lists. Used by the Verifier. - ''' + """ def __init__(self, root_dir, workspace_dir): - ''' + """ Initialzie the Gatherer. :param str root_dir: Path to the testing directory. :param str workspace_dir: Path to the gecko checkout. - ''' + """ self.root_dir = root_dir self.workspace_dir = workspace_dir self._perfdocs_tree = [] @@ -38,14 +38,14 @@ def __init__(self, root_dir, workspace_dir): @property def perfdocs_tree(self): - ''' + """ Returns the perfdocs_tree, and computes it if it doesn't exist. :return dict: The perfdocs tree containing all framework perfdoc entries. See `fetch_perfdocs_tree` for information on the data structure. - ''' + """ if self._perfdocs_tree: return self._perfdocs_tree else: @@ -53,7 +53,7 @@ def perfdocs_tree(self): return self._perfdocs_tree def fetch_perfdocs_tree(self): - ''' + """ Creates the perfdocs tree with the following structure: [ { @@ -66,11 +66,11 @@ def fetch_perfdocs_tree(self): This method doesn't return anything. The result can be found in the perfdocs_tree attribute. - ''' + """ for dirpath, dirname, files in os.walk(self.root_dir): # Walk through the testing directory tree - if dirpath.endswith('/perfdocs'): + if dirpath.endswith("/perfdocs"): matched = {"path": dirpath, "yml": "", "rst": "", "static": []} for file in files: # Add the yml/rst/static file to its key if re finds the searched file @@ -85,11 +85,14 @@ def fetch_perfdocs_tree(self): if all(val for val in matched.values() if not type(val) == list): self._perfdocs_tree.append(matched) - logger.log("Found {} perfdocs directories in {}" - .format(len(self._perfdocs_tree), self.root_dir)) + logger.log( + "Found {} perfdocs directories in {}".format( + len(self._perfdocs_tree), self.root_dir + ) + ) def get_test_list(self, sdt_entry): - ''' + """ Use a perfdocs_tree entry to find the test list for the framework that was found. @@ -99,28 +102,29 @@ def get_test_list(self, sdt_entry): 'name': Name of framework, 'test_list': Test list found for the framework } - ''' + """ # If it was computed before, return it - yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml']) + yaml_path = os.path.join(sdt_entry["path"], sdt_entry["yml"]) for entry in self._test_list: - if entry['yml_path'] == yaml_path: + if entry["yml_path"] == yaml_path: return entry # Set up framework entry with meta data yaml_content = read_yaml(yaml_path) framework = { - 'yml_content': yaml_content, - 'yml_path': yaml_path, - 'name': yaml_content["name"], + "yml_content": yaml_content, + "yml_path": yaml_path, + "name": yaml_content["name"], } # Get and then store the frameworks tests self.framework_gatherers[framework["name"]] = frameworks[framework["name"]]( - framework["yml_path"], - self.workspace_dir + framework["yml_path"], self.workspace_dir ) - framework["test_list"] = self.framework_gatherers[framework["name"]].get_test_list() + framework["test_list"] = self.framework_gatherers[ + framework["name"] + ].get_test_list() self._test_list.append(framework) return framework diff --git a/tools/lint/perfdocs/generator.py b/tools/lint/perfdocs/generator.py index d8060d57d8f26..ae62333e44c60 100644 --- a/tools/lint/perfdocs/generator.py +++ b/tools/lint/perfdocs/generator.py @@ -9,47 +9,45 @@ import tempfile from perfdocs.logger import PerfDocLogger -from perfdocs.utils import ( - are_dirs_equal, - read_file, - read_yaml, - save_file -) +from perfdocs.utils import are_dirs_equal, read_file, read_yaml, save_file logger = PerfDocLogger() class Generator(object): - ''' + """ After each perfdocs directory was validated, the generator uses the templates for each framework, fills them with the test descriptions in config and saves the perfdocs in the form index.rst as index file and suite_name.rst for each suite of tests in the framework. - ''' + """ + def __init__(self, verifier, workspace, generate=False): - ''' + """ Initialize the Generator. :param verifier: Verifier object. It should not be a fresh Verifier object, but an initialized one with validate_tree() method already called :param workspace: Path to the top-level checkout directory. :param generate: Flag for generating the documentation - ''' + """ self._workspace = workspace if not self._workspace: raise Exception("PerfDocs Generator requires a workspace directory.") # Template documents without added information reside here self.templates_path = os.path.join( - self._workspace, 'tools', 'lint', 'perfdocs', 'templates') + self._workspace, "tools", "lint", "perfdocs", "templates" + ) self.perfdocs_path = os.path.join( - self._workspace, 'testing', 'perfdocs', 'generated') + self._workspace, "testing", "perfdocs", "generated" + ) self._generate = generate self._verifier = verifier self._perfdocs_tree = self._verifier._gatherer.perfdocs_tree def build_perfdocs_from_tree(self): - ''' + """ Builds up a document for each framework that was found. :return dict: A dictionary containing a mapping from each framework @@ -58,83 +56,84 @@ def build_perfdocs_from_tree(self): framework_name: framework_document, ... } - ''' + """ + def _append_rst_section(title, content, documentation, type=None): - ''' + """ Adds a section to the documentation with the title as the type mentioned and paragraph as content mentioned. :param title: title of the section :param content: content of section paragraph :param documentation: documentation object to add section to :param type: type of the title heading - ''' - heading_map = { - 'H4': '-', - 'H5': '^' - } - heading_symbol = heading_map.get(type, '-') - documentation.extend([title, heading_symbol * len(title), content, '']) + """ + heading_map = {"H4": "-", "H5": "^"} + heading_symbol = heading_map.get(type, "-") + documentation.extend([title, heading_symbol * len(title), content, ""]) # Using the verified `perfdocs_tree`, build up the documentation. frameworks_info = {} for framework in self._perfdocs_tree: - yaml_content = read_yaml(os.path.join(framework['path'], framework['yml'])) + yaml_content = read_yaml(os.path.join(framework["path"], framework["yml"])) rst_content = read_file( - os.path.join(framework['path'], framework['rst']), - stringify=True) + os.path.join(framework["path"], framework["rst"]), stringify=True + ) # Gather all tests and descriptions and format them into # documentation content documentation = [] - suites = yaml_content['suites'] + suites = yaml_content["suites"] for suite_name in sorted(suites.keys()): suite_info = suites[suite_name] # Add the suite with an H4 heading _append_rst_section( - suite_name.capitalize(), suite_info['description'], documentation, type="H4") - tests = suite_info.get('tests', {}) + suite_name.capitalize(), + suite_info["description"], + documentation, + type="H4", + ) + tests = suite_info.get("tests", {}) for test_name in sorted(tests.keys()): documentation.extend( - self._verifier - ._gatherer - .framework_gatherers[yaml_content["name"]] - .build_test_description( - test_name, tests[test_name] - ) + self._verifier._gatherer.framework_gatherers[ + yaml_content["name"] + ].build_test_description(test_name, tests[test_name]) ) - documentation.append('') + documentation.append("") # Insert documentation into `.rst` file framework_rst = re.sub( - r'{documentation}', - os.linesep.join(documentation), - rst_content + r"{documentation}", os.linesep.join(documentation), rst_content ) - frameworks_info[yaml_content["name"]] = {"dynamic": framework_rst, "static": []} + frameworks_info[yaml_content["name"]] = { + "dynamic": framework_rst, + "static": [], + } # For static `.rst` file for static_file in framework["static"]: - frameworks_info[yaml_content["name"]]["static"].append({ - "file": static_file, - "content": read_file( - os.path.join(framework["path"], static_file), - stringify=True - ) - }) + frameworks_info[yaml_content["name"]]["static"].append( + { + "file": static_file, + "content": read_file( + os.path.join(framework["path"], static_file), stringify=True + ), + } + ) return frameworks_info def _create_temp_dir(self): - ''' + """ Create a temp directory as preparation of saving the documentation tree. :return: str the location of perfdocs_tmpdir - ''' + """ # Build the directory that will contain the final result (a tmp dir # that will be moved to the final location afterwards) try: tmpdir = tempfile.mkdtemp() - perfdocs_tmpdir = os.path.join(tmpdir, 'generated') + perfdocs_tmpdir = os.path.join(tmpdir, "generated") os.mkdir(perfdocs_tmpdir) except OSError as e: logger.critical("Error creating temp file: {}".format(e)) @@ -146,10 +145,10 @@ def _create_temp_dir(self): return success def _create_perfdocs(self): - ''' + """ Creates the perfdocs documentation. :return: str path of the temp dir it is saved in - ''' + """ # All directories that are kept in the perfdocs tree are valid, # so use it to build up the documentation. framework_docs = self.build_perfdocs_from_tree() @@ -161,29 +160,33 @@ def _create_perfdocs(self): frameworks.append(framework_name) save_file( framework_docs[framework_name]["dynamic"], - os.path.join(perfdocs_tmpdir, framework_name) + os.path.join(perfdocs_tmpdir, framework_name), ) for static_name in framework_docs[framework_name]["static"]: save_file( static_name["content"], - os.path.join(perfdocs_tmpdir, static_name["file"].split(".")[0]) + os.path.join(perfdocs_tmpdir, static_name["file"].split(".")[0]), ) # Get the main page and add the framework links to it - mainpage = read_file(os.path.join(self.templates_path, "index.rst"), stringify=True) - fmt_frameworks = os.linesep.join([' * :doc:`%s`' % name for name in frameworks]) + mainpage = read_file( + os.path.join(self.templates_path, "index.rst"), stringify=True + ) + fmt_frameworks = os.linesep.join( + [" * :doc:`%s`" % name for name in frameworks] + ) fmt_mainpage = re.sub(r"{test_documentation}", fmt_frameworks, mainpage) - save_file(fmt_mainpage, os.path.join(perfdocs_tmpdir, 'index')) + save_file(fmt_mainpage, os.path.join(perfdocs_tmpdir, "index")) return perfdocs_tmpdir def _save_perfdocs(self, perfdocs_tmpdir): - ''' + """ Copies the perfdocs tree after it was saved into the perfdocs_tmpdir :param perfdocs_tmpdir: str location of the temp dir where the perfdocs was saved - ''' + """ # Remove the old docs and copy the new version there without # checking if they need to be regenerated. logger.log("Regenerating perfdocs...") @@ -194,13 +197,18 @@ def _save_perfdocs(self, perfdocs_tmpdir): try: saved = shutil.copytree(perfdocs_tmpdir, self.perfdocs_path) if saved: - logger.log("Documentation saved to {}/".format( - re.sub(".*testing", "testing", self.perfdocs_path))) + logger.log( + "Documentation saved to {}/".format( + re.sub(".*testing", "testing", self.perfdocs_path) + ) + ) except Exception as e: - logger.critical("There was an error while saving the documentation: {}".format(e)) + logger.critical( + "There was an error while saving the documentation: {}".format(e) + ) def generate_perfdocs(self): - ''' + """ Generate the performance documentation. If `self._generate` is True, then the documentation will be regenerated @@ -212,21 +220,23 @@ def generate_perfdocs(self): docs were regenerated. If `self._generate` is False, then True will mean that the docs should be regenerated, and False means that they do not need to be regenerated. - ''' + """ def get_possibly_changed_files(): - ''' + """ Returns files that might have been modified (used to output a linter warning for regeneration) :return: list - files that might have been modified - ''' + """ # Returns files that might have been modified # (used to output a linter warning for regeneration) files = [] for entry in self._perfdocs_tree: files.extend( - [os.path.join(entry['path'], entry['yml']), - os.path.join(entry['path'], entry['rst'])] + [ + os.path.join(entry["path"], entry["yml"]), + os.path.join(entry["path"], entry["rst"]), + ] ) return files @@ -235,8 +245,7 @@ def get_possibly_changed_files(): # If they don't exist and we are not generating, then throw # a linting error and exit. logger.warning( - "PerfDocs need to be regenerated.", - files=get_possibly_changed_files() + "PerfDocs need to be regenerated.", files=get_possibly_changed_files() ) return True @@ -249,5 +258,5 @@ def get_possibly_changed_files(): if not are_dirs_equal(perfdocs_tmpdir, self.perfdocs_path): logger.warning( "PerfDocs are outdated, run ./mach lint -l perfdocs --fix` to update them.", - files=get_possibly_changed_files() + files=get_possibly_changed_files(), ) diff --git a/tools/lint/perfdocs/logger.py b/tools/lint/perfdocs/logger.py index 3905a6cd01376..6a0733f864ff1 100644 --- a/tools/lint/perfdocs/logger.py +++ b/tools/lint/perfdocs/logger.py @@ -7,16 +7,17 @@ class PerfDocLogger(object): - ''' + """ Logger for the PerfDoc tooling. Handles the warnings by outputting them into through the StructuredLogger provided by lint. - ''' + """ + PATHS = [] LOGGER = None FAILED = False def __init__(self): - '''Initializes the PerfDocLogger.''' + """Initializes the PerfDocLogger.""" # Set up class attributes for all logger instances if not PerfDocLogger.LOGGER: @@ -24,28 +25,26 @@ def __init__(self): "Missing linting LOGGER instance for PerfDocLogger initialization" ) if not PerfDocLogger.PATHS: - raise Exception( - "Missing PATHS for PerfDocLogger initialization" - ) + raise Exception("Missing PATHS for PerfDocLogger initialization") self.logger = PerfDocLogger.LOGGER def log(self, msg): - ''' + """ Log an info message. :param str msg: Message to log. - ''' + """ self.logger.info(msg) def warning(self, msg, files): - ''' + """ Logs a validation warning message. The warning message is used as the error message that is output in the reviewbot. :param str msg: Message to log, it's also used as the error message for the issue that is output by the reviewbot. :param list/str files: The file(s) that this warning is about. - ''' + """ if type(files) != list: files = [files] @@ -67,17 +66,17 @@ def warning(self, msg, files): lineno=0, column=None, path=fpath, - linter='perfdocs', - rule="Flawless performance docs." + linter="perfdocs", + rule="Flawless performance docs.", ) PerfDocLogger.FAILED = True break def critical(self, msg): - ''' + """ Log a critical message. :param str msg: Message to log. - ''' + """ self.logger.critical(msg) diff --git a/tools/lint/perfdocs/perfdocs.py b/tools/lint/perfdocs/perfdocs.py index d462b46edb188..4650e5934209a 100644 --- a/tools/lint/perfdocs/perfdocs.py +++ b/tools/lint/perfdocs/perfdocs.py @@ -8,7 +8,7 @@ def run_perfdocs(config, logger=None, paths=None, generate=True): - ''' + """ Build up performance testing documentation dynamically by combining text data from YAML files that reside in `perfdoc` folders across the `testing` directory. Each directory is expected to have @@ -42,13 +42,13 @@ def run_perfdocs(config, logger=None, paths=None, generate=True): :param list paths: The paths that are being tested. Used to filter out errors from files outside of these paths. :param bool generate: If true, the docs will be (re)generated. - ''' + """ from perfdocs.logger import PerfDocLogger - top_dir = os.environ.get('WORKSPACE', None) + top_dir = os.environ.get("WORKSPACE", None) if not top_dir: floc = os.path.abspath(__file__) - top_dir = floc.split('tools')[0] + top_dir = floc.split("tools")[0] PerfDocLogger.LOGGER = logger # Convert all the paths to relative ones @@ -56,7 +56,7 @@ def run_perfdocs(config, logger=None, paths=None, generate=True): PerfDocLogger.PATHS = rel_paths # TODO: Expand search to entire tree rather than just the testing directory - testing_dir = os.path.join(top_dir, 'testing') + testing_dir = os.path.join(top_dir, "testing") if not os.path.exists(testing_dir): raise Exception("Cannot locate testing directory at %s" % testing_dir) diff --git a/tools/lint/perfdocs/utils.py b/tools/lint/perfdocs/utils.py index 51f3f0c4698d2..99b304260b32e 100644 --- a/tools/lint/perfdocs/utils.py +++ b/tools/lint/perfdocs/utils.py @@ -12,50 +12,48 @@ def save_file(file_content, path, extension="rst"): - ''' + """ Saves data into a file. :param str path: Location and name of the file being saved (without an extension). :param str data: Content to write into the file. :param str extension: Extension to save the file as. - ''' - with open("{}.{}".format(path, extension), 'w') as f: + """ + with open("{}.{}".format(path, extension), "w") as f: f.write(file_content) def read_file(path, stringify=False): - ''' + """ Opens a file and returns its contents. :param str path: Path to the file. :return list: List containing the lines in the file. - ''' - with open(path, 'r') as f: + """ + with open(path, "r") as f: return f.read() if stringify else f.readlines() def read_yaml(yaml_path): - ''' + """ Opens a YAML file and returns the contents. :param str yaml_path: Path to the YAML to open. :return dict: Dictionary containing the YAML content. - ''' + """ contents = {} try: - with open(yaml_path, 'r') as f: + with open(yaml_path, "r") as f: contents = yaml.safe_load(f) except Exception as e: - logger.warning( - "Error opening file {}: {}".format(yaml_path, str(e)), yaml_path - ) + logger.warning("Error opening file {}: {}".format(yaml_path, str(e)), yaml_path) return contents def are_dirs_equal(dir_1, dir_2): - ''' + """ Compare two directories to see if they are equal. Files in each directory are assumed to be equal if their names and contents are equal. @@ -63,7 +61,7 @@ def are_dirs_equal(dir_1, dir_2): :param dir_1: First directory path :param dir_2: Second directory path :return: True if the directory trees are the same and False otherwise. - ''' + """ dirs_cmp = filecmp.dircmp(dir_1, dir_2) if dirs_cmp.left_only or dirs_cmp.right_only or dirs_cmp.funny_files: diff --git a/tools/lint/perfdocs/verifier.py b/tools/lint/perfdocs/verifier.py index 2b0bb55ce4a5b..4c02a7446aae5 100644 --- a/tools/lint/perfdocs/verifier.py +++ b/tools/lint/perfdocs/verifier.py @@ -13,7 +13,7 @@ logger = PerfDocLogger() -''' +""" Schema for the config.yml file. Expecting a YAML file with a format such as this: @@ -31,7 +31,7 @@ tests: wasm: "All wasm tests." -''' +""" CONFIG_SCHEMA = { "type": "object", "properties": { @@ -45,46 +45,38 @@ "properties": { "tests": { "type": "object", - "properties": { - "test_name": {"type": "string"}, - } + "properties": {"test_name": {"type": "string"}}, }, "description": {"type": "string"}, }, - "required": [ - "description" - ] + "required": ["description"], } - } - } + }, + }, }, - "required": [ - "name", - "manifest", - "suites" - ] + "required": ["name", "manifest", "suites"], } class Verifier(object): - ''' + """ Verifier is used for validating the perfdocs folders/tree. In the future, the generator will make use of this class to obtain a validated set of descriptions that can be used to build up a document. - ''' + """ def __init__(self, root_dir, workspace_dir): - ''' + """ Initialize the Verifier. :param str root_dir: Path to the 'testing' directory. :param str workspace_dir: Path to the top-level checkout directory. - ''' + """ self.workspace_dir = workspace_dir self._gatherer = Gatherer(root_dir, workspace_dir) def validate_descriptions(self, framework_info): - ''' + """ Cross-validate the tests found in the manifests and the YAML test definitions. This function doesn't return a valid flag. Instead, the StructDocLogger.VALIDATION_LOG is used to determine validity. @@ -101,12 +93,12 @@ def validate_descriptions(self, framework_info): :param dict framework_info: Contains information about the framework. See `Gatherer.get_test_list` for information about its structure. - ''' - yaml_content = framework_info['yml_content'] + """ + yaml_content = framework_info["yml_content"] # Check for any bad test/suite names in the yaml config file global_descriptions = {} - for suite, ytests in yaml_content['suites'].items(): + for suite, ytests in yaml_content["suites"].items(): # Find the suite, then check against the tests within it if framework_info["test_list"].get(suite): global_descriptions[suite] = [] @@ -116,7 +108,7 @@ def validate_descriptions(self, framework_info): # Suite found - now check if any tests in YAML # definitions doesn't exist - ytests = ytests['tests'] + ytests = ytests["tests"] for test_name in ytests: foundtest = False for t in framework_info["test_list"][suite]: @@ -137,12 +129,14 @@ def validate_descriptions(self, framework_info): "Could not find an existing test for {} - bad test name?".format( test_name ), - framework_info["yml_path"] + framework_info["yml_path"], ) else: logger.warning( - "Could not find an existing suite for {} - bad suite name?".format(suite), - framework_info["yml_path"] + "Could not find an existing suite for {} - bad suite name?".format( + suite + ), + framework_info["yml_path"], ) # Check for any missing tests/suites @@ -151,14 +145,14 @@ def validate_descriptions(self, framework_info): # Description doesn't exist for the suite logger.warning( "Missing suite description for {}".format(suite), - yaml_content['manifest'] + yaml_content["manifest"], ) continue # If only a description is provided for the suite, assume # that this is a suite-wide description and don't check for # it's tests - stests = yaml_content['suites'][suite].get('tests', None) + stests = yaml_content["suites"][suite].get("tests", None) if not stests: continue @@ -197,67 +191,66 @@ def validate_descriptions(self, framework_info): for test_name in new_mtests: logger.warning( "Could not find a test description for {}".format(test_name), - test_to_manifest[test_name] + test_to_manifest[test_name], ) def validate_yaml(self, yaml_path): - ''' + """ Validate that the YAML file has all the fields that are required and parse the descriptions into strings in case some are give as relative file paths. :param str yaml_path: Path to the YAML to validate. :return bool: True/False => Passed/Failed Validation - ''' + """ + def _get_description(desc): - ''' + """ Recompute the description in case it's a file. - ''' + """ desc_path = os.path.join(self.workspace_dir, desc) if os.path.exists(desc_path): - with open(desc_path, 'r') as f: + with open(desc_path, "r") as f: desc = f.readlines() return desc def _parse_descriptions(content): for suite, sinfo in content.items(): - desc = sinfo['description'] - sinfo['description'] = _get_description(desc) + desc = sinfo["description"] + sinfo["description"] = _get_description(desc) # It's possible that the suite has no tests and # only a description. If they exist, then parse them. - if 'tests' in sinfo: - for test, desc in sinfo['tests'].items(): - sinfo['tests'][test] = _get_description(desc) + if "tests" in sinfo: + for test, desc in sinfo["tests"].items(): + sinfo["tests"][test] = _get_description(desc) valid = False yaml_content = read_yaml(yaml_path) try: jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) - _parse_descriptions(yaml_content['suites']) + _parse_descriptions(yaml_content["suites"]) valid = True except Exception as e: - logger.warning( - "YAML ValidationError: {}".format(str(e)), yaml_path - ) + logger.warning("YAML ValidationError: {}".format(str(e)), yaml_path) return valid def validate_rst_content(self, rst_path): - ''' + """ Validate that the index file given has a {documentation} entry so that the documentation can be inserted there. :param str rst_path: Path to the RST file. :return bool: True/False => Passed/Failed Validation - ''' + """ rst_content = read_file(rst_path) # Check for a {documentation} entry in some line, # if we can't find one, then the validation fails. valid = False - docs_match = re.compile('.*{documentation}.*') + docs_match = re.compile(".*{documentation}.*") for line in rst_content: if docs_match.search(line): valid = True @@ -265,36 +258,36 @@ def validate_rst_content(self, rst_path): if not valid: logger.warning( "Cannot find a '{documentation}' entry in the given index file", - rst_path + rst_path, ) return valid def _check_framework_descriptions(self, item): - ''' + """ Helper method for validating descriptions - ''' + """ framework_info = self._gatherer.get_test_list(item) self.validate_descriptions(framework_info) def validate_tree(self): - ''' + """ Validate the `perfdocs` directory that was found. Returns True if it is good, false otherwise. :return bool: True/False => Passed/Failed Validation - ''' + """ found_good = 0 # For each framework, check their files and validate descriptions for matched in self._gatherer.perfdocs_tree: # Get the paths to the YAML and RST for this framework - matched_yml = os.path.join(matched['path'], matched['yml']) - matched_rst = os.path.join(matched['path'], matched['rst']) + matched_yml = os.path.join(matched["path"], matched["yml"]) + matched_rst = os.path.join(matched["path"], matched["rst"]) _valid_files = { "yml": self.validate_yaml(matched_yml), - "rst": self.validate_rst_content(matched_rst) + "rst": self.validate_rst_content(matched_rst), } # Log independently the errors found for the matched files diff --git a/tools/lint/python/check_compat.py b/tools/lint/python/check_compat.py index 01fea613bb008..25a15fcedc36d 100755 --- a/tools/lint/python/check_compat.py +++ b/tools/lint/python/check_compat.py @@ -9,18 +9,18 @@ def parse_file(f): - with open(f, 'rb') as fh: + with open(f, "rb") as fh: content = fh.read() try: return ast.parse(content) except SyntaxError as e: err = { - 'path': f, - 'message': e.msg, - 'lineno': e.lineno, - 'column': e.offset, - 'source': e.text, - 'rule': 'is-parseable', + "path": f, + "message": e.msg, + "lineno": e.lineno, + "column": e.offset, + "source": e.text, + "rule": "is-parseable", } print(json.dumps(err)) @@ -39,7 +39,7 @@ def check_compat_py2(f): may_have_relative_imports = False for node in ast.walk(root): if isinstance(node, ast.ImportFrom): - if node.module == '__future__': + if node.module == "__future__": future_lineno = node.lineno futures |= set(n.name for n in node.names) else: @@ -50,19 +50,19 @@ def check_compat_py2(f): haveprint = True err = { - 'path': f, - 'lineno': future_lineno, - 'column': 1, + "path": f, + "lineno": future_lineno, + "column": 1, } - if 'absolute_import' not in futures and may_have_relative_imports: - err['rule'] = 'require absolute_import' - err['message'] = 'Missing from __future__ import absolute_import' + if "absolute_import" not in futures and may_have_relative_imports: + err["rule"] = "require absolute_import" + err["message"] = "Missing from __future__ import absolute_import" print(json.dumps(err)) - if haveprint and 'print_function' not in futures: - err['rule'] = 'require print_function' - err['message'] = 'Missing from __future__ import print_function' + if haveprint and "print_function" not in futures: + err["rule"] = "require print_function" + err["message"] = "Missing from __future__ import print_function" print(json.dumps(err)) @@ -71,14 +71,14 @@ def check_compat_py3(f): parse_file(f) -if __name__ == '__main__': +if __name__ == "__main__": if sys.version_info[0] == 2: fn = check_compat_py2 else: fn = check_compat_py3 manifest = sys.argv[1] - with open(manifest, 'r') as fh: + with open(manifest, "r") as fh: files = fh.read().splitlines() for f in files: diff --git a/tools/lint/python/compat.py b/tools/lint/python/compat.py index 5389bb56d24c5..dd541bd66e9c3 100644 --- a/tools/lint/python/compat.py +++ b/tools/lint/python/compat.py @@ -20,17 +20,19 @@ class PyCompatProcess(ProcessHandlerMixin): def __init__(self, config, *args, **kwargs): self.config = config - kwargs['processOutputLine'] = [self.process_line] + kwargs["processOutputLine"] = [self.process_line] ProcessHandlerMixin.__init__(self, *args, **kwargs) def process_line(self, line): try: res = json.loads(line) except ValueError: - print('Non JSON output from {} linter: {}'.format(self.config['name'], line)) + print( + "Non JSON output from {} linter: {}".format(self.config["name"], line) + ) return - res['level'] = 'error' + res["level"] = "error" results.append(result.from_config(self.config, **res)) @@ -41,27 +43,27 @@ def setup(python): binary = find_executable(python) if not binary: # TODO Bootstrap python2/python3 if not available - print('warning: {} not detected, skipping py-compat check'.format(python)) + print("warning: {} not detected, skipping py-compat check".format(python)) def run_linter(python, paths, config, **lintargs): - log = lintargs['log'] + log = lintargs["log"] binary = find_executable(python) if not binary: # If we're in automation, this is fatal. Otherwise, the warning in the # setup method was already printed. - if 'MOZ_AUTOMATION' in os.environ: + if "MOZ_AUTOMATION" in os.environ: return 1 return [] - files = expand_exclusions(paths, config, lintargs['root']) + files = expand_exclusions(paths, config, lintargs["root"]) - with mozfile.NamedTemporaryFile(mode='w') as fh: - fh.write('\n'.join(files)) + with mozfile.NamedTemporaryFile(mode="w") as fh: + fh.write("\n".join(files)) fh.flush() - cmd = [binary, os.path.join(here, 'check_compat.py'), fh.name] - log.debug("Command: {}".format(' '.join(cmd))) + cmd = [binary, os.path.join(here, "check_compat.py"), fh.name] + log.debug("Command: {}".format(" ".join(cmd))) proc = PyCompatProcess(config, cmd) proc.run() @@ -74,16 +76,16 @@ def run_linter(python, paths, config, **lintargs): def setuppy2(**lintargs): - return setup('python2') + return setup("python2") def lintpy2(*args, **kwargs): - return run_linter('python2', *args, **kwargs) + return run_linter("python2", *args, **kwargs) def setuppy3(**lintargs): - return setup('python3') + return setup("python3") def lintpy3(*args, **kwargs): - return run_linter('python3', *args, **kwargs) + return run_linter("python3", *args, **kwargs) diff --git a/tools/lint/python/flake8.py b/tools/lint/python/flake8.py index ddfaac51e76fe..bd90960ccf696 100644 --- a/tools/lint/python/flake8.py +++ b/tools/lint/python/flake8.py @@ -15,38 +15,42 @@ from mozlint.util import pip here = os.path.abspath(os.path.dirname(__file__)) -FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt') +FLAKE8_REQUIREMENTS_PATH = os.path.join(here, "flake8_requirements.txt") FLAKE8_NOT_FOUND = """ Could not find flake8! Install flake8 and try again. $ pip install -U --require-hashes -r {} -""".strip().format(FLAKE8_REQUIREMENTS_PATH) +""".strip().format( + FLAKE8_REQUIREMENTS_PATH +) FLAKE8_INSTALL_ERROR = """ Unable to install correct version of flake8 Try to install it manually with: $ pip install -U --require-hashes -r {} -""".strip().format(FLAKE8_REQUIREMENTS_PATH) +""".strip().format( + FLAKE8_REQUIREMENTS_PATH +) LINE_OFFSETS = { # continuation line under-indented for hanging indent - 'E121': (-1, 2), + "E121": (-1, 2), # continuation line missing indentation or outdented - 'E122': (-1, 2), + "E122": (-1, 2), # continuation line over-indented for hanging indent - 'E126': (-1, 2), + "E126": (-1, 2), # continuation line over-indented for visual indent - 'E127': (-1, 2), + "E127": (-1, 2), # continuation line under-indented for visual indent - 'E128': (-1, 2), + "E128": (-1, 2), # continuation line unaligned for hanging indend - 'E131': (-1, 2), + "E131": (-1, 2), # expected 1 blank line, found 0 - 'E301': (-1, 2), + "E301": (-1, 2), # expected 2 blank lines, found 1 - 'E302': (-2, 3), + "E302": (-2, 3), } """Maps a flake8 error to a lineoffset tuple. @@ -56,10 +60,10 @@ # We use sys.prefix to find executables as that gets modified with # virtualenv's activate_this.py, whereas sys.executable doesn't. -if platform.system() == 'Windows': - bindir = os.path.join(sys.prefix, 'Scripts') +if platform.system() == "Windows": + bindir = os.path.join(sys.prefix, "Scripts") else: - bindir = os.path.join(sys.prefix, 'bin') + bindir = os.path.join(sys.prefix, "bin") class NothingToLint(Exception): @@ -77,19 +81,21 @@ def setup(root, **lintargs): def lint(paths, config, **lintargs): from flake8.main.application import Application - log = lintargs['log'] - root = lintargs['root'] - config_path = os.path.join(root, '.flake8') + log = lintargs["log"] + root = lintargs["root"] + config_path = os.path.join(root, ".flake8") - if lintargs.get('fix'): + if lintargs.get("fix"): fix_cmd = [ - os.path.join(bindir, 'autopep8'), - '--global-config', config_path, - '--in-place', '--recursive', + os.path.join(bindir, "autopep8"), + "--global-config", + config_path, + "--in-place", + "--recursive", ] - if config.get('exclude'): - fix_cmd.extend(['--exclude', ','.join(config['exclude'])]) + if config.get("exclude"): + fix_cmd.extend(["--exclude", ",".join(config["exclude"])]) subprocess.call(fix_cmd + paths) @@ -97,15 +103,19 @@ def lint(paths, config, **lintargs): app = Application() log.debug("flake8 version={}".format(app.version)) - output_file = mozfile.NamedTemporaryFile(mode='r') + output_file = mozfile.NamedTemporaryFile(mode="r") flake8_cmd = [ - '--config', config_path, - '--output-file', output_file.name, - '--format', '{"path":"%(path)s","lineno":%(row)s,' - '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', - '--filename', ','.join(['*.{}'.format(e) for e in config['extensions']]), + "--config", + config_path, + "--output-file", + output_file.name, + "--format", + '{"path":"%(path)s","lineno":%(row)s,' + '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', + "--filename", + ",".join(["*.{}".format(e) for e in config["extensions"]]), ] - log.debug("Command: {}".format(' '.join(flake8_cmd))) + log.debug("Command: {}".format(" ".join(flake8_cmd))) orig_make_file_checker_manager = app.make_file_checker_manager @@ -119,16 +129,18 @@ def wrap_make_file_checker_manager(self): tools/lint/mach_commands.py. """ # Ignore exclude rules if `--no-filter` was passed in. - config.setdefault('exclude', []) - if lintargs.get('use_filters', True): - config['exclude'].extend(self.options.exclude) + config.setdefault("exclude", []) + if lintargs.get("use_filters", True): + config["exclude"].extend(self.options.exclude) # Since we use the root .flake8 file to store exclusions, we haven't # properly filtered the paths through mozlint's `filterpaths` function # yet. This mimics that though there could be other edge cases that are # different. Maybe we should call `filterpaths` directly, though for # now that doesn't appear to be necessary. - filtered = [p for p in paths if not any(p.startswith(e) for e in config['exclude'])] + filtered = [ + p for p in paths if not any(p.startswith(e) for e in config["exclude"]) + ] self.args = self.args + list(expand_exclusions(filtered, config, root)) @@ -136,7 +148,9 @@ def wrap_make_file_checker_manager(self): raise NothingToLint return orig_make_file_checker_manager() - app.make_file_checker_manager = wrap_make_file_checker_manager.__get__(app, Application) + app.make_file_checker_manager = wrap_make_file_checker_manager.__get__( + app, Application + ) # Make sure to run from repository root so exclusions are joined to the # repository root and not the current working directory. @@ -153,15 +167,15 @@ def wrap_make_file_checker_manager(self): def process_line(line): # Escape slashes otherwise JSON conversion will not work - line = line.replace('\\', '\\\\') + line = line.replace("\\", "\\\\") try: res = json.loads(line) except ValueError: - print('Non JSON output from linter, will not be processed: {}'.format(line)) + print("Non JSON output from linter, will not be processed: {}".format(line)) return - if res.get('code') in LINE_OFFSETS: - res['lineoffset'] = LINE_OFFSETS[res['code']] + if res.get("code") in LINE_OFFSETS: + res["lineoffset"] = LINE_OFFSETS[res["code"]] results.append(result.from_config(config, **res)) diff --git a/tools/lint/python/l10n_lint.py b/tools/lint/python/l10n_lint.py index 85b2cbcce0483..fb79d41a2f857 100644 --- a/tools/lint/python/l10n_lint.py +++ b/tools/lint/python/l10n_lint.py @@ -16,7 +16,7 @@ from compare_locales.paths import TOMLParser, ProjectFiles -LOCALE = 'gecko-strings' +LOCALE = "gecko-strings" PULL_AFTER = timedelta(days=2) @@ -24,9 +24,9 @@ def lint(paths, lintconfig, **lintargs): l10n_base = mb_util.get_state_dir() - root = lintargs['root'] - exclude = lintconfig.get('exclude') - extensions = lintconfig.get('extensions') + root = lintargs["root"] + exclude = lintconfig.get("exclude") + extensions = lintconfig.get("extensions") # Load l10n.toml configs l10nconfigs = load_configs(lintconfig, root, l10n_base) @@ -35,9 +35,9 @@ def lint(paths, lintconfig, **lintargs): # Only the l10n.yml will show up here, but if the l10n.toml files # change, we also get the l10n.yml as the toml files are listed as # support files. - if lintconfig['path'] in paths: + if lintconfig["path"] in paths: results = validate_linter_includes(lintconfig, l10nconfigs, lintargs) - paths.remove(lintconfig['path']) + paths.remove(lintconfig["path"]) else: results = [] @@ -53,18 +53,21 @@ def lint(paths, lintconfig, **lintargs): # explicitly excluded in the l10n.yml configuration. # `browser/locales/en-US/firefox-l10n.js` is a good example. all_files, _ = pathutils.filterpaths( - lintargs['root'], all_files, lintconfig['include'], - exclude=exclude, extensions=extensions + lintargs["root"], + all_files, + lintconfig["include"], + exclude=exclude, + extensions=extensions, ) # These should be excluded in l10n.yml skips = {p for p in all_files if not parser.hasParser(p)} results.extend( result.from_config( lintconfig, - level='warning', + level="warning", path=path, - message="file format not supported in compare-locales" - ) + message="file format not supported in compare-locales", + ) for path in skips ) all_files = [p for p in all_files if p not in skips] @@ -79,7 +82,7 @@ def lint(paths, lintconfig, **lintargs): def gecko_strings_setup(**lint_args): gs = mozpath.join(mb_util.get_state_dir(), LOCALE) - marker = mozpath.join(gs, '.hg', 'l10n_pull_marker') + marker = mozpath.join(gs, ".hg", "l10n_pull_marker") try: last_pull = datetime.fromtimestamp(os.stat(marker).st_mtime) skip_clone = datetime.now() < last_pull + PULL_AFTER @@ -87,27 +90,21 @@ def gecko_strings_setup(**lint_args): skip_clone = False if skip_clone: return - hg = mozversioncontrol.get_tool_path('hg') + hg = mozversioncontrol.get_tool_path("hg") mozversioncontrol.repoupdate.update_mercurial_repo( - hg, - 'https://hg.mozilla.org/l10n/gecko-strings', - gs + hg, "https://hg.mozilla.org/l10n/gecko-strings", gs ) - with open(marker, 'w') as fh: + with open(marker, "w") as fh: fh.flush() def load_configs(lintconfig, root, l10n_base): - '''Load l10n configuration files specified in the linter configuration.''' + """Load l10n configuration files specified in the linter configuration.""" configs = [] - env = { - 'l10n_base': l10n_base - } - for toml in lintconfig['l10n_configs']: + env = {"l10n_base": l10n_base} + for toml in lintconfig["l10n_configs"]: cfg = TOMLParser().parse( - mozpath.join(root, toml), - env=env, - ignore_missing_includes=True + mozpath.join(root, toml), env=env, ignore_missing_includes=True ) cfg.set_locales([LOCALE], deep=True) configs.append(cfg) @@ -115,9 +112,9 @@ def load_configs(lintconfig, root, l10n_base): def validate_linter_includes(lintconfig, l10nconfigs, lintargs): - '''Check l10n.yml config against l10n.toml configs.''' + """Check l10n.yml config against l10n.toml configs.""" reference_paths = set( - mozpath.relpath(p['reference'].prefix, lintargs['root']) + mozpath.relpath(p["reference"].prefix, lintargs["root"]) for project in l10nconfigs for config in project.configs for p in config.paths @@ -125,24 +122,29 @@ def validate_linter_includes(lintconfig, l10nconfigs, lintargs): # Just check for directories reference_dirs = sorted(p for p in reference_paths if os.path.isdir(p)) missing_in_yml = [ - refd for refd in reference_dirs if refd not in lintconfig['include'] + refd for refd in reference_dirs if refd not in lintconfig["include"] ] # These might be subdirectories in the config, though missing_in_yml = [ - d for d in missing_in_yml - if not any(d.startswith(parent + '/') for parent in lintconfig['include']) + d + for d in missing_in_yml + if not any(d.startswith(parent + "/") for parent in lintconfig["include"]) ] if missing_in_yml: - dirs = ', '.join(missing_in_yml) - return [result.from_config( - lintconfig, path=lintconfig['path'], - message='l10n.yml out of sync with l10n.toml, add: ' + dirs - )] + dirs = ", ".join(missing_in_yml) + return [ + result.from_config( + lintconfig, + path=lintconfig["path"], + message="l10n.yml out of sync with l10n.toml, add: " + dirs, + ) + ] return [] class MozL10nLinter(L10nLinter): - '''Subclass linter to generate the right result type.''' + """Subclass linter to generate the right result type.""" + def __init__(self, lintconfig): super(MozL10nLinter, self).__init__() self.lintconfig = lintconfig diff --git a/tools/lint/python/pylint.py b/tools/lint/python/pylint.py index c919b6cdc3549..ded6b8869beb1 100644 --- a/tools/lint/python/pylint.py +++ b/tools/lint/python/pylint.py @@ -15,20 +15,24 @@ from mozlint.util import pip here = os.path.abspath(os.path.dirname(__file__)) -PYLINT_REQUIREMENTS_PATH = os.path.join(here, 'pylint_requirements.txt') +PYLINT_REQUIREMENTS_PATH = os.path.join(here, "pylint_requirements.txt") PYLINT_NOT_FOUND = """ Could not find pylint! Install pylint and try again. $ pip install -U --require-hashes -r {} -""".strip().format(PYLINT_REQUIREMENTS_PATH) +""".strip().format( + PYLINT_REQUIREMENTS_PATH +) PYLINT_INSTALL_ERROR = """ Unable to install correct version of pylint Try to install it manually with: $ pip install -U --require-hashes -r {} -""".strip().format(PYLINT_REQUIREMENTS_PATH) +""".strip().format( + PYLINT_REQUIREMENTS_PATH +) class PylintProcess(ProcessHandler): @@ -65,7 +69,7 @@ def run_process(config, cmd): return proc.output -PYLINT_FORMAT_REGEX = re.compile(r'(.*):(.*): [(.*)] (.*)$') +PYLINT_FORMAT_REGEX = re.compile(r"(.*):(.*): [(.*)] (.*)$") def parse_issues(log, config, issues_json, path): @@ -92,25 +96,27 @@ def parse_issues(log, config, issues_json, path): def lint(paths, config, **lintargs): - log = lintargs['log'] + log = lintargs["log"] binary = get_pylint_binary() - log = lintargs['log'] - paths = list(expand_exclusions(paths, config, lintargs['root'])) + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) cmd_args = [binary] results = [] # list from https://code.visualstudio.com/docs/python/linting#_pylint # And ignore a bit more elements - cmd_args += ["-fjson", - "--disable=all", - "--enable=F,E,unreachable,duplicate-key,unnecessary-semicolon,global-variable-not-assigned,unused-variable,binary-op-exception,bad-format-string,anomalous-backslash-in-string,bad-open-mode,no-else-return", # NOQA: E501 - "--disable=import-error,no-member"] + cmd_args += [ + "-fjson", + "--disable=all", + "--enable=F,E,unreachable,duplicate-key,unnecessary-semicolon,global-variable-not-assigned,unused-variable,binary-op-exception,bad-format-string,anomalous-backslash-in-string,bad-open-mode,no-else-return", # NOQA: E501 + "--disable=import-error,no-member", + ] base_command = cmd_args + paths - log.debug("Command: {}".format(' '.join(cmd_args))) + log.debug("Command: {}".format(" ".join(cmd_args))) output = " ".join(run_process(config, base_command)) results = parse_issues(log, config, str(output), []) diff --git a/tools/lint/rst/__init__.py b/tools/lint/rst/__init__.py index 49f67ffa09509..409699a2e6b64 100644 --- a/tools/lint/rst/__init__.py +++ b/tools/lint/rst/__init__.py @@ -19,7 +19,7 @@ # (4, 'severe') abspath = os.path.abspath(os.path.dirname(__file__)) -rstcheck_requirements_file = os.path.join(abspath, 'requirements.txt') +rstcheck_requirements_file = os.path.join(abspath, "requirements.txt") results = [] @@ -27,15 +27,19 @@ Could not find rstcheck! Install rstcheck and try again. $ pip install -U --require-hashes -r {} -""".strip().format(rstcheck_requirements_file) +""".strip().format( + rstcheck_requirements_file +) RSTCHECK_INSTALL_ERROR = """ Unable to install required version of rstcheck Try to install it manually with: $ pip install -U --require-hashes -r {} -""".strip().format(rstcheck_requirements_file) +""".strip().format( + rstcheck_requirements_file +) -RSTCHECK_FORMAT_REGEX = re.compile(r'(.*):(.*): \(.*/([0-9]*)\) (.*)$') +RSTCHECK_FORMAT_REGEX = re.compile(r"(.*):(.*): \(.*/([0-9]*)\) (.*)$") def setup(root, **lintargs): @@ -49,11 +53,11 @@ def get_rstcheck_binary(): Returns the path of the first rstcheck binary available if not found returns None """ - binary = os.environ.get('RSTCHECK') + binary = os.environ.get("RSTCHECK") if binary: return binary - return which('rstcheck') + return which("rstcheck") def parse_with_split(errors): @@ -64,24 +68,21 @@ def parse_with_split(errors): def lint(files, config, **lintargs): - log = lintargs['log'] - config['root'] = lintargs['root'] - paths = expand_exclusions(files, config, config['root']) + log = lintargs["log"] + config["root"] = lintargs["root"] + paths = expand_exclusions(files, config, config["root"]) paths = list(paths) chunk_size = 50 binary = get_rstcheck_binary() rstcheck_options = "--ignore-language=cpp,json" while paths: - cmdargs = [ - which('python'), - binary, - rstcheck_options, - ] + paths[:chunk_size] - log.debug("Command: {}".format(' '.join(cmdargs))) + cmdargs = [which("python"), binary, rstcheck_options] + paths[:chunk_size] + log.debug("Command: {}".format(" ".join(cmdargs))) proc = subprocess.Popen( - cmdargs, stdout=subprocess.PIPE, + cmdargs, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=os.environ, universal_newlines=True, @@ -91,10 +92,10 @@ def lint(files, config, **lintargs): if len(errors) > 1: filename, lineno, level, message = parse_with_split(errors) res = { - 'path': filename, - 'message': message, - 'lineno': lineno, - 'level': "error" if int(level) >= 2 else "warning", + "path": filename, + "message": message, + "lineno": lineno, + "level": "error" if int(level) >= 2 else "warning", } results.append(result.from_config(config, **res)) paths = paths[chunk_size:] diff --git a/tools/lint/rust/__init__.py b/tools/lint/rust/__init__.py index 2518494d31066..13bf44bc4126a 100644 --- a/tools/lint/rust/__init__.py +++ b/tools/lint/rust/__init__.py @@ -117,20 +117,18 @@ def get_rustfmt_version(binary): """ try: output = subprocess.check_output( - [binary, "--version"], - stderr=subprocess.STDOUT, - universal_newlines=True, + [binary, "--version"], stderr=subprocess.STDOUT, universal_newlines=True, ) except subprocess.CalledProcessError as e: output = e.output - version = re.findall(r'\d.\d+.\d+', output)[0] + version = re.findall(r"\d.\d+.\d+", output)[0] return StrictVersion(version) def lint(paths, config, fix=None, **lintargs): - log = lintargs['log'] - paths = list(expand_exclusions(paths, config, lintargs['root'])) + log = lintargs["log"] + paths = list(expand_exclusions(paths, config, lintargs["root"])) # An empty path array can occur when the user passes in `-n`. If we don't # return early in this case, rustfmt will attempt to read stdin and hang. @@ -145,7 +143,7 @@ def lint(paths, config, fix=None, **lintargs): return 1 return [] - min_version_str = config.get('min_rustfmt_version') + min_version_str = config.get("min_rustfmt_version") min_version = StrictVersion(min_version_str) actual_version = get_rustfmt_version(binary) log.debug( @@ -162,7 +160,7 @@ def lint(paths, config, fix=None, **lintargs): if not fix: cmd_args.append("--check") base_command = cmd_args + paths - log.debug("Command: {}".format(' '.join(cmd_args))) + log.debug("Command: {}".format(" ".join(cmd_args))) output = run_process(config, base_command) if fix: diff --git a/tools/lint/shell/__init__.py b/tools/lint/shell/__init__.py index a0a93614a6752..0c4a2ed96bcf2 100644 --- a/tools/lint/shell/__init__.py +++ b/tools/lint/shell/__init__.py @@ -24,22 +24,21 @@ class ShellcheckProcess(LintProcess): - def process_line(self, line): try: data = json.loads(line) except JSONDecodeError as e: - print('Unable to load shellcheck output ({}): {}'.format(e, line)) + print("Unable to load shellcheck output ({}): {}".format(e, line)) return for entry in data: res = { - 'path': entry['file'], - 'message': entry['message'], - 'level': 'error', - 'lineno': entry['line'], - 'column': entry['column'], - 'rule': entry['code'], + "path": entry["file"], + "message": entry["message"], + "level": "error", + "lineno": entry["line"], + "column": entry["column"], + "rule": entry["code"], } results.append(result.from_config(self.config, **res)) @@ -54,22 +53,22 @@ def determine_shell_from_script(path): #!/bin/bash #!/usr/bin/env bash """ - with open(path, 'r') as f: + with open(path, "r") as f: head = f.readline() - if not head.startswith('#!'): + if not head.startswith("#!"): return # allow for parameters to the shell shebang = head.split()[0] # if the first entry is a variant of /usr/bin/env - if 'env' in shebang: + if "env" in shebang: shebang = head.split()[1] - if shebang.endswith('sh'): + if shebang.endswith("sh"): # Strip first to avoid issues with #!bash - return shebang.strip('#!').split('/')[-1] + return shebang.strip("#!").split("/")[-1] # make it clear we return None, rather than fall through. return @@ -77,19 +76,22 @@ def determine_shell_from_script(path): def find_shell_scripts(config, paths): found = dict() - root = config['root'] - exclude = [mozpath.join(root, e) for e in config.get('exclude', [])] + root = config["root"] + exclude = [mozpath.join(root, e) for e in config.get("exclude", [])] - if config.get('extensions'): - pattern = '**/*.{}'.format(config.get('extensions')[0]) + if config.get("extensions"): + pattern = "**/*.{}".format(config.get("extensions")[0]) else: - pattern = '**/*.sh' + pattern = "**/*.sh" files = [] for path in paths: path = mozpath.normsep(path) - ignore = [e[len(path):].lstrip('/') for e in exclude - if mozpath.commonprefix((path, e)) == path] + ignore = [ + e[len(path) :].lstrip("/") + for e in exclude + if mozpath.commonprefix((path, e)) == path + ] finder = FileFinder(path, ignore=ignore) files.extend([os.path.join(path, p) for p, f in finder.find(pattern)]) @@ -114,34 +116,34 @@ def get_shellcheck_binary(): Returns the path of the first shellcheck binary available if not found returns None """ - binary = os.environ.get('SHELLCHECK') + binary = os.environ.get("SHELLCHECK") if binary: return binary - return which('shellcheck') + return which("shellcheck") def lint(paths, config, **lintargs): - log = lintargs['log'] + log = lintargs["log"] binary = get_shellcheck_binary() if not binary: print(SHELLCHECK_NOT_FOUND) - if 'MOZ_AUTOMATION' in os.environ: + if "MOZ_AUTOMATION" in os.environ: return 1 return [] - config['root'] = lintargs['root'] + config["root"] = lintargs["root"] files = find_shell_scripts(config, paths) - base_command = [binary, '-f', 'json'] - if config.get('excludecodes'): - base_command.extend(['-e', ','.join(config.get('excludecodes'))]) + base_command = [binary, "-f", "json"] + if config.get("excludecodes"): + base_command.extend(["-e", ",".join(config.get("excludecodes"))]) for f in files: cmd = list(base_command) - cmd.extend(['-s', files[f], f]) + cmd.extend(["-s", files[f], f]) log.debug("Command: {}".format(cmd)) run_process(config, cmd) return results diff --git a/tools/lint/spell/__init__.py b/tools/lint/spell/__init__.py index 6c492fb4deb7a..d5d0ecb9b57f1 100644 --- a/tools/lint/spell/__init__.py +++ b/tools/lint/spell/__init__.py @@ -18,46 +18,49 @@ from mozlint.util.implementation import LintProcess here = os.path.abspath(os.path.dirname(__file__)) -CODESPELL_REQUIREMENTS_PATH = os.path.join(here, 'codespell_requirements.txt') +CODESPELL_REQUIREMENTS_PATH = os.path.join(here, "codespell_requirements.txt") CODESPELL_NOT_FOUND = """ Could not find codespell! Install codespell and try again. $ pip install -U --require-hashes -r {} -""".strip().format(CODESPELL_REQUIREMENTS_PATH) +""".strip().format( + CODESPELL_REQUIREMENTS_PATH +) CODESPELL_INSTALL_ERROR = """ Unable to install correct version of codespell Try to install it manually with: $ pip install -U --require-hashes -r {} -""".strip().format(CODESPELL_REQUIREMENTS_PATH) +""".strip().format( + CODESPELL_REQUIREMENTS_PATH +) results = [] -CODESPELL_FORMAT_REGEX = re.compile(r'(.*):(.*): (.*) ==> (.*)$') +CODESPELL_FORMAT_REGEX = re.compile(r"(.*):(.*): (.*) ==> (.*)$") class CodespellProcess(LintProcess): - def process_line(self, line): try: match = CODESPELL_FORMAT_REGEX.match(line) abspath, line, typo, correct = match.groups() except AttributeError: - print('Unable to match regex against output: {}'.format(line)) + print("Unable to match regex against output: {}".format(line)) return # Ignore false positive like aParent (which would be fixed to apparent) # See https://github.com/lucasdemarchi/codespell/issues/314 - m = re.match(r'^[a-z][A-Z][a-z]*', typo) + m = re.match(r"^[a-z][A-Z][a-z]*", typo) if m: return res = { - 'path': abspath, - 'message': typo.strip() + " ==> " + correct, - 'level': 'error', - 'lineno': line, + "path": abspath, + "message": typo.strip() + " ==> " + correct, + "level": "error", + "lineno": line, } results.append(result.from_config(self.config, **res)) @@ -76,11 +79,11 @@ def get_codespell_binary(): Returns the path of the first codespell binary available if not found returns None """ - binary = os.environ.get('CODESPELL') + binary = os.environ.get("CODESPELL") if binary: return binary - return which('codespell') + return which("codespell") def setup(root, **lintargs): @@ -90,36 +93,38 @@ def setup(root, **lintargs): def lint(paths, config, fix=None, **lintargs): - log = lintargs['log'] + log = lintargs["log"] binary = get_codespell_binary() if not binary: print(CODESPELL_NOT_FOUND) - if 'MOZ_AUTOMATION' in os.environ: + if "MOZ_AUTOMATION" in os.environ: return 1 return [] - config['root'] = lintargs['root'] - - skip_files = '' - if 'exclude' in config: - skip_files = '--skip=*.dic,{}'.format(','.join(config['exclude'])) - - exclude_list = os.path.join(here, 'exclude-list.txt') - cmd_args = [which('python'), - binary, - '--disable-colors', - # Silence some warnings: - # 1: disable warnings about wrong encoding - # 2: disable warnings about binary file - # 4: shut down warnings about automatic fixes - # that were disabled in dictionary. - '--quiet-level=7', - '--ignore-words=' + exclude_list, - skip_files] + config["root"] = lintargs["root"] + + skip_files = "" + if "exclude" in config: + skip_files = "--skip=*.dic,{}".format(",".join(config["exclude"])) + + exclude_list = os.path.join(here, "exclude-list.txt") + cmd_args = [ + which("python"), + binary, + "--disable-colors", + # Silence some warnings: + # 1: disable warnings about wrong encoding + # 2: disable warnings about binary file + # 4: shut down warnings about automatic fixes + # that were disabled in dictionary. + "--quiet-level=7", + "--ignore-words=" + exclude_list, + skip_files, + ] if fix: - cmd_args.append('--write-changes') - log.debug("Command: {}".format(' '.join(cmd_args))) + cmd_args.append("--write-changes") + log.debug("Command: {}".format(" ".join(cmd_args))) base_command = cmd_args + paths diff --git a/tools/lint/test/conftest.py b/tools/lint/test/conftest.py index b81eb143a459c..637d8ce52a818 100644 --- a/tools/lint/test/conftest.py +++ b/tools/lint/test/conftest.py @@ -20,18 +20,20 @@ logger = logging.getLogger("mozlint") -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def root(request): """Return the root directory for the files of the linter under test. For example, with LINTER=flake8 this would be tools/lint/test/files/flake8. """ - if not hasattr(request.module, 'LINTER'): - pytest.fail("'root' fixture used from a module that didn't set the LINTER variable") - return os.path.join(here, 'files', request.module.LINTER) + if not hasattr(request.module, "LINTER"): + pytest.fail( + "'root' fixture used from a module that didn't set the LINTER variable" + ) + return os.path.join(here, "files", request.module.LINTER) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def paths(root): """Return a function that can resolve file paths relative to the linter under test. @@ -39,10 +41,12 @@ def paths(root): Can be used like `paths('foo.py', 'bar/baz')`. This will return a list of absolute paths under the `root` files directory. """ + def _inner(*paths): if not paths: return [root] return [os.path.normpath(os.path.join(root, p)) for p in paths] + return _inner @@ -55,11 +59,13 @@ def config(request): used to test a single linter. If no LINTER variable is defined, the test will fail. """ - if not hasattr(request.module, 'LINTER'): - pytest.fail("'config' fixture used from a module that didn't set the LINTER variable") + if not hasattr(request.module, "LINTER"): + pytest.fail( + "'config' fixture used from a module that didn't set the LINTER variable" + ) name = request.module.LINTER - config_path = os.path.join(lintdir, '{}.yml'.format(name)) + config_path = os.path.join(lintdir, "{}.yml".format(name)) parser = Parser(build.topsrcdir) # TODO Don't assume one linter per yaml file return parser.parse(config_path)[0] @@ -70,10 +76,10 @@ def run_setup(config): """Make sure that if the linter named in the LINTER global variable has a setup function, it gets called before running the tests. """ - if 'setup' not in config: + if "setup" not in config: return - func = findobject(config['setup']) + func = findobject(config["setup"]) func(build.topsrcdir) @@ -86,18 +92,19 @@ def lint(config, root): specified. """ try: - func = findobject(config['payload']) + func = findobject(config["payload"]) except (ImportError, ValueError): - pytest.fail("could not resolve a lint function from '{}'".format(config['payload'])) + pytest.fail( + "could not resolve a lint function from '{}'".format(config["payload"]) + ) ResultSummary.root = root def wrapper(paths, config=config, root=root, collapse_results=False, **lintargs): logger.setLevel(logging.DEBUG) - lintargs['log'] = logging.LoggerAdapter(logger, { - "lintname": config.get("name"), - "pid": os.getpid() - }) + lintargs["log"] = logging.LoggerAdapter( + logger, {"lintname": config.get("name"), "pid": os.getpid()} + ) results = func(paths, config, root=root, **lintargs) if not collapse_results: return results @@ -113,8 +120,9 @@ def wrapper(paths, config=config, root=root, collapse_results=False, **lintargs) @pytest.fixture def create_temp_file(tmpdir): def inner(contents, name=None): - name = name or 'temp.py' + name = name or "temp.py" path = tmpdir.join(name) path.write(contents) return path.strpath + return inner diff --git a/tools/lint/test/test_clang_format.py b/tools/lint/test/test_clang_format.py index 7e94db8c668d0..a04fde6267bcf 100644 --- a/tools/lint/test/test_clang_format.py +++ b/tools/lint/test/test_clang_format.py @@ -2,7 +2,7 @@ from conftest import build -LINTER = 'clang-format' +LINTER = "clang-format" def test_good(lint, config, paths): @@ -21,7 +21,7 @@ def test_basic(lint, config, paths): assert results[0].lineno == 1 assert results[0].column == 4 assert "bad.cpp" in results[0].path - assert 'int main ( ) {' in results[0].diff + assert "int main ( ) {" in results[0].diff def test_dir(lint, config, paths): @@ -34,7 +34,7 @@ def test_dir(lint, config, paths): assert results[0].lineno == 1 assert results[0].column == 4 assert "bad.cpp" in results[0].path - assert 'int main ( ) {' in results[0].diff + assert "int main ( ) {" in results[0].diff assert "Reformat C/C++" in results[5].message assert results[5].level == "warning" @@ -44,5 +44,5 @@ def test_dir(lint, config, paths): assert "#include" in results[5].diff -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_clippy.py b/tools/lint/test/test_clippy.py index 4fe1e98518058..1548c8837eb5b 100644 --- a/tools/lint/test/test_clippy.py +++ b/tools/lint/test/test_clippy.py @@ -1,7 +1,7 @@ import mozunit import os -LINTER = 'clippy' +LINTER = "clippy" def test_basic(lint, config, paths): @@ -9,7 +9,9 @@ def test_basic(lint, config, paths): print(results) assert len(results) > 7 - assert "is never read" in results[0].message or 'but never used' in results[0].message + assert ( + "is never read" in results[0].message or "but never used" in results[0].message + ) assert results[0].level == "warning" assert results[0].lineno == 7 assert results[0].column == 9 @@ -97,5 +99,5 @@ def test_cleanup(lint, paths, root): assert os.path.exists(os.path.join(root, "test2/Cargo.lock")) -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_codespell.py b/tools/lint/test/test_codespell.py index 86226966bd3dd..41912e01a7926 100644 --- a/tools/lint/test/test_codespell.py +++ b/tools/lint/test/test_codespell.py @@ -2,14 +2,14 @@ import mozunit -LINTER = 'codespell' +LINTER = "codespell" def test_lint_codespell(lint, paths): results = lint(paths()) assert len(results) == 2 - assert results[0].message == 'informations ==> information' + assert results[0].message == "informations ==> information" assert results[0].level == "error" assert results[0].lineno == 1 assert results[0].relpath == "ignore.rst" @@ -20,5 +20,5 @@ def test_lint_codespell(lint, paths): assert results[1].relpath == "ignore.rst" -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_eslint.py b/tools/lint/test/test_eslint.py index 2ce7a7c222ea5..01456a406392e 100644 --- a/tools/lint/test/test_eslint.py +++ b/tools/lint/test/test_eslint.py @@ -2,29 +2,29 @@ from conftest import build -LINTER = 'eslint' +LINTER = "eslint" def test_lint_with_global_exclude(lint, config, paths): - config['exclude'] = ['subdir', 'import'] + config["exclude"] = ["subdir", "import"] results = lint(paths(), config=config, root=build.topsrcdir) assert len(results) == 0 def test_no_files_to_lint(lint, config, paths): # A directory with no files to lint. - results = lint(paths('nolint'), root=build.topsrcdir) + results = lint(paths("nolint"), root=build.topsrcdir) assert results == [] # Errors still show up even when a directory with no files is passed in. - results = lint(paths('nolint', 'subdir/bad.js'), root=build.topsrcdir) + results = lint(paths("nolint", "subdir/bad.js"), root=build.topsrcdir) assert len(results) == 1 def test_bad_import(lint, config, paths): - results = lint(paths('import'), config=config, root=build.topsrcdir) + results = lint(paths("import"), config=config, root=build.topsrcdir) assert results == 1 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_file_license.py b/tools/lint/test/test_file_license.py index 3c109d4fa2885..6c02cd146af56 100644 --- a/tools/lint/test/test_file_license.py +++ b/tools/lint/test/test_file_license.py @@ -2,7 +2,7 @@ import mozunit -LINTER = 'license' +LINTER = "license" def test_lint_license(lint, paths): @@ -21,5 +21,5 @@ def test_lint_license(lint, paths): assert "bad.js" in results[2].relpath -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_file_perm.py b/tools/lint/test/test_file_perm.py index 21a098cf7b76b..5929889b9c60d 100644 --- a/tools/lint/test/test_file_perm.py +++ b/tools/lint/test/test_file_perm.py @@ -2,7 +2,7 @@ import mozunit -LINTER = 'file-perm' +LINTER = "file-perm" def test_lint_file_perm(lint, paths): @@ -19,5 +19,5 @@ def test_lint_file_perm(lint, paths): assert results[1].relpath == "bad.js" -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_file_whitespace.py b/tools/lint/test/test_file_whitespace.py index 02305280d5876..deac3ee283122 100644 --- a/tools/lint/test/test_file_whitespace.py +++ b/tools/lint/test/test_file_whitespace.py @@ -2,7 +2,7 @@ import mozunit -LINTER = 'file-whitespace' +LINTER = "file-whitespace" def test_lint_file_whitespace(lint, paths): @@ -25,5 +25,5 @@ def test_lint_file_whitespace(lint, paths): assert results[2].lineno == 2 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_flake8.py b/tools/lint/test/test_flake8.py index d2fc56fadbf18..66befab4e7b84 100644 --- a/tools/lint/test/test_flake8.py +++ b/tools/lint/test/test_flake8.py @@ -2,26 +2,26 @@ import mozunit -LINTER = 'flake8' +LINTER = "flake8" def test_lint_single_file(lint, paths): - results = lint(paths('bad.py')) + results = lint(paths("bad.py")) assert len(results) == 2 - assert results[0].rule == 'F401' - assert results[1].rule == 'E501' + assert results[0].rule == "F401" + assert results[1].rule == "E501" assert results[1].lineno == 5 # run lint again to make sure the previous results aren't counted twice - results = lint(paths('bad.py')) + results = lint(paths("bad.py")) assert len(results) == 2 def test_lint_custom_config_ignored(lint, paths): - results = lint(paths('custom')) + results = lint(paths("custom")) assert len(results) == 2 - results = lint(paths('custom/good.py')) + results = lint(paths("custom/good.py")) assert len(results) == 2 @@ -33,7 +33,7 @@ def foobar(): pass """.lstrip() - path = create_temp_file(contents, name='bad.py') + path = create_temp_file(contents, name="bad.py") results = lint([path]) assert len(results) == 2 @@ -42,11 +42,11 @@ def foobar(): assert len(results) == 1 # Also test with a directory - path = os.path.dirname(create_temp_file(contents, name='bad2.py')) + path = os.path.dirname(create_temp_file(contents, name="bad2.py")) results = lint([path], fix=True) # There should now be two files with 2 combined errors assert len(results) == 2 - assert all(r.rule != 'E501' for r in results) + assert all(r.rule != "E501" for r in results) def test_lint_fix_uses_config(lint, create_temp_file): @@ -54,56 +54,56 @@ def test_lint_fix_uses_config(lint, create_temp_file): foo = ['A list of strings', 'that go over 80 characters', 'to test if autopep8 fixes it'] """.lstrip() - path = create_temp_file(contents, name='line_length.py') + path = create_temp_file(contents, name="line_length.py") lint([path], fix=True) # Make sure autopep8 reads the global config under lintargs['root']. If it # didn't, then the line-length over 80 would get fixed. - with open(path, 'r') as fh: + with open(path, "r") as fh: assert fh.read() == contents def test_lint_excluded_file(lint, paths, config): # First file is globally excluded, second one is from .flake8 config. - files = paths('bad.py', 'subdir/exclude/bad.py', 'subdir/exclude/exclude_subdir') - config['exclude'] = paths('bad.py') + files = paths("bad.py", "subdir/exclude/bad.py", "subdir/exclude/exclude_subdir") + config["exclude"] = paths("bad.py") results = lint(files, config) print(results) assert len(results) == 0 # Make sure excludes also apply when running from a different cwd. - cwd = paths('subdir')[0] + cwd = paths("subdir")[0] os.chdir(cwd) - results = lint(paths('subdir/exclude')) + results = lint(paths("subdir/exclude")) print(results) assert len(results) == 0 def test_lint_excluded_file_with_glob(lint, paths, config): - config['exclude'] = paths('ext/*.configure') + config["exclude"] = paths("ext/*.configure") - files = paths('ext') + files = paths("ext") results = lint(files, config) print(results) assert len(results) == 0 - files = paths('ext/bad.configure') + files = paths("ext/bad.configure") results = lint(files, config) print(results) assert len(results) == 0 def test_lint_excluded_file_with_no_filter(lint, paths, config): - results = lint(paths('subdir/exclude'), use_filters=False) + results = lint(paths("subdir/exclude"), use_filters=False) print(results) assert len(results) == 4 def test_lint_uses_custom_extensions(lint, paths): - assert len(lint(paths('ext'))) == 1 - assert len(lint(paths('ext/bad.configure'))) == 1 + assert len(lint(paths("ext"))) == 1 + assert len(lint(paths("ext/bad.configure"))) == 1 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_lintpref.py b/tools/lint/test/test_lintpref.py index 959259da6695e..4fb3b8d23ce46 100644 --- a/tools/lint/test/test_lintpref.py +++ b/tools/lint/test/test_lintpref.py @@ -2,17 +2,17 @@ import mozunit -LINTER = 'lintpref' +LINTER = "lintpref" def test_lintpref(lint, paths): results = lint(paths()) assert len(results) == 1 - assert results[0].level == 'error' + assert results[0].level == "error" assert 'pref("dom.webidl.test1", true);' in results[0].message - assert 'bad.js' in results[0].relpath + assert "bad.js" in results[0].relpath assert results[0].lineno == 2 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_pylint.py b/tools/lint/test/test_pylint.py index 880fb2e41e89e..954575c851a5d 100644 --- a/tools/lint/test/test_pylint.py +++ b/tools/lint/test/test_pylint.py @@ -1,24 +1,24 @@ import mozunit -LINTER = 'pylint' +LINTER = "pylint" def test_lint_single_file(lint, paths): - results = lint(paths('bad.py')) + results = lint(paths("bad.py")) assert len(results) == 3 - assert results[0].rule == 'E0602' - assert results[1].rule == 'W0101' + assert results[0].rule == "E0602" + assert results[1].rule == "W0101" assert results[1].lineno == 5 # run lint again to make sure the previous results aren't counted twice - results = lint(paths('bad.py')) + results = lint(paths("bad.py")) assert len(results) == 3 def test_lint_single_file_good(lint, paths): - results = lint(paths('good.py')) + results = lint(paths("good.py")) assert len(results) == 0 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_rst.py b/tools/lint/test/test_rst.py index d9e7070dea489..532ac332d68b6 100644 --- a/tools/lint/test/test_rst.py +++ b/tools/lint/test/test_rst.py @@ -2,8 +2,10 @@ import mozunit from mozfile import which -LINTER = 'rst' -pytestmark = pytest.mark.skipif(not which('rstcheck'), reason="rstcheck is not installed") +LINTER = "rst" +pytestmark = pytest.mark.skipif( + not which("rstcheck"), reason="rstcheck is not installed" +) def test_basic(lint, paths): @@ -19,5 +21,5 @@ def test_basic(lint, paths): assert results[1].relpath == "bad2.rst" -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_rustfmt.py b/tools/lint/test/test_rustfmt.py index 4745b9c60d6ca..4eaf333825c8b 100644 --- a/tools/lint/test/test_rustfmt.py +++ b/tools/lint/test/test_rustfmt.py @@ -1,7 +1,7 @@ import mozunit -LINTER = 'rustfmt' +LINTER = "rustfmt" def test_good(lint, config, paths): @@ -40,5 +40,5 @@ def test_dir(lint, config, paths): assert "Print text to the console" in results[1].diff -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_shellcheck.py b/tools/lint/test/test_shellcheck.py index c9456ad33214d..aa6fdc4494475 100644 --- a/tools/lint/test/test_shellcheck.py +++ b/tools/lint/test/test_shellcheck.py @@ -2,8 +2,10 @@ import mozunit from mozfile import which -LINTER = 'shellcheck' -pytestmark = pytest.mark.skipif(not which('shellcheck'), reason="shellcheck is not installed") +LINTER = "shellcheck" +pytestmark = pytest.mark.skipif( + not which("shellcheck"), reason="shellcheck is not installed" +) def test_basic(lint, paths): @@ -20,5 +22,5 @@ def test_basic(lint, paths): assert results[1].relpath == "bad.sh" -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/test/test_yaml.py b/tools/lint/test/test_yaml.py index 0170d5ea94296..96c1334dfa0a7 100644 --- a/tools/lint/test/test_yaml.py +++ b/tools/lint/test/test_yaml.py @@ -2,8 +2,10 @@ import mozunit from mozfile import which -LINTER = 'yaml' -pytestmark = pytest.mark.skipif(not which('yamllint'), reason="yamllint is not installed") +LINTER = "yaml" +pytestmark = pytest.mark.skipif( + not which("yamllint"), reason="yamllint is not installed" +) def test_basic(lint, paths): @@ -27,5 +29,5 @@ def test_basic(lint, paths): assert results[2].lineno == 9 -if __name__ == '__main__': +if __name__ == "__main__": mozunit.main() diff --git a/tools/lint/wpt/wpt.py b/tools/lint/wpt/wpt.py index d16bd2fd30969..2dcbb39132a70 100644 --- a/tools/lint/wpt/wpt.py +++ b/tools/lint/wpt/wpt.py @@ -16,8 +16,8 @@ def lint(files, config, **kwargs): - log = kwargs['log'] - tests_dir = os.path.join(kwargs['root'], 'testing', 'web-platform', 'tests') + log = kwargs["log"] + tests_dir = os.path.join(kwargs["root"], "testing", "web-platform", "tests") def process_line(line): try: @@ -26,27 +26,34 @@ def process_line(line): return data["level"] = "error" - data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), kwargs['root']) + data["path"] = os.path.relpath( + os.path.join(tests_dir, data["path"]), kwargs["root"] + ) data.setdefault("lineno", 0) results.append(result.from_config(config, **data)) if files == [tests_dir]: - print("No specific files specified, running the full wpt lint" - " (this is slow)", file=sys.stderr) + print( + "No specific files specified, running the full wpt lint" " (this is slow)", + file=sys.stderr, + ) files = ["--all"] - cmd = ['python2', os.path.join(tests_dir, 'wpt'), 'lint', '--json'] + files - log.debug("Command: {}".format(' '.join(cmd))) + cmd = ["python2", os.path.join(tests_dir, "wpt"), "lint", "--json"] + files + log.debug("Command: {}".format(" ".join(cmd))) - proc = ProcessHandler(cmd, env=os.environ, processOutputLine=process_line, - universal_newlines=True) + proc = ProcessHandler( + cmd, env=os.environ, processOutputLine=process_line, universal_newlines=True + ) proc.run() try: proc.wait() if proc.returncode != 0: results.append( - result.from_config(config, - message="Lint process exited with return code %s" % - proc.returncode)) + result.from_config( + config, + message="Lint process exited with return code %s" % proc.returncode, + ) + ) except KeyboardInterrupt: proc.kill() diff --git a/tools/lint/yamllint_/__init__.py b/tools/lint/yamllint_/__init__.py index 4a220fc95c0b1..ebcea1b69f800 100644 --- a/tools/lint/yamllint_/__init__.py +++ b/tools/lint/yamllint_/__init__.py @@ -14,28 +14,28 @@ from mozlint.util.implementation import LintProcess -YAMLLINT_FORMAT_REGEX = re.compile('(.*):(.*):(.*): \[(error|warning)\] (.*) \((.*)\)$') +YAMLLINT_FORMAT_REGEX = re.compile("(.*):(.*):(.*): \[(error|warning)\] (.*) \((.*)\)$") results = [] class YAMLLintProcess(LintProcess): - def process_line(self, line): try: match = YAMLLINT_FORMAT_REGEX.match(line) abspath, line, col, level, message, code = match.groups() except AttributeError: - print('Unable to match yaml regex against output: {}'.format(line)) + print("Unable to match yaml regex against output: {}".format(line)) return - res = {'path': os.path.relpath(str(abspath), self.config['root']), - 'message': str(message), - 'level': 'error', - 'lineno': line, - 'column': col, - 'rule': code, - } + res = { + "path": os.path.relpath(str(abspath), self.config["root"]), + "message": str(message), + "level": "error", + "lineno": line, + "column": col, + "rule": code, + } results.append(result.from_config(self.config, **res)) @@ -45,19 +45,21 @@ def get_yamllint_binary(mc_root): Returns the path of the first yamllint binary available if not found returns None """ - binary = os.environ.get('YAMLLINT') + binary = os.environ.get("YAMLLINT") if binary: return binary # yamllint is vendored in mozilla-central: let's use this # if no environment variable is found. - return os.path.join(mc_root, 'third_party', 'python', 'yamllint', 'yamllint') + return os.path.join(mc_root, "third_party", "python", "yamllint", "yamllint") def get_yamllint_version(binary): - return subprocess.check_output([which('python'), binary, "--version"], - universal_newlines=True, - stderr=subprocess.STDOUT) + return subprocess.check_output( + [which("python"), binary, "--version"], + universal_newlines=True, + stderr=subprocess.STDOUT, + ) def _run_pip(*args): @@ -65,8 +67,7 @@ def _run_pip(*args): Helper function that runs pip with subprocess """ try: - subprocess.check_output(['pip'] + list(args), - stderr=subprocess.STDOUT) + subprocess.check_output(["pip"] + list(args), stderr=subprocess.STDOUT) return True except subprocess.CalledProcessError as e: print(e.output) @@ -86,37 +87,35 @@ def gen_yamllint_args(cmdargs, paths=None, conf_file=None): args = cmdargs[:] if isinstance(paths, str): paths = [paths] - if conf_file and conf_file != 'default': - return args + ['-c', conf_file] + paths + if conf_file and conf_file != "default": + return args + ["-c", conf_file] + paths return args + paths def lint(files, config, **lintargs): - log = lintargs['log'] + log = lintargs["log"] - binary = get_yamllint_binary(lintargs['root']) + binary = get_yamllint_binary(lintargs["root"]) log.debug("Version: {}".format(get_yamllint_version(binary))) - cmdargs = [ - which('python'), - binary, - '-f', 'parsable' - ] - log.debug("Command: {}".format(' '.join(cmdargs))) + cmdargs = [which("python"), binary, "-f", "parsable"] + log.debug("Command: {}".format(" ".join(cmdargs))) config = config.copy() - config['root'] = lintargs['root'] + config["root"] = lintargs["root"] # Run any paths with a .yamllint file in the directory separately so # it gets picked up. This means only .yamllint files that live in # directories that are explicitly included will be considered. paths_by_config = defaultdict(list) for f in files: - conf_files = get_ancestors_by_name('.yamllint', f, config['root']) - paths_by_config[conf_files[0] if conf_files else 'default'].append(f) + conf_files = get_ancestors_by_name(".yamllint", f, config["root"]) + paths_by_config[conf_files[0] if conf_files else "default"].append(f) for conf_file, paths in paths_by_config.items(): - run_process(config, gen_yamllint_args(cmdargs, conf_file=conf_file, paths=paths)) + run_process( + config, gen_yamllint_args(cmdargs, conf_file=conf_file, paths=paths) + ) return results