ci: Use upstream envoy.code.check (envoyproxy#19737)

This PR updates CI to use an upstream tool that integrates 4 of the current format CI checks: - flake8 - glint - shellcheck - yapf The tool can be run with: $ bazel run //tools/code:check # -- -c glint shellcheck python_yapf python_flake8 The -- --fix flag works for yapf only Also worth mentioning is that the following will only check changes against eg main (commits/branches/etc should work) $ bazel run //tools/code:check -- -s main This should allow us to use the tool in git hooks once enough of the checks have been integrated We can add more of the format checks to the tool after There are also various other planned improvements See https://github.com/envoyproxy/pytooling/milestones for general roadmap I would also like to rationalize the format check part of CI as we go (the pytest/tooling ci jobs can be removed fairly imminently) As the new tool produces quite a bit of logging ive had to make it run first - so as it doesnt bury the logs from other checkers For that reason, i added a message at the bottom of the log output to advise searching above A failing check run can be seen here: https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7560 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7433 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7292 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=124 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=251 https://dev.azure.com/cncf/envoy/_build/results?buildId=103005&view=logs&j=c5dd2866-6ab3-5f3c-3a44-4cef0ec909b5&t=a9eb66d6-8944-5769-b3f7-476949dadcb8&l=7050 In addition to adding the envoy.code.check tool this PR also brings in newer versions of core libs that have been fixed/optimized/cleaned up significantly in this release release info/milestone for this PR https://github.com/envoyproxy/pytooling/milestone/1 https://github.com/envoyproxy/pytooling/releases/tag/2022-03-04.0 Next milestone is the next iteration of the dep.checker - mostly cleanups/fixes i think https://github.com/envoyproxy/pytooling/milestone/2 Signed-off-by: Ryan Northey <ryan@synca.io>
LeonGao91 · Mar 9, 2022 · 4abc226 · 4abc226
1 parent 1e20ccd
commit 4abc226
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 251 deletions.
diff --git a/.flake8 b/.flake8
@@ -5,4 +5,4 @@
 ignore = W503,W504,E121,E126,E241,E125,E127,E129,E251,E265,E303,E306,E402,E501,E502,E711,E713,E722,E741,F523,F541,F841,N803,N806,N817,W605
 
 # TODO(phlax): exclude less
-exclude = build_docs,.git,generated,test,examples,venv
+exclude = build_docs,.git,generated,test,examples,venv,tools/dev
diff --git a/ci/format_pre.sh b/ci/format_pre.sh
@@ -26,12 +26,12 @@ trap_errors () {
             FAILED+=("  > ${sub}@ ${file} :${line}")
         else
             FAILED+=("${sub}@ ${file} :${line}${command}")
-            if [[ "$CURRENT" == "glint" ]]; then
+            if [[ "$CURRENT" == "check" ]]; then
+                # shellcheck disable=SC2016
                 FAILED+=(
-                    "    Please fix your editor to ensure:"
-                    "      - no trailing whitespace"
-                    "      - no mixed tabs/spaces"
-                    "      - all files end with a newline")
+                    ""
+                    '   *Code formatting check failed*: please search above logs for `CodeChecker ERROR`'
+                    "")
             fi
         fi
         ((frame++))
@@ -42,19 +42,12 @@ trap_errors () {
 trap trap_errors ERR
 trap exit 1 INT
 
-# TODO: move these to bazel
-CURRENT=glint
-"${ENVOY_SRCDIR}"/tools/code_format/glint.sh
-
-CURRENT=shellcheck
-"${ENVOY_SRCDIR}"/tools/code_format/check_shellcheck_format.sh check
+CURRENT=check
+time bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code:check
 
 CURRENT=configs
 bazel run "${BAZEL_BUILD_OPTIONS[@]}" //configs:example_configs_validation
 
-CURRENT=python
-bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/code_format:python_check -- --diff-file="$DIFF_OUTPUT" -werror --fix
-
 CURRENT=extensions
 bazel run "${BAZEL_BUILD_OPTIONS[@]}" //tools/extensions:extensions_check
 

diff --git a/tools/base/requirements.in b/tools/base/requirements.in
@@ -2,22 +2,22 @@ abstracts>=0.0.12
 aio.api.bazel
 aio.api.github>=0.0.5
 aio.api.nist
-aio.core>=0.5.8
-aio.run.checker>=0.3.5
-aio.run.runner>=0.2.6
+aio.core>=0.8.1
+aio.run.checker>=0.5.1
+aio.run.runner>=0.3.2
 colorama
 coloredlogs
 coverage
 envoy.base.utils>=0.1.0
-envoy.code_format.python_check>=0.0.8
+envoy.code.check
 envoy.dependency.check>=0.0.5
 envoy.dependency.pip_check>=0.1.2
 envoy.distribution.release>=0.0.7
 envoy.distribution.repo>=0.0.5
 envoy.distribution.verify>=0.0.8
-envoy.docs.sphinx-runner>=0.0.8
-envoy.gpg.identity>=0.0.6
-envoy.gpg.sign>=0.0.11
+envoy.docs.sphinx-runner>=0.1.0
+envoy.gpg.identity>=0.1.0
+envoy.gpg.sign>=0.1.0
 flake8
 frozendict
 gitpython

diff --git a/tools/base/requirements.txt b/tools/base/requirements.txt
@@ -12,10 +12,9 @@ abstracts==0.0.12 \
     #   aio-api-github
     #   aio-api-nist
     #   aio-core
-    #   aio-run-checker
     #   aio-run-runner
     #   envoy-base-utils
-    #   envoy-code-format-python-check
+    #   envoy-code-check
     #   envoy-dependency-check
     #   envoy-dependency-pip-check
     #   envoy-distribution-release
@@ -38,34 +37,35 @@ aio-api-nist==0.0.1 \
     # via
     #   -r requirements.in
     #   envoy-dependency-check
-aio-core==0.5.8 \
-    --hash=sha256:3e027710040ce43548f5704a8cd45fbb9c2a0a609a3d65735db0da3f79fa8d3f \
-    --hash=sha256:a5f09e74b726accba91abf3bd32d2072d9add833bd5d092a6d9545e5efe26e7c
+aio-core==0.8.1 \
+    --hash=sha256:085fb6573b7b8f07876f6f45d10692f20445cfe686ed43c46ed05d33379556b3 \
+    --hash=sha256:136544fcc2f04a48fc9dd8d4413381bb1a6dd226c9eab733477167762a677854
     # via
     #   -r requirements.in
     #   aio-api-github
     #   aio-api-nist
     #   aio-run-runner
     #   envoy-base-utils
-    #   envoy-code-format-python-check
+    #   envoy-code-check
     #   envoy-dependency-check
     #   envoy-distribution-release
     #   envoy-distribution-repo
     #   envoy-github-abstract
     #   envoy-github-release
-aio-run-checker==0.3.5 \
-    --hash=sha256:3ac33c5a8d5168d457c1fa2e874f76032d27615f57937c7a939123dec20cc8bf \
-    --hash=sha256:3c8e09b4514dd33afdce21cae98b7fae90a9983973714d9935ba38726b084f07
+    #   envoy-gpg-identity
+aio-run-checker==0.5.1 \
+    --hash=sha256:07e0c39031afa5370740e98a7e1628123e56646610fd8b9cf983ad85205d3adf \
+    --hash=sha256:c9dc570aa1bf16355a35aec2417d897ecb77d03b82bd5a4f56986966437ab947
     # via
     #   -r requirements.in
-    #   envoy-code-format-python-check
+    #   envoy-code-check
     #   envoy-dependency-check
     #   envoy-dependency-pip-check
     #   envoy-distribution-distrotest
     #   envoy-distribution-verify
-aio-run-runner==0.2.6 \
-    --hash=sha256:5512fbef70a1eae793455caa29653d95da850cabb8063a74241d94b110de32fe \
-    --hash=sha256:fb350bc70a998ca07f9185c277d4d114cd6cc8de3eb4d95906407b49abaa26b1
+aio-run-runner==0.3.2 \
+    --hash=sha256:75ad35f9090d7dade92846d6700b6afbfe5751afdd97c7cf5fd8e2cf4b45d8f9 \
+    --hash=sha256:a19aa3607fb0b585f695c264df72d043658c5cf7d19b4f34f29cdf4af8b39fa1
     # via
     #   -r requirements.in
     #   aio-run-checker
@@ -299,6 +299,7 @@ envoy-base-utils==0.1.0 \
     --hash=sha256:f5ca5e147e60af5c2f755cc61dcfc507b9253d4657869f923e56dfd7ac03b835
     # via
     #   -r requirements.in
+    #   envoy-code-check
     #   envoy-dependency-check
     #   envoy-dependency-pip-check
     #   envoy-distribution-distrotest
@@ -308,9 +309,9 @@ envoy-base-utils==0.1.0 \
     #   envoy-docs-sphinx-runner
     #   envoy-github-release
     #   envoy-gpg-sign
-envoy-code-format-python-check==0.0.8 \
-    --hash=sha256:2257f718567e469993bacb6aaa08d1af8d5195fd1f6936dde621b4449da45c49 \
-    --hash=sha256:acfd00e3bbd6615eaf50b3d6bad0510b967afd687911fa01da5322a3911005d0
+envoy-code-check==0.0.1 \
+    --hash=sha256:cc8094e2451ba88a24af2bde05765253391b10125d13947f1bd9ca7afdc07f20 \
+    --hash=sha256:e8da3d11a9a17beae55ba82c9b7107142d92742d64cbe772df860d1259cd9eb9
     # via -r requirements.in
 envoy-dependency-check==0.0.5 \
     --hash=sha256:39a8b25ae88e0305eaf38a90c3c7956db3b3c53e967464368319e41f3309df36 \
@@ -339,9 +340,9 @@ envoy-distribution-verify==0.0.8 \
 envoy-docker-utils==0.0.2 \
     --hash=sha256:a12cb57f0b6e204d646cbf94f927b3a8f5a27ed15f60d0576176584ec16a4b76
     # via envoy-distribution-distrotest
-envoy-docs-sphinx-runner==0.0.8 \
-    --hash=sha256:b9d051aa7ee995b647d00ea5a86fe3be30f399e344837185edd53ff3b6fc6988 \
-    --hash=sha256:fb2dc69067d9593adebda90794e912d84bb9a1c0e9c5d23ac6da608b79a8fc98
+envoy-docs-sphinx-runner==0.1.0 \
+    --hash=sha256:8be22a71038dac272f20c43589155aa0adca64223d0c85ef8864d8b1ca272b4e \
+    --hash=sha256:c0031fc261621d00cc03ed1de47d947a720a4649d6be34d3dcd34986137bdc6a
     # via -r requirements.in
 envoy-github-abstract==0.0.21 \
     --hash=sha256:243dae9457243fb42e4643b1f45006c3b0d3151c808884217447d29a081b26a1 \
@@ -353,22 +354,22 @@ envoy-github-release==0.0.11 \
     --hash=sha256:1bed7a829bd77391c33107a58057b89de34a3045895f276816e07ee0119d844f \
     --hash=sha256:a3c117892b822b9f1d29bbc92f9c82c733d972316fbe2611c4895cf1e4b1113d
     # via envoy-distribution-release
-envoy-gpg-identity==0.0.6 \
-    --hash=sha256:2195e09aaacbaa8d6588378ad96d206708ffaf863aead38445eb3e54e4223c34 \
-    --hash=sha256:93032dd82c9c422c2145dbf426fd93c3673c53ba03cca2eab13f5fb8cf24f47c
+envoy-gpg-identity==0.1.0 \
+    --hash=sha256:1df6989e1a6a3a9f5809ac056829b840f57326d41ae1181db415a722216bff48 \
+    --hash=sha256:9e11625e25ad6a031f1c85c57067673f50c0427b0e71ef0a7e926ce77ebb75a9
     # via
     #   -r requirements.in
     #   envoy-gpg-sign
-envoy-gpg-sign==0.0.11 \
-    --hash=sha256:6e7d1defb15eea4a64b0042271e1206a1e3892000e53990c5988c2b6e726734b \
-    --hash=sha256:95c86ddd7031abf5249b292986b5de1279ca9ccc0dd1a5a5d977cd90a1a48122
+envoy-gpg-sign==0.1.0 \
+    --hash=sha256:14b8efee80916fa78857198fb98357c9ddfaa9dec34eb8a453bda1a364f4b973 \
+    --hash=sha256:36bb56534a6c5947c18bd10c43185345f092664de86b49318e2afc6cac7d9158
     # via -r requirements.in
 flake8==4.0.1 \
     --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \
     --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d
     # via
     #   -r requirements.in
-    #   envoy-code-format-python-check
+    #   envoy-code-check
     #   flake8-polyfill
     #   pep8-naming
 flake8-polyfill==1.0.2 \
@@ -641,7 +642,9 @@ psutil==5.9.0 \
     --hash=sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3 \
     --hash=sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c \
     --hash=sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3
-    # via envoy-dependency-check
+    # via
+    #   aio-core
+    #   envoy-dependency-check
 py==1.10.0 \
     --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \
     --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a
@@ -896,7 +899,9 @@ types-orjson==3.6.2 \
 types-psutil==5.8.20 \
     --hash=sha256:35e16c5d58c21cd638b7a74a8b451b3c099e0f8140121f1d084174c60625a5c8 \
     --hash=sha256:b9c5d6c1f8bd6154f845a79dfbe46e7628f8d58978c6cadde0afa4abe6400294
-    # via envoy-dependency-check
+    # via
+    #   aio-core
+    #   envoy-dependency-check
 typing-extensions==3.10.0.2 \
     --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \
     --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \
@@ -950,7 +955,7 @@ yapf==0.32.0 \
     --hash=sha256:a3f5085d37ef7e3e004c4ba9f9b3e40c54ff1901cd111f05145ae313a7c67d1b
     # via
     #   -r requirements.in
-    #   envoy-code-format-python-check
+    #   envoy-code-check
 yarl==1.6.3 \
     --hash=sha256:00d7ad91b6583602eb9c1d085a2cf281ada267e9a197e8b7cae487dadbfa293e \
     --hash=sha256:0355a701b3998dcd832d0dc47cc5dedf3874f966ac7f870e0f3a6788d802d434 \

diff --git a/tools/code/BUILD b/tools/code/BUILD
@@ -0,0 +1,13 @@
+load("//bazel:envoy_build_system.bzl", "envoy_package")
+load("//tools/base:envoy_python.bzl", "envoy_entry_point")
+load("@envoy_repo//:path.bzl", "PATH")
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_entry_point(
+    name = "check",
+    args = ["--path=%s" % PATH],
+    pkg = "envoy.code.check",
+)
diff --git a/tools/code_format/BUILD b/tools/code_format/BUILD
@@ -1,5 +1,3 @@
-load("@base_pip3//:requirements.bzl", "requirement")
-load("@rules_python//python:defs.bzl", "py_binary")
 load("//bazel:envoy_build_system.bzl", "envoy_package")
 
 licenses(["notice"])  # Apache 2
@@ -11,12 +9,3 @@ exports_files([
     "header_order.py",
     "envoy_build_fixer.py",
 ])
-
-py_binary(
-    name = "python_check",
-    srcs = ["python_check.py"],
-    deps = [
-        "@envoy_repo",
-        requirement("envoy.code_format.python_check"),
-    ],
-)
diff --git a/tools/code_format/check_shellcheck_format.sh b/tools/code_format/check_shellcheck_format.sh
diff --git a/tools/code_format/glint.sh b/tools/code_format/glint.sh