diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02b616f78d452..ed4694ff44156 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,21 +88,5 @@ jobs: - name: yarn install if: steps.cache.outputs.cache-hit != 'true' run: yarn install - - name: Install httpserver - run: volta install http-server - - name: Install linkcheck - if: steps.cache.outputs.cache-hit != 'true' - run: |- - wget -nc https://github.com/filiph/linkcheck/releases/download/v2.0.12/linkcheck-linux-x64.exe -O ./linkcheck - chmod +x ./linkcheck - - name: Build - run: yarn build - - name: Run linkcheck (search for "HTTP 404" in logs to get to errors) - run: |- - http-server public/ > /dev/null & - printf "Waiting for server to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null "http://localhost:8080"); do printf '.'; sleep 0.5; done' - - # exitcode=1 -- only warnings (about broken anchors) - # exitcode=2 -- hard 404s - # We only fail the build if there are hard 404s - ./linkcheck http://localhost:8080 || [ $? -eq 1 ] + - name: yarn linkcheck + run: yarn linkcheck diff --git a/.gitignore b/.gitignore index 37abe8174a365..d0ebe5c3da811 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,6 @@ static/_platforms/ # generated nginx file nginx.out.conf + +# linkchecker binary +linkcheck diff --git a/bin/linkchecker b/bin/linkchecker new file mode 100755 index 0000000000000..065adf4db9ff9 --- /dev/null +++ b/bin/linkchecker @@ -0,0 +1,77 @@ +#!/bin/bash +set -e +set -o pipefail + +LINKCHECK_VERSION=2.0.12 + +# This linkchecker written in Dart is the fastest we've found and finds the +# most links. It is still very slow, requires a separate HTTP server, and is +# barely configurable wrt output verbosity. Hence this script exists. +# +# Considering lack of performant, correct alternatives we *assume* that +# building our own would be hard. + +if [ "$(uname)" = "Darwin" ]; then + url="https://github.com/filiph/linkcheck/releases/download/v$LINKCHECK_VERSION/linkcheck-mac-x64" +else + url="https://github.com/filiph/linkcheck/releases/download/v$LINKCHECK_VERSION/linkcheck-linux-x64.exe" +fi + +if [ ! -f ./linkcheck ]; then + echo ">>> Downloading linkcheck" + wget -nc "$url" -O ./linkcheck + chmod +x ./linkcheck +fi + +echo ">>> Building docs" +yarn build + +echo ">>> Starting HTTP server" +yarn http-server public/ > /dev/null & +httpserver_pid=$! +trap "kill -9 $httpserver_pid" SIGINT SIGTERM EXIT + +echo ">>> Waiting for server to be up" +timeout 60 bash -c 'until $(curl -Isf -o /dev/null "http://localhost:8080"); do printf '.'; sleep 0.5; done' + +# exitcode=1 -- only warnings (about broken anchors) +# exitcode=2 -- hard 404s +# We only fail the build if there are hard 404s +echo ">>> Checking some links (started at $(date))" +echo " This step takes 5-10 minutes and does not produce intermediate output." +echo " If your fans are spinning it's working." +(./linkcheck http://localhost:8080 || [ $? -eq 1 ]) | \ + + # Post-process the output and remove all anchor warnings + grep -v "(HTTP 200 but missing anchor)" | \ + + # Clean up output some more, launch in subshell because of namespace + # pollution (and because I want to keep the bash-isms to a minimum) + ( + prev_line="" + while IFS= read line; do + # XXX: Bashism + if [ -z "$line" ] && [[ "$prev_line" = http://localhost:8080/* ]]; then + # We are in a section for which we report nothing (because we grepped + # away anchor warnings): + # + # http://... <-- prev_line + # <-- line + # http://... + + true + else + echo "$prev_line" + fi + + prev_line="$line" + done + + # Since the loop always outputs the prev_line, we need to fix the off-by-one error here + echo "$prev_line" + ) | \ + + # Remove repeated blank lines + cat -s + +kill -9 $httpserver_pid diff --git a/package.json b/package.json index b0157728afd6a..17f011f10bb40 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,8 @@ "prettier:fix:all": "prettier --write \"./{src,plugins}/**/*.{md,mdx,js,jsx}\"", "start": "yarn run develop", "format": "prettier --write \"src/**/*.js\"", - "test": "jest" + "test": "jest", + "linkcheck": "bin/linkchecker" }, "devDependencies": { "@testing-library/jest-dom": "^5.11.4", @@ -106,6 +107,7 @@ "eslint-plugin-react": "^7.20.6", "est": "^0.2.1-alpha", "gh-pages": "^2.2.0", + "http-server": "^0.12.3", "identity-obj-proxy": "^3.0.0", "jest": "^26.4.2", "jest-dom": "^4.0.0", diff --git a/yarn.lock b/yarn.lock index 5da530750bf3a..374cdc9c90b4b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4102,6 +4102,11 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" +basic-auth@^1.0.3: + version "1.1.0" + resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-1.1.0.tgz#45221ee429f7ee1e5035be3f51533f1cdfd29884" + integrity sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ= + batch@0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/batch/-/batch-0.6.1.tgz#dc34314f4e679318093fc760272525f94bf25c16" @@ -5210,7 +5215,7 @@ colorette@^1.2.1: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.1.tgz#4d0b921325c14faf92633086a536db6e89564b1b" integrity sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw== -colors@^1.1.2: +colors@^1.1.2, colors@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== @@ -5526,6 +5531,11 @@ cors@^2.8.5: object-assign "^4" vary "^1" +corser@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87" + integrity sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c= + cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a" @@ -6601,6 +6611,16 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" +ecstatic@^3.3.2: + version "3.3.2" + resolved "https://registry.yarnpkg.com/ecstatic/-/ecstatic-3.3.2.tgz#6d1dd49814d00594682c652adb66076a69d46c48" + integrity sha512-fLf9l1hnwrHI2xn9mEDT7KIi22UDqA2jaCwyCbSUJh9a1V+LEUSL/JO/6TIz/QyuBURWUHrFL5Kg2TtO1bkkog== + dependencies: + he "^1.1.1" + mime "^1.6.0" + minimist "^1.1.0" + url-join "^2.0.5" + ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" @@ -9542,6 +9562,11 @@ hastscript@^5.0.0: property-information "^5.0.0" space-separated-tokens "^1.0.0" +he@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" + integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== + header-case@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/header-case/-/header-case-1.0.1.tgz#9535973197c144b09613cd65d317ef19963bd02d" @@ -9767,7 +9792,7 @@ http-proxy-middleware@0.19.1: lodash "^4.17.11" micromatch "^3.1.10" -http-proxy@^1.17.0, http-proxy@^1.18.1: +http-proxy@^1.17.0, http-proxy@^1.18.0, http-proxy@^1.18.1: version "1.18.1" resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== @@ -9776,6 +9801,22 @@ http-proxy@^1.17.0, http-proxy@^1.18.1: follow-redirects "^1.0.0" requires-port "^1.0.0" +http-server@^0.12.3: + version "0.12.3" + resolved "https://registry.yarnpkg.com/http-server/-/http-server-0.12.3.tgz#ba0471d0ecc425886616cb35c4faf279140a0d37" + integrity sha512-be0dKG6pni92bRjq0kvExtj/NrrAd28/8fCXkaI/4piTwQMSDSLMhWyW0NI1V+DBI3aa1HMlQu46/HjVLfmugA== + dependencies: + basic-auth "^1.0.3" + colors "^1.4.0" + corser "^2.0.1" + ecstatic "^3.3.2" + http-proxy "^1.18.0" + minimist "^1.2.5" + opener "^1.5.1" + portfinder "^1.0.25" + secure-compare "3.0.1" + union "~0.5.0" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -12389,7 +12430,7 @@ mime-types@^2.1.12, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24: dependencies: mime-db "1.44.0" -mime@1.6.0, mime@^1.3.4: +mime@1.6.0, mime@^1.3.4, mime@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== @@ -13154,6 +13195,11 @@ open@^6.4.0: dependencies: is-wsl "^1.1.0" +opener@^1.5.1: + version "1.5.2" + resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" + integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== + opentracing@^0.14.4: version "0.14.4" resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.4.tgz#a113408ea740da3a90fde5b3b0011a375c2e4268" @@ -13903,7 +13949,7 @@ pnp-webpack-plugin@^1.6.4: dependencies: ts-pnp "^1.1.6" -portfinder@^1.0.26: +portfinder@^1.0.25, portfinder@^1.0.26: version "1.0.28" resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778" integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA== @@ -14561,7 +14607,7 @@ qs@6.7.0: resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== -qs@^6.5.2: +qs@^6.4.0, qs@^6.5.2: version "6.9.4" resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.4.tgz#9090b290d1f91728d3c22e54843ca44aea5ab687" integrity sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ== @@ -15887,6 +15933,11 @@ section-matter@^1.0.0: extend-shallow "^2.0.1" kind-of "^6.0.0" +secure-compare@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3" + integrity sha1-8aAymzCLIh+uN7mXTz1XjQypmeM= + seek-bzip@^1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.6.tgz#35c4171f55a680916b52a07859ecf3b5857f21c4" @@ -17794,6 +17845,13 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" +union@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075" + integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA== + dependencies: + qs "^6.4.0" + uniq@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff" @@ -18060,6 +18118,11 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= +url-join@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-2.0.5.tgz#5af22f18c052a000a48d7b82c5e9c2e2feeda728" + integrity sha1-WvIvGMBSoACkjXuCxenC4v7tpyg= + url-loader@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-1.1.2.tgz#b971d191b83af693c5e3fea4064be9e1f2d7f8d8"