Merge branch 'stable' of https://github.com/chong-he/lighthouse into …

…stable

.config/nextest.toml

@@ -0,0 +1,113 @@
+# This is the default config used by nextest. It is embedded in the binary at
+# build time. It may be used as a template for .config/nextest.toml.
+
+[store]
+# The directory under the workspace root at which nextest-related files are
+# written. Profile-specific storage is currently written to dir/<profile-name>.
+dir = "target/nextest"
+
+# This section defines the default nextest profile. Custom profiles are layered
+# on top of the default profile.
+[profile.default]
+# "retries" defines the number of times a test should be retried. If set to a
+# non-zero value, tests that succeed on a subsequent attempt will be marked as
+# non-flaky. Can be overridden through the `--retries` option.
+# Examples
+# * retries = 3
+# * retries = { backoff = "fixed", count = 2, delay = "1s" }
+# * retries = { backoff = "exponential", count = 10, delay = "1s", jitter = true, max-delay = "10s" }
+retries = 0
+
+# The number of threads to run tests with. Supported values are either an integer or
+# the string "num-cpus". Can be overridden through the `--test-threads` option.
+test-threads = 8
+
+# The number of threads required for each test. This is generally used in overrides to
+# mark certain tests as heavier than others. However, it can also be set as a global parameter.
+threads-required = 1
+
+# Show these test statuses in the output.
+#
+# The possible values this can take are:
+# * none: no output
+# * fail: show failed (including exec-failed) tests
+# * retry: show flaky and retried tests
+# * slow: show slow tests
+# * pass: show passed tests
+# * skip: show skipped tests (most useful for CI)
+# * all: all of the above
+#
+# Each value includes all the values above it; for example, "slow" includes
+# failed and retried tests.
+#
+# Can be overridden through the `--status-level` flag.
+status-level = "pass"
+
+# Similar to status-level, show these test statuses at the end of the run.
+final-status-level = "flaky"
+
+# "failure-output" defines when standard output and standard error for failing tests are produced.
+# Accepted values are
+# * "immediate": output failures as soon as they happen
+# * "final": output failures at the end of the test run
+# * "immediate-final": output failures as soon as they happen and at the end of
+#   the test run; combination of "immediate" and "final"
+# * "never": don't output failures at all
+#
+# For large test suites and CI it is generally useful to use "immediate-final".
+#
+# Can be overridden through the `--failure-output` option.
+failure-output = "immediate"
+
+# "success-output" controls production of standard output and standard error on success. This should
+# generally be set to "never".
+success-output = "never"
+
+# Cancel the test run on the first failure. For CI runs, consider setting this
+# to false.
+fail-fast = true
+
+# Treat a test that takes longer than the configured 'period' as slow, and print a message.
+# See <https://nexte.st/book/slow-tests> for more information.
+#
+# Optional: specify the parameter 'terminate-after' with a non-zero integer,
+# which will cause slow tests to be terminated after the specified number of
+# periods have passed.
+# Example: slow-timeout = { period = "60s", terminate-after = 2 }
+slow-timeout = { period = "120s" }
+
+# Treat a test as leaky if after the process is shut down, standard output and standard error
+# aren't closed within this duration.
+#
+# This usually happens in case of a test that creates a child process and lets it inherit those
+# handles, but doesn't clean the child process up (especially when it fails).
+#
+# See <https://nexte.st/book/leaky-tests> for more information.
+leak-timeout = "100ms"
+
+[profile.default.junit]
+# Output a JUnit report into the given file inside 'store.dir/<profile-name>'.
+# If unspecified, JUnit is not written out.
+
+# path = "junit.xml"
+
+# The name of the top-level "report" element in JUnit report. If aggregating
+# reports across different test runs, it may be useful to provide separate names
+# for each report.
+report-name = "lighthouse-run"
+
+# Whether standard output and standard error for passing tests should be stored in the JUnit report.
+# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
+store-success-output = false
+
+# Whether standard output and standard error for failing tests should be stored in the JUnit report.
+# Output is stored in the <system-out> and <system-err> elements of the <testcase> element.
+#
+# Note that if a description can be extracted from the output, it is always stored in the
+# <description> element.
+store-failure-output = true
+
+# This profile is activated if MIRI_SYSROOT is set.
+[profile.default-miri]
+# Miri tests take up a lot of memory, so only run 1 test at a time by default.
+test-threads = 4

.editorconfig

@@ -6,4 +6,4 @@ end_of_line=lf
 charset=utf-8
 trim_trailing_whitespace=true
 max_line_length=100
-insert_final_newline=false
+insert_final_newline=true

.github/mergify.yml

@@ -0,0 +1,19 @@
+queue_rules:
+  - name: default
+    batch_size: 8
+    batch_max_wait_time: 60 s
+    checks_timeout: 10800 s
+    merge_method: squash
+    commit_message_template: |
+      {{ title }} (#{{ number }})
+      
+      {% for commit in commits %}
+      * {{ commit.commit_message }}
+      {% endfor %}
+    queue_conditions:
+      - "#approved-reviews-by >= 1"
+      - "check-success=license/cla"
+      - "check-success=target-branch-check"
+    merge_conditions:
+      - "check-success=test-suite-success"
+      - "check-success=local-testnet-success"

.github/workflows/book.yml

@@ -5,6 +5,10 @@ on:
     branches:
       - unstable
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build-and-upload-to-s3:
     runs-on: ubuntu-20.04

.github/workflows/docker.yml

@@ -8,11 +8,17 @@ on:
         tags:
             - v*
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
     IMAGE_NAME: ${{ github.repository_owner}}/lighthouse
     LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli
+    # Enable self-hosted runners for the sigp repo only.
+    SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }}
 
 jobs:
     # Extract the VERSION which is either `latest` or `vX.Y.Z`, and the VERSION_SUFFIX
@@ -44,7 +50,8 @@ jobs:
             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
     build-docker-single-arch:
         name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }}
-        runs-on: ubuntu-22.04
+        # Use self-hosted runners only on the sigp repo.
+        runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04'  }}
         strategy:
             matrix:
                 binary: [aarch64,
@@ -60,14 +67,13 @@ jobs:
 
         needs: [extract-version]
         env:
-            # We need to enable experimental docker features in order to use `docker buildx`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
             FEATURE_SUFFIX: ${{ matrix.features.version_suffix }}
         steps:
             - uses: actions/checkout@v3
             - name: Update Rust
+              if: env.SELF_HOSTED_RUNNERS == 'false'
               run: rustup update stable
             - name: Dockerhub login
               run: |
@@ -76,16 +82,14 @@ jobs:
               run: |
                   cargo install cross
                   env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }}
+            - name: Make bin dir
+              run: mkdir ./bin
             - name: Move cross-built binary into Docker scope (if ARM)
               if: startsWith(matrix.binary, 'aarch64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
+              run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Move cross-built binary into Docker scope (if x86_64)
               if: startsWith(matrix.binary, 'x86_64')
-              run: |
-                  mkdir ./bin;
-                  mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin;
+              run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin
             - name: Map aarch64 to arm64 short arch
               if: startsWith(matrix.binary, 'aarch64')
               run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV
@@ -95,17 +99,24 @@ jobs:
             - name: Set modernity suffix
               if: endsWith(matrix.binary, '-portable') != true
               run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV;
-            # Install dependencies for emulation. Have to create a new builder to pick up emulation support.
-            - name: Build Dockerfile and push
-              run: |
-                  docker run --privileged --rm tonistiigi/binfmt --install ${SHORT_ARCH}
-                  docker buildx create --use --name cross-builder
-                  docker buildx build \
-                      --platform=linux/${SHORT_ARCH} \
-                      --file ./Dockerfile.cross . \
-                      --tag ${IMAGE_NAME}:${VERSION}-${SHORT_ARCH}${VERSION_SUFFIX}${MODERNITY_SUFFIX}${FEATURE_SUFFIX} \
-                      --provenance=false \
-                      --push
+
+            - name: Install QEMU
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              run: sudo apt-get update && sudo apt-get install -y qemu-user-static
+
+            - name: Set up Docker Buildx
+              if: env.SELF_HOSTED_RUNNERS == 'false'
+              uses: docker/setup-buildx-action@v2
+
+            - name: Build and push
+              uses: docker/build-push-action@v4
+              with:
+                file: ./Dockerfile.cross
+                context: .
+                platforms: linux/${{ env.SHORT_ARCH }}
+                push: true
+                tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }}
+
     build-docker-multiarch:
         name: build-docker-multiarch${{ matrix.modernity }}
         runs-on: ubuntu-22.04
@@ -114,20 +125,22 @@ jobs:
             matrix:
                 modernity: ["", "-modern"]
         env:
-            # We need to enable experimental docker features in order to use `docker manifest`
-            DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
         steps:
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v2
+
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
+
             - name: Create and push multiarch manifest
               run: |
-                  docker manifest create ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
-                      --amend ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
-                  docker manifest push ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }}
+                  docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \
+                      ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }};
+
     build-docker-lcli:
         runs-on: ubuntu-22.04
         needs: [extract-version]
@@ -139,10 +152,12 @@ jobs:
             - name: Dockerhub login
               run: |
                   echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin
-            - name: Build lcli dockerfile (with push)
-              run: |
-                  docker build \
-                      --build-arg PORTABLE=true \
-                      --tag ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX} \
-                      --file ./lcli/Dockerfile .
-                  docker push ${LCLI_IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}
+            - name: Build lcli and push
+              uses: docker/build-push-action@v5
+              with:
+                  build-args: |
+                      FEATURES=portable
+                  context: .
+                  push: true
+                  file: ./lcli/Dockerfile
+                  tags: ${{ env.LCLI_IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }}

.github/workflows/linkcheck.yml

@@ -9,6 +9,10 @@ on:
       - 'book/**'
   merge_group:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   linkcheck:
     name: Check broken links
@@ -18,14 +22,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Create docker network
-        run: docker network create book
-
       - name: Run mdbook server
-        run: docker run -v ${{ github.workspace }}/book:/book --network book --name book -p 3000:3000 -d peaceiris/mdbook:v0.4.20-rust serve --hostname 0.0.0.0
+        run: |
+          docker run -v ${{ github.workspace }}/book:/book --name book -p 3000:3000 -d peaceiris/mdbook:latest serve --hostname 0.0.0.0
+          sleep 5
 
       - name: Print logs
         run: docker logs book
 
       - name: Run linkcheck
-        run: docker run  --network book tennox/linkcheck:latest book:3000
+        run: |
+          curl -sL https://github.com/filiph/linkcheck/releases/download/3.0.0/linkcheck-3.0.0-linux-x64.tar.gz | tar xvzf - linkcheck/linkcheck --strip 1
+          ./linkcheck localhost:3000 -d