Gate PRs on the full test suite, not just after merge

.github/workflows/full-tests-trigger.yml

@@ -0,0 +1,124 @@
+name: Full test suite trigger
+
+# `full-tests.yml` ("full-tests / make test-all") is a required status check
+# but never runs on its own — see the comment there for why. This workflow
+# is the trigger: a PR comment whose body starts with `/ci-run-full-tests`,
+# and if that commenter actually holds `maintain` or `admin` permission on
+# this repo, this sets a pending status on the PR's current head commit and
+# dispatches full-tests.yml against it.
+#
+# Uses issue_comment (a plain PR comment on the Conversation tab), not
+# pull_request_review — a submitted review pins naturally to a specific
+# commit (github.event.review.commit_id); a plain comment doesn't carry a
+# commit sha at all, so this fetches the PR's current head sha itself. That
+# means there's a small window between commenting and this job running
+# where a new push could move the head sha out from under the comment —
+# accepted as a UX tradeoff for "just comment normally" over requiring the
+# Review-changes flow.
+#
+# Deliberately checks the real collaborator permission level via the API
+# rather than `github.event.comment.author_association` — that field can
+# only tell you OWNER/MEMBER/COLLABORATOR/etc, none of which distinguish
+# "has write access" from "has maintain/admin access". As of 2026-08-17,
+# quickwit-oss/quickwit has 57 collaborators with at least write access but
+# only 24 with maintain/admin — author_association would have let all 57
+# trigger this, not just the intended 24.
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: read
+  statuses: write
+  actions: write
+
+concurrency:
+  group: full-tests-trigger-${{ github.event.comment.id }}
+  cancel-in-progress: false
+
+jobs:
+  trigger:
+    name: Trigger full test suite
+    runs-on: ubuntu-latest
+    # github.event.issue.pull_request only exists when the comment is on a
+    # PR (issue_comment fires for both issues and PRs).
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/ci-run-full-tests') }}
+    timeout-minutes: 5
+    steps:
+      - name: Resolve PR head sha
+        id: pr
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            });
+            core.setOutput('sha', pr.head.sha);
+
+      # Bound to github.event.comment.user.login (who actually posted the
+      # comment), not context.actor (whoever triggered this workflow *run*
+      # — those differ on a manual re-run, where actor becomes the
+      # re-runner while the comment payload stays frozen from the
+      # original event).
+      - name: Check commenter has maintain/admin permission
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          COMMENTER: ${{ github.event.comment.user.login }}
+        with:
+          script: |
+            const commenter = process.env.COMMENTER;
+            const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              username: commenter,
+            });
+            core.info(`${commenter} has permission: ${data.permission}`);
+            if (!['maintain', 'admin'].includes(data.permission)) {
+              core.setFailed(
+                `@${commenter} has '${data.permission}' permission on this repo, ` +
+                `but triggering the full test suite requires 'maintain' or 'admin'.`
+              );
+            }
+
+      - name: Set commit status to pending
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          COMMENTER: ${{ github.event.comment.user.login }}
+          SHA: ${{ steps.pr.outputs.sha }}
+        with:
+          script: |
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: process.env.SHA,
+              state: 'pending',
+              context: 'full-tests / make test-all',
+              description: `Triggered by @${process.env.COMMENTER}`,
+              target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/pull/${context.issue.number}`,
+            })
+
+      # Dispatch against `main` (a ref guaranteed to exist in this repo),
+      # not the PR's own branch: for fork PRs, the head branch name only
+      # exists in the fork, not here, and workflow_dispatch's `ref` must
+      # name a branch/tag in *this* repo. full-tests.yml's own checkout
+      # step is what actually fetches `inputs.sha` — that's the PR head
+      # commit resolved above, regardless of which ref ran the dispatch.
+      - name: Dispatch full-tests.yml
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          SHA: ${{ steps.pr.outputs.sha }}
+          PR_NUMBER: ${{ github.event.issue.number }}
+        with:
+          script: |
+            await github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'full-tests.yml',
+              ref: 'main',
+              inputs: {
+                sha: process.env.SHA,
+                pr_number: process.env.PR_NUMBER,
+              },
+            })

.github/workflows/full-tests.yml

@@ -0,0 +1,233 @@
+name: Full test suite
+
+# Runs the full test suite (`make test-all`: --all-features + failpoints)
+# against every broker/backend service (Kafka, Pulsar, Azurite, fake GCS,
+# Pub/Sub emulator, LocalStack, Postgres).
+#
+# This is a REQUIRED status check for merging, but it never runs on its
+# own — it only runs via workflow_dispatch, triggered by
+# full-tests-trigger.yml when a reviewer with `maintain`/`admin` permission
+# on the repo submits a PR review containing `/ci-run-full-tests`.
+#
+# Why gate it at all instead of running automatically on every PR: only
+# `ci.yml` (`cargo nextest --features=postgres,metrics`) and
+# `datafusion-ci.yml` run automatically. None of the other optional
+# features (kafka, pulsar, sqs, gcp-pubsub, azure, gcs, failpoints, ...) or
+# broker-backed integration tests get compiled/exercised otherwise pre-merge
+# — the only workflow that already runs them (`coverage.yml`) triggers on
+# `push` to `main`, i.e. after merge. Running the full suite unconditionally
+# on every PR push was the first design here, but it doesn't match how
+# quickwit's team already runs the sibling `vector` repo's CI (broker/full
+# suites there are merge-queue- or maintainer-comment-gated, never
+# automatic-for-everyone) — this mirrors that instead.
+#
+# Job layout is deliberately split so that the only job which checks out
+# and executes untrusted PR content (`full-tests`) never holds
+# `statuses: write`. `inputs.sha`/`inputs.pr_number` are workflow_dispatch
+# inputs — free-text fields anyone with plain repo write access can set to
+# anything via the Actions UI/API, bypassing full-tests-trigger.yml's
+# permission check entirely — so they're validated to a strict shape
+# (`validate-inputs`) before any other job trusts them, and passed to
+# github-script via `env:`/`process.env` rather than templated into script
+# source, so a crafted input can't break out of a string literal.
+# TEMPORARY: pull_request: added to validate the redesigned job graph
+# (validate-inputs/set-pending/full-tests/report-status split, the protoc
+# fix, docker-compose services) actually runs green before wiring up the
+# real workflow_dispatch-only trigger. Remove this trigger, and the
+# `|| github.event.pull_request...` fallbacks below, before merging.
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: "Commit SHA to check out and report the status against"
+        required: true
+        type: string
+      pr_number:
+        description: "PR number (used only for the failure notification link)"
+        required: true
+        type: string
+  pull_request:
+
+permissions:
+  contents: read
+
+env:
+  STATUS_CONTEXT: "full-tests / make test-all"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.sha || github.event.pull_request.head.sha }}
+  cancel-in-progress: true
+
+jobs:
+  validate-inputs:
+    name: Validate inputs
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions: {}
+    steps:
+      - name: Validate sha and pr_number are well-formed
+        env:
+          SHA: ${{ inputs.sha || github.event.pull_request.head.sha }}
+          PR_NUMBER: ${{ inputs.pr_number || github.event.pull_request.number }}
+        run: |
+          if ! [[ "$SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then
+            echo "::error::sha must be a 40-character hex commit SHA, got: $SHA"
+            exit 1
+          fi
+          if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then
+            echo "::error::pr_number must be numeric, got: $PR_NUMBER"
+            exit 1
+          fi
+
+  set-pending:
+    name: Set pending status
+    needs: [validate-inputs]
+    # TEMPORARY (see `on:` above): while pull_request: is active, don't
+    # post a real commit status on every PR push in the repo — that could
+    # auto-satisfy this exact check's context if it's already required
+    # elsewhere. Remove this condition along with the trigger before merge.
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions:
+      statuses: write
+    steps:
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          SHA: ${{ inputs.sha || github.event.pull_request.head.sha }}
+        with:
+          script: |
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: process.env.SHA,
+              state: 'pending',
+              context: '${{ env.STATUS_CONTEXT }}',
+              description: 'Running full test suite...',
+              target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+            })
+
+  full-tests:
+    name: All-features tests + failpoints (make test-all)
+    needs: [set-pending]
+    # A skipped set-pending (see TEMPORARY note there) would otherwise
+    # cascade into skipping this job too, since `needs` defaults to
+    # requiring success, not skipped-or-success.
+    if: ${{ !failure() && !cancelled() }}
+    # gh-ubuntu-arm64 (used by coverage.yml too) reproducibly killed this
+    # job at ~8 minutes across two separate runs — not our timeout-minutes,
+    # something infra-side. Vector's CI (test.yml) uses plain GitHub-hosted
+    # runners throughout (ubuntu-24.04, ubuntu-24.04-8core for heavy jobs);
+    # matching that here with the standard tier first since we can't
+    # confirm quickwit-oss has GitHub's larger-runner tier provisioned.
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.sha || github.event.pull_request.head.sha }}
+          persist-credentials: false
+
+      - name: Install Ubuntu packages
+        run: |
+          sudo apt-get update
+          sudo apt-get -y install libsasl2-dev libcurl4-openssl-dev
+
+      # apt's protobuf-compiler is too old to support proto3 optional fields
+      # by default, which the `substrait` crate (pulled in by --all-features
+      # via the `datafusion` feature) requires. coverage.yml hits the same
+      # constraint and installs protoc this way for the same reason.
+      - name: Install protoc
+        uses: taiki-e/install-action@7769b73c2ec98c38dfcf2e18c83cfd4880c038c1
+        with:
+          tool: protoc
+
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v.6.2.0
+        with:
+          python-version: '3.11'
+
+      - name: Setup stable Rust Toolchain
+        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master
+        with:
+          toolchain: stable
+
+      - name: Setup cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        with:
+          workspaces: "./quickwit -> target"
+          shared-key: "quickwit-cargo-full"
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@7769b73c2ec98c38dfcf2e18c83cfd4880c038c1
+        with:
+          tool: cargo-nextest
+
+      - name: Start Docker services
+        run: make docker-compose-up
+
+      - name: Install python packages
+        run: |
+          pip install --user --require-hashes -r ${{ github.workspace }}/.github/workflows/requirements.txt
+          pipenv install --deploy --ignore-pipfile
+        working-directory: ./quickwit/quickwit-cli/tests
+
+      - name: Prepare LocalStack S3
+        run: pipenv run ./prepare_tests.sh
+        working-directory: ./quickwit/quickwit-cli/tests
+
+      - name: make test-all
+        run: make -C quickwit test-all
+        env:
+          QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
+
+  report-status:
+    name: Report final status
+    needs: [validate-inputs, full-tests]
+    # TEMPORARY (see `on:` above): only post a real status for a
+    # workflow_dispatch run, same reasoning as set-pending.
+    if: ${{ always() && needs.validate-inputs.result == 'success' && github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions:
+      statuses: write
+    steps:
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        env:
+          SHA: ${{ inputs.sha || github.event.pull_request.head.sha }}
+          RESULT: ${{ needs.full-tests.result }}
+        with:
+          script: |
+            const state = process.env.RESULT === 'success' ? 'success' : 'failure';
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: process.env.SHA,
+              state,
+              context: '${{ env.STATUS_CONTEXT }}',
+              description: state === 'success' ? 'Full test suite passed' : 'Full test suite failed',
+              target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+            })
+
+  on-failure:
+    if: ${{ github.repository_owner == 'quickwit-oss' && needs.full-tests.result == 'failure' }}
+    name: On Failure
+    needs: [full-tests]
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions: {}
+    steps:
+      - name: Send Message
+        uses: sarisia/actions-status-discord@eb045afee445dc055c18d3d90bd0f244fd062708 # v1.16.0
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+          nodetail: true
+          color: "#FF0000"
+          title: ""
+          description: |
+            ### ❌ [PR #${{ inputs.pr_number || github.event.pull_request.number }}](https://github.com/${{ github.repository }}/pull/${{ inputs.pr_number || github.event.pull_request.number }})
+
+            The full test suite (`make test-all`) failed.
+
+            **[View logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})**