@@ -659,6 +659,58 @@ jobs:
659659 echo 'Items on this shard (a package name, or a package plus a k/n file-level slice):'
660660 cat "$RUNNER_TEMP/shard-packages.txt"
661661
662+ # ⛔ THIS SHARD'S `^build` CLOSURE IS BUILT HERE, ON THE TURBO REMOTE
663+ # CACHE (#22077), so `Run this shard's tests` REPLAYS it from the local
664+ # cache instead of building it. Before this step the closure was built
665+ # inside the test step's own turbo run (the slice step below runs zero
666+ # iterations since #21487). Merge-queue run 37623284168, `Test Core
667+ # (3/6)`: `Tasks: 72 successful, 72 total` / `Cached: 1 cached, 72
668+ # total` / `Time: 11m43.373s` for the tests AND their closure in one
669+ # run. That run cannot read the remote, because `test` / `test:repo`
670+ # stay off it (Build Core's PINS), so the closure moves here.
671+ #
672+ # `--filter=PKG^...` selects PKG's dependencies without PKG: the
673+ # `^build` closure that `test` and `test:repo` schedule. Measured on
674+ # 3d9188502e, turbo 2.11.5, all six shards of the local partition: this
675+ # plan equals the test plan's build tasks, 32 to 63 per shard, with
676+ # identical hashes, 0 differing and 0 extra, and every one of them has
677+ # Build Core's hash. A `test` task that `dependsOn: ["build"]` (cli and
678+ # metadata in turbo.json) also needs its OWN package's build. This filter
679+ # leaves that out on purpose, so a shard never builds a package its tests
680+ # do not need; the test step builds it as before. Today that is one task,
681+ # `cli#build` on shard 1/6.
682+ #
683+ # Its own guarded step, for the reason the slice step below gives: a
684+ # guarded SITE is (file, job, step). No `--summarize`: `.turbo/runs/`
685+ # stays the test step's alone, and that is what the drift check and the
686+ # timings capture read. The four TURBO_* lines are Build Core's,
687+ # verbatim; its "Turbo remote cache (#21186)" comment holds the write
688+ # rule, the signing and the PINS. ⛔ An empty FILTERS must exit here:
689+ # `turbo run build` with no filter builds the whole workspace.
690+ - name : Build this shard's dependency closure
691+ env :
692+ NODE_OPTIONS : --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports
693+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
694+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
695+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
696+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
697+ run : |
698+ FILTERS=""
699+ if [ -s "$RUNNER_TEMP/shard-packages.txt" ]; then
700+ while read -r PKG _; do
701+ [ -n "$PKG" ] || continue
702+ FILTERS="$FILTERS --filter=$PKG^..."
703+ done < "$RUNNER_TEMP/shard-packages.txt"
704+ fi
705+ if [ -z "$FILTERS" ]; then
706+ echo "No packages on this shard — nothing to build."
707+ exit 0
708+ fi
709+ mkdir -p "$RUNNER_TEMP/stall-reports"
710+ node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core-closure-build.log" --stall-minutes 10 \
711+ --report-dir "$RUNNER_TEMP/stall-reports" -- \
712+ pnpm turbo run build $FILTERS --concurrency=4 --log-order=stream
713+
662714 # ⛔ A FILE-LEVEL SLICE BUILDS ITS DEPENDENCY CLOSURE HERE, IN ITS OWN
663715 # GUARDED STEP, so the slice leg in the next step REPLAYS it. Since #19278
664716 # that leg carries its slice in `OS_TEST_SHARD`, which only the `test`
@@ -1503,7 +1555,18 @@ jobs:
15031555 - name : Install dependencies
15041556 run : pnpm install --frozen-lockfile
15051557
1558+ # Turbo remote cache (#22077), on all four of this job's build steps: the
1559+ # four TURBO_* lines are Build Core's, verbatim, and Build Core's "Turbo
1560+ # remote cache (#21186)" comment holds the write rule, the signing and the
1561+ # PINS. Each of these plans is all `build` tasks with Build Core's hashes
1562+ # (8, 18, 14 and 31 nodes, measured). The suites between them run vitest
1563+ # through `pnpm --filter`, not turbo, and carry no credentials.
15061564 - name : Build driver-sql and its dependencies
1565+ env :
1566+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
1567+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
1568+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
1569+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
15071570 run : pnpm exec turbo run build --filter=@objectstack/driver-sql... --concurrency=4
15081571
15091572 # The whole driver-sql suite runs under the skewed process zone — not
@@ -1561,6 +1624,11 @@ jobs:
15611624 # which is what a ratchet is for — it makes the property enforced rather
15621625 # than incidental.
15631626 - name : Build the non-SQL temporal backends
1627+ env :
1628+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
1629+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
1630+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
1631+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
15641632 run : >-
15651633 pnpm exec turbo run build
15661634 --filter=@objectstack/service-analytics...
@@ -1640,6 +1708,11 @@ jobs:
16401708 # single shorter substring (`live-`) would select the same set today and
16411709 # silently widen with the next unrelated file that happens to spell it.
16421710 - name : Build metadata-protocol and its dependencies
1711+ env :
1712+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
1713+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
1714+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
1715+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
16431716 run : pnpm exec turbo run build --filter=@objectstack/metadata-protocol... --concurrency=4
16441717
16451718 - name : Run the metadata-protocol migration statements against live MySQL and PostgreSQL
@@ -1697,6 +1770,11 @@ jobs:
16971770 # speaks up when a named path selects no tests — a second net, not a
16981771 # licence to change the form.)
16991772 - name : Build runtime and its dependencies
1773+ env :
1774+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
1775+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
1776+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
1777+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
17001778 run : pnpm exec turbo run build --filter=@objectstack/runtime... --concurrency=4
17011779
17021780 - name : Run the runtime cascade-delete matrix against live PostgreSQL
@@ -1820,9 +1898,20 @@ jobs:
18201898 # own `--stall-minutes` and its own headroom row. The step is NOT allowed
18211899 # to buy itself room by raising this job's `timeout-minutes`: that budget
18221900 # is the instrument that shows this fix working (#16886).
1901+ #
1902+ # Turbo remote cache (#22077): the four TURBO_* lines are Build Core's,
1903+ # verbatim, and Build Core's "Turbo remote cache (#21186)" comment holds
1904+ # the write rule, the signing and the PINS. This plan is 66 `build` nodes
1905+ # (63 with a command), each with Build Core's hash (measured). ⛔ They stay on THIS step: the
1906+ # test step below runs `dogfood#test`, which must never read the remote,
1907+ # and replays this closure from the local cache this step fills.
18231908 - name : Build the dogfood package's dependency closure
18241909 env :
18251910 NODE_OPTIONS : --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports
1911+ TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
1912+ TURBO_TOKEN : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY != '' && secrets.TURBO_TOKEN || '' }}
1913+ TURBO_TEAM : ${{ vars.TURBO_TEAM }}
1914+ TURBO_REMOTE_CACHE_SIGNATURE_KEY : ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
18261915 run : |
18271916 mkdir -p "$RUNNER_TEMP/stall-reports"
18281917 node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/dogfood-build.log" --stall-minutes 10 \
@@ -2189,10 +2278,15 @@ jobs:
21892278 # in both places (72 of 72 build tasks, read from both CI logs at one
21902279 # SHA), so a pin bump there can replay what this step built on `main`.
21912280 #
2192- # Scope: THIS step only, i.e. `build` tasks. `test` / `test:repo` stay
2193- # off the remote: their cross-package inputs are hand-declared, and a
2194- # wrong hash replayed from a remote reaches every run and cloud too.
2195- # Build Docs forces execution and carries no credentials.
2281+ # Scope: `turbo run build` steps only, i.e. `build` tasks. Since #22077
2282+ # that is this step plus the build steps of the Test Core shards, the
2283+ # dogfood shards, Temporal Conformance and lint.yml's three Type Check
2284+ # build lanes, each carrying the four lines below verbatim; the PINS
2285+ # paragraph at the end says what keeps that set honest. `test` /
2286+ # `test:repo` stay off the remote: their cross-package inputs are
2287+ # hand-declared, and a wrong hash replayed from a remote reaches every
2288+ # run and cloud too. Build Docs forces execution and carries no
2289+ # credentials.
21962290 #
21972291 # TURBO_CACHE is the env form of `--cache`. Writes happen on
21982292 # `merge_group` and on `push` / `workflow_dispatch` against main; every
@@ -2211,6 +2305,30 @@ jobs:
22112305 # upload and verify every download with that key. Any of the three
22122306 # unset: remote caching is off and the build is unchanged; the summary
22132307 # then reads `Remote caching disabled (remote cache requested — …)`.
2308+ #
2309+ # PINS, for every step that carries these four lines (#22077). Each
2310+ # carrier points back here.
2311+ # - BUILD TASKS ONLY. #19086's lesson: a step that writes artifacts from
2312+ # a cache-served dist must not itself be served. Each carrier runs one
2313+ # `turbo run build` whose `--dry=json` plan is all `#build` tasks.
2314+ # ⛔ Never put these lines on a step that runs `test`, `test:repo`,
2315+ # `typecheck` or a `gen:*` task (turbo.json caches `gen:schema` and
2316+ # `gen:skill-refs`), or on a step that writes tracked files.
2317+ # - SAME HASHES AS THIS STEP. Every carrier's build tasks are a subset
2318+ # of this step's plan with identical hashes (measured on 3d9188502e,
2319+ # turbo 2.11.5: 8 to 77 tasks per carrier, 0 differing), so a PR read
2320+ # hits what this step wrote on `main` or in the queue. Filters and
2321+ # `--concurrency` do not enter a task hash; an env var a carrier sets
2322+ # that turbo.json hashes would, so do not add one to a carrier.
2323+ # - A HIT IS ONLY AS CORRECT AS THE HASH (#21193). Every root file a
2324+ # build reads is a `$TURBO_ROOT$` input in turbo.json. Re-measured for
2325+ # #22077 on 3d9188502e: one appended line in each of the 17 declared
2326+ # root inputs moves the build hashes (72, 71 or 6 of 72, #21193's
2327+ # numbers); the root tsup.config.ts control moves 72/72; an unrelated
2328+ # script, ci.yml and lint.yml move 0. The static import closure of
2329+ # every root script a build command or tsup config names is inside
2330+ # those declarations. A build that starts reading a new root file
2331+ # must declare it in turbo.json before this cache can be trusted.
22142332 - name : Build packages (excluding docs)
22152333 env :
22162334 TURBO_CACHE : ${{ (github.event_name == 'merge_group' || ((github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref == 'refs/heads/main')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}
0 commit comments