fix(sandbox): supervise preview source sync (#213)

## Why

A transient Daytona FUSE read denial could terminate the
durable-to-native source synchronizer while the Vite child stayed
healthy. Process reuse then saw a live port and returned a permanently
stale preview.

## What changed

- Retry transient durable-source read failures immediately and within a
bounded grace period.
- Treat the app and synchronizer as one managed process unit; a dead
synchronizer terminates the app so the existing bounded supervisor
restarts both.
- Extend immutable snapshot smoke checks to cover transient recovery,
live source propagation, and sync-child failure.
- Document the managed-preview liveness contract at the sandbox and
agent-worker boundaries.

## Verification

- `pnpm lint`
- `pnpm typecheck`
- `pnpm turbo build --force`
- `pnpm deadcode`
- `pnpm architecture:check`
- `pnpm turbo skills:build`
- Local lifecycle smoke: simulated the observed `EPERM`, verified source
recovery, verified live update propagation, killed the sync child, and
confirmed the preview parent exited non-zero.

All repository commands ran with Node 24.18.0 and pnpm 11.15.0.
Production browser QA follows immutable snapshot publication and
promotion.

Author: iamjr15

.github/workflows/build-snapshot.yml

@@ -149,6 +149,69 @@ jobs:
             test -x /opt/cheatcode/project-source-sync.py
             test -x /opt/cheatcode/pnpm-build-policy.mjs
             python3 -c "import ast,pathlib;ast.parse(pathlib.Path(\"/opt/cheatcode/project-source-sync.py\").read_text())"
+            python3 - <<PY
+          import errno
+          import importlib.util
+          import multiprocessing
+          import os
+          import pathlib
+          import sys
+          import tempfile
+          import time
+
+          sys.dont_write_bytecode = True
+          script = "/opt/cheatcode/project-source-sync.py"
+          spec = importlib.util.spec_from_file_location("project_source_sync", script)
+          if spec is None or spec.loader is None:
+              raise RuntimeError("source synchronizer module could not be loaded")
+          synchronizer = importlib.util.module_from_spec(spec)
+          spec.loader.exec_module(synchronizer)
+
+          with tempfile.TemporaryDirectory() as root:
+              source = os.path.join(root, "durable")
+              target = os.path.join(root, "mirror")
+              lock = os.path.join(root, "project.lock")
+              os.makedirs(source)
+              pathlib.Path(source, "index.html").write_text("sync recovered", encoding="utf-8")
+              original_sync = synchronizer.sync_directory
+              failures_remaining = [4]
+
+              def transient_sync(current_source, current_target):
+                  if failures_remaining[0] > 0:
+                      failures_remaining[0] -= 1
+                      raise PermissionError(
+                          errno.EPERM,
+                          "simulated durable source contention",
+                          os.path.join(source, "index.html"),
+                      )
+                  original_sync(current_source, current_target)
+
+              synchronizer.sync_directory = transient_sync
+              process = multiprocessing.get_context("fork").Process(
+                  target=synchronizer.preview_sync,
+                  args=(source, target, "preview-loop", lock),
+              )
+              process.start()
+              try:
+                  deadline = time.monotonic() + 5
+                  output = pathlib.Path(target, "index.html")
+                  while time.monotonic() < deadline and not output.exists():
+                      if not process.is_alive():
+                          raise RuntimeError("preview sync exited after transient source contention")
+                      time.sleep(0.05)
+                  if not output.is_file() or output.read_text(encoding="utf-8") != "sync recovered":
+                      raise RuntimeError("preview sync did not recover after transient source contention")
+                  if not process.is_alive():
+                      raise RuntimeError("preview sync stopped after recovering source access")
+              finally:
+                  process.terminate()
+                  process.join(timeout=5)
+                  if process.is_alive():
+                      process.kill()
+                      process.join(timeout=5)
+                  if process.is_alive():
+                      raise RuntimeError("preview sync smoke process did not stop")
+          PY
             node --check /opt/cheatcode/pnpm-build-policy.mjs
             policy_test_dir="$(mktemp -d)"
             printf "packages:\\n  - .\\nallowBuilds:\\n  sharp: false\\n" > "$policy_test_dir/pnpm-workspace.yaml"
@@ -175,10 +238,12 @@ jobs:
 
             wait_for_preview() {
               preview_port="$1"
-              preview_log="$2"
+              preview_text="$2"
+              preview_log="$3"
+              preview_limit="$4"
               preview_attempt=0
-              while [ "$preview_attempt" -lt 90 ]; do
-                if curl --fail --silent "http://127.0.0.1:$preview_port" | grep --fixed-strings --quiet "reviewed lifecycle policy works"; then
+              while [ "$preview_attempt" -lt "$preview_limit" ]; do
+                if curl --fail --silent "http://127.0.0.1:$preview_port" | grep --fixed-strings --quiet "$preview_text"; then
                   return 0
                 fi
                 if ! kill -0 "$preview_pid" 2>/dev/null; then
@@ -194,7 +259,7 @@ jobs:
             cold_started_at="$(date +%s)"
             PORT=5199 /opt/cheatcode/project-source-sync.py preview-run "$preview_source" "$preview_mirror" "$preview_lock" "$preview_mirror" - -- pnpm run dev --host 0.0.0.0 --port 5199 > "$preview_test_dir/cold.log" 2>&1 &
             preview_pid="$!"
-            wait_for_preview 5199 "$preview_test_dir/cold.log"
+            wait_for_preview 5199 "reviewed lifecycle policy works" "$preview_test_dir/cold.log" 90
             cold_elapsed=$(( $(date +%s) - cold_started_at ))
             kill "$preview_pid" 2>/dev/null || true
             wait "$preview_pid" 2>/dev/null || true
@@ -209,18 +274,37 @@ jobs:
             warm_started_at="$(date +%s)"
             PORT=5200 /opt/cheatcode/project-source-sync.py preview-run "$preview_source" "$preview_mirror" "$preview_lock" "$preview_mirror" - -- pnpm run dev --host 0.0.0.0 --port 5200 > "$preview_test_dir/warm.log" 2>&1 &
             preview_pid="$!"
-            wait_for_preview 5200 "$preview_test_dir/warm.log"
+            wait_for_preview 5200 "reviewed lifecycle policy works" "$preview_test_dir/warm.log" 10
             warm_elapsed=$(( $(date +%s) - warm_started_at ))
-            kill "$preview_pid" 2>/dev/null || true
-            wait "$preview_pid" 2>/dev/null || true
-            preview_pid=""
             test "$warm_elapsed" -le 10
             test ! -e "$preview_source/pnpm-workspace.yaml"
             test ! -e "$preview_mirror/pnpm-workspace.yaml"
             if grep --fixed-strings --quiet "Progress:" "$preview_test_dir/warm.log"; then
               echo "Warm preview unexpectedly reinstalled dependencies." >&2
               exit 1
             fi
+            printf "%s\n" "<main id=\"app\">live source synchronization works</main>" > "$preview_source/index.html"
+            wait_for_preview 5200 "live source synchronization works" "$preview_test_dir/warm.log" 10
+            sync_pid="$(pgrep -P "$preview_pid" -f "project-source-sync.py preview-loop" | head -n 1)"
+            if [ -z "$sync_pid" ]; then
+              echo "Managed preview did not expose a source synchronizer child." >&2
+              exit 1
+            fi
+            kill "$sync_pid"
+            preview_stop_attempt=0
+            while kill -0 "$preview_pid" 2>/dev/null && [ "$preview_stop_attempt" -lt 20 ]; do
+              sleep 1
+              preview_stop_attempt=$((preview_stop_attempt + 1))
+            done
+            if kill -0 "$preview_pid" 2>/dev/null; then
+              echo "Managed preview stayed alive without its source synchronizer." >&2
+              exit 1
+            fi
+            if wait "$preview_pid"; then
+              echo "Managed preview reported success after its source synchronizer failed." >&2
+              exit 1
+            fi
+            preview_pid=""
             echo "Vite preview smoke passed: cold=${cold_elapsed}s warm=${warm_elapsed}s"
             cleanup_preview_test
             trap - EXIT

apps/agent-worker/README.md

@@ -167,7 +167,11 @@ webpack compilation even after the listening socket opens. A baked Python synchr
 content identities rather than unrelated FUSE/native-disk timestamps, performs atomic replacement
 on native disk and checksum-verified direct overwrites on the non-POSIX durable mount, and mirrors
 subsequent writes and deletions within 250 ms, including equal-size and
-shell-based edits. The local source, dependency tree, and build cache are disposable;
+shell-based edits. Transient read contention on the durable source is retried within a bounded grace
+period; sustained source failure or a terminated synchronizer fails the whole managed preview so the
+existing bounded process restart policy restores both the app and its source feed instead of reusing
+a stale listening port. The local source,
+dependency tree, and build cache are disposable;
 wake and restart reconstruct them from the durable project without changing the Files surface.
 Persisted pnpm-backed preview commands restore a missing sandbox-local dependency tree before the
 server starts. A package/lock/config digest skips the install entirely when the local dependency tree

infra/containers/sandbox/README.md

@@ -106,7 +106,11 @@ Direct pnpm invocations execute inside one `flock`-guarded transaction; process
 releases that lock automatically, and successful source changes commit only after a
 three-way conflict check against the durable tree. Long-lived previews invoke the same helper as
 direct argv: it synchronizes source, restores dependencies under the project lock, supervises the
-native-disk app and sync-loop children, and forwards termination signals. A dependency-state digest
+native-disk app and sync-loop children, and forwards termination signals. Transient read contention
+from the durable FUSE source is retried during a bounded grace period without interrupting the app.
+If source access does not recover or the synchronizer otherwise exits, the managed preview exits as
+one failed process unit so its existing bounded restart policy cannot leave a healthy port backed by
+stale source. A dependency-state digest
 skips unchanged reinstalls, while projects without a durable lockfile install without creating one
 as a preview side effect. Dependency restoration temporarily merges the image's reviewed build
 policy into the sandbox-local workspace, currently permitting `esbuild` so Vite can install its

infra/containers/sandbox/scripts/project-source-sync.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 """Synchronize durable project source with its sandbox-local native-disk mirror."""
 
+import errno
 import fcntl
 import hashlib
 import os
@@ -16,8 +17,20 @@
 IGNORED_DIRS = {".expo", ".next", ".turbo", "build", "coverage", "dist", "node_modules", "out"}
 LOCAL_ONLY_NAMES = IGNORED_DIRS | {"next-env.d.ts"}
 PACKAGE_CONFLICT_EXIT = 74
+PREVIEW_SYNC_FAILURE_EXIT = 75
 DEPENDENCY_STATE_FILENAME = ".cheatcode-dependency-state"
 PNPM_BUILD_POLICY_BINARY = "/opt/cheatcode/pnpm-build-policy.mjs"
+PREVIEW_SYNC_INTERVAL_SECONDS = 0.25
+SOURCE_FAILURE_GRACE_SECONDS = 10
+SOURCE_READ_RETRY_DELAYS_SECONDS = (0.05, 0.1, 0.2)
+SOURCE_WARNING_INTERVAL_SECONDS = 5
+TRANSIENT_SOURCE_ERRNOS = {
+    errno.EACCES,
+    errno.EBUSY,
+    errno.EIO,
+    errno.EPERM,
+    errno.ESTALE,
+}
 
 
 def remove_path(path):
@@ -136,18 +149,68 @@ def try_lock(lock):
         return False
 
 
+def is_transient_source_error(error, source):
+    if error.errno not in TRANSIENT_SOURCE_ERRNOS or not error.filename:
+        return False
+    source_root = os.path.abspath(source)
+    failed_path = os.path.abspath(error.filename)
+    try:
+        return os.path.commonpath([source_root, failed_path]) == source_root
+    except ValueError:
+        return False
+
+
+def sync_source_once(source, target):
+    for delay in (*SOURCE_READ_RETRY_DELAYS_SECONDS, None):
+        try:
+            sync_directory(source, target)
+            return
+        except OSError as error:
+            if delay is None or not is_transient_source_error(error, source):
+                raise
+            time.sleep(delay)
+
+
 def preview_sync(source, target, mode, lock_path):
     is_once = mode == "preview-once"
+    source_failure_started_at = None
+    last_warning_at = 0
     while True:
-        with open_lock(lock_path) as lock:
-            if is_once:
-                fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
-                sync_directory(source, target)
-            elif try_lock(lock):
-                sync_directory(source, target)
+        did_attempt_sync = False
+        did_sync = False
+        try:
+            with open_lock(lock_path) as lock:
+                if is_once:
+                    fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
+                    did_attempt_sync = True
+                    sync_source_once(source, target)
+                    did_sync = True
+                elif try_lock(lock):
+                    did_attempt_sync = True
+                    sync_source_once(source, target)
+                    did_sync = True
+        except OSError as error:
+            if is_once or not is_transient_source_error(error, source):
+                raise
+            now = time.monotonic()
+            if source_failure_started_at is None:
+                source_failure_started_at = now
+            elif now - source_failure_started_at >= SOURCE_FAILURE_GRACE_SECONDS:
+                raise
+            if now - last_warning_at >= SOURCE_WARNING_INTERVAL_SECONDS:
+                print(
+                    f"durable source temporarily unreadable; preview sync is retrying: {error}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                last_warning_at = now
+        if did_sync:
+            source_failure_started_at = None
+        elif not did_attempt_sync:
+            source_failure_started_at = None
         if is_once:
             return
-        time.sleep(0.25)
+        time.sleep(PREVIEW_SYNC_INTERVAL_SECONDS)
 
 
 def overwrite_durable_file(source, target):
@@ -415,7 +478,7 @@ def preview_run(source, target, lock_path, local_cwd, dependency_template, comma
         raise ValueError("preview-run requires an app command")
     with open_lock(lock_path) as lock:
         fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
-        sync_directory(source, target)
+        sync_source_once(source, target)
         dependency_status = restore_pnpm_dependencies(local_cwd, target, dependency_template)
         if dependency_status != 0:
             return dependency_status
@@ -442,7 +505,21 @@ def terminate_children(_signum=None, _frame=None):
             env=app_environment,
             start_new_session=True,
         )
-        return app_process.wait()
+        while True:
+            app_status = app_process.poll()
+            if app_status is not None:
+                return app_status
+            sync_status = sync_process.poll()
+            if sync_status is not None:
+                print(
+                    f"preview source synchronizer exited unexpectedly with status {sync_status}",
+                    file=sys.stderr,
+                    flush=True,
+                )
+                stop_process(app_process)
+                reap_process(app_process)
+                return sync_status or PREVIEW_SYNC_FAILURE_EXIT
+            time.sleep(PREVIEW_SYNC_INTERVAL_SECONDS)
     finally:
         terminate_children()
         reap_process(app_process)