Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions apps/cli/src/command-internal/docker-lifecycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ function parseContainerState(stdout: string): {
readonly running: boolean;
readonly status: string;
readonly exitCode: number;
readonly oomKilled: boolean;
readonly health?: string;
} {
const trimmed = stdout.trim();
Expand All @@ -265,12 +266,15 @@ function parseContainerState(stdout: string): {
const status = typeof state["Status"] === "string" ? state["Status"] : "";
const running = state["Running"] === true;
const exitCode = typeof state["ExitCode"] === "number" ? state["ExitCode"] : 0;
// Docker sets `OOMKilled` only when the container's own memory limit was hit; a host-level
// out-of-memory kill reports the same exit code with `OOMKilled: false`.
const oomKilled = state["OOMKilled"] === true;
const health = state["Health"];
const healthStatus =
isJsonRecord(health) && typeof health["Status"] === "string" ? health["Status"] : undefined;
return healthStatus !== undefined
? { running, status, exitCode, health: healthStatus }
: { running, status, exitCode };
? { running, status, exitCode, oomKilled, health: healthStatus }
: { running, status, exitCode, oomKilled };
}

function isJsonRecord(value: unknown): value is { readonly [key: string]: unknown } {
Expand Down
32 changes: 27 additions & 5 deletions apps/cli/src/command-internal/docker-lifecycle.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ describe("inspectContainerState", () => {
running: true,
status: "running",
exitCode: 0,
oomKilled: false,
health: "healthy",
});
expect(mock.spawned).toEqual([
Expand All @@ -248,7 +249,28 @@ describe("inspectContainerState", () => {
});
return inspectContainerState(mock.spawner, "supabase_kong_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({ running: true, status: "running", exitCode: 0 });
expect(state).toEqual({ running: true, status: "running", exitCode: 0, oomKilled: false });
}),
);
});

it.live("reports a container killed for exceeding its memory limit", () => {
const mock = mockSpawner({
stdout: JSON.stringify({
Status: "exited",
Running: false,
ExitCode: 137,
OOMKilled: true,
}),
});
return inspectContainerState(mock.spawner, "supabase_edge_runtime_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({
running: false,
status: "exited",
exitCode: 137,
oomKilled: true,
});
}),
);
});
Expand All @@ -259,7 +281,7 @@ describe("inspectContainerState", () => {
});
return inspectContainerState(mock.spawner, "supabase_kong_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({ running: false, status: "exited", exitCode: 1 });
expect(state).toEqual({ running: false, status: "exited", exitCode: 1, oomKilled: false });
}),
);
});
Expand All @@ -272,7 +294,7 @@ describe("inspectContainerState", () => {
});
return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({ running: true, status: "paused", exitCode: 0 });
expect(state).toEqual({ running: true, status: "paused", exitCode: 0, oomKilled: false });
}),
);
},
Expand Down Expand Up @@ -334,7 +356,7 @@ describe("inspectContainerState", () => {
const mock = mockSpawner({ stdout: "" });
return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({ running: false, status: "", exitCode: 0 });
expect(state).toEqual({ running: false, status: "", exitCode: 0, oomKilled: false });
}),
);
});
Expand All @@ -343,7 +365,7 @@ describe("inspectContainerState", () => {
const mock = mockSpawner({ stdout: "null" });
return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe(
Effect.map((state) => {
expect(state).toEqual({ running: false, status: "", exitCode: 0 });
expect(state).toEqual({ running: false, status: "", exitCode: 0, oomKilled: false });
}),
);
});
Expand Down
4 changes: 4 additions & 0 deletions apps/cli/src/command-internal/docker-suggest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ export const SUGGEST_DOCKER_INSTALL =
export const SUGGEST_DOCKER_START =
"Docker is no longer reachable. Start Docker, then rerun `supabase functions serve`.";

/** Remediation hint shown when the OOM killer kills a container. */
export const SUGGEST_CONTAINER_MEMORY_LIMIT =
"The container ran out of memory and was killed. Raise the memory allocated to your container runtime (Docker Desktop: Settings > Resources), or free up memory on the host, and try again.";

/**
* Whether a container-CLI stderr indicates the daemon is unreachable. Matches the docker/podman
* "cannot connect"/"is the docker daemon running" messages, a socket permission-denied message,
Expand Down
27 changes: 15 additions & 12 deletions apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,17 +61,20 @@ back to local keys. No scheme/host validation is performed on the discovered URL

## Exit Codes

| Code | Condition |
| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `0` | clean shutdown after `SIGINT` or `SIGTERM` |
| `0` | the edge-runtime container stops on its own with exit code `0` |
| `0` | the edge-runtime container is torn down by an external supervisor — exit `129`/`130`/`131`/`143` (`SIGHUP`/`SIGINT`/`SIGQUIT`/`SIGTERM`), e.g. `supabase stop` run in another terminal |
| `0` | the edge-runtime container is already gone by the time a follow-up `docker container inspect` runs after the log stream ended |
| `0` | an edge-runtime startup failure or log-stream failure lands within the shutdown grace period (~50ms) of a `SIGINT`/`SIGTERM` |
| `1` | local DB container is not running, or the Docker daemon is unreachable (surfaces from the DB inspect as `failed to inspect service: …` plus the Docker Desktop install suggestion) |
| `1` | invalid inspect flag combination, or a `Config.Validate` failure anywhere in `config.toml` (not just project/auth config) |
| `1` | env file, signing key, import map, or function bind resolution failure |
| `1` | edge-runtime container startup, log streaming, or restart loop failure — including the edge-runtime container crashing with any exit code other than `0`, `137`, or `129`/`130`/`131`/`143` |
| Code | Condition |
| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `0` | clean shutdown after `SIGINT` or `SIGTERM` |
| `0` | the edge-runtime container stops on its own with exit code `0` |
| `0` | the edge-runtime container is torn down by an external supervisor — exit `129`/`130`/`131`/`143` (`SIGHUP`/`SIGINT`/`SIGQUIT`/`SIGTERM`), e.g. `supabase stop` run in another terminal |
| `0` | the edge-runtime container is already gone by the time a follow-up `docker container inspect` runs after the log stream ended |
| `0` | the edge-runtime container is force-killed (exit `137`, no `State.OOMKilled`) and pruned before the re-inspect that follows a non-OOM `137` completes, e.g. by `supabase stop` |
| `0` | an edge-runtime startup failure or log-stream failure lands within the shutdown grace period (~50ms) of a `SIGINT`/`SIGTERM` |
| `1` | local DB container is not running, or the Docker daemon is unreachable (surfaces from the DB inspect as `failed to inspect service: …` plus the Docker Desktop install suggestion) |
| `1` | invalid inspect flag combination, or a `Config.Validate` failure anywhere in `config.toml` (not just project/auth config) |
| `1` | env file, signing key, import map, or function bind resolution failure |
| `1` | edge-runtime container startup, log streaming, or restart loop failure — including the edge-runtime container crashing with any exit code other than `0` or `129`/`130`/`131`/`143` |
| `1` | the edge-runtime container is killed for exceeding its memory limit — `State.OOMKilled`, typically with exit `137` |
| `1` | the edge-runtime container is killed from outside the CLI and is still present after a re-inspect — exit `137` without `State.OOMKilled` |

## Telemetry Events Fired

Expand Down Expand Up @@ -136,7 +139,7 @@ Per-function TOML `env` remains the final override for that function.
- Config, project dotenv discovery, and function discovery all resolve from `<workdir>` with no ancestor search (CLI-2285), so they can never disagree.
- Before each container (re)start, resolves the edge-runtime image through the same registry-candidate pull-with-retry every native `functions` Docker path uses: `docker image inspect <candidate>` (ECR, then GHCR, then Docker Hub) to check the local cache, then `docker pull <candidate>` with 2 retries (4s/8s backoff) on a miss, after `assertLocalDbRunning` — resolving it earlier would hijack the down-daemon error message that DB-inspect step is responsible for producing.
- Runs the full `Config.Validate` pipeline (`resolveLocalConfigValues`, same one `start`/`stop`/`status` use) on every startup/restart, before `assertLocalDbRunning` — an invalid config now fails `serve` up front even for fields this command never otherwise reads (e.g. a bad `db.major_version` or malformed auth hook).
- A container that stops on its own with exit code `0`, or that is torn down by an external supervisor (exit `129`/`130`/`131`/`143`, e.g. `supabase stop` in another terminal), or that is already gone by the time a follow-up inspect runs, all end the command successfully — each prints its own distinct line (see Output above) rather than the user-initiated `Stopped serving …` line, so scrollback can tell "I stopped it" from "the runtime walked out" or "a supervisor tore it down". In a `functions serve &` CI step this means a runtime that exits on its own does not fail the step; the distinct message is the only signal, and a downstream failure otherwise only surfaces later as connection-refused. Exit `137` (SIGKILL, e.g. an OOM kill) is retried by re-attaching to the log stream rather than failing the command. Any other non-zero container exit fails the command; the error message includes the container id. Only a watched-file change restarts the container itself — none of these outcomes ever restart it.
- A container that stops on its own with exit code `0`, or that is torn down by an external supervisor (exit `129`/`130`/`131`/`143`, e.g. `supabase stop` in another terminal), or that is already gone by the time a follow-up inspect runs, all end the command successfully — each prints its own distinct line (see Output above) rather than the user-initiated `Stopped serving …` line, so scrollback can tell "I stopped it" from "the runtime walked out" or "a supervisor tore it down". In a `functions serve &` CI step this means a runtime that exits on its own does not fail the step; the distinct message is the only signal, and a downstream failure otherwise only surfaces later as connection-refused. Exit `137` (SIGKILL) fails the command: `State.OOMKilled` separates a container that hit its memory limit, which reports the memory-allocation remediation, from one killed by something the CLI cannot identify. A non-OOM `137` gets one re-inspect first — if `supabase stop` force-killed a runtime that ignored `SIGTERM` and the follow-up prune has already removed the container, the command still ends successfully instead of failing on a container that is already gone. Any other non-zero container exit fails the command; the error message includes the container id. Only a watched-file change restarts the container itself — none of these outcomes ever restart it.
- A `docker logs -f` re-attach (the daemon can close the stream while the container keeps running) resumes with `--since <last forwarded log timestamp>` instead of replaying the full log history, and is capped at 5 consecutive re-attaches that forward no new output; exceeding the cap fails the command with a tagged error instead of looping forever.
- On the log-stream path, the Docker-daemon-unreachable classification comes from a follow-up `docker container inspect` failure, not from `docker logs -f`'s own stderr text.
- The worker bootstrap template (`serve.main.ts`) is bundled into a single self-contained module with `jose` and the local path/status helpers inlined, so the edge-runtime worker boots without any network access (supabase/supabase#45570). The bundle is embedded at build time for shipped binaries and produced on demand (esbuild) when running from source. It is delivered into the created (not yet started) container as a `docker cp` stdin tar archive at `/root/index.ts` — never a single-file host bind mount, which materializes as an empty directory on daemons that cannot see the client's filesystem (remote `DOCKER_HOST`/Docker-context daemons, podman machines) and breaks bring-up with edge-runtime's "failed to determine entrypoint" (supabase/cli#6254). Only this bootstrap template is daemon-independent: user function sources, import maps, static files, and the multiline-env script directory (present only when an env value contains a newline) still arrive by host bind mounts, so they require a daemon that can see the project directory.
Expand Down
Loading
Loading