Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 51 additions & 43 deletions apps/server/src/mcp/McpHttpServer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ it.effect("tells the agent how to fall back when no desktop app can run the snap
);

it.effect.each([
{ mode: "default", input: {}, images: true },
{ mode: "default", input: {}, images: false },
{ mode: "explicit image", input: { includeImage: true }, images: true },
{ mode: "text only", input: { includeImage: false }, images: false },
])("returns fresh $mode snapshots on repeated MCP calls", ({ input, images }) =>
Expand Down Expand Up @@ -336,12 +336,7 @@ it.effect.each([
Effect.provideService(McpInvocationContext.McpInvocationContext, invocation),
Effect.provideService(McpSchema.McpServerClient, client),
);
expect(nextDefault.content.map((content) => content.type)).toEqual([
"text",
"text",
"text",
"image",
]);
expect(nextDefault.content.map((content) => content.type)).toEqual(["text", "text", "text"]);
expect(nextDefault.structuredContent).toMatchObject({ title: "Snapshot 7", screenshot });
expect(nextDefault.structuredContent).not.toHaveProperty("accessibilityTree");
expect(requests).toBe(7);
Expand Down Expand Up @@ -371,43 +366,56 @@ it.effect("rejects non-boolean snapshot image options before selecting a browser
}).pipe(Effect.provide(TestLayer)),
);

it.effect("saves the snapshot PNG on request and reports its path", () =>
Effect.scoped(
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult);
it.effect.each([{}, { includeImage: false }, { includeImage: true }])(
"saves the snapshot PNG independently of inline image inclusion %#",
(imageOptions) =>
Effect.scoped(
Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fileSystem = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const inputs = yield* serveSnapshots("mcp-save-client", snapshotResult);

const snapshot = yield* callSnapshot({ save: true });
const snapshot = yield* callSnapshot({ save: true, ...imageOptions });
const includesImage = "includeImage" in imageOptions && imageOptions.includeImage === true;
expect(snapshot.content.some((part) => part.type === "image")).toBe(includesImage);

expect(snapshot.isError).toBe(false);
// The browser never receives the server-only `save` flag.
expect(inputs).toEqual([{}]);
const structured = snapshot.structuredContent as { readonly screenshotPath?: string };
const screenshotPath = structured.screenshotPath;
expect(typeof screenshotPath).toBe("string");
expect(path.dirname(screenshotPath!)).toBe(config.browserArtifactsDir);
expect(path.basename(screenshotPath!)).toMatch(
/^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
);
expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
const [, text] = snapshot.content;
expect(text?.type === "text" ? text.text : "").toContain(screenshotPath);

const unsaved = yield* callSnapshot({});
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");

// A save without the image skips the page dump.
const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe("png");
const [only, ...others] = pathOnly.content;
expect(others).toEqual([]);
expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
}),
).pipe(Effect.provide(TestLayer)),
expect(snapshot.isError).toBe(false);
// The browser never receives the server-only `save` flag.
expect(inputs).toEqual([{}]);
const structured = snapshot.structuredContent as { readonly screenshotPath?: string };
const screenshotPath = structured.screenshotPath;
expect(typeof screenshotPath).toBe("string");
expect(path.dirname(screenshotPath!)).toBe(config.browserArtifactsDir);
expect(path.basename(screenshotPath!)).toMatch(
/^browser-screenshot-example-test-[0-9a-z]+-[0-9a-f]{8}\.png$/,
);
expect(Buffer.from(yield* fileSystem.readFile(screenshotPath!)).toString()).toBe("png");
expect(
snapshot.content.some(
(part) => part.type === "text" && part.text.includes(screenshotPath!),
),
).toBe(true);
if (!includesImage) {
expect(snapshot.structuredContent).toEqual({ url: snapshotResult.url, screenshotPath });
expect(snapshot.content).toHaveLength(1);
}

const unsaved = yield* callSnapshot({});
expect(unsaved.structuredContent).not.toHaveProperty("screenshotPath");

// A save without the image skips the page dump.
const pathOnly = yield* callSnapshot({ save: true, includeImage: false });
const saved = pathOnly.structuredContent as { readonly screenshotPath: string };
expect(saved).toEqual({ url: snapshotResult.url, screenshotPath: expect.any(String) });
expect(Buffer.from(yield* fileSystem.readFile(saved.screenshotPath)).toString()).toBe(
"png",
);
const [only, ...others] = pathOnly.content;
expect(others).toEqual([]);
expect(only?.type === "text" ? decodeJsonText(only.text) : null).toEqual(saved);
}),
).pipe(Effect.provide(TestLayer)),
);

it.effect("reports a tagged error when the screenshot cannot be saved", () =>
Expand Down Expand Up @@ -821,7 +829,7 @@ it.effect("registers annotated tools and preserves authenticated request context
Effect.provideService(McpSchema.McpServerClient, client),
);
expect(snapshot.isError).toBe(false);
expect(snapshot.content.some((content) => content.type === "image")).toBe(true);
expect(snapshot.content.some((content) => content.type === "image")).toBe(false);
expect(snapshot.structuredContent).toMatchObject({
screenshot: { mimeType: "image/png", width: 10, height: 5 },
});
Expand Down
8 changes: 4 additions & 4 deletions apps/server/src/mcp/McpHttpServer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,7 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
const png = new Uint8Array(Buffer.from(screenshot.data, "base64"));
const screenshotPath =
payload?.save === true ? yield* saveScreenshot(snapshot.url, png) : undefined;
if (screenshotPath !== undefined && payload?.includeImage === false) {
if (screenshotPath !== undefined && payload?.includeImage !== true) {
// The agent only wants a file to show the user. The url keeps the site icon on the tool row.
const saved = {
url: cutText(snapshot.url, MAX_SNAPSHOT_IDENTIFIER_CHARS),
Expand Down Expand Up @@ -465,9 +465,9 @@ const registerPreviewSnapshot = Effect.fn("McpHttpServer.registerPreviewSnapshot
text: `Snapshot text was bounded. Omitted: ${bounded.omitted.join("; ")}.`,
},
]),
...(payload?.includeImage === false
? []
: [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]),
...(payload?.includeImage === true
? [{ type: "image" as const, data: png, mimeType: screenshot.mimeType }]
: []),
],
});
}),
Expand Down
6 changes: 3 additions & 3 deletions apps/server/src/mcp/toolkits/preview/tools.ts
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,19 @@ const PreviewSetAppearanceTool = safeBrowserTool(
export const PreviewSnapshotTool = readonlyBrowserTool(
Tool.make("preview_snapshot", {
description:
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=false for text-only output with the same page metadata. Set save=true to also write the PNG to disk and get screenshotPath back; with includeImage=false, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
"Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, and action history as text by default. The text is capped near 20 KB and lists what it omitted; use preview_evaluate to read more. Set includeImage=true only when you need to inspect the screenshot visually; inline images remain in provider history. Set save=true to also write the PNG to disk and get screenshotPath back; without includeImage=true, save=true returns only the url and screenshotPath. Embed that path in your reply as ![alt](screenshotPath) so the user sees it. This is the only way to show the user a screenshot; the image in the tool result is not saved anywhere.",
parameters: Schema.Struct({
...PreviewAutomationTabTargetInput.fields,
includeImage: Schema.optional(
Schema.Boolean.annotate({
description:
"Include the PNG image in the tool response. Defaults to true. Set false for text-only output.",
"Include the PNG image in the tool response. Defaults to false. Set true to include the image.",
}),
),
save: Schema.optional(
Schema.Boolean.annotate({
description:
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. With includeImage=false, return only the url and screenshotPath. Defaults to false.",
"Write the screenshot PNG to disk and return its absolute path as screenshotPath. Without includeImage=true, return only the url and screenshotPath. Defaults to false.",
}),
),
}),
Expand Down
Loading