Skip to content

Commit c0192e2

Browse files
committed
chore(mcp): cap image size
1 parent 3f67e71 commit c0192e2

File tree

5 files changed

+213
-38
lines changed

5 files changed

+213
-38
lines changed

packages/playwright-core/src/server/utils/comparators.ts

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ import pixelmatch from '../../third_party/pixelmatch';
2121
import { jpegjs } from '../../utilsBundle';
2222
import { colors, diff } from '../../utilsBundle';
2323
import { PNG } from '../../utilsBundle';
24+
import { padImageToSize } from './imageUtils';
25+
26+
import type { ImageData } from './imageUtils';
2427

2528
export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number, comparator?: string };
2629
export type ComparatorResult = { diff?: Buffer; errorMessage: string; } | null;
@@ -48,8 +51,6 @@ export function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedB
4851
return null;
4952
}
5053

51-
type ImageData = { width: number, height: number, data: Buffer };
52-
5354
function compareImages(mimeType: string, actualBuffer: Buffer | string, expectedBuffer: Buffer, options: ImageComparatorOptions = {}): ComparatorResult {
5455
if (!actualBuffer || !(actualBuffer instanceof Buffer))
5556
return { errorMessage: 'Actual result should be a Buffer.' };
@@ -61,8 +62,8 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expected
6162
let sizesMismatchError = '';
6263
if (expected.width !== actual.width || expected.height !== actual.height) {
6364
sizesMismatchError = `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `;
64-
actual = resizeImage(actual, size);
65-
expected = resizeImage(expected, size);
65+
actual = padImageToSize(actual, size);
66+
expected = padImageToSize(expected, size);
6667
}
6768
const diff = new PNG({ width: size.width, height: size.height });
6869
let count;
@@ -131,27 +132,3 @@ function compareText(actual: Buffer | string, expectedBuffer: Buffer): Comparato
131132
const errorMessage = coloredLines.join('\n');
132133
return { errorMessage };
133134
}
134-
135-
function resizeImage(image: ImageData, size: { width: number, height: number }): ImageData {
136-
if (image.width === size.width && image.height === size.height)
137-
return image;
138-
const buffer = new Uint8Array(size.width * size.height * 4);
139-
for (let y = 0; y < size.height; y++) {
140-
for (let x = 0; x < size.width; x++) {
141-
const to = (y * size.width + x) * 4;
142-
if (y < image.height && x < image.width) {
143-
const from = (y * image.width + x) * 4;
144-
buffer[to] = image.data[from];
145-
buffer[to + 1] = image.data[from + 1];
146-
buffer[to + 2] = image.data[from + 2];
147-
buffer[to + 3] = image.data[from + 3];
148-
} else {
149-
buffer[to] = 0;
150-
buffer[to + 1] = 0;
151-
buffer[to + 2] = 0;
152-
buffer[to + 3] = 0;
153-
}
154-
}
155-
}
156-
return { data: Buffer.from(buffer), width: size.width, height: size.height };
157-
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/**
2+
* Copyright (c) Microsoft Corporation.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
export type ImageData = { width: number, height: number, data: Buffer };
18+
19+
export function padImageToSize(image: ImageData, size: { width: number, height: number }): ImageData {
20+
if (image.width === size.width && image.height === size.height)
21+
return image;
22+
const buffer = new Uint8Array(size.width * size.height * 4);
23+
for (let y = 0; y < size.height; y++) {
24+
for (let x = 0; x < size.width; x++) {
25+
const to = (y * size.width + x) * 4;
26+
if (y < image.height && x < image.width) {
27+
const from = (y * image.width + x) * 4;
28+
buffer[to] = image.data[from];
29+
buffer[to + 1] = image.data[from + 1];
30+
buffer[to + 2] = image.data[from + 2];
31+
buffer[to + 3] = image.data[from + 3];
32+
} else {
33+
buffer[to] = 0;
34+
buffer[to + 1] = 0;
35+
buffer[to + 2] = 0;
36+
buffer[to + 3] = 0;
37+
}
38+
}
39+
}
40+
return { data: Buffer.from(buffer), width: size.width, height: size.height };
41+
}
42+
43+
export function scaleImageToSize(image: ImageData, size: { width: number; height: number }): ImageData {
44+
const { data: src, width: w1, height: h1 } = image;
45+
const w2 = size.width | 0, h2 = size.height | 0;
46+
if (w1 === w2 && h1 === h2)
47+
return image;
48+
49+
const clamp = (v: number, lo: number, hi: number) => (v < lo ? lo : v > hi ? hi : v);
50+
51+
// Catmull–Rom weights
52+
const weights = (t: number, o: Float32Array) => {
53+
const t2 = t * t, t3 = t2 * t;
54+
o[0] = -0.5 * t + 1.0 * t2 - 0.5 * t3;
55+
o[1] = 1.0 - 2.5 * t2 + 1.5 * t3;
56+
o[2] = 0.5 * t + 2.0 * t2 - 1.5 * t3;
57+
o[3] = -0.5 * t2 + 0.5 * t3;
58+
};
59+
60+
const srcRowStride = w1 * 4;
61+
const dstRowStride = w2 * 4;
62+
63+
// Precompute X: indices, weights, and byte offsets (idx*4)
64+
const xIdx = new Int32Array(w2 * 4);
65+
const xOff = new Int32Array(w2 * 4); // byte offsets = xIdx*4
66+
const xW = new Float32Array(w2 * 4);
67+
const wx = new Float32Array(4);
68+
const xScale = w1 / w2;
69+
for (let x = 0; x < w2; x++) {
70+
const sx = (x + 0.5) * xScale - 0.5;
71+
const sxi = Math.floor(sx);
72+
const t = sx - sxi;
73+
weights(t, wx);
74+
const b = x * 4;
75+
const i0 = clamp(sxi - 1, 0, w1 - 1);
76+
const i1 = clamp(sxi + 0, 0, w1 - 1);
77+
const i2 = clamp(sxi + 1, 0, w1 - 1);
78+
const i3 = clamp(sxi + 2, 0, w1 - 1);
79+
xIdx[b + 0] = i0; xIdx[b + 1] = i1; xIdx[b + 2] = i2; xIdx[b + 3] = i3;
80+
xOff[b + 0] = i0 << 2; xOff[b + 1] = i1 << 2; xOff[b + 2] = i2 << 2; xOff[b + 3] = i3 << 2;
81+
xW[b + 0] = wx[0]; xW[b + 1] = wx[1]; xW[b + 2] = wx[2]; xW[b + 3] = wx[3];
82+
}
83+
84+
// Precompute Y: indices, weights, and row-base byte offsets (y*rowStride)
85+
const yIdx = new Int32Array(h2 * 4);
86+
const yRow = new Int32Array(h2 * 4); // row base in bytes
87+
const yW = new Float32Array(h2 * 4);
88+
const wy = new Float32Array(4);
89+
const yScale = h1 / h2;
90+
for (let y = 0; y < h2; y++) {
91+
const sy = (y + 0.5) * yScale - 0.5;
92+
const syi = Math.floor(sy);
93+
const t = sy - syi;
94+
weights(t, wy);
95+
const b = y * 4;
96+
const j0 = clamp(syi - 1, 0, h1 - 1);
97+
const j1 = clamp(syi + 0, 0, h1 - 1);
98+
const j2 = clamp(syi + 1, 0, h1 - 1);
99+
const j3 = clamp(syi + 2, 0, h1 - 1);
100+
yIdx[b + 0] = j0; yIdx[b + 1] = j1; yIdx[b + 2] = j2; yIdx[b + 3] = j3;
101+
yRow[b + 0] = j0 * srcRowStride;
102+
yRow[b + 1] = j1 * srcRowStride;
103+
yRow[b + 2] = j2 * srcRowStride;
104+
yRow[b + 3] = j3 * srcRowStride;
105+
yW[b + 0] = wy[0]; yW[b + 1] = wy[1]; yW[b + 2] = wy[2]; yW[b + 3] = wy[3];
106+
}
107+
108+
const dst = new Uint8Array(w2 * h2 * 4);
109+
110+
for (let y = 0; y < h2; y++) {
111+
const yb = y * 4;
112+
const rb0 = yRow[yb + 0], rb1 = yRow[yb + 1], rb2 = yRow[yb + 2], rb3 = yRow[yb + 3];
113+
const wy0 = yW[yb + 0], wy1 = yW[yb + 1], wy2 = yW[yb + 2], wy3 = yW[yb + 3];
114+
const dstBase = y * dstRowStride;
115+
116+
for (let x = 0; x < w2; x++) {
117+
const xb = x * 4;
118+
const xo0 = xOff[xb + 0], xo1 = xOff[xb + 1], xo2 = xOff[xb + 2], xo3 = xOff[xb + 3];
119+
const wx0 = xW[xb + 0], wx1 = xW[xb + 1], wx2 = xW[xb + 2], wx3 = xW[xb + 3];
120+
const di = dstBase + (x << 2);
121+
122+
// unrolled RGBA
123+
for (let c = 0; c < 4; c++) {
124+
const r0 = src[rb0 + xo0 + c] * wx0 + src[rb0 + xo1 + c] * wx1 + src[rb0 + xo2 + c] * wx2 + src[rb0 + xo3 + c] * wx3;
125+
const r1 = src[rb1 + xo0 + c] * wx0 + src[rb1 + xo1 + c] * wx1 + src[rb1 + xo2 + c] * wx2 + src[rb1 + xo3 + c] * wx3;
126+
const r2 = src[rb2 + xo0 + c] * wx0 + src[rb2 + xo1 + c] * wx1 + src[rb2 + xo2 + c] * wx2 + src[rb2 + xo3 + c] * wx3;
127+
const r3 = src[rb3 + xo0 + c] * wx0 + src[rb3 + xo1 + c] * wx1 + src[rb3 + xo2 + c] * wx2 + src[rb3 + xo3 + c] * wx3;
128+
const v = r0 * wy0 + r1 * wy1 + r2 * wy2 + r3 * wy3;
129+
dst[di + c] = v < 0 ? 0 : v > 255 ? 255 : v | 0;
130+
}
131+
}
132+
}
133+
134+
return { data: Buffer.from(dst), width: w2, height: h2 };
135+
}

packages/playwright-core/src/utils.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ export * from './server/utils/expectUtils';
4343
export * from './server/utils/fileUtils';
4444
export * from './server/utils/hostPlatform';
4545
export * from './server/utils/httpServer';
46+
export * from './server/utils/imageUtils';
4647
export * from './server/utils/network';
4748
export * from './server/utils/nodePlatform';
4849
export * from './server/utils/processLauncher';

packages/playwright/src/mcp/browser/tools/screenshot.ts

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
* limitations under the License.
1515
*/
1616

17+
import fs from 'fs';
18+
19+
import { mkdirIfNeeded, scaleImageToSize } from 'playwright-core/lib/utils';
20+
import { jpegjs, PNG } from 'playwright-core/lib/utilsBundle';
21+
1722
import { z } from '../../sdk/bundle';
1823
import { defineTabTool } from './tool';
1924
import * as javascript from '../codegen';
@@ -51,7 +56,6 @@ const screenshot = defineTabTool({
5156
type: fileType,
5257
quality: fileType === 'png' ? undefined : 90,
5358
scale: 'css',
54-
path: fileName,
5559
...(params.fullPage !== undefined && { fullPage: params.fullPage })
5660
};
5761
const isElementScreenshot = params.element && params.ref;
@@ -68,19 +72,36 @@ const screenshot = defineTabTool({
6872
response.addCode(`await page.screenshot(${javascript.formatObject(options)});`);
6973

7074
const buffer = ref ? await ref.locator.screenshot(options) : await tab.page.screenshot(options);
75+
76+
await mkdirIfNeeded(fileName);
77+
await fs.promises.writeFile(fileName, buffer);
78+
7179
response.addResult(`Took the ${screenshotTarget} screenshot and saved it as ${fileName}`);
7280

73-
// https://github.com/microsoft/playwright-mcp/issues/817
74-
// Never return large images to LLM, saving them to the file system is enough.
75-
if (!params.fullPage) {
76-
response.addImage({
77-
contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',
78-
data: buffer
79-
});
80-
}
81+
response.addImage({
82+
contentType: fileType === 'png' ? 'image/png' : 'image/jpeg',
83+
data: scaleImageToFitMessage(buffer, fileType)
84+
});
8185
}
8286
});
8387

88+
export function scaleImageToFitMessage(buffer: Buffer, imageType: 'png' | 'jpeg'): Buffer {
89+
// https://docs.claude.com/en/docs/build-with-claude/vision#evaluate-image-size
90+
// Not more than 1.15 megapixel, linear size not more than 1568.
91+
92+
const image = imageType === 'png' ? PNG.sync.read(buffer) : jpegjs.decode(buffer, { maxMemoryUsageInMB: 5 * 1024 });
93+
const pixels = image.width * image.height;
94+
95+
const shrink = Math.min(1568 / image.width, 1568 / image.height, Math.sqrt(1.15 * 1024 * 1024 / pixels));
96+
if (shrink > 1)
97+
return buffer;
98+
99+
const width = image.width * shrink | 0;
100+
const height = image.height * shrink | 0;
101+
const scaledImage = scaleImageToSize(image, { width, height });
102+
return imageType === 'png' ? PNG.sync.write(scaledImage as any) : jpegjs.encode(scaledImage, 80).data;
103+
}
104+
84105
export default [
85106
screenshot,
86107
];

tests/mcp/screenshot.spec.ts

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import fs from 'fs';
1818

1919
import { test, expect } from './fixtures';
20+
import { jpegjs, PNG } from 'packages/playwright-core/lib/utilsBundle';
2021

2122
test('browser_take_screenshot (viewport)', async ({ startClient, server }, testInfo) => {
2223
const { client } = await startClient({
@@ -264,11 +265,51 @@ test('browser_take_screenshot (fullPage: true)', async ({ startClient, server },
264265
{
265266
text: expect.stringContaining('fullPage: true'),
266267
type: 'text',
267-
}
268+
},
269+
{
270+
data: expect.any(String),
271+
mimeType: 'image/png',
272+
type: 'image',
273+
},
268274
],
269275
});
270276
});
271277

278+
test('browser_take_screenshot size cap', async ({ startClient, server }, testInfo) => {
279+
const { client } = await startClient({
280+
config: { outputDir: testInfo.outputPath('output') },
281+
});
282+
283+
const expectations = [
284+
{ title: '2000x500', pageWidth: 2000, pageHeight: 500, expectedWidth: 1568, expectedHeight: 500 * 1568 / 2000 | 0 },
285+
{ title: '2000x2000', pageWidth: 2000, pageHeight: 2000, expectedWidth: 1098, expectedHeight: 1098 },
286+
{ title: '1280x800', pageWidth: 1280, pageHeight: 800, expectedWidth: 1280, expectedHeight: 800 },
287+
];
288+
289+
for (const expectation of expectations) {
290+
await test.step(expectation.title, async () => {
291+
server.setContent('/', `<body style="width: ${expectation.pageWidth}px; height: ${expectation.pageHeight}px; background: red; margin: 0;"></body>`, 'text/html');
292+
await client.callTool({ name: 'browser_navigate', arguments: { url: server.PREFIX } });
293+
294+
const pngResult = await client.callTool({
295+
name: 'browser_take_screenshot',
296+
arguments: { fullPage: true },
297+
});
298+
const png = PNG.sync.read(Buffer.from(pngResult.content?.[1]?.data, 'base64'));
299+
expect(png.width).toBe(expectation.expectedWidth);
300+
expect(png.height).toBe(expectation.expectedHeight);
301+
302+
const jpegResult = await client.callTool({
303+
name: 'browser_take_screenshot',
304+
arguments: { fullPage: true, type: 'jpeg' },
305+
});
306+
const jpeg = jpegjs.decode(Buffer.from(jpegResult.content?.[1]?.data, 'base64'));
307+
expect(jpeg.width).toBe(expectation.expectedWidth);
308+
expect(jpeg.height).toBe(expectation.expectedHeight);
309+
});
310+
}
311+
});
312+
272313
test('browser_take_screenshot (fullPage with element should error)', async ({ startClient, server }, testInfo) => {
273314
const { client } = await startClient({
274315
config: { outputDir: testInfo.outputPath('output') },

0 commit comments

Comments
 (0)