chore: copy perform snapshtos on write

docs/src/api/params.md

@@ -372,10 +372,10 @@ Emulates consistent window screen size available inside web page via `window.scr
 
 ## context-option-agent
 - `agent` <[Object]>
-  - `provider` <[string]> LLM provider to use.
-  - `model` <[string]> Model identifier within provider.
+  - `provider` ?<[string]> LLM provider to use. Required in non-cache mode.
+  - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode.
   - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
-  - `cacheMode` ?<[CacheMode]<"force"|"ignore"|"update"|"auto">> Cache control, defaults to 'auto'.
+  - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
   - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.
   - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.
   - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.

docs/src/test-api/class-testoptions.md

@@ -49,11 +49,11 @@ export default defineConfig({
 ## property: TestOptions.agent
 * since: v1.58
 - type: <[Object]>
+  - `provider` ?<[string]> LLM provider to use. Required in non-cache mode.
+  - `model` ?<[string]> Model identifier within the provider. Required in non-cache mode.
   - `cachePathTemplate` ?<[string]> Cache file template to use/generate code for performed actions into.
   - `maxTurns` ?<[int]> Maximum number of agentic turns to take per call. Defaults to 10.
   - `maxTokens` ?<[int]> Maximum number of tokens to consume per call. The agentic loop will stop after input + output tokens exceed this value. Defaults on unlimited.
-  - `model` <[string]> Model identifier within provider.
-  - `provider` <[string]> LLM provider to use.
   - `secrets` ?<[Object]<[string], [string]>> Secrets to hide from the LLM.
 
 ## property: TestOptions.baseURL = %%-context-option-baseURL-%%

packages/playwright-client/types/types.d.ts

@@ -22226,24 +22226,24 @@ export interface BrowserContextOptions {
    */
   agent?: {
     /**
-     * LLM provider to use.
+     * LLM provider to use. Required in non-cache mode.
      */
-    provider: string;
+    provider?: string;
 
     /**
-     * Model identifier within provider.
+     * Model identifier within the provider. Required in non-cache mode.
      */
-    model: string;
+    model?: string;
 
     /**
      * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
      */
     cacheFile?: string;
 
     /**
-     * Cache control, defaults to 'auto'.
+     * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
      */
-    cacheMode?: "force"|"ignore"|"update"|"auto";
+    cacheOutFile?: string;
 
     /**
      * Secrets to hide from the LLM.

packages/playwright-core/src/protocol/validator.ts

@@ -603,10 +603,10 @@ scheme.BrowserTypeLaunchPersistentContextParams = tObject({
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
   agent: tOptional(tObject({
-    provider: tString,
-    model: tString,
+    provider: tOptional(tString),
+    model: tOptional(tString),
     cacheFile: tOptional(tString),
-    cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
+    cacheOutFile: tOptional(tString),
     secrets: tOptional(tArray(tType('NameValue'))),
     maxTurns: tOptional(tInt),
     maxTokens: tOptional(tInt),
@@ -704,10 +704,10 @@ scheme.BrowserNewContextParams = tObject({
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
   agent: tOptional(tObject({
-    provider: tString,
-    model: tString,
+    provider: tOptional(tString),
+    model: tOptional(tString),
     cacheFile: tOptional(tString),
-    cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
+    cacheOutFile: tOptional(tString),
     secrets: tOptional(tArray(tType('NameValue'))),
     maxTurns: tOptional(tInt),
     maxTokens: tOptional(tInt),
@@ -784,10 +784,10 @@ scheme.BrowserNewContextForReuseParams = tObject({
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
   agent: tOptional(tObject({
-    provider: tString,
-    model: tString,
+    provider: tOptional(tString),
+    model: tOptional(tString),
     cacheFile: tOptional(tString),
-    cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
+    cacheOutFile: tOptional(tString),
     secrets: tOptional(tArray(tType('NameValue'))),
     maxTurns: tOptional(tInt),
     maxTokens: tOptional(tInt),
@@ -909,10 +909,10 @@ scheme.BrowserContextInitializer = tObject({
     selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
     testIdAttributeName: tOptional(tString),
     agent: tOptional(tObject({
-      provider: tString,
-      model: tString,
+      provider: tOptional(tString),
+      model: tOptional(tString),
       cacheFile: tOptional(tString),
-      cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
+      cacheOutFile: tOptional(tString),
       secrets: tOptional(tArray(tType('NameValue'))),
       maxTurns: tOptional(tInt),
       maxTokens: tOptional(tInt),
@@ -2837,10 +2837,10 @@ scheme.AndroidDeviceLaunchBrowserParams = tObject({
   selectorEngines: tOptional(tArray(tType('SelectorEngine'))),
   testIdAttributeName: tOptional(tString),
   agent: tOptional(tObject({
-    provider: tString,
-    model: tString,
+    provider: tOptional(tString),
+    model: tOptional(tString),
     cacheFile: tOptional(tString),
-    cacheMode: tOptional(tEnum(['ignore', 'force', 'update', 'auto'])),
+    cacheOutFile: tOptional(tString),
     secrets: tOptional(tArray(tType('NameValue'))),
     maxTurns: tOptional(tInt),
     maxTokens: tOptional(tInt),

packages/playwright-core/src/server/agent/agent.ts

@@ -66,8 +66,8 @@ async function perform(progress: Progress, context: Context, userTask: string, r
 }> {
   const { page } = context;
   const browserContext = page.browserContext;
-  if (!browserContext._options.agent)
-    throw new Error(`page.perform() and page.extract() require the agent to be set on the browser context`);
+  if (!browserContext._options.agent?.provider || !browserContext._options.agent?.model)
+    throw new Error(`This action requires the agent provider and model to be set on the browser context`);
 
   const { full } = await page.snapshotForAI(progress);
   const { tools, callTool } = toolsForLoop(context);
@@ -137,20 +137,15 @@ type CachedActions = Record<string, {
   actions: actions.Action[],
 }>;
 
-const allCaches = new Map<string, CachedActions>();
-
 async function cachedPerform(progress: Progress, context: Context, options: channels.PagePerformParams): Promise<boolean> {
-  if (!context.options?.cacheFile || context.options.cacheMode === 'ignore' || context.options.cacheMode === 'update')
+  if (!context.options?.cacheFile)
     return false;
 
-  const cache = await cachedActions(context.options.cacheFile);
+  const cache = await cachedActions(context, context.options?.cacheFile);
   const cacheKey = (options.key ?? options.task).trim();
-  const entry = cache[cacheKey];
-  if (!entry) {
-    if (context.options.cacheMode === 'force')
-      throw new Error(`No cached actions for key "${cacheKey}", but cache mode is set to "force"`);
+  const entry = cache.actions[cacheKey];
+  if (!entry)
     return false;
-  }
 
   for (const action of entry.actions)
     await runAction(progress, 'run', context.page, action, context.options.secrets ?? []);
@@ -159,24 +154,41 @@ async function cachedPerform(progress: Progress, context: Context, options: chan
 
 async function updateCache(context: Context, options: channels.PagePerformParams) {
   const cacheFile = context.options?.cacheFile;
-  if (!cacheFile)
-    return;
-  const cache = await cachedActions(cacheFile);
+  const cacheOutFile = context.options?.cacheOutFile;
+
+  const cache = cacheFile ? await cachedActions(context, cacheFile) : { actions: {}, newActions: {} };
   const cacheKey = (options.key ?? options.task).trim();
-  cache[cacheKey] = {
+  const newEntry = {
     timestamp: Date.now(),
     actions: context.actions,
   };
-  const entries = Object.entries(cache);
-  entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
-  await fs.promises.writeFile(cacheFile, JSON.stringify(Object.fromEntries(entries), undefined, 2));
+  cache.actions[cacheKey] = newEntry;
+  cache.newActions[cacheKey] = newEntry;
+
+  if (cacheOutFile) {
+    const entries = Object.entries(cache.newActions);
+    entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
+    await fs.promises.writeFile(cacheOutFile, JSON.stringify(Object.fromEntries(entries), undefined, 2));
+  } else if (cacheFile) {
+    const entries = Object.entries(cache.actions);
+    entries.sort((e1, e2) => e1[0].localeCompare(e2[0]));
+    await fs.promises.writeFile(cacheFile, JSON.stringify(Object.fromEntries(entries), undefined, 2));
+  }
 }
 
-async function cachedActions(cacheFile: string): Promise<CachedActions> {
-  let cache = allCaches.get(cacheFile);
+type Cache = {
+  actions: CachedActions;
+  newActions: CachedActions;
+};
+
+async function cachedActions(context: Context, cacheFile: string): Promise<Cache> {
+  let cache = (context as any)[agentCacheSymbol] as Cache | undefined;
   if (!cache) {
-    cache = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions;
-    allCaches.set(cacheFile, cache);
+    const actions = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions;
+    cache = { actions, newActions: {} };
+    (context as any)[agentCacheSymbol] = cache;
   }
   return cache;
 }
+
+const agentCacheSymbol = Symbol('agentCache');

packages/playwright-core/types/types.d.ts

@@ -22226,24 +22226,24 @@ export interface BrowserContextOptions {
    */
   agent?: {
     /**
-     * LLM provider to use.
+     * LLM provider to use. Required in non-cache mode.
      */
-    provider: string;
+    provider?: string;
 
     /**
-     * Model identifier within provider.
+     * Model identifier within the provider. Required in non-cache mode.
      */
-    model: string;
+    model?: string;
 
     /**
      * Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
      */
     cacheFile?: string;
 
     /**
-     * Cache control, defaults to 'auto'.
+     * When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
      */
-    cacheMode?: "force"|"ignore"|"update"|"auto";
+    cacheOutFile?: string;
 
     /**
      * Secrets to hide from the LLM.

packages/playwright/src/common/ipc.ts

@@ -102,7 +102,8 @@ export type CloneStoragePayload = {
 };
 
 export type UpstreamStoragePayload = {
-  workerFile: string;
+  storageFile: string;
+  storageOutFile: string;
 };
 
 export type CustomMessageRequestPayload = {

packages/playwright/src/index.ts

@@ -645,6 +645,8 @@ class ArtifactsRecorder {
   private _screenshotRecorder: SnapshotRecorder;
   private _pageSnapshot: string | undefined;
   private _agent: PlaywrightTestOptions['agent'];
+  private _agentCacheFile: string | undefined;
+  private _agentCacheOutFile: string | undefined;
 
   constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, agent: PlaywrightTestOptions['agent']) {
     this._playwright = playwright;
@@ -711,19 +713,23 @@ class ArtifactsRecorder {
       return;
 
     const cachePathTemplate = this._agent.cachePathTemplate ?? '{testDir}/{testFilePath}-cache.json';
-    const cacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json');
-    const workerFile = await this._testInfo._cloneStorage(cacheFile);
+    this._agentCacheFile = this._testInfo._applyPathTemplate(cachePathTemplate, '', '.json');
+    this._agentCacheOutFile = path.join(this._testInfo.artifactsDir(), 'agent-cache-' + createGuid() + '.json');
+
+    const cacheFile = this._testInfo.config.runAgents ? undefined : await this._testInfo._cloneStorage(this._agentCacheFile);
     options.agent = {
       ...this._agent,
-      cacheFile: workerFile,
-      cacheMode: this._testInfo.config.runAgents ? 'update' : 'force',
+      cacheFile,
+      cacheOutFile: this._agentCacheOutFile,
     };
   }
 
   private async _upstreamAgentCache(context: BrowserContextImpl) {
-    const agent = context._options.agent;
-    if (this._testInfo.status === 'passed' && agent?.cacheFile)
-      await this._testInfo._upstreamStorage(agent.cacheFile);
+    if (!this._agentCacheFile || !this._agentCacheOutFile)
+      return;
+    if (this._testInfo.status !== 'passed')
+      return;
+    await this._testInfo._upstreamStorage(this._agentCacheFile, this._agentCacheOutFile);
   }
 
   async didCreateRequestContext(context: APIRequestContextImpl) {

packages/playwright/src/runner/dispatcher.ts

@@ -263,10 +263,10 @@ export class Dispatcher {
       this._producedEnvByProjectId.set(testGroup.projectId, { ...producedEnv, ...worker.producedEnv() });
     });
     worker.onRequest('cloneStorage', async (params: ipc.CloneStoragePayload) => {
-      return await Storage.clone(params.storageFile, worker.artifactsDir());
+      return await Storage.clone(params.storageFile, outputDir);
     });
     worker.onRequest('upstreamStorage', async (params: ipc.UpstreamStoragePayload) => {
-      await Storage.upstream(params.workerFile);
+      await Storage.upstream(params.storageFile, params.storageOutFile);
     });
     return worker;
   }