fix(core): only send png, jpeg and webp images to xAI

packages/core/src/session/model-request.ts

@@ -126,10 +126,24 @@ const mimeToModality = (mime: string) => {
   if (mime === "application/pdf") return "pdf"
 }
 
-const unsupportedMedia = (mime: string, name: string | undefined, capabilities: Model.Capabilities) => {
+// xAI rejects any other image type (e.g. GIF) with invalid_image, and the stored image would fail every later turn.
+const XAI_IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/webp"])
+
+const unsupportedMedia = (
+  mime: string,
+  name: string | undefined,
+  capabilities: Model.Capabilities,
+  provider: string | undefined,
+) => {
   const modality = mimeToModality(mime)
-  if (!modality || capabilities.input.some((item) => item.startsWith(modality))) return
-  return `ERROR: Cannot read ${name ? `"${name}"` : modality} (this model does not support ${modality} input). Inform the user.`
+  if (!modality) return
+  const unsupported = !capabilities.input.some((item) => item.startsWith(modality))
+    ? modality
+    : provider === "xai" && modality === "image" && !XAI_IMAGE_MIMES.has(mime.toLowerCase())
+      ? mime
+      : undefined
+  if (!unsupported) return
+  return `ERROR: Cannot read ${name ? `"${name}"` : modality} (this model does not support ${unsupported} input). Inform the user.`
 }
 
 // Remote and provider-referenced media carry no local payload and never count toward the inline budget.
@@ -166,8 +180,11 @@ const replaceMedia = (
       : new Message({ ...message, content })
   })
 
-export const unsupportedParts = (messages: LLMRequest["messages"], capabilities: Model.Capabilities) =>
-  replaceMedia(messages, (media) => unsupportedMedia(media.mime, media.name, capabilities))
+export const unsupportedParts = (
+  messages: LLMRequest["messages"],
+  capabilities: Model.Capabilities,
+  provider?: string,
+) => replaceMedia(messages, (media) => unsupportedMedia(media.mime, media.name, capabilities, provider))
 
 export const boundImages = (messages: LLMRequest["messages"]) => {
   const isImage = (mime: string) => mime.toLowerCase().startsWith("image/")
@@ -277,7 +294,7 @@ export const layer = Layer.effect(
         // TODO: Persist cache lineage so nested forks reuse the root session's cache key.
         promptCacheKey: /^ses_[0-9a-f]{64}$/.test(affinity) ? affinity.slice(4) : affinity,
         system: shaped.system,
-        messages: boundImages(unsupportedParts(shaped.messages, model.capabilities)),
+        messages: boundImages(unsupportedParts(shaped.messages, model.capabilities, model.model.provider)),
         tools: Array.from(hooked, ([name, t]) => ({ ...t, name })),
         toolChoice: input.toolChoice,
         generation: Object.keys(generation).length === 0 ? undefined : generation,

packages/core/test/session-model-request.test.ts

@@ -59,6 +59,51 @@ describe("SessionModelRequest.unsupportedParts", () => {
     })
   })
 
+  test("replaces images xAI cannot decode and keeps png, jpeg and webp", () => {
+    const image = (mime: string, name: string) => ({
+      type: "media" as const,
+      media: Media.base64("aGVsbG8=", mime),
+      filename: name,
+    })
+    const user = [image("image/png", "a.png"), image("image/jpeg", "b.jpg"), image("image/webp", "c.webp")]
+    const messages = [
+      Message.user([...user, image("image/gif", "d.gif")]),
+      Message.tool(
+        ToolResultPart.make({
+          id: "call_1",
+          name: "read",
+          result: {
+            type: "content",
+            value: [
+              { type: "text", text: "Image read successfully" },
+              { type: "file", uri: "data:image/gif;base64,R0lGODlh", mime: "image/gif", name: "e.gif" },
+            ],
+          },
+        }),
+      ),
+    ]
+    const result = unsupportedParts(messages, capabilities(["text", "image"]), "xai")
+
+    expect(result[0]?.content).toEqual([
+      ...user,
+      Message.text('ERROR: Cannot read "d.gif" (this model does not support image/gif input). Inform the user.'),
+    ])
+    expect(result[1]?.content[0]).toMatchObject({
+      type: "tool-result",
+      result: {
+        type: "content",
+        value: [
+          { type: "text", text: "Image read successfully" },
+          {
+            type: "text",
+            text: 'ERROR: Cannot read "e.gif" (this model does not support image/gif input). Inform the user.',
+          },
+        ],
+      },
+    })
+    expect(unsupportedParts(messages, capabilities(["text", "image"]), "openai")).toEqual(messages)
+  })
+
   test("preserves supported media", () => {
     const message = Message.user({ type: "media", media: Media.base64("aGVsbG8=", "image/png") })
     expect(unsupportedParts([message], capabilities(["text", "image"]))[0]?.content).toEqual(message.content)