feat(ai): add vision support for multimodal messages

packages/ai/src/shared/providers/anthropic.ts

@@ -23,6 +23,7 @@ import {
   AIError,
   AuthenticationError,
   ContextLengthError,
+  extractTextContent,
   ModelNotFoundError,
   RateLimitError,
 } from '../types';
@@ -413,13 +414,14 @@ export class AnthropicProvider implements AIInterface {
     }> = [];
 
     for (const message of messages) {
+      const textContent = extractTextContent(message.content);
       if (message.role === 'system') {
         // Combine multiple system messages
-        system = system ? `${system}\n\n${message.content}` : message.content;
+        system = system ? `${system}\n\n${textContent}` : textContent;
       } else {
         anthropicMessages.push({
           role: message.role === 'assistant' ? 'assistant' : 'user',
-          content: message.content,
+          content: textContent,
         });
       }
     }

packages/ai/src/shared/providers/bedrock.ts

@@ -19,6 +19,7 @@ import {
   AIError,
   AuthenticationError,
   ContextLengthError,
+  extractTextContent,
   ModelNotFoundError,
   RateLimitError,
 } from '../types';
@@ -378,12 +379,13 @@ export class BedrockProvider implements AIInterface {
     }> = [];
 
     for (const message of messages) {
+      const textContent = extractTextContent(message.content);
       if (message.role === 'system') {
-        system = system ? `${system}\n\n${message.content}` : message.content;
+        system = system ? `${system}\n\n${textContent}` : textContent;
       } else {
         anthropicMessages.push({
           role: message.role === 'assistant' ? 'assistant' : 'user',
-          content: message.content,
+          content: textContent,
         });
       }
     }

packages/ai/src/shared/providers/claude-cli.ts

@@ -32,6 +32,7 @@ import {
   AIError,
   AuthenticationError,
   ContextLengthError,
+  extractTextContent,
   RateLimitError,
 } from '../types';
 
@@ -354,14 +355,15 @@ export class ClaudeCliProvider implements AIInterface {
     const conversationParts: string[] = [];
 
     for (const message of messages) {
+      const textContent = extractTextContent(message.content);
       if (message.role === 'system') {
         systemPrompt = systemPrompt
-          ? `${systemPrompt}\n\n${message.content}`
-          : message.content;
+          ? `${systemPrompt}\n\n${textContent}`
+          : textContent;
       } else if (message.role === 'user') {
-        conversationParts.push(`User: ${message.content}`);
+        conversationParts.push(`User: ${textContent}`);
       } else if (message.role === 'assistant') {
-        conversationParts.push(`Assistant: ${message.content}`);
+        conversationParts.push(`Assistant: ${textContent}`);
       }
     }
 

packages/ai/src/shared/providers/gemini.ts

@@ -20,6 +20,7 @@ import type {
 import {
   AIError,
   AuthenticationError,
+  extractTextContent,
   ModelNotFoundError,
   RateLimitError,
 } from '../types';
@@ -376,15 +377,16 @@ export class GeminiProvider implements AIInterface {
     // The new SDK expects a string for the contents field
     return messages
       .map((message) => {
+        const textContent = extractTextContent(message.content);
         switch (message.role) {
           case 'system':
-            return `Instructions: ${message.content}`;
+            return `Instructions: ${textContent}`;
           case 'user':
-            return `Human: ${message.content}`;
+            return `Human: ${textContent}`;
           case 'assistant':
-            return `Assistant: ${message.content}`;
+            return `Assistant: ${textContent}`;
           default:
-            return message.content;
+            return textContent;
         }
       })
       .join('\n\n');

packages/ai/src/shared/providers/huggingface.ts

@@ -19,6 +19,7 @@ import {
   AIError,
   AuthenticationError,
   ContextLengthError,
+  extractTextContent,
   ModelNotFoundError,
   RateLimitError,
 } from '../types';
@@ -285,15 +286,16 @@ export class HuggingFaceProvider implements AIInterface {
     // Convert chat messages to a single prompt format
     return `${messages
       .map((message) => {
+        const textContent = extractTextContent(message.content);
         switch (message.role) {
           case 'system':
-            return `System: ${message.content}`;
+            return `System: ${textContent}`;
           case 'user':
-            return `Human: ${message.content}`;
+            return `Human: ${textContent}`;
           case 'assistant':
-            return `Assistant: ${message.content}`;
+            return `Assistant: ${textContent}`;
           default:
-            return message.content;
+            return textContent;
         }
       })
       .join('\n')}\nAssistant:`;

packages/ai/src/shared/providers/openai.ts

@@ -17,6 +17,7 @@ import type {
   AIResponse,
   ChatOptions,
   CompletionOptions,
+  ContentPart,
   EmbeddingOptions,
   EmbeddingResponse,
   MessageOptions,
@@ -414,10 +415,32 @@ export class OpenAIProvider implements AIInterface {
     messages: AIMessage[],
   ): OpenAI.Chat.ChatCompletionMessageParam[] {
     return messages.map((message) => {
+      // Handle content that can be string or ContentPart[]
+      let content: string | OpenAI.Chat.ChatCompletionContentPart[];
+
+      if (typeof message.content === 'string') {
+        content = message.content;
+      } else {
+        // Array of content parts - map to OpenAI format
+        content = message.content.map((part: ContentPart) => {
+          if (part.type === 'text') {
+            return { type: 'text' as const, text: part.text };
+          }
+          // Image content part
+          return {
+            type: 'image_url' as const,
+            image_url: {
+              url: part.image_url.url,
+              detail: part.image_url.detail,
+            },
+          };
+        });
+      }
+
       // Build message based on role and content
       const baseMessage = {
         role: message.role as OpenAI.Chat.ChatCompletionRole,
-        content: message.content,
+        content,
       };
 
       // Add optional fields based on role and availability

packages/ai/src/shared/types.ts

@@ -2,8 +2,76 @@
  * Core types and interfaces for the AI library
  */
 
+/**
+ * Text content part for multimodal messages
+ */
+export interface TextContentPart {
+  type: 'text';
+  text: string;
+}
+
+/**
+ * Image content part for vision-capable models
+ */
+export interface ImageContentPart {
+  type: 'image_url';
+  image_url: {
+    /** Image URL (http/https) or base64 data URL */
+    url: string;
+    /** Image detail level for processing */
+    detail?: 'auto' | 'low' | 'high';
+  };
+}
+
+/**
+ * Union type for all content parts in multimodal messages
+ */
+export type ContentPart = TextContentPart | ImageContentPart;
+
+/**
+ * Extract text content from a message content field.
+ *
+ * Handles both simple string content and multimodal content arrays,
+ * extracting only the text parts and concatenating them.
+ *
+ * @param content - The message content (string or ContentPart array)
+ * @returns The extracted text content
+ */
+export function extractTextContent(content: string | ContentPart[]): string {
+  if (typeof content === 'string') {
+    return content;
+  }
+  // Extract text from content parts
+  return content
+    .filter((part): part is TextContentPart => part.type === 'text')
+    .map((part) => part.text)
+    .join('\n');
+}
+
 /**
  * AI message structure for chat interactions
+ *
+ * Supports both simple string content and multimodal content arrays
+ * for vision-capable models.
+ *
+ * @example Simple text message
+ * ```typescript
+ * const message: AIMessage = {
+ *   role: 'user',
+ *   content: 'Hello, how are you?'
+ * };
+ * ```
+ *
+ * @example Multimodal message with image
+ * ```typescript
+ * const message: AIMessage = {
+ *   role: 'user',
+ *   content: [
+ *     { type: 'text', text: 'What is in this image?' },
+ *     { type: 'image_url', image_url: { url: 'data:image/png;base64,...' } }
+ *   ]
+ * };
+ * ```
  */
 export interface AIMessage {
   /**
@@ -12,9 +80,12 @@ export interface AIMessage {
   role: 'system' | 'user' | 'assistant' | 'function' | 'tool';
 
   /**
-   * Content of the message
+   * Content of the message.
+   *
+   * Can be a simple string for text-only messages, or an array of content parts
+   * for multimodal messages (e.g., text + images for vision models).
    */
-  content: string;
+  content: string | ContentPart[];
 
   /**
    * Optional name for the message sender