feat(browser): add image attachment for browser mode

browser/src/components/ChatSender/index.tsx

@@ -18,6 +18,7 @@ import { QuillContext } from '../QuillEditor/QuillContext';
 import SuggestionList, { type ISuggestionListRef } from '../SuggestionList';
 import SenderFooter from './SenderFooter';
 import SenderHeader from './SenderHeader';
+import type { FilePart, ImagePart } from '@/types/chat';
 
 const useStyle = createStyles(({ token, css }) => {
   const maxWidth = 800;
@@ -54,6 +55,7 @@ const ChatSender: React.FC = () => {
   const { t } = useTranslation();
   const { status } = useSnapshot(state);
   const { prompt } = useSnapshot(sender.state);
+  const { attachments } = useSnapshot(context.state);
 
   const [openPopup, setOpenPopup] = useState(false);
   const [inputText, setInputText] = useState<string>('');
@@ -73,7 +75,10 @@ const ChatSender: React.FC = () => {
   } = useSuggestion();
 
   const handleSubmit = () => {
-    actions.send(prompt, sender.state.delta);
+    actions.send(prompt, {
+      delta: sender.state.delta,
+      attachments: attachments as (ImagePart | FilePart)[],
+    });
     setInputText('');
     sender.actions.updatePrompt('');
     sender.actions.updateDelta(new Delta());

browser/src/state/chat.ts

@@ -76,13 +76,17 @@ interface ChatActions {
     sessionId: string;
     messages: Message[];
   }): Promise<() => void>;
-  send(message: string, delta?: Delta): void;
+  send(
+    message: string,
+    extra: { delta?: Delta; attachments?: (FilePart | ImagePart)[] },
+  ): void;
   addMessage(message: UIMessage | UIMessage[]): void;
   destroy(): void;
   sendMessage(opts: {
     message: string | null;
     planMode?: boolean;
     model?: string;
+    attachments?: (FilePart | ImagePart)[];
   }): Promise<LoopResult | { success: false; error: Error }>;
   getSlashCommands(): Promise<CommandEntry[]>;
   getFiles(opts: { query?: string }): Promise<FileItem[]>;
@@ -255,7 +259,7 @@ export const actions: ChatActions = {
     };
   },
 
-  async send(message, delta: Delta) {
+  async send(message, { delta, attachments }) {
     const { cwd, sessionId } = state;
 
     const isDelta = BLOT_NAME_CONTENT_REGEX.test(message);
@@ -267,13 +271,13 @@ export const actions: ChatActions = {
     });
 
     if (!isDelta) {
-      const result = await this.sendMessage({ message });
+      const result = await this.sendMessage({ message, attachments });
       await this.setSummary({ userPrompt: message, result });
       return;
     }
 
     const isCommand = SLASH_COMMAND_REGEX.test(message);
-    const prompt = getPrompt(delta);
+    const prompt = delta ? getPrompt(delta) : '';
 
     if (!isCommand) {
       await clientActions.request('session.addMessages', {
@@ -287,7 +291,7 @@ export const actions: ChatActions = {
           },
         ],
       });
-      const result = await this.sendMessage({ message: null });
+      const result = await this.sendMessage({ message: null, attachments });
       await this.setSummary({ userPrompt: message, result });
       return;
     }
@@ -349,7 +353,7 @@ export const actions: ChatActions = {
             sessionId,
             messages: messages,
           });
-          await this.sendMessage({ message: null });
+          await this.sendMessage({ message: null, attachments });
         } else if (isLocal) {
           const parsedMessages = messages.map((message) => {
             if (message.role === 'user') {
@@ -385,21 +389,21 @@ export const actions: ChatActions = {
     message: string | null;
     planMode?: boolean;
     model?: string;
+    attachments?: (FilePart | ImagePart)[];
   }) {
     try {
       state.status = 'processing';
       state.processingTokens = 0;
       state.loading = true;
       const { cwd, sessionId } = state;
-      let attachments: Array<FilePart | ImagePart> = [];
 
       const response = (await clientActions.request('session.send', {
         message: opts.message,
         planMode: opts.planMode,
         model: opts.model,
         cwd,
         sessionId,
-        attachments,
+        attachments: opts.attachments,
       })) as LoopResult;
 
       if (response.success) {

browser/src/state/context.ts

@@ -1,15 +1,18 @@
 import { proxy } from 'valtio';
 import { ContextType } from '@/constants/context';
 import * as sender from '@/state/sender';
-import type { FileItem, SlashCommand } from '@/types/chat';
+import type { FileItem, FilePart, ImagePart, SlashCommand } from '@/types/chat';
 import type { ContextItem } from '@/types/context';
+import type { ImageItem } from '@/api/model';
 
 interface ContextState {
   contexts: {
     files: Omit<FileItem, 'name'>[];
     slashCommands: Pick<SlashCommand, 'name' | 'description'>[];
   };
 
+  attachments: (ImagePart | FilePart)[];
+
   attachedContexts: ContextItem[];
 
   loading: boolean;
@@ -53,6 +56,23 @@ export const state = proxy<ContextState>({
       slashCommands,
     };
   },
+
+  get attachments() {
+    // images
+    return this.attachedContexts
+      .filter(
+        (contextItem: ContextItem) => contextItem.type === ContextType.IMAGE,
+      )
+      .map((contextItem: ContextItem) => {
+        const context = contextItem.context as ImageItem;
+
+        return {
+          type: 'image',
+          data: context.src,
+          mimeType: context.mime,
+        } as ImagePart;
+      });
+  },
 });
 
 export const actions = {

browser/src/types/context.ts

@@ -1,8 +1,8 @@
-import type { FileItem, ImageItem } from '@/api/model';
+import type { ImageItem } from '@/api/model';
 import type { ContextType } from '@/constants/context';
-import type { SlashCommand } from '@/types/chat';
+import type { FileItem, SlashCommand } from '@/types/chat';
 
-export type ContextStoreValue = FileItem | ImageItem | SlashCommand;
+export type ContextStoreValue = ImageItem | SlashCommand | FileItem;
 
 export interface ContextItem {
   type: ContextType;