✨ feat: support upload images to chat with gpt4-vision model (lobehub…

…#440)

.eslintrc.js

@@ -13,5 +13,6 @@ config.rules['no-extra-boolean-cast'] = 0;
 config.rules['unicorn/no-useless-undefined'] = 0;
 config.rules['react/no-unknown-property'] = 0;
 config.rules['unicorn/prefer-ternary'] = 0;
+config.rules['unicorn/prefer-spread'] = 0;
 
 module.exports = config;

__mocks__/zustand/traditional.ts

@@ -0,0 +1,25 @@
+import { act } from 'react-dom/test-utils';
+import { beforeEach } from 'vitest';
+import { createWithEqualityFn as actualCreate } from 'zustand/traditional';
+
+// a variable to hold reset functions for all stores declared in the app
+const storeResetFns = new Set<() => void>();
+
+// when creating a store, we get its initial state, create a reset function and add it in the set
+const createImpl = (createState: any) => {
+  const store = actualCreate(createState, Object.is);
+  const initialState = store.getState();
+  storeResetFns.add(() => store.setState(initialState, true));
+  return store;
+};
+
+// Reset all stores after each test run
+beforeEach(() => {
+  act(() =>
+    { for (const resetFn of storeResetFns) {
+      resetFn();
+    } },
+  );
+});
+
+export const createWithEqualityFn = (f: any) => (f === undefined ? createImpl : createImpl(f));

package.json

@@ -73,13 +73,14 @@
     "@lobehub/ui": "latest",
     "@vercel/analytics": "^1",
     "ahooks": "^3",
-    "ai": "2.2.20",
+    "ai": "^2.2.22",
     "antd": "^5",
     "antd-style": "^3",
     "brotli-wasm": "^1",
     "chroma-js": "^2",
     "copy-to-clipboard": "^3",
     "dayjs": "^1",
+    "dexie": "^3",
     "fast-deep-equal": "^3",
     "gpt-tokenizer": "^2",
     "i18next": "^23",
@@ -93,7 +94,7 @@
     "modern-screenshot": "^4",
     "nanoid": "^5",
     "next": "14.0.1",
-    "openai": "~4.15.0",
+    "openai": "^4.17.3",
     "polished": "^4",
     "posthog-js": "^1",
     "query-string": "^8",
@@ -104,6 +105,7 @@
     "react-intersection-observer": "^9",
     "react-layout-kit": "^1",
     "react-lazy-load": "^4",
+    "react-spring-lightbox": "^1",
     "remark": "^14",
     "remark-gfm": "^3",
     "remark-html": "^15",
@@ -116,6 +118,7 @@
     "use-merge-value": "^1",
     "utility-types": "^3",
     "uuid": "^9",
+    "zod": "^3",
     "zustand": "^4.4",
     "zustand-utils": "^1"
   },
@@ -144,6 +147,7 @@
     "consola": "^3",
     "dpdm": "^3",
     "eslint": "^8",
+    "fake-indexeddb": "^5",
     "husky": "^8",
     "jsdom": "^22",
     "lint-staged": "^15",

src/app/api/files/image/imgur.ts

@@ -0,0 +1,72 @@
+import { getServerConfig } from '@/config/server';
+
+interface UploadResponse {
+  data: UploadData;
+  status: number;
+  success: boolean;
+}
+
+interface UploadData {
+  account_id: any;
+  account_url: any;
+  ad_type: any;
+  ad_url: any;
+  animated: boolean;
+  bandwidth: number;
+  datetime: number;
+  deletehash: string;
+  description: any;
+  favorite: boolean;
+  has_sound: boolean;
+  height: number;
+  hls: string;
+  id: string;
+  in_gallery: boolean;
+  in_most_viral: boolean;
+  is_ad: boolean;
+  link: string;
+  mp4: string;
+  name: string;
+  nsfw: any;
+  section: any;
+  size: number;
+  tags: any[];
+  title: any;
+  type: string;
+  views: number;
+  vote: any;
+  width: number;
+}
+
+export class Imgur {
+  clientId: string;
+  api = 'https://api.imgur.com/3';
+
+  constructor() {
+    this.clientId = getServerConfig().IMGUR_CLIENT_ID;
+  }
+
+  async upload(image: Blob) {
+    const formData = new FormData();
+
+    formData.append('image', image, 'image.png');
+
+    const res = await fetch(`${this.api}/upload`, {
+      body: formData,
+      headers: {
+        Authorization: `Client-ID ${this.clientId}`,
+      },
+      method: 'POST',
+    });
+
+    if (!res.ok) {
+      console.log(await res.text());
+    }
+
+    const data: UploadResponse = await res.json();
+    if (data.success) {
+      return data.data.link;
+    }
+    return undefined;
+  }
+}

src/app/api/files/image/route.ts

@@ -0,0 +1,42 @@
+import { Imgur } from './imgur';
+
+const updateByImgur = async ({ url, blob }: { blob?: Blob; url?: string }) => {
+  let imageBlob: Blob;
+
+  if (url) {
+    const res = await fetch(url);
+    imageBlob = await res.blob();
+  } else if (blob) {
+    imageBlob = blob;
+  } else {
+    // TODO: error handle
+    return;
+  }
+
+  const imgur = new Imgur();
+
+  return await imgur.upload(imageBlob);
+};
+
+export const POST = async (req: Request) => {
+  const { url } = await req.json();
+
+  const cdnUrl = await updateByImgur({ url });
+
+  return new Response(JSON.stringify({ url: cdnUrl }));
+};
+// import { Imgur } from './imgur';
+
+export const runtime = 'edge';
+
+// export const POST = async (req: Request) => {
+//   const { url } = await req.json();
+//
+//   const imgur = new Imgur();
+//
+//   const image = await fetch(url);
+//
+//   const cdnUrl = await imgur.upload(await image.blob());
+//
+//   return new Response(JSON.stringify({ url: cdnUrl }));
+// };

src/app/api/openai/chat/createChatCompletion.ts

@@ -14,11 +14,12 @@ export const createChatCompletion = async ({ payload, openai }: CreateChatComple
   // ============  1. preprocess messages   ============ //
   const { messages, ...params } = payload;
 
+  // remove unnecessary fields like `plugins` or `files` by lobe-chat
   const formatMessages = messages.map((m) => ({
     content: m.content,
     name: m.name,
     role: m.role,
-  }));
+  })) as OpenAI.ChatCompletionMessageParam[];
 
   // ============  2. send api   ============ //
 

src/app/chat/(desktop)/features/ChatInput/DragUpload.tsx

@@ -0,0 +1,166 @@
+import { Icon } from '@lobehub/ui';
+import { createStyles } from 'antd-style';
+import { FileImage, FileText, FileUpIcon } from 'lucide-react';
+import { rgba } from 'polished';
+import { memo, useEffect, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Center, Flexbox } from 'react-layout-kit';
+
+import { useFileStore } from '@/store/files';
+
+const useStyles = createStyles(({ css, token, stylish }) => {
+  return {
+    container: css`
+      width: 300px;
+      height: 300px;
+      padding: 16px;
+
+      color: ${token.colorWhite};
+
+      background: ${token.geekblue};
+      border-radius: 16px;
+      box-shadow:
+        ${rgba(token.geekblue, 0.1)} 0 1px 1px 0 inset,
+        ${rgba(token.geekblue, 0.1)} 0 50px 100px -20px,
+        ${rgba(token.geekblue, 0.3)} 0 30px 60px -30px;
+    `,
+    content: css`
+      width: 100%;
+      height: 100%;
+      padding: 16px;
+
+      border: 2px dashed ${token.colorWhite};
+      border-radius: 12px;
+    `,
+    desc: css`
+      color: ${rgba(token.colorTextLightSolid, 0.6)};
+    `,
+    title: css`
+      font-size: 24px;
+      font-weight: bold;
+    `,
+    wrapper: css`
+      position: fixed;
+      z-index: 10000000;
+      top: 0;
+      left: 0;
+
+      width: 100%;
+      height: 100%;
+
+      transition: all 0.3s ease-in-out;
+
+      background: ${token.colorBgMask};
+      ${stylish.blur};
+    `,
+  };
+});
+
+const handleDragOver = (e: DragEvent) => {
+  e.preventDefault();
+};
+
+const DragUpload = memo(() => {
+  const { styles } = useStyles();
+  const { t } = useTranslation('chat');
+  const [isDragging, setIsDragging] = useState(false);
+  // When a file is dragged to a different area, the 'dragleave' event may be triggered,
+  // causing isDragging to be mistakenly set to false.
+  // to fix this issue, use a counter to ensure the status change only when drag event left the browser window .
+  const dragCounter = useRef(0);
+
+  const uploadFile = useFileStore((s) => s.uploadFile);
+
+  const uploadImages = async (fileList: FileList | undefined) => {
+    if (!fileList || fileList.length === 0) return;
+
+    const pools = Array.from(fileList).map(async (file) => {
+      // skip none-file items
+      if (!file.type.startsWith('image')) return;
+      await uploadFile(file);
+    });
+
+    await Promise.all(pools);
+  };
+
+  const handleDragEnter = (e: DragEvent) => {
+    e.preventDefault();
+
+    dragCounter.current += 1;
+    if (e.dataTransfer?.items && e.dataTransfer.items.length > 0) {
+      setIsDragging(true);
+    }
+  };
+
+  const handleDragLeave = (e: DragEvent) => {
+    e.preventDefault();
+
+    // reset counter
+    dragCounter.current -= 1;
+
+    if (dragCounter.current === 0) {
+      setIsDragging(false);
+    }
+  };
+
+  const handleDrop = async (e: DragEvent) => {
+    e.preventDefault();
+    // reset counter
+    dragCounter.current = 0;
+
+    setIsDragging(false);
+
+    // get filesList
+    // TODO: support folder files upload
+    const files = e.dataTransfer?.files;
+
+    // upload files
+    uploadImages(files);
+  };
+
+  const handlePaste = (event: ClipboardEvent) => {
+    // get files from clipboard
+
+    const files = event.clipboardData?.files;
+
+    uploadImages(files);
+  };
+
+  useEffect(() => {
+    window.addEventListener('dragenter', handleDragEnter);
+    window.addEventListener('dragover', handleDragOver);
+    window.addEventListener('dragleave', handleDragLeave);
+    window.addEventListener('drop', handleDrop);
+    window.addEventListener('paste', handlePaste);
+
+    return () => {
+      window.removeEventListener('dragenter', handleDragEnter);
+      window.removeEventListener('dragover', handleDragOver);
+      window.removeEventListener('dragleave', handleDragLeave);
+      window.removeEventListener('drop', handleDrop);
+      window.removeEventListener('paste', handlePaste);
+    };
+  }, []);
+
+  return (
+    isDragging && (
+      <Center className={styles.wrapper}>
+        <div className={styles.container}>
+          <Center className={styles.content} gap={40}>
+            <Flexbox horizontal>
+              <Icon icon={FileImage} size={{ fontSize: 64, strokeWidth: 1 }} />
+              <Icon icon={FileUpIcon} size={{ fontSize: 64, strokeWidth: 1 }} />
+              <Icon icon={FileText} size={{ fontSize: 64, strokeWidth: 1 }} />
+            </Flexbox>
+            <Flexbox align={'center'} gap={8} style={{ textAlign: 'center' }}>
+              <Flexbox className={styles.title}>{t('upload.dragTitle')}</Flexbox>
+              <Flexbox className={styles.desc}>{t('upload.dragDesc')}</Flexbox>
+            </Flexbox>
+          </Center>
+        </div>
+      </Center>
+    )
+  );
+});
+
+export default DragUpload;