feat: Supports chatting with files/images #1880 (#1943)

### What problem does this PR solve?

feat: Supports chatting with files/images #1880

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/components/indented-tree/modal.tsx

@@ -0,0 +1,31 @@
+import { useFetchKnowledgeGraph } from '@/hooks/chunk-hooks';
+import { Modal } from 'antd';
+import { useTranslation } from 'react-i18next';
+import IndentedTree from './indented-tree';
+
+import { IModalProps } from '@/interfaces/common';
+
+const IndentedTreeModal = ({
+  documentId,
+  visible,
+  hideModal,
+}: IModalProps<any> & { documentId: string }) => {
+  const { data } = useFetchKnowledgeGraph(documentId);
+  const { t } = useTranslation();
+
+  return (
+    <Modal
+      title={t('chunk.graph')}
+      open={visible}
+      onCancel={hideModal}
+      width={'90vw'}
+      footer={null}
+    >
+      <section>
+        <IndentedTree data={data?.data?.mind_map} show></IndentedTree>
+      </section>
+    </Modal>
+  );
+};
+
+export default IndentedTreeModal;

web/src/components/message-input/index.tsx

@@ -0,0 +1,129 @@
+import { Authorization } from '@/constants/authorization';
+import { useTranslate } from '@/hooks/common-hooks';
+import { getAuthorization } from '@/utils/authorization-util';
+import { PlusOutlined } from '@ant-design/icons';
+import type { GetProp, UploadFile } from 'antd';
+import { Button, Flex, Input, Upload, UploadProps } from 'antd';
+import get from 'lodash/get';
+import { ChangeEventHandler, useCallback, useState } from 'react';
+
+type FileType = Parameters<GetProp<UploadProps, 'beforeUpload'>>[0];
+
+interface IProps {
+  disabled: boolean;
+  value: string;
+  sendDisabled: boolean;
+  sendLoading: boolean;
+  onPressEnter(documentIds: string[]): Promise<any>;
+  onInputChange: ChangeEventHandler<HTMLInputElement>;
+  conversationId: string;
+}
+
+const getBase64 = (file: FileType): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file as any);
+    reader.onload = () => resolve(reader.result as string);
+    reader.onerror = (error) => reject(error);
+  });
+
+const MessageInput = ({
+  disabled,
+  value,
+  onPressEnter,
+  sendDisabled,
+  sendLoading,
+  onInputChange,
+  conversationId,
+}: IProps) => {
+  const { t } = useTranslate('chat');
+
+  const [fileList, setFileList] = useState<UploadFile[]>([
+    // {
+    //   uid: '-1',
+    //   name: 'image.png',
+    //   status: 'done',
+    //   url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
+    // },
+    // {
+    //   uid: '-xxx',
+    //   percent: 50,
+    //   name: 'image.png',
+    //   status: 'uploading',
+    //   url: 'https://zos.alipayobjects.com/rmsportal/jkjgkEfvpUPVyRjUImniVslZfWPnJuuZ.png',
+    // },
+    // {
+    //   uid: '-5',
+    //   name: 'image.png',
+    //   status: 'error',
+    // },
+  ]);
+
+  const handlePreview = async (file: UploadFile) => {
+    if (!file.url && !file.preview) {
+      file.preview = await getBase64(file.originFileObj as FileType);
+    }
+
+    // setPreviewImage(file.url || (file.preview as string));
+    // setPreviewOpen(true);
+  };
+
+  const handleChange: UploadProps['onChange'] = ({ fileList: newFileList }) => {
+    console.log('🚀 ~ newFileList:', newFileList);
+    setFileList(newFileList);
+  };
+
+  const handlePressEnter = useCallback(async () => {
+    const ids = fileList.reduce((pre, cur) => {
+      return pre.concat(get(cur, 'response.data', []));
+    }, []);
+
+    await onPressEnter(ids);
+    setFileList([]);
+  }, [fileList, onPressEnter]);
+
+  const uploadButton = (
+    <button style={{ border: 0, background: 'none' }} type="button">
+      <PlusOutlined />
+      <div style={{ marginTop: 8 }}>Upload</div>
+    </button>
+  );
+
+  return (
+    <Flex gap={10} vertical>
+      <Input
+        size="large"
+        placeholder={t('sendPlaceholder')}
+        value={value}
+        disabled={disabled}
+        suffix={
+          <Button
+            type="primary"
+            onClick={handlePressEnter}
+            loading={sendLoading}
+            disabled={sendDisabled}
+          >
+            {t('send')}
+          </Button>
+        }
+        onPressEnter={handlePressEnter}
+        onChange={onInputChange}
+      />
+      <Upload
+        action="/v1/document/upload_and_parse"
+        listType="picture-card"
+        fileList={fileList}
+        onPreview={handlePreview}
+        onChange={handleChange}
+        multiple
+        headers={{ [Authorization]: getAuthorization() }}
+        data={{ conversation_id: conversationId }}
+        method="post"
+      >
+        {fileList.length >= 8 ? null : uploadButton}
+      </Upload>
+    </Flex>
+  );
+};
+
+export default MessageInput;

web/src/components/message-item/index.tsx

@@ -1,15 +1,17 @@
 import { ReactComponent as AssistantIcon } from '@/assets/svg/assistant.svg';
 import { MessageType } from '@/constants/chat';
-import { useTranslate } from '@/hooks/common-hooks';
+import { useSetModalState, useTranslate } from '@/hooks/common-hooks';
 import { useSelectFileThumbnails } from '@/hooks/knowledge-hooks';
 import { IReference, Message } from '@/interfaces/database/chat';
 import { IChunk } from '@/interfaces/database/knowledge';
 import classNames from 'classnames';
-import { useMemo } from 'react';
+import { memo, useCallback, useEffect, useMemo, useState } from 'react';
 
+import { useFetchDocumentInfosByIds } from '@/hooks/document-hooks';
 import MarkdownContent from '@/pages/chat/markdown-content';
-import { getExtension } from '@/utils/document-util';
-import { Avatar, Flex, List } from 'antd';
+import { getExtension, isImage } from '@/utils/document-util';
+import { Avatar, Button, Flex, List } from 'antd';
+import IndentedTreeModal from '../indented-tree/modal';
 import NewDocumentLink from '../new-document-link';
 import SvgIcon from '../svg-icon';
 import styles from './index.less';
@@ -32,8 +34,13 @@ const MessageItem = ({
   clickDocumentButton,
 }: IProps) => {
   const isAssistant = item.role === MessageType.Assistant;
+  const isUser = item.role === MessageType.User;
   const { t } = useTranslate('chat');
   const fileThumbnails = useSelectFileThumbnails();
+  const { data: documentList, setDocumentIds } = useFetchDocumentInfosByIds();
+  console.log('🚀 ~ documentList:', documentList);
+  const { visible, hideModal, showModal } = useSetModalState();
+  const [clickedDocumentId, setClickedDocumentId] = useState('');
 
   const referenceDocumentList = useMemo(() => {
     return reference?.doc_aggs ?? [];
@@ -47,6 +54,21 @@ const MessageItem = ({
     return loading ? text?.concat('~~2$$') : text;
   }, [item.content, loading, t]);
 
+  const handleUserDocumentClick = useCallback(
+    (id: string) => () => {
+      setClickedDocumentId(id);
+      showModal();
+    },
+    [showModal],
+  );
+
+  useEffect(() => {
+    const ids = item?.doc_ids ?? [];
+    if (ids.length) {
+      setDocumentIds(ids);
+    }
+  }, [item.doc_ids, setDocumentIds]);
+
   return (
     <div
       className={classNames(styles.messageItem, {
@@ -124,11 +146,62 @@ const MessageItem = ({
                 }}
               />
             )}
+            {isUser && documentList.length > 0 && (
+              <List
+                bordered
+                dataSource={documentList}
+                renderItem={(item) => {
+                  const fileThumbnail = fileThumbnails[item.id];
+                  const fileExtension = getExtension(item.name);
+                  return (
+                    <List.Item>
+                      <Flex gap={'small'} align="center">
+                        {fileThumbnail ? (
+                          <img
+                            src={fileThumbnail}
+                            className={styles.thumbnailImg}
+                          ></img>
+                        ) : (
+                          <SvgIcon
+                            name={`file-icon/${fileExtension}`}
+                            width={24}
+                          ></SvgIcon>
+                        )}
+
+                        {isImage(fileExtension) ? (
+                          <NewDocumentLink
+                            documentId={item.id}
+                            documentName={item.name}
+                            prefix="document"
+                          >
+                            {item.name}
+                          </NewDocumentLink>
+                        ) : (
+                          <Button
+                            type={'text'}
+                            onClick={handleUserDocumentClick(item.id)}
+                          >
+                            {item.name}
+                          </Button>
+                        )}
+                      </Flex>
+                    </List.Item>
+                  );
+                }}
+              />
+            )}
           </Flex>
         </div>
       </section>
+      {visible && (
+        <IndentedTreeModal
+          visible={visible}
+          hideModal={hideModal}
+          documentId={clickedDocumentId}
+        ></IndentedTreeModal>
+      )}
     </div>
   );
 };
 
-export default MessageItem;
+export default memo(MessageItem);

web/src/hooks/chunk-hooks.ts

@@ -207,12 +207,13 @@ export const useFetchChunk = (chunkId?: string): ResponseType<any> => {
   return data;
 };
 
-export const useFetchKnowledgeGraph = (): ResponseType<any> => {
-  const { documentId } = useGetKnowledgeSearchParams();
-
+export const useFetchKnowledgeGraph = (
+  documentId: string,
+): ResponseType<any> => {
   const { data } = useQuery({
     queryKey: ['fetchKnowledgeGraph', documentId],
     initialData: true,
+    enabled: !!documentId,
     gcTime: 0,
     queryFn: async () => {
       const data = await kbService.knowledge_graph({

web/src/hooks/common-hooks.tsx

@@ -7,16 +7,16 @@ import { useTranslation } from 'react-i18next';
 export const useSetModalState = () => {
   const [visible, setVisible] = useState(false);
 
-  const showModal = () => {
+  const showModal = useCallback(() => {
     setVisible(true);
-  };
-  const hideModal = () => {
+  }, []);
+  const hideModal = useCallback(() => {
     setVisible(false);
-  };
+  }, []);
 
-  const switchVisible = () => {
+  const switchVisible = useCallback(() => {
     setVisible(!visible);
-  };
+  }, [visible]);
 
   return { visible, showModal, hideModal, switchVisible };
 };

web/src/hooks/document-hooks.ts

@@ -1,7 +1,10 @@
+import { IDocumentInfo } from '@/interfaces/database/document';
 import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
+import kbService from '@/services/knowledge-service';
 import { api_host } from '@/utils/api';
 import { buildChunkHighlights } from '@/utils/document-util';
+import { useQuery } from '@tanstack/react-query';
 import { UploadFile } from 'antd';
 import { useCallback, useMemo, useState } from 'react';
 import { IHighlight } from 'react-pdf-highlighter';
@@ -253,3 +256,37 @@ export const useSelectRunDocumentLoading = () => {
   const loading = useOneNamespaceEffectsLoading('kFModel', ['document_run']);
   return loading;
 };
+
+export const useFetchDocumentInfosByIds = () => {
+  const [ids, setDocumentIds] = useState<string[]>([]);
+  const { data } = useQuery<IDocumentInfo[]>({
+    queryKey: ['fetchDocumentInfos', ids],
+    enabled: ids.length > 0,
+    initialData: [],
+    queryFn: async () => {
+      const { data } = await kbService.document_infos({ doc_ids: ids });
+      if (data.retcode === 0) {
+        return data.data;
+      }
+
+      return [];
+    },
+  });
+
+  return { data, setDocumentIds };
+};
+
+export const useFetchDocumentThumbnailsByIds = () => {
+  const [ids, setDocumentIds] = useState<string[]>([]);
+  const { data } = useQuery({
+    queryKey: ['fetchDocumentThumbnails', ids],
+    initialData: [],
+    queryFn: async () => {
+      const { data } = await kbService.document_thumbnails({ doc_ids: ids });
+
+      return data;
+    },
+  });
+
+  return { data, setDocumentIds };
+};

web/src/interfaces/database/chat.ts

@@ -66,6 +66,7 @@ export interface IConversation {
 export interface Message {
   content: string;
   role: MessageType;
+  doc_ids?: string[];
 }
 
 export interface IReference {