Allow switching between RAG and native file input in GUI

AttachFileMenu is now used for all endpoints, and RAG file uploads are
sent with the `file_search` tool.

Based on the tool, the upload handler can now select between vectorDB or
local files, without need to check for an empty `RAG_API_URL`.

api/server/services/Config/loadAsyncEndpoints.js

@@ -1,4 +1,4 @@
-const { EModelEndpoint } = require('librechat-data-provider');
+const { EModelEndpoint, BaseCapabilities } = require('librechat-data-provider');
 const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs');
 const { availableTools } = require('~/app/clients/tools');
 const { isUserProvided } = require('~/server/utils');
@@ -37,21 +37,25 @@ async function loadAsyncEndpoints(req) {
   }
   const plugins = transformToolsToMap(tools);
 
-  const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false;
+  const google =
+    serviceKey || googleKey
+      ? { userProvide: googleUserProvides, capabilities: [BaseCapabilities.file_search] }
+      : false;
 
   const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins;
   const gptPlugins =
     useAzure || openAIApiKey || azureOpenAIApiKey
       ? {
-        plugins,
-        availableAgents: ['classic', 'functions'],
-        userProvide: useAzure ? false : userProvidedOpenAI,
-        userProvideURL: useAzure
-          ? false
-          : config[EModelEndpoint.openAI]?.userProvideURL ||
+          plugins,
+          availableAgents: ['classic', 'functions'],
+          userProvide: useAzure ? false : userProvidedOpenAI,
+          userProvideURL: useAzure
+            ? false
+            : config[EModelEndpoint.openAI]?.userProvideURL ||
               config[EModelEndpoint.azureOpenAI]?.userProvideURL,
-        azure: useAzurePlugins || useAzure,
-      }
+          azure: useAzurePlugins || useAzure,
+          capabilities: [BaseCapabilities.file_search],
+        }
       : false;
 
   return { google, gptPlugins };

api/server/services/Files/process.js

@@ -379,10 +379,12 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true })
  */
 const processFileUpload = async ({ req, res, metadata }) => {
   const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint);
-  const has_rag = !!process.env.RAG_API_URL;
+  const isSearch = metadata.tool_resource === EToolResources.file_search;
+  console.log('Processing upload', metadata);
+
+  const localSource = isSearch ? FileSources.vectordb : req.app.locals.fileStrategy;
   const assistantSource =
     metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai;
-  const localSource = has_rag ? FileSources.vectordb : req.app.locals.fileStrategy;
 
   const source = isAssistantUpload ? assistantSource : localSource;
 

api/server/utils/handleText.js

@@ -1,6 +1,7 @@
 const path = require('path');
 const crypto = require('crypto');
 const {
+  BaseCapabilities,
   Capabilities,
   EModelEndpoint,
   isAgentsEndpoint,
@@ -181,6 +182,8 @@ function generateConfig(key, baseURL, endpoint) {
     config.userProvideURL = isUserProvided(baseURL);
   }
 
+  // default capabilities:
+  config.capabilities = [BaseCapabilities.file_search];
   const assistants = isAssistantsEndpoint(endpoint);
   const agents = isAgentsEndpoint(endpoint);
   if (assistants) {

client/src/components/Chat/Input/Files/AttachFileMenu.tsx

@@ -1,55 +1,89 @@
 import * as Ariakit from '@ariakit/react';
 import React, { useRef, useState, useMemo } from 'react';
-import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react';
-import { EToolResources, EModelEndpoint } from 'librechat-data-provider';
+import { FileSearch, ImageUpIcon, FileUpIcon, TerminalSquareIcon } from 'lucide-react';
+import {
+  EToolResources,
+  AgentCapabilities,
+  BaseCapabilities,
+  EModelEndpoint,
+  mergeFileConfig,
+  supportsGenericFiles,
+  fileConfig as defaultFileConfig,
+} from 'librechat-data-provider';
 import { useGetEndpointsQuery } from 'librechat-data-provider/react-query';
+import { useGetFileConfig } from '~/data-provider';
 import { FileUpload, TooltipAnchor, DropdownPopup } from '~/components/ui';
 import { AttachmentIcon } from '~/components/svg';
 import { useLocalize } from '~/hooks';
 import { cn } from '~/utils';
 
 interface AttachFileProps {
+  endpoint: EModelEndpoint | null;
   isRTL: boolean;
   disabled?: boolean | null;
   handleFileChange: (event: React.ChangeEvent<HTMLInputElement>) => void;
   setToolResource?: React.Dispatch<React.SetStateAction<string | undefined>>;
 }
 
-const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: AttachFileProps) => {
+const AttachFileMenu = ({
+  endpoint,
+  isRTL,
+  disabled,
+  setToolResource,
+  handleFileChange,
+}: AttachFileProps) => {
   const localize = useLocalize();
   const isUploadDisabled = disabled ?? false;
   const inputRef = useRef<HTMLInputElement>(null);
   const [isPopoverActive, setIsPopoverActive] = useState(false);
   const { data: endpointsConfig } = useGetEndpointsQuery();
+  const { data: fileConfig = defaultFileConfig } = useGetFileConfig({
+    select: (data) => mergeFileConfig(data),
+  });
+
+  const _endpoint = endpoint ?? '';
+
+  const genericFiles = useMemo(() => supportsGenericFiles[_endpoint] ?? false, [endpoint]);
 
   const capabilities = useMemo(
-    () => endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [],
-    [endpointsConfig],
+    () => endpointsConfig?.[_endpoint]?.capabilities ?? [],
+    [endpointsConfig, _endpoint],
+  );
+
+  const fileFilter = useMemo(
+    () => fileConfig.endpoints[_endpoint]?.fileFilter ?? '',
+    [fileConfig, _endpoint],
   );
 
-  const handleUploadClick = (isImage?: boolean) => {
+  const handleUploadClick = (isTool: boolean = true) => {
     if (!inputRef.current) {
       return;
     }
     inputRef.current.value = '';
-    inputRef.current.accept = isImage === true ? 'image/*' : '';
+    inputRef.current.accept = isTool ? '' : fileFilter;
     inputRef.current.click();
     inputRef.current.accept = '';
   };
 
   const dropdownItems = useMemo(() => {
     const items = [
       {
-        label: localize('com_ui_upload_image_input'),
+        label: genericFiles
+          ? localize('com_ui_upload_file_input')
+          : localize('com_ui_upload_image_input'),
         onClick: () => {
           setToolResource?.(undefined);
-          handleUploadClick(true);
+          handleUploadClick(false);
         },
-        icon: <ImageUpIcon className="icon-md" />,
+        icon: genericFiles ? (
+          <FileUpIcon className="icon-md" />
+        ) : (
+          <ImageUpIcon className="icon-md" />
+        ),
       },
     ];
 
-    if (capabilities.includes(EToolResources.file_search)) {
+    if (capabilities.includes(BaseCapabilities.file_search)) {
       items.push({
         label: localize('com_ui_upload_file_search'),
         onClick: () => {
@@ -60,7 +94,7 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta
       });
     }
 
-    if (capabilities.includes(EToolResources.execute_code)) {
+    if (capabilities.includes(AgentCapabilities.execute_code)) {
       items.push({
         label: localize('com_ui_upload_code_files'),
         onClick: () => {
@@ -114,4 +148,4 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta
   );
 };
 
-export default React.memo(AttachFile);
+export default React.memo(AttachFileMenu);

client/src/components/Chat/Input/Files/FileFormWrapper.tsx

@@ -43,21 +43,17 @@ function FileFormWrapper({
   const isUploadDisabled = (disableInputs || endpointFileConfig?.disabled) ?? false;
 
   const renderAttachFile = () => {
-    if (isAgents) {
+    if (isAgents || (endpointSupportsFiles && !isUploadDisabled)) {
       return (
         <AttachFileMenu
+          endpoint={_endpoint}
           isRTL={isRTL}
           disabled={disableInputs}
           setToolResource={setToolResource}
           handleFileChange={handleFileChange}
         />
       );
     }
-    if (endpointSupportsFiles && !isUploadDisabled) {
-      return (
-        <AttachFile isRTL={isRTL} disabled={disableInputs} handleFileChange={handleFileChange} />
-      );
-    }
 
     return null;
   };

client/src/hooks/Files/useFileHandling.ts

@@ -183,14 +183,15 @@ const useFileHandling = (params?: UseFileHandling) => {
       }
     }
 
+    const tool_resource = extendedFile.tool_resource ?? toolResource;
+    if (tool_resource != null) {
+      formData.append('tool_resource', tool_resource);
+    }
+
     if (isAgentsEndpoint(endpoint)) {
       if (!agent_id) {
         formData.append('message_file', 'true');
       }
-      const tool_resource = extendedFile.tool_resource ?? toolResource;
-      if (tool_resource != null) {
-        formData.append('tool_resource', tool_resource);
-      }
       if (conversation?.agent_id != null && formData.get('agent_id') == null) {
         formData.append('agent_id', conversation.agent_id);
       }

client/src/localization/languages/Eng.ts

@@ -205,6 +205,7 @@ export default {
   com_ui_stop: 'Stop',
   com_ui_upload_files: 'Upload files',
   com_ui_upload_type: 'Select Upload Type',
+  com_ui_upload_file_input: 'Upload to provider',
   com_ui_upload_image_input: 'Upload Image',
   com_ui_upload_file_search: 'Upload for File Search',
   com_ui_upload_code_files: 'Upload for Code Interpreter',

packages/data-provider/src/config.ts

@@ -140,11 +140,15 @@ export enum Capabilities {
   tools = 'tools',
 }
 
+export enum BaseCapabilities {
+  file_search = 'file_search',
+}
+
 export enum AgentCapabilities {
   hide_sequential_outputs = 'hide_sequential_outputs',
   end_after_tools = 'end_after_tools',
   execute_code = 'execute_code',
-  file_search = 'file_search',
+  file_search = BaseCapabilities.file_search,
   actions = 'actions',
   tools = 'tools',
 }
@@ -251,6 +255,10 @@ export const endpointSchema = baseEndpointSchema.merge(
     customOrder: z.number().optional(),
     directEndpoint: z.boolean().optional(),
     titleMessageRole: z.string().optional(),
+    capabilities: z
+      .array(z.nativeEnum(BaseCapabilities))
+      .optional()
+      .default([BaseCapabilities.file_search]),
   }),
 );
 

packages/data-provider/src/file-config.ts

@@ -15,6 +15,19 @@ export const supportsFiles = {
   [EModelEndpoint.bedrock]: true,
 };
 
+// Has support for file types other than images
+export const supportsGenericFiles = {
+  [EModelEndpoint.openAI]: false,
+  [EModelEndpoint.google]: true,
+  [EModelEndpoint.assistants]: false,
+  [EModelEndpoint.azureAssistants]: false,
+  [EModelEndpoint.agents]: false,
+  [EModelEndpoint.azureOpenAI]: false,
+  [EModelEndpoint.anthropic]: true,
+  [EModelEndpoint.custom]: false,
+  [EModelEndpoint.bedrock]: false,
+};
+
 export const excelFileTypes = [
   'application/vnd.ms-excel',
   'application/msexcel',
@@ -135,9 +148,11 @@ export const codeInterpreterMimeTypes = [
   imageMimeTypes,
 ];
 
-export const googleMimeTypes = [textMimeTypes, pdfMimeType, imageMimeTypes, audioMimeTypes];
+const googleMimeTypes = [textMimeTypes, pdfMimeType, imageMimeTypes, audioMimeTypes];
+const googleFileFilter = 'text/*,application/pdf,image/*,audio/*';
 
-export const anthropicMimeTypes = [pdfMimeType, imageMimeTypes];
+const anthropicMimeTypes = [pdfMimeType, imageMimeTypes];
+const anthropicFileFilter = 'application/pdf,image/*';
 
 export const codeTypeMapping: { [key: string]: string } = {
   c: 'text/x-c',
@@ -165,11 +180,14 @@ export const megabyte = 1024 * 1024;
 export const mbToBytes = (mb: number): number => mb * megabyte;
 
 const defaultSizeLimit = mbToBytes(512);
+const defaultFileFilter = 'image/*';
+
 const assistantsFileConfig = {
   fileLimit: 10,
   fileSizeLimit: defaultSizeLimit,
   totalSizeLimit: defaultSizeLimit,
   supportedMimeTypes,
+  fileFilter: defaultFileFilter,
   disabled: false,
 };
 
@@ -178,6 +196,7 @@ const googleFileConfig = {
   fileSizeLimit: mbToBytes(20), // Limit for inline files
   totalSizeLimit: mbToBytes(20),
   supportedMimeTypes: googleMimeTypes,
+  fileFilter: googleFileFilter,
   disabled: false,
 };
 
@@ -186,6 +205,7 @@ const anthropicFileConfig = {
   fileSizeLimit: mbToBytes(20), // Limit for inline files
   totalSizeLimit: mbToBytes(20),
   supportedMimeTypes: anthropicMimeTypes,
+  fileFilter: anthropicFileFilter,
   disabled: false,
 };
 
@@ -201,6 +221,7 @@ export const fileConfig = {
       fileSizeLimit: defaultSizeLimit,
       totalSizeLimit: defaultSizeLimit,
       supportedMimeTypes,
+      fileFilter: defaultFileFilter,
       disabled: false,
     },
   },