From a60c0581ef3722289d51fdd7986d1cfe98b9bfc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=80lex=20torregrosa?= Date: Mon, 30 Dec 2024 17:44:50 +0100 Subject: [PATCH] Allow switching between RAG and native file input in GUI AttachFileMenu is now used for all endpoints, and RAG file uploads are sent with the `file_search` tool. Based on the tool, the upload handler can now select between vectorDB or local files, without need to check for an empty `RAG_API_URL`. --- .../services/Config/loadAsyncEndpoints.js | 24 ++++---- api/server/services/Files/process.js | 6 +- api/server/utils/handleText.js | 3 + .../Chat/Input/Files/AttachFileMenu.tsx | 60 +++++++++++++++---- .../Chat/Input/Files/FileFormWrapper.tsx | 8 +-- client/src/hooks/Files/useFileHandling.ts | 9 +-- client/src/localization/languages/Eng.ts | 1 + packages/data-provider/src/config.ts | 10 +++- packages/data-provider/src/file-config.ts | 25 +++++++- 9 files changed, 108 insertions(+), 38 deletions(-) diff --git a/api/server/services/Config/loadAsyncEndpoints.js b/api/server/services/Config/loadAsyncEndpoints.js index 409b9485de2..b730919a691 100644 --- a/api/server/services/Config/loadAsyncEndpoints.js +++ b/api/server/services/Config/loadAsyncEndpoints.js @@ -1,4 +1,4 @@ -const { EModelEndpoint } = require('librechat-data-provider'); +const { EModelEndpoint, BaseCapabilities } = require('librechat-data-provider'); const { addOpenAPISpecs } = require('~/app/clients/tools/util/addOpenAPISpecs'); const { availableTools } = require('~/app/clients/tools'); const { isUserProvided } = require('~/server/utils'); @@ -37,21 +37,25 @@ async function loadAsyncEndpoints(req) { } const plugins = transformToolsToMap(tools); - const google = serviceKey || googleKey ? { userProvide: googleUserProvides } : false; + const google = + serviceKey || googleKey + ? { userProvide: googleUserProvides, capabilities: [BaseCapabilities.file_search] } + : false; const useAzure = req.app.locals[EModelEndpoint.azureOpenAI]?.plugins; const gptPlugins = useAzure || openAIApiKey || azureOpenAIApiKey ? { - plugins, - availableAgents: ['classic', 'functions'], - userProvide: useAzure ? false : userProvidedOpenAI, - userProvideURL: useAzure - ? false - : config[EModelEndpoint.openAI]?.userProvideURL || + plugins, + availableAgents: ['classic', 'functions'], + userProvide: useAzure ? false : userProvidedOpenAI, + userProvideURL: useAzure + ? false + : config[EModelEndpoint.openAI]?.userProvideURL || config[EModelEndpoint.azureOpenAI]?.userProvideURL, - azure: useAzurePlugins || useAzure, - } + azure: useAzurePlugins || useAzure, + capabilities: [BaseCapabilities.file_search], + } : false; return { google, gptPlugins }; diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 1656b34c603..2ade66457c0 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -379,10 +379,12 @@ const uploadImageBuffer = async ({ req, context, metadata = {}, resize = true }) */ const processFileUpload = async ({ req, res, metadata }) => { const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint); - const has_rag = !!process.env.RAG_API_URL; + const isSearch = metadata.tool_resource === EToolResources.file_search; + console.log('Processing upload', metadata); + + const localSource = isSearch ? FileSources.vectordb : req.app.locals.fileStrategy; const assistantSource = metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai; - const localSource = has_rag ? FileSources.vectordb : req.app.locals.fileStrategy; const source = isAssistantUpload ? assistantSource : localSource; diff --git a/api/server/utils/handleText.js b/api/server/utils/handleText.js index 92f8253fc73..39eb5609f73 100644 --- a/api/server/utils/handleText.js +++ b/api/server/utils/handleText.js @@ -1,6 +1,7 @@ const path = require('path'); const crypto = require('crypto'); const { + BaseCapabilities, Capabilities, EModelEndpoint, isAgentsEndpoint, @@ -181,6 +182,8 @@ function generateConfig(key, baseURL, endpoint) { config.userProvideURL = isUserProvided(baseURL); } + // default capabilities: + config.capabilities = [BaseCapabilities.file_search]; const assistants = isAssistantsEndpoint(endpoint); const agents = isAgentsEndpoint(endpoint); if (assistants) { diff --git a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx index 088cdfaa91a..7d852482db1 100644 --- a/client/src/components/Chat/Input/Files/AttachFileMenu.tsx +++ b/client/src/components/Chat/Input/Files/AttachFileMenu.tsx @@ -1,38 +1,66 @@ import * as Ariakit from '@ariakit/react'; import React, { useRef, useState, useMemo } from 'react'; -import { FileSearch, ImageUpIcon, TerminalSquareIcon } from 'lucide-react'; -import { EToolResources, EModelEndpoint } from 'librechat-data-provider'; +import { FileSearch, ImageUpIcon, FileUpIcon, TerminalSquareIcon } from 'lucide-react'; +import { + EToolResources, + AgentCapabilities, + BaseCapabilities, + EModelEndpoint, + mergeFileConfig, + supportsGenericFiles, + fileConfig as defaultFileConfig, +} from 'librechat-data-provider'; import { useGetEndpointsQuery } from 'librechat-data-provider/react-query'; +import { useGetFileConfig } from '~/data-provider'; import { FileUpload, TooltipAnchor, DropdownPopup } from '~/components/ui'; import { AttachmentIcon } from '~/components/svg'; import { useLocalize } from '~/hooks'; import { cn } from '~/utils'; interface AttachFileProps { + endpoint: EModelEndpoint | null; isRTL: boolean; disabled?: boolean | null; handleFileChange: (event: React.ChangeEvent) => void; setToolResource?: React.Dispatch>; } -const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: AttachFileProps) => { +const AttachFileMenu = ({ + endpoint, + isRTL, + disabled, + setToolResource, + handleFileChange, +}: AttachFileProps) => { const localize = useLocalize(); const isUploadDisabled = disabled ?? false; const inputRef = useRef(null); const [isPopoverActive, setIsPopoverActive] = useState(false); const { data: endpointsConfig } = useGetEndpointsQuery(); + const { data: fileConfig = defaultFileConfig } = useGetFileConfig({ + select: (data) => mergeFileConfig(data), + }); + + const _endpoint = endpoint ?? ''; + + const genericFiles = useMemo(() => supportsGenericFiles[_endpoint] ?? false, [endpoint]); const capabilities = useMemo( - () => endpointsConfig?.[EModelEndpoint.agents]?.capabilities ?? [], - [endpointsConfig], + () => endpointsConfig?.[_endpoint]?.capabilities ?? [], + [endpointsConfig, _endpoint], + ); + + const fileFilter = useMemo( + () => fileConfig.endpoints[_endpoint]?.fileFilter ?? '', + [fileConfig, _endpoint], ); - const handleUploadClick = (isImage?: boolean) => { + const handleUploadClick = (isTool: boolean = true) => { if (!inputRef.current) { return; } inputRef.current.value = ''; - inputRef.current.accept = isImage === true ? 'image/*' : ''; + inputRef.current.accept = isTool ? '' : fileFilter; inputRef.current.click(); inputRef.current.accept = ''; }; @@ -40,16 +68,22 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta const dropdownItems = useMemo(() => { const items = [ { - label: localize('com_ui_upload_image_input'), + label: genericFiles + ? localize('com_ui_upload_file_input') + : localize('com_ui_upload_image_input'), onClick: () => { setToolResource?.(undefined); - handleUploadClick(true); + handleUploadClick(false); }, - icon: , + icon: genericFiles ? ( + + ) : ( + + ), }, ]; - if (capabilities.includes(EToolResources.file_search)) { + if (capabilities.includes(BaseCapabilities.file_search)) { items.push({ label: localize('com_ui_upload_file_search'), onClick: () => { @@ -60,7 +94,7 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta }); } - if (capabilities.includes(EToolResources.execute_code)) { + if (capabilities.includes(AgentCapabilities.execute_code)) { items.push({ label: localize('com_ui_upload_code_files'), onClick: () => { @@ -114,4 +148,4 @@ const AttachFile = ({ isRTL, disabled, setToolResource, handleFileChange }: Atta ); }; -export default React.memo(AttachFile); +export default React.memo(AttachFileMenu); diff --git a/client/src/components/Chat/Input/Files/FileFormWrapper.tsx b/client/src/components/Chat/Input/Files/FileFormWrapper.tsx index a0310cf7f2a..daef2e797eb 100644 --- a/client/src/components/Chat/Input/Files/FileFormWrapper.tsx +++ b/client/src/components/Chat/Input/Files/FileFormWrapper.tsx @@ -43,9 +43,10 @@ function FileFormWrapper({ const isUploadDisabled = (disableInputs || endpointFileConfig?.disabled) ?? false; const renderAttachFile = () => { - if (isAgents) { + if (isAgents || (endpointSupportsFiles && !isUploadDisabled)) { return ( ); } - if (endpointSupportsFiles && !isUploadDisabled) { - return ( - - ); - } return null; }; diff --git a/client/src/hooks/Files/useFileHandling.ts b/client/src/hooks/Files/useFileHandling.ts index d1e71d8a6d0..88603a73484 100644 --- a/client/src/hooks/Files/useFileHandling.ts +++ b/client/src/hooks/Files/useFileHandling.ts @@ -183,14 +183,15 @@ const useFileHandling = (params?: UseFileHandling) => { } } + const tool_resource = extendedFile.tool_resource ?? toolResource; + if (tool_resource != null) { + formData.append('tool_resource', tool_resource); + } + if (isAgentsEndpoint(endpoint)) { if (!agent_id) { formData.append('message_file', 'true'); } - const tool_resource = extendedFile.tool_resource ?? toolResource; - if (tool_resource != null) { - formData.append('tool_resource', tool_resource); - } if (conversation?.agent_id != null && formData.get('agent_id') == null) { formData.append('agent_id', conversation.agent_id); } diff --git a/client/src/localization/languages/Eng.ts b/client/src/localization/languages/Eng.ts index 3b309af9fd1..716cb9e7a2f 100644 --- a/client/src/localization/languages/Eng.ts +++ b/client/src/localization/languages/Eng.ts @@ -205,6 +205,7 @@ export default { com_ui_stop: 'Stop', com_ui_upload_files: 'Upload files', com_ui_upload_type: 'Select Upload Type', + com_ui_upload_file_input: 'Upload to provider', com_ui_upload_image_input: 'Upload Image', com_ui_upload_file_search: 'Upload for File Search', com_ui_upload_code_files: 'Upload for Code Interpreter', diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 7a524682745..16f883df66f 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -140,11 +140,15 @@ export enum Capabilities { tools = 'tools', } +export enum BaseCapabilities { + file_search = 'file_search', +} + export enum AgentCapabilities { hide_sequential_outputs = 'hide_sequential_outputs', end_after_tools = 'end_after_tools', execute_code = 'execute_code', - file_search = 'file_search', + file_search = BaseCapabilities.file_search, actions = 'actions', tools = 'tools', } @@ -251,6 +255,10 @@ export const endpointSchema = baseEndpointSchema.merge( customOrder: z.number().optional(), directEndpoint: z.boolean().optional(), titleMessageRole: z.string().optional(), + capabilities: z + .array(z.nativeEnum(BaseCapabilities)) + .optional() + .default([BaseCapabilities.file_search]), }), ); diff --git a/packages/data-provider/src/file-config.ts b/packages/data-provider/src/file-config.ts index b2bfaacbf21..a49a95b8083 100644 --- a/packages/data-provider/src/file-config.ts +++ b/packages/data-provider/src/file-config.ts @@ -15,6 +15,19 @@ export const supportsFiles = { [EModelEndpoint.bedrock]: true, }; +// Has support for file types other than images +export const supportsGenericFiles = { + [EModelEndpoint.openAI]: false, + [EModelEndpoint.google]: true, + [EModelEndpoint.assistants]: false, + [EModelEndpoint.azureAssistants]: false, + [EModelEndpoint.agents]: false, + [EModelEndpoint.azureOpenAI]: false, + [EModelEndpoint.anthropic]: true, + [EModelEndpoint.custom]: false, + [EModelEndpoint.bedrock]: false, +}; + export const excelFileTypes = [ 'application/vnd.ms-excel', 'application/msexcel', @@ -135,9 +148,11 @@ export const codeInterpreterMimeTypes = [ imageMimeTypes, ]; -export const googleMimeTypes = [textMimeTypes, pdfMimeType, imageMimeTypes, audioMimeTypes]; +const googleMimeTypes = [textMimeTypes, pdfMimeType, imageMimeTypes, audioMimeTypes]; +const googleFileFilter = 'text/*,application/pdf,image/*,audio/*'; -export const anthropicMimeTypes = [pdfMimeType, imageMimeTypes]; +const anthropicMimeTypes = [pdfMimeType, imageMimeTypes]; +const anthropicFileFilter = 'application/pdf,image/*'; export const codeTypeMapping: { [key: string]: string } = { c: 'text/x-c', @@ -165,11 +180,14 @@ export const megabyte = 1024 * 1024; export const mbToBytes = (mb: number): number => mb * megabyte; const defaultSizeLimit = mbToBytes(512); +const defaultFileFilter = 'image/*'; + const assistantsFileConfig = { fileLimit: 10, fileSizeLimit: defaultSizeLimit, totalSizeLimit: defaultSizeLimit, supportedMimeTypes, + fileFilter: defaultFileFilter, disabled: false, }; @@ -178,6 +196,7 @@ const googleFileConfig = { fileSizeLimit: mbToBytes(20), // Limit for inline files totalSizeLimit: mbToBytes(20), supportedMimeTypes: googleMimeTypes, + fileFilter: googleFileFilter, disabled: false, }; @@ -186,6 +205,7 @@ const anthropicFileConfig = { fileSizeLimit: mbToBytes(20), // Limit for inline files totalSizeLimit: mbToBytes(20), supportedMimeTypes: anthropicMimeTypes, + fileFilter: anthropicFileFilter, disabled: false, }; @@ -201,6 +221,7 @@ export const fileConfig = { fileSizeLimit: defaultSizeLimit, totalSizeLimit: defaultSizeLimit, supportedMimeTypes, + fileFilter: defaultFileFilter, disabled: false, }, },