From 8602e2a11765cff401f87f7a64cc98c6b986b876 Mon Sep 17 00:00:00 2001 From: matiasfnunez Date: Fri, 22 Mar 2024 23:06:35 -0300 Subject: [PATCH] Add images support for Claude 3 Vision (#1562) * Add images support for Claude 3 Vision * Fix condition logic for image_url field in api route --------- Co-authored-by: Mckay Wrigley --- app/api/chat/anthropic/route.ts | 112 +++++++++++++++++++-------- components/chat/chat-input.tsx | 2 +- lib/models/llm/anthropic-llm-list.ts | 6 +- lib/utils.ts | 10 +++ 4 files changed, 93 insertions(+), 37 deletions(-) diff --git a/app/api/chat/anthropic/route.ts b/app/api/chat/anthropic/route.ts index aac392ae86..8de8326bf3 100644 --- a/app/api/chat/anthropic/route.ts +++ b/app/api/chat/anthropic/route.ts @@ -1,27 +1,63 @@ -import { CHAT_SETTING_LIMITS } from "@/lib/chat-setting-limits"; -import { checkApiKey, getServerProfile } from "@/lib/server/server-chat-helpers"; -import { ChatSettings } from "@/types"; -import Anthropic from "@anthropic-ai/sdk"; -import { AnthropicStream, StreamingTextResponse } from "ai"; -import { NextRequest, NextResponse } from "next/server"; +import { CHAT_SETTING_LIMITS } from "@/lib/chat-setting-limits" +import { checkApiKey, getServerProfile } from "@/lib/server/server-chat-helpers" +import { getBase64FromDataURL, getMediaTypeFromDataURL } from "@/lib/utils" +import { ChatSettings } from "@/types" +import Anthropic from "@anthropic-ai/sdk" +import { AnthropicStream, StreamingTextResponse } from "ai" +import { NextRequest, NextResponse } from "next/server" -export const runtime = "edge"; +export const runtime = "edge" export async function POST(request: NextRequest) { - const json = await request.json(); + const json = await request.json() const { chatSettings, messages } = json as { - chatSettings: ChatSettings; - messages: any[]; - }; + chatSettings: ChatSettings + messages: any[] + } try { - const profile = await getServerProfile(); - checkApiKey(profile.anthropic_api_key, "Anthropic"); - let ANTHROPIC_FORMATTED_MESSAGES: any = messages.slice(1); + const profile = await getServerProfile() + + checkApiKey(profile.anthropic_api_key, "Anthropic") + + let ANTHROPIC_FORMATTED_MESSAGES: any = messages.slice(1) + + ANTHROPIC_FORMATTED_MESSAGES = ANTHROPIC_FORMATTED_MESSAGES?.map( + (message: any) => { + const messageContent = + typeof message?.content === "string" + ? [message.content] + : message?.content + + return { + ...message, + content: messageContent.map((content: any) => { + if (typeof content === "string") { + // Handle the case where content is a string + return { type: "text", text: content } + } else if ( + content?.type === "image_url" && + content?.image_url?.length + ) { + return { + type: "image", + source: { + type: "base64", + media_type: getMediaTypeFromDataURL(content.image_url), + data: getBase64FromDataURL(content.image_url) + } + } + } else { + return content + } + }) + } + } + ) const anthropic = new Anthropic({ - apiKey: profile.anthropic_api_key || "", - }); + apiKey: profile.anthropic_api_key || "" + }) try { const response = await anthropic.messages.create({ @@ -29,37 +65,47 @@ export async function POST(request: NextRequest) { messages: ANTHROPIC_FORMATTED_MESSAGES, temperature: chatSettings.temperature, system: messages[0].content, - max_tokens: CHAT_SETTING_LIMITS[chatSettings.model].MAX_TOKEN_OUTPUT_LENGTH, - stream: true, - }); + max_tokens: + CHAT_SETTING_LIMITS[chatSettings.model].MAX_TOKEN_OUTPUT_LENGTH, + stream: true + }) try { - const stream = AnthropicStream(response); - return new StreamingTextResponse(stream); + const stream = AnthropicStream(response) + return new StreamingTextResponse(stream) } catch (error: any) { - console.error("Error parsing Anthropic API response:", error); + console.error("Error parsing Anthropic API response:", error) return new NextResponse( - JSON.stringify({ message: "An error occurred while parsing the Anthropic API response" }), + JSON.stringify({ + message: + "An error occurred while parsing the Anthropic API response" + }), { status: 500 } - ); + ) } } catch (error: any) { - console.error("Error calling Anthropic API:", error); + console.error("Error calling Anthropic API:", error) return new NextResponse( - JSON.stringify({ message: "An error occurred while calling the Anthropic API" }), + JSON.stringify({ + message: "An error occurred while calling the Anthropic API" + }), { status: 500 } - ); + ) } } catch (error: any) { - let errorMessage = error.message || "An unexpected error occurred"; - const errorCode = error.status || 500; + let errorMessage = error.message || "An unexpected error occurred" + const errorCode = error.status || 500 + if (errorMessage.toLowerCase().includes("api key not found")) { - errorMessage = "Anthropic API Key not found. Please set it in your profile settings."; + errorMessage = + "Anthropic API Key not found. Please set it in your profile settings." } else if (errorCode === 401) { - errorMessage = "Anthropic API Key is incorrect. Please fix it in your profile settings."; + errorMessage = + "Anthropic API Key is incorrect. Please fix it in your profile settings." } + return new NextResponse(JSON.stringify({ message: errorMessage }), { - status: errorCode, - }); + status: errorCode + }) } } diff --git a/components/chat/chat-input.tsx b/components/chat/chat-input.tsx index b51b356700..761c6cdcf0 100644 --- a/components/chat/chat-input.tsx +++ b/components/chat/chat-input.tsx @@ -11,6 +11,7 @@ import { import Image from "next/image" import { FC, useContext, useEffect, useRef, useState } from "react" import { useTranslation } from "react-i18next" +import { toast } from "sonner" import { Input } from "../ui/input" import { TextareaAutosize } from "../ui/textarea-autosize" import { ChatCommandInput } from "./chat-command-input" @@ -19,7 +20,6 @@ import { useChatHandler } from "./chat-hooks/use-chat-handler" import { useChatHistoryHandler } from "./chat-hooks/use-chat-history" import { usePromptAndCommand } from "./chat-hooks/use-prompt-and-command" import { useSelectFileHandler } from "./chat-hooks/use-select-file-handler" -import { toast } from "sonner" interface ChatInputProps {} diff --git a/lib/models/llm/anthropic-llm-list.ts b/lib/models/llm/anthropic-llm-list.ts index eeb1ac265e..36f735fab1 100644 --- a/lib/models/llm/anthropic-llm-list.ts +++ b/lib/models/llm/anthropic-llm-list.ts @@ -32,7 +32,7 @@ const CLAUDE_3_HAIKU: LLM = { provider: "anthropic", hostedId: "claude-3-haiku-20240307", platformLink: ANTHROPIC_PLATFORM_LINK, - imageInput: false + imageInput: true } // Claude 3 Sonnet (UPDATED 03/04/24) @@ -42,7 +42,7 @@ const CLAUDE_3_SONNET: LLM = { provider: "anthropic", hostedId: "claude-3-sonnet-20240229", platformLink: ANTHROPIC_PLATFORM_LINK, - imageInput: false + imageInput: true } // Claude 3 Opus (UPDATED 03/04/24) @@ -52,7 +52,7 @@ const CLAUDE_3_OPUS: LLM = { provider: "anthropic", hostedId: "claude-3-opus-20240229", platformLink: ANTHROPIC_PLATFORM_LINK, - imageInput: false + imageInput: true } export const ANTHROPIC_LLM_LIST: LLM[] = [ diff --git a/lib/utils.ts b/lib/utils.ts index faed745045..d7c69c4d88 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -13,3 +13,13 @@ export function formatDate(input: string | number | Date): string { year: "numeric" }) } + +export function getMediaTypeFromDataURL(dataURL: string): string | null { + const matches = dataURL.match(/^data:([A-Za-z-+\/]+);base64/) + return matches ? matches[1] : null +} + +export function getBase64FromDataURL(dataURL: string): string | null { + const matches = dataURL.match(/^data:[A-Za-z-+\/]+;base64,(.*)$/) + return matches ? matches[1] : null +}