diff --git a/index.ts b/index.ts index 87940538..a7136379 100644 --- a/index.ts +++ b/index.ts @@ -55,7 +55,8 @@ const AskHandle: Middleware = async (ctx) => { ctx.body = {error: `${site} not support model ${model}`} as AskRes; return; } - ctx.body = await chat.ask({prompt: PromptToString(prompt, tokenLimit), model}); + const [content, messages] = PromptToString(prompt, tokenLimit); + ctx.body = await chat.ask({prompt: content, messages, model}); } const AskStreamHandle: (ESType: new () => EventStream) => Middleware = (ESType) => async (ctx) => { @@ -88,7 +89,8 @@ const AskStreamHandle: (ESType: new () => EventStream) => Middleware = (ESType) es.end(); return; } - await chat.askStream({prompt: PromptToString(prompt, tokenLimit), model}, es); + const [content, messages] = PromptToString(prompt, tokenLimit); + await chat.askStream({prompt: content, messages, model}, es); ctx.body = es.stream(); } @@ -111,7 +113,7 @@ router.get('/supports', (ctx) => { const site = Site[key]; //@ts-ignore const chat = chatModel.get(site); - const support: Support = {site:site, models: []} + const support: Support = {site: site, models: []} for (const mKey in ModelType) { //@ts-ignore const model = ModelType[mKey]; @@ -145,14 +147,14 @@ const openAIHandle: Middleware = async (ctx, next) => { "index": 0, "message": { "role": "assistant", - "content": ctx.body.content, + "content": ctx.body.content || ctx.body.error, }, "finish_reason": "stop" }], "usage": { "prompt_tokens": 100, - "completion_tokens": getTokenSize(ctx.body.content), - "total_tokens": 100 + getTokenSize(ctx.body.content) + "completion_tokens": getTokenSize(ctx.body.content || ''), + "total_tokens": 100 + getTokenSize(ctx.body.content || '') } } }; diff --git a/model/base.ts b/model/base.ts index 55fddab9..0fe50628 100644 --- a/model/base.ts +++ b/model/base.ts @@ -17,33 +17,34 @@ export enum ModelType { GPT3p5Turbo = 'gpt-3.5-turbo', GPT3p5_16k = 'gpt-3.5-turbo-16k', GPT4 = 'gpt-4', + GPT4_32k = 'gpt-4-32k', + Sage = 'sage', NetGpt3p5 = 'net-gpt3.5-turbo', + ClaudeInstance = 'claude-instance', + Claude = 'claude', + Claude100k = 'claude-100k', + Claude2_100k = 'claude-2-100k', + Gpt4free = 'gpt4free', + GooglePalm = 'google-palm', } export interface ChatRequest { prompt: string; model: ModelType; + messages: Message[]; } -export function PromptToString(prompt: string, limit: number): string { +export function PromptToString(prompt: string, limit: number): [string, Message[]] { try { const messages: Message[] = JSON.parse(prompt); - let result: Message[] = []; - let tokenSize = 0; - for (let i = messages.length - 1; i >= 0; i--) { - const item = messages[i]; - const {role, content} = item; - tokenSize += getTokenSize(content); - if (tokenSize > limit) { - break; - } - result.push(item); + const res = `${messages.map(item => `${item.role}: ${item.content}`).join('\n')}\nassistant: `; + console.log(prompt.length, limit, getTokenSize(res)); + if (getTokenSize(res) >= limit && messages.length > 1) { + return PromptToString(JSON.stringify(messages.slice(1, messages.length)), limit); } - return `${result.reverse().map(item => `${item.role} - : ${item.content} - `).join('\n')}\nassistant: `; + return [res, messages]; } catch (e) { - return prompt; + return [prompt, [{role: 'user', content: prompt}]]; } } diff --git a/model/index.ts b/model/index.ts index e3efb95e..315d08ba 100644 --- a/model/index.ts +++ b/model/index.ts @@ -6,6 +6,7 @@ import {Phind} from "./phind"; import {Vita} from "./vita"; import {FakeOpen} from "./fakeopen"; import {Better} from "./better"; +import {Xun} from "./xun"; export enum Site { // define new model here @@ -16,6 +17,7 @@ export enum Site { Vita = 'vita', FakeOpen = 'fakeopen', Better = 'better', + Xun = 'xun', } export class ChatModelFactory { @@ -37,6 +39,7 @@ export class ChatModelFactory { this.modelMap.set(Site.Vita, new Vita(this.options)) this.modelMap.set(Site.FakeOpen, new FakeOpen(this.options)) this.modelMap.set(Site.Better, new Better(this.options)) + this.modelMap.set(Site.Xun, new Xun(this.options)) } get(model: Site): Chat | undefined { diff --git a/model/xun/index.ts b/model/xun/index.ts new file mode 100644 index 00000000..a8b2f243 --- /dev/null +++ b/model/xun/index.ts @@ -0,0 +1,110 @@ +import {Chat, ChatOptions, ChatRequest, ChatResponse, Message, ModelType} from "../base"; +import {AxiosInstance, AxiosRequestConfig, CreateAxiosDefaults} from "axios"; +import {CreateAxiosProxy} from "../../utils/proxyAgent"; +import es from "event-stream"; +import {ErrorData, Event, EventStream, MessageData, parseJSON} from "../../utils"; + +interface RealReq { + messages: Message[]; + model: string; + temperature: number; + presence_penalty: number; + top_p: number; + frequency_penalty: number; + stream: boolean; +} + +export class Xun extends Chat { + private client: AxiosInstance; + + constructor(options?: ChatOptions) { + super(options); + this.client = CreateAxiosProxy({ + baseURL: 'https://gpt4.xunika.uk/api/openai', + headers: { + 'Content-Type': 'application/json', + "Cache-Control": "no-cache", + "Proxy-Connection": "keep-alive" + } + } as CreateAxiosDefaults); + } + + support(model: ModelType): number { + switch (model) { + case ModelType.GPT3p5Turbo: + return 3000; + case ModelType.GPT3p5_16k: + return 15000; + default: + return 0; + } + } + + public async ask(req: ChatRequest): Promise { + const stream = new EventStream(); + const res = await this.askStream(req, stream); + const result: ChatResponse = { + content: '', + } + return new Promise(resolve => { + stream.read((event, data) => { + switch (event) { + case Event.done: + break; + case Event.message: + result.content += (data as MessageData).content || ''; + break; + case Event.error: + result.error = (data as ErrorData).error; + break; + } + }, () => { + resolve(result); + }) + }) + + } + + public async askStream(req: ChatRequest, stream: EventStream) { + const data: RealReq = { + "messages": req.messages, + "model": req.model, + "temperature": 1, + "presence_penalty": 0, + "top_p": 1, + "frequency_penalty": 0, + "stream": true + }; + try { + const res = await this.client.post('/v1/chat/completions', data, { + responseType: 'stream', + } as AxiosRequestConfig); + res.data.pipe(es.split(/\r?\n\r?\n/)).pipe(es.map(async (chunk: any, cb: any) => { + const dataStr = chunk.replace('data: ', ''); + if (!dataStr) { + return; + } + if (dataStr === '[DONE]') { + stream.write(Event.done, {content: ""}) + stream.end(); + return; + } + const data = parseJSON(dataStr, {} as any); + if (!data?.choices) { + stream.write(Event.error, {error: 'not found data.choices'}) + stream.end(); + return; + } + const [{delta: {content = ""}, finish_reason}] = data.choices; + if (finish_reason === 'stop') { + return; + } + stream.write(Event.message, {content}); + })) + } catch (e: any) { + console.error(e); + stream.write(Event.error, {error: e.message}) + stream.end(); + } + } +}