forked from vercel/ai
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
ccea4b8
commit 559be1f
Showing
26 changed files
with
562 additions
and
234 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,4 +19,4 @@ | |
| "sidebar": false | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,6 @@ | ||
| { | ||
| "index": "Introduction", | ||
| "getting-started": "Getting Started", | ||
| "guides": "Guides", | ||
| "api": "API Reference" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| --- | ||
| title: API Reference | ||
| --- | ||
|
|
||
| # API Reference | ||
|
|
||
| ## `OpenAIStream(res: Response, cb: AIStreamCallbacks): ReadableStream` | ||
|
|
||
| A transform that will extract the text from all chat and completion OpenAI models as returned as a `ReadableStream`. | ||
|
|
||
| ```tsx | ||
| // app/api/generate/route.ts | ||
| import { Configuration, OpenAIApi } from 'openai-edge'; | ||
| import { OpenAITextStream, StreamingTextResponse } from '@vercel/ai-utils'; | ||
|
|
||
| const config = new Configuration({ | ||
| apiKey: process.env.OPENAI_API_KEY, | ||
| }); | ||
| const openai = new OpenAIApi(config); | ||
|
|
||
| export const runtime = 'edge'; | ||
|
|
||
| export async function POST() { | ||
| const response = await openai.createChatCompletion({ | ||
| model: 'gpt-4', | ||
| stream: true, | ||
| messages: [{ role: 'user', content: 'What is love?' }], | ||
| }); | ||
| const stream = OpenAITextStream(response, { | ||
| async onStart() { | ||
| console.log('streamin yo') | ||
| }, | ||
| async onToken(token) { | ||
| console.log('token: ' + token) | ||
| }, | ||
| async onCompletion(content) { | ||
| console.log('full text: ' + ) | ||
| // await prisma.messages.create({ content }) or something | ||
| } | ||
| }); | ||
| return new StreamingTextResponse(stream); | ||
| } | ||
| ``` | ||
|
|
||
| ## `HuggingFaceStream(iter: AsyncGenerator<any>, cb?: AIStreamCallbacks): ReadableStream` | ||
|
|
||
| A transform that will extract the text from _most_ chat and completion HuggingFace models and return them as a `ReadableStream`. | ||
|
|
||
| ```tsx | ||
| // app/api/generate/route.ts | ||
| import { HfInference } from '@huggingface/inference'; | ||
| import { HuggingFaceStream, StreamingTextResponse } from '@vercel/ai-utils'; | ||
|
|
||
| export const runtime = 'edge'; | ||
|
|
||
| const Hf = new HfInference(process.env.HUGGINGFACE_API_KEY); | ||
|
|
||
| export async function POST() { | ||
| const response = await Hf.textGenerationStream({ | ||
| model: 'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5', | ||
| inputs: `<|prompter|>What's the Earth total population?<|endoftext|><|assistant|>`, | ||
| parameters: { | ||
| max_new_tokens: 200, | ||
| // @ts-ignore | ||
| typical_p: 0.2, // you'll need this for OpenAssistant | ||
| repetition_penalty: 1, | ||
| truncate: 1000, | ||
| return_full_text: false, | ||
| }, | ||
| }); | ||
| const stream = HuggingFaceStream(response); | ||
| return new StreamingTextResponse(stream); | ||
| } | ||
| ``` | ||
|
|
||
| ## `StreamingTextResponse(res: ReadableStream, init?: ResponseInit)` | ||
|
|
||
| This is a tiny wrapper around `Response` class that makes returning `ReadableStreams` of text a one liner. Status is automatically set to `200`, with `'Content-Type': 'text/plain; charset=utf-8'` set as `headers`. | ||
|
|
||
| ```tsx | ||
| // app/api/generate/route.ts | ||
| import { OpenAITextStream, StreamingTextResponse } from '@vercel/ai-utils'; | ||
|
|
||
| export const runtime = 'edge'; | ||
|
|
||
| export async function POST() { | ||
| const response = await openai.createChatCompletion({ | ||
| model: 'gpt-4', | ||
| stream: true, | ||
| messages: { role: 'user', content: 'What is love?' }, | ||
| }); | ||
| const stream = OpenAITextStream(response); | ||
| return new StreamingTextResponse(stream, { | ||
| 'X-RATE-LIMIT': 'lol', | ||
| }); // => new Response(stream, { status: 200, headers: { 'Content-Type': 'text/plain; charset=utf-8', 'X-RATE-LIMIT': 'lol' }}) | ||
| } | ||
| ``` |
Oops, something went wrong.