diff --git a/assets/box-icon.png b/assets/box-icon.png new file mode 100644 index 0000000..50f7470 Binary files /dev/null and b/assets/box-icon.png differ diff --git a/package.json b/package.json index 95b181f..84b530b 100755 --- a/package.json +++ b/package.json @@ -176,12 +176,19 @@ "description": "Check for extension updates from the official GitHub source", "icon": "update-icon.png", "mode": "view" + }, + { + "name": "configureG4FLocalApi", + "title": "Configure GPT4Free Local API", + "description": "Configure GPT4Free Local API settings", + "icon": "box-icon.png", + "mode": "view" } ], "preferences": [ { "name": "gptProvider", - "title": "Default GPT Provider", + "title": "Default Provider", "description": "The default provider and model used in this extension.", "required": false, "type": "dropdown", @@ -241,6 +248,10 @@ { "title": "Google Gemini (requires API Key)", "value": "GoogleGemini" + }, + { + "title": "GPT4Free Local API", + "value": "G4FLocal" } ], "default": "GPT35" diff --git a/src/aiChat.jsx b/src/aiChat.jsx index d4fb314..c700fe2 100755 --- a/src/aiChat.jsx +++ b/src/aiChat.jsx @@ -285,7 +285,7 @@ export default function Chat({ launchContext }) { } > - + {providers.ChatProvidersReact} @@ -416,7 +416,7 @@ export default function Chat({ launchContext }) { second: "2-digit", })}`} /> - + {providers.ChatProvidersReact} diff --git a/src/api/Providers/deepinfra.jsx b/src/api/Providers/deepinfra.jsx index e67fa05..6a2e52e 100644 --- a/src/api/Providers/deepinfra.jsx +++ b/src/api/Providers/deepinfra.jsx @@ -49,19 +49,17 @@ export const getDeepInfraResponse = async function* (chat, options, max_retries const reader = response.body; for await (let chunk of reader) { const str = chunk.toString(); - let lines = str.split("\n"); + for (let i = 0; i < lines.length; i++) { let line = lines[i]; if (line.startsWith("data: ")) { let chunk = line.substring(6); if (chunk.trim() === "[DONE]") return; // trim() is important - let data = JSON.parse(chunk); - let choice = data["choices"][0]; - // python: if "content" in choice["delta"] and choice["delta"]["content"]: - if ("delta" in choice && "content" in choice["delta"] && choice["delta"]["content"]) { - let delta = choice["delta"]["content"]; + try { + let data = JSON.parse(chunk); + let delta = data["choices"][0]["delta"]["content"]; if (first) { delta = delta.trimStart(); } @@ -69,9 +67,9 @@ export const getDeepInfraResponse = async function* (chat, options, max_retries first = false; yield delta; } - } + } catch (e) {} // eslint-disable-line } - } // readline + } } } catch (e) { if (max_retries > 0) { diff --git a/src/api/Providers/g4f.jsx b/src/api/Providers/g4f.jsx index 099dfa5..b0b208a 100644 --- a/src/api/Providers/g4f.jsx +++ b/src/api/Providers/g4f.jsx @@ -1,5 +1,6 @@ import { G4F } from "g4f"; const g4f = new G4F(); +import { messages_to_json } from "../../classes/message"; export const G4FProvider = { GPT: g4f.providers.GPT, @@ -10,5 +11,8 @@ export const getG4FResponse = async (chat, options) => { // replace "g4f_provider" key in options with "provider" options.provider = options.g4f_provider; // this is a member of G4FProvider enum delete options.g4f_provider; + + chat = messages_to_json(chat); + return await g4f.chatCompletion(chat, options); }; diff --git a/src/api/Providers/g4f_local.jsx b/src/api/Providers/g4f_local.jsx new file mode 100644 index 0000000..dfa013a --- /dev/null +++ b/src/api/Providers/g4f_local.jsx @@ -0,0 +1,136 @@ +// This module allows communication and requests to the local G4F API. +// Read more here: https://github.com/xtekky/gpt4free/blob/main/docs/interference.md + +import { exec } from "child_process"; +import fetch from "node-fetch"; + +import { Storage } from "../storage"; +import { messages_to_json } from "../../classes/message"; + +import { environment, Form } from "@raycast/api"; + +// constants +const DEFAULT_MODEL = "meta-ai"; +export const DEFAULT_TIMEOUT = "900"; + +const BASE_URL = "http://localhost:1337/v1"; +const API_URL = "http://localhost:1337/v1/chat/completions"; +const MODELS_URL = "http://localhost:1337/v1/models"; + +// main function +export const G4FLocalProvider = "G4FLocalProvider"; +export const getG4FLocalResponse = async function* (chat, options) { + if (!(await isG4FRunning())) { + await startG4F(); + } + + chat = messages_to_json(chat); + const model = await getSelectedG4FModel(); + + const response = await fetch(API_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: model, + stream: options.stream, + messages: chat, + }), + }); + + // Important! we assume that the response is a stream, as this is true for most G4F models. + // If in the future this is not the case, we should add separate handling for non-streaming responses. + const reader = response.body; + for await (let chunk of reader) { + const str = chunk.toString(); + let lines = str.split("\n"); + for (let i = 0; i < lines.length; i++) { + let line = lines[i]; + if (line.startsWith("data: ")) { + let chunk = line.substring(6); + if (chunk.trim() === "[DONE]") return; // trim() is important + + try { + let data = JSON.parse(chunk); + let delta = data["choices"][0]["delta"]["content"]; + if (delta) { + yield delta; + } + } catch (e) {} // eslint-disable-line + } + } + } +}; + +/// utilities + +// check if the G4F API is running +// with a request timeout of 0.5 seconds (since it's localhost) +const isG4FRunning = async () => { + try { + const controller = new AbortController(); + setTimeout(() => controller.abort(), 500); + const response = await fetch(BASE_URL, { signal: controller.signal }); + return response.ok; + } catch (e) { + return false; + } +}; + +// get available models +const getG4FModels = async () => { + try { + const response = await fetch(MODELS_URL); + return (await response.json()).data || []; + } catch (e) { + return []; + } +}; + +// get available models as dropdown component +export const getG4FModelsDropdown = async () => { + const models = await getG4FModels(); + return ( + + {models.map((model) => { + return ; + })} + + ); +}; + +// get G4F executable path from storage +export const getG4FExecutablePath = async () => { + return await Storage.read("g4f_executable", "g4f"); +}; + +// get the currently selected G4F model from storage +const getSelectedG4FModel = async () => { + return await Storage.read("g4f_model", DEFAULT_MODEL); +}; + +// get G4F API timeout (in seconds) from storage +export const getG4FTimeout = async () => { + return parseInt(await Storage.read("g4f_timeout", DEFAULT_TIMEOUT)) || parseInt(DEFAULT_TIMEOUT); +}; + +// start the G4F API +const startG4F = async () => { + const exe = await getG4FExecutablePath(); + const timeout_s = await getG4FTimeout(); + const START_COMMAND = `export PATH="/opt/homebrew/bin:$PATH"; ( ${exe} api ) & sleep ${timeout_s} ; kill $!`; + const dirPath = environment.supportPath; + try { + const child = exec(START_COMMAND, { cwd: dirPath }); + console.log("G4F API Process ID:", child.pid); + child.stderr.on("data", (data) => { + console.log("g4f >", data); + }); + // sleep for some time to allow the API to start + await new Promise((resolve) => setTimeout(resolve, 2000)); + console.log(`G4F API started with timeout ${timeout_s}`); + } catch (e) { + console.log(e); + } +}; diff --git a/src/api/Providers/google_gemini.jsx b/src/api/Providers/google_gemini.jsx index 40f3fea..9268bed 100644 --- a/src/api/Providers/google_gemini.jsx +++ b/src/api/Providers/google_gemini.jsx @@ -43,9 +43,7 @@ export const getGoogleGeminiResponse = async (chat, options, stream_update, max_ response = await geminiChat.ask(query, { safetySettings: safetySettings }); return response; } - } catch (e) { - continue; - } + } catch (e) {} // eslint-disable-line } } catch (e) { // if all API keys fail, we allow a few retries diff --git a/src/api/gpt.jsx b/src/api/gpt.jsx index 7ee3b53..b376bc9 100755 --- a/src/api/gpt.jsx +++ b/src/api/gpt.jsx @@ -426,6 +426,9 @@ export const chatCompletion = async (chat, options, stream_update = null, status } else if (provider === providers.G4FProvider) { // G4F response = await providers.getG4FResponse(chat, options); + } else if (provider === providers.G4FLocalProvider) { + // G4F Local + response = await providers.getG4FLocalResponse(chat, options); } // stream = false diff --git a/src/api/providers.jsx b/src/api/providers.jsx index 691d76b..1d1dfd4 100644 --- a/src/api/providers.jsx +++ b/src/api/providers.jsx @@ -33,8 +33,12 @@ export { ReplicateProvider, getReplicateResponse }; import { GeminiProvider, getGoogleGeminiResponse } from "./Providers/google_gemini"; export { GeminiProvider, getGoogleGeminiResponse }; +// G4F Local module +import { G4FLocalProvider, getG4FLocalResponse } from "./Providers/g4f_local"; +export { G4FLocalProvider, getG4FLocalResponse }; + /// All providers info -// {provider, model, stream, extra options} +// { provider internal name, {provider, model, stream, extra options} } // prettier-ignore export const providers_info = { GPT35: { provider: NexraProvider, model: "chatgpt", stream: true }, @@ -51,6 +55,7 @@ export const providers_info = { ReplicateLlama3_70B: { provider: ReplicateProvider, model: "meta/meta-llama-3-70b-instruct", stream: true }, ReplicateMixtral_8x7B: { provider: ReplicateProvider, model: "mistralai/mixtral-8x7b-instruct-v0.1", stream: true }, GoogleGemini: { provider: GeminiProvider, model: "gemini-1.5-flash-latest", stream: true }, + G4FLocal: { provider: G4FLocalProvider, stream: true }, }; /// Chat providers (user-friendly names) @@ -69,6 +74,7 @@ export const chat_providers = [ ["Replicate (meta-llama-3-70b)", "ReplicateLlama3_70B"], ["Replicate (mixtral-8x7b)", "ReplicateMixtral_8x7B"], ["Google Gemini (requires API Key)", "GoogleGemini"], + ["GPT4Free Local API", "G4FLocal"], ]; export const ChatProvidersReact = chat_providers.map((x) => { diff --git a/src/api/storage.jsx b/src/api/storage.jsx index 50c76b8..a50d09a 100644 --- a/src/api/storage.jsx +++ b/src/api/storage.jsx @@ -109,7 +109,8 @@ export const Storage = { await Storage.run_sync(); }, - // combined read function - read from local storage, fallback to file storage + // combined read function - read from local storage, fallback to file storage. + // also writes the default value to local storage if it is provided and key is not found read: async (key, default_value = undefined) => { let value; if (await Storage.localStorage_has(key)) { diff --git a/src/configureG4FLocalApi.jsx b/src/configureG4FLocalApi.jsx new file mode 100644 index 0000000..942d090 --- /dev/null +++ b/src/configureG4FLocalApi.jsx @@ -0,0 +1,64 @@ +import { getG4FExecutablePath, getG4FTimeout, DEFAULT_TIMEOUT, getG4FModelsDropdown } from "./api/Providers/g4f_local"; +import { Storage } from "./api/storage"; +import { help_action } from "./helpers/helpPage"; + +import { Form, ActionPanel, Action, useNavigation, showToast, Toast } from "@raycast/api"; +import { useState, useEffect } from "react"; + +export default function ConfigureG4FLocalApi() { + const [executablePath, setExecutablePath] = useState(""); + const [timeout, setTimeout] = useState(""); + const [modelsDropdown, setModelsDropdown] = useState([]); + const [rendered, setRendered] = useState(false); + + const { pop } = useNavigation(); + + useEffect(() => { + (async () => { + setExecutablePath(await getG4FExecutablePath()); + setTimeout((await getG4FTimeout()).toString()); + setModelsDropdown(await getG4FModelsDropdown()); + setRendered(true); + })(); + }, []); + + return ( +
+ { + pop(); + await Storage.write("g4f_executable", values.g4f_executable); + await Storage.write("g4f_timeout", values.g4f_timeout || DEFAULT_TIMEOUT); + await Storage.write("g4f_model", values.model); + await showToast(Toast.Style.Success, "Configuration Saved"); + }} + /> + {help_action("g4fLocal")} + + } + > + + { + if (rendered) setExecutablePath(x); + }} + /> + { + if (rendered) setTimeout(x); + }} + /> + {modelsDropdown} + + ); +} diff --git a/src/genImage.jsx b/src/genImage.jsx index 46b6823..2dca514 100644 --- a/src/genImage.jsx +++ b/src/genImage.jsx @@ -120,7 +120,7 @@ export default function genImage() { if (err) { toast(Toast.Style.Failure, "Error saving image"); console.log("Error saving image. Current path: " + __dirname); - console.error(err); + console.log(err); imagePath = ""; } }); diff --git a/src/helpers/helpPage.jsx b/src/helpers/helpPage.jsx index 0db5991..808ce94 100644 --- a/src/helpers/helpPage.jsx +++ b/src/helpers/helpPage.jsx @@ -5,6 +5,7 @@ const helpPages = { aiChat: "https://github.com/XInTheDark/raycast-g4f/wiki/Help-page:-AI-Chat", customAICommands: "https://github.com/XInTheDark/raycast-g4f/wiki/Help-page:-Custom-AI-Commands", genImage: "https://github.com/XInTheDark/raycast-g4f/wiki/Help-page:-Generate-Images", + g4fLocal: "https://github.com/XInTheDark/raycast-g4f/wiki/Help-page:-GPT4Free-Local-API", default: "https://github.com/XInTheDark/raycast-g4f/blob/main/README.md", };