Feat/vercel ai sdk

dev-docs.json

@@ -1,9 +1,7 @@
 {
   "gitHubApp": {
     "approvalWorkflow": true,
-    "userDocsWorkflows": [
-      "generateUserDocs"
-    ],
+    "userDocsWorkflows": ["generateUserDocs"],
     "issues": true
   }
-}
+}

index.ts

@@ -1,58 +1,2 @@
-import "isomorphic-fetch";
-import AudioApis from "./src/audio/audio";
-import General from "./src/general";
-import PromptEngine from "./src/prompt_engine";
-import { RequestClient } from "./src/request";
-import Search from "./src/search/search";
-import { File } from "./src/store/file";
-import Validate from "./src/validate";
-import Vision from "./src/vision/vision";
-import Web from "./src/web/web";
-import { BaseConfig } from "./types";
-
-const JigsawStack = (config?: BaseConfig) => {
-  const _apiKey = config?.apiKey || process?.env?.JIGSAWSTACK_API_KEY;
-
-  if (!_apiKey) {
-    throw new Error("No JigsawStack API key provided");
-  }
-
-  const client = new RequestClient({ ...config, apiKey: _apiKey });
-  const general = new General(client);
-  const web = new Web(client);
-  const search = new Search(client);
-  const vision = new Vision(client);
-  const audio = new AudioApis(client);
-  const promptengine = new PromptEngine(client);
-  const file = new File(client);
-  const validate = new Validate(client);
-  const store = {
-    upload: file.upload,
-    retrieve: file.retrieve,
-    delete: file.delete,
-  };
-
-  return {
-    fetch: client.fetchJSS,
-    sentiment: general.sentiment,
-    translate: general.translate,
-    image_generation: general.image_generation,
-    summary: general.summary,
-    prediction: general.prediction,
-    text_to_sql: general.text_to_sql,
-    embedding: general.embedding,
-    audio,
-    vision,
-    web: {
-      ai_scrape: web.ai_scrape,
-      html_to_any: web.html_to_any,
-      search: search.search,
-      search_suggestions: search.suggestion,
-    },
-    prompt_engine: promptengine,
-    store,
-    validate,
-  };
-};
-
-export { JigsawStack };
+export { JigsawStack } from "./src/jigsawstack";
+export { JigsawStackToolSet } from "./src/vercel-ai-toolkit";

package.json

@@ -36,7 +36,9 @@
     "format": "biome check --write ."
   },
   "dependencies": {
-    "isomorphic-fetch": "^3.0.0"
+    "ai": "^4.3.16",
+    "isomorphic-fetch": "^3.0.0",
+    "zod": "^3.25.67"
   },
   "devDependencies": {
     "@biomejs/biome": "^1.9.4",

src/audio/audio.ts

@@ -14,7 +14,7 @@ class Audio {
 
   text_to_speech = async (params: TextToSpeechParams) => {
     const resp = await this.client.fetchJSS("/ai/tts", "POST", params);
-    return respToFileChoice(resp);
+    return respToFileChoice({ resp, return_type: params.return_type });
   };
 
   speaker_voice_accents = async () => {

src/audio/interfaces.ts

@@ -36,6 +36,7 @@ export interface TextToSpeechParams {
   text: string;
   accent?: SupportedAccents; // see https://docs.jigsawstack.com/additional-resources/speaker-voices for the list of supported accents.
   voice_clone_id?: string;
+  return_type?: "url" | "binary" | "base64";
 }
 
 export interface TTSCloneParams {
@@ -54,7 +55,7 @@ export interface TextToSpeechResponse {
   id: string;
 }
 
-type SupportedAccents =
+export type SupportedAccents =
   | "af-ZA-female-1"
   | "af-ZA-male-1"
   | "am-ET-female-1"

src/general/index.ts

@@ -37,10 +37,10 @@ class General {
     ): Promise<ReturnType<typeof respToFileChoice>> => {
       if (params instanceof Blob || params instanceof Buffer) {
         const resp: Response = await this.client.fetchJSS("/ai/translate/image", "POST", params, options);
-        return respToFileChoice(resp);
+        return respToFileChoice({ resp, return_type: options?.return_type });
       }
       const resp: Response = await this.client.fetchJSS("/ai/translate/image", "POST", params);
-      return respToFileChoice(resp);
+      return respToFileChoice({ resp, return_type: params.return_type });
     },
   };
 
@@ -55,6 +55,7 @@ class General {
     height?: number;
     steps?: number;
     output_format?: "png" | "svg";
+    return_type?: "url" | "binary" | "base64";
     advance_config?: {
       negative_prompt?: string;
       guidance?: number;
@@ -64,7 +65,8 @@ class General {
     file_store_key?: string;
   }) => {
     const resp: Response = await this.client.fetchJSS("/ai/image_generation", "POST", params);
-    return respToFileChoice(resp);
+
+    return respToFileChoice({ resp, return_type: params.return_type });
   };
 
   text_to_sql = async (params: {

src/general/interfaces.ts

@@ -28,6 +28,7 @@ export interface TranslateImageParams {
   target_language: string;
   url?: string;
   file_store_key?: string;
+  return_type?: "url" | "binary" | "base64";
 }
 
 export interface SpeechToTextResponse extends BaseResponse {

src/helpers.ts

@@ -1,4 +1,8 @@
-export const respToFileChoice = (resp: Response) => {
+export const respToFileChoice = ({ resp, return_type }: { resp: Response; return_type?: "url" | "binary" | "base64" }) => {
+  if (return_type === "url") {
+    return resp;
+  }
+
   return {
     blob: () => resp.blob(),
     buffer: async () => {

src/jigsawstack.ts

@@ -0,0 +1,59 @@
+import "isomorphic-fetch";
+import { BaseConfig } from "../types";
+import AudioApis from "./audio/audio";
+import General from "./general";
+import ObjectDetection from "./object_detection";
+import PromptEngine from "./prompt_engine";
+import { RequestClient } from "./request";
+import Search from "./search/search";
+import { File } from "./store/file";
+import Validate from "./validate";
+import Vision from "./vision/vision";
+import Web from "./web/web";
+
+export const JigsawStack = (config?: BaseConfig) => {
+  const _apiKey = config?.apiKey || process?.env?.JIGSAWSTACK_API_KEY;
+
+  if (!_apiKey) {
+    throw new Error("No JigsawStack API key provided");
+  }
+
+  const client = new RequestClient({ ...config, apiKey: _apiKey });
+  const general = new General(client);
+  const web = new Web(client);
+  const search = new Search(client);
+  const vision = new Vision(client);
+  const audio = new AudioApis(client);
+  const promptengine = new PromptEngine(client);
+  const file = new File(client);
+  const validate = new Validate(client);
+  const object_detection = new ObjectDetection(client);
+  const store = {
+    upload: file.upload,
+    retrieve: file.retrieve,
+    delete: file.delete,
+  };
+
+  return {
+    fetch: client.fetchJSS,
+    sentiment: general.sentiment,
+    translate: general.translate,
+    image_generation: general.image_generation,
+    summary: general.summary,
+    prediction: general.prediction,
+    text_to_sql: general.text_to_sql,
+    embedding: general.embedding,
+    object_detection,
+    audio,
+    vision,
+    web: {
+      ai_scrape: web.ai_scrape,
+      html_to_any: web.html_to_any,
+      search: search.search,
+      search_suggestions: search.suggestion,
+    },
+    prompt_engine: promptengine,
+    store,
+    validate,
+  };
+};

src/object_detection/index.ts

@@ -0,0 +1,83 @@
+import { RequestClient } from "../request";
+
+interface ObjectDetectionParams {
+  url?: string;
+  file_store_key?: string;
+  prompts?: string[];
+  features?: ("object_detection" | "gui")[];
+  annotated_image?: boolean;
+  return_type?: "url" | "base64";
+}
+
+// .object({
+//     url: z
+//       .string()
+//       .refine(isValidURL, {
+//         message: "Invalid url",
+//       })
+//       .optional(),
+//     file_store_key: z.string().optional(),
+//     prompts: z.array(z.string().min(1).max(150, "Max 150 characters")).optional().nullable(),
+//     features: z
+//       .array(z.enum(["object_detection", "gui"]))
+//       .min(1)
+//       .default(["object_detection", "gui"]),
+//     annotated_image: z.boolean().default(false),
+//     return_type: z.enum(["url", "base64"]).default("url"),
+//   })
+//   .refine((params) => params.url || params.file_store_key, {
+//     message: "Either url or file_store_key is required",
+//   });
+
+interface ObjectDetectionResponse {
+  // Optional annotated image - included only if annotated_image=true and objects/gui_elements exist
+  annotated_image?: string; // URL or base64 string depending on return_type
+
+  // Optional GUI elements - included only if features includes "gui"
+  gui_elements?: GuiElement[];
+
+  // Optional detected objects - included only if features includes "object_detection"
+  objects?: DetectedObject[];
+
+  // Optional usage statistics
+  _usage?: UsageStats;
+}
+
+interface GuiElement {
+  bounds: BoundingBox;
+  content: string | null; // Can be null if no object detected
+}
+
+interface DetectedObject {
+  bounds: BoundingBox;
+  mask?: string; // URL or base64 string depending on return_type - only present for some objects
+}
+
+interface BoundingBox {
+  top_left: Point;
+  top_right: Point;
+  bottom_left: Point;
+  bottom_right: Point;
+  width: number;
+  height: number;
+}
+
+interface Point {
+  x: number;
+  y: number;
+}
+
+interface UsageStats {
+  // The exact structure depends on the RunPod response
+  // You may need to examine actual responses to define this precisely
+  [key: string]: any;
+}
+
+class ObjectDetection {
+  constructor(private readonly client: RequestClient) {}
+  detect = async (params: ObjectDetectionParams): Promise<ObjectDetectionResponse> => {
+    return await this.client.fetchJSS("/ai/object_detection", "POST", params);
+  };
+}
+
+export default ObjectDetection;

src/store/file.ts

@@ -23,6 +23,7 @@ export class File {
 
   retrieve = async (key: string) => {
     const resp = await this.client.fetchJSS(`/store/file/read/${key}`, "GET");
+
     return respToFileChoice(resp);
   };