Dalle integration PR

src/App.svelte

@@ -4,7 +4,7 @@
   <script>
     import { DeepChat } from "deep-chat-dev";
     import { onMount } from 'svelte';
-    import { BIDARA_SYS, PAPER_SEARCH_FUNC } from './bidara';
+    import { BIDARA_SYS, PAPER_SEARCH_FUNC, GEN_IMAGE_FUNC } from './bidara';
     import { funcCalling } from './bidaraFunctions';
     import { getKeyAndAsst } from './openaiUtils';
     import hljs from "highlight.js";
@@ -120,7 +120,11 @@
                 { 
                   type: "function",
                   function: PAPER_SEARCH_FUNC
-                }
+                },
+                {
+                  type: "function",
+                  function: GEN_IMAGE_FUNC
+                },
               ]
             },
             function_handler: funcCalling

src/bidaraFunctions.js

@@ -1,3 +1,5 @@
+import { getDalleImageGeneration } from "./openaiUtils";
+
 export function getCurrentWeather(location) {
   location = location.toLowerCase();
   if (location.includes('tokyo')) {
@@ -39,11 +41,43 @@ export async function ssSearch(params) {
   return JSON.stringify(papers);
 }
 
+async function genImage(params) {
+
+  var imageParams = JSON.parse(params);
+
+  if ("parameters" in imageParams) {
+    imageParams = imageParams.parameters;
+  }
+
+  var imageDescription = JSON.stringify(imageParams.description) + " Realistic depiction of the object and its environment. Stay true to science, engineering, and biology. DO NOT INCLUDE ANY WORDS OR BRANDING."
+
+  const res = await getDalleImageGeneration(imageDescription);
+
+  if (!res) {
+    return "We are having trouble generating images at this time.";
+  }
+
+  const imageURL = res.data[0].url;
+
+  const imageWidth = 256;
+  const imageHeight = 256;
+  const imageTag = `<img src=${imageURL} alt='Image of: ${imageDescription}. Generated by AI.' width='${imageWidth}' height='${imageHeight}'/>`;
+  const message = {html: imageTag, role: "ai"};
+
+  const deepChatRef = document.getElementById('chat-element');
+  deepChatRef._addMessage(message);
+
+  return "The image has been inserted into the chat, respond by tieing this image back into the Biomimicry Design Process with a short question or response. DO NOT RESPOND WITH AN IMAGE, URL, OR MARKDOWN.";
+}
+
 export async function callFunc(functionDetails) {
   let tmp = '';
   if(functionDetails.name == "get_graph_paper_relevance_search") {
     tmp = await ssSearch(functionDetails.arguments);
   }
+  else if(functionDetails.name == "generate_image_from_description") {
+    tmp = await genImage(functionDetails.arguments);
+  }
   else if(functionDetails.name == "get_weather") {
     tmp = getCurrentWeather(functionDetails.arguments);
   }
@@ -56,4 +90,4 @@ export async function callFunc(functionDetails) {
 export async function funcCalling(functionsDetails) {
   let tmp = await Promise.all(functionsDetails.map(callFunc));
   return tmp;
-}
+}

src/openaiUtils.js

@@ -94,4 +94,47 @@ export async function getKeyAndAsst() {
 
   let asst = await getAsst(key);
   return [key, asst]
-}
+}
+
+export async function getDalleImageGeneration(prompt, image_size = null, image_quality = null, num_images = null) {
+  if (!image_size) image_size = "1024x1024";
+  if (!image_quality) image_quality = "standard";
+  if (!num_images) num_images = 1;
+
+  try {
+    const key = await getOpenAIKey();
+    if (!key) {
+      return null;
+    }
+
+    const requestURL = "https://api.openai.com/v1/images/generations";
+
+    const request = {
+      method: "POST",
+      headers: {
+        'Authorization': 'Bearer ' + key,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        model: "dall-e-3",
+        prompt: prompt,
+        n: num_images,
+        size: image_size,
+        quality: image_quality
+      })
+    }
+
+    const response = await fetch(requestURL, request);
+
+    const r = await response.json();
+    if (r.error && r.error.type === 'invalid_request_error') {
+      return null;
+    }
+
+    return r;
+
+  } catch (e) {
+
+    return null;
+  }
+}