Merge pull request langchain-ai#189 from langchain-ai/staging

Merge `staging` to `main`

.env.example

@@ -16,6 +16,7 @@ NEXT_PUBLIC_FIREWORKS_ENABLED=true
 NEXT_PUBLIC_GEMINI_ENABLED=false
 NEXT_PUBLIC_ANTHROPIC_ENABLED=true
 NEXT_PUBLIC_OPENAI_ENABLED=true
+NEXT_PUBLIC_AZURE_ENABLED=false
 
 # LangGraph Deployment, or local development server via LangGraph Studio.
 # If running locally, this URL should be set in the `constants.ts` file.
@@ -25,3 +26,12 @@ NEXT_PUBLIC_OPENAI_ENABLED=true
 # Public keys
 NEXT_PUBLIC_SUPABASE_URL=
 NEXT_PUBLIC_SUPABASE_ANON_KEY=
+
+# Azure OpenAI Configuration
+# ENSURE THEY ARE PREFIXED WITH AN UNDERSCORE.
+_AZURE_OPENAI_API_KEY=your-azure-openai-api-key
+_AZURE_OPENAI_API_INSTANCE_NAME=your-instance-name
+_AZURE_OPENAI_API_DEPLOYMENT_NAME=your-deployment-name
+_AZURE_OPENAI_API_VERSION=2024-08-01-preview
+# Optional: Azure OpenAI Base Path (if using a different domain)
+# _AZURE_OPENAI_API_BASE_PATH=https://your-custom-domain.com/openai/deployments

package.json

@@ -69,8 +69,9 @@
     "langsmith": "^0.1.61",
     "lodash": "^4.17.21",
     "lucide-react": "^0.441.0",
-    "next": "14.2.7",
+    "next": "14.2.10",
     "react": "^18",
+    "react-colorful": "^5.6.1",
     "react-dom": "^18",
     "react-icons": "^5.3.0",
     "react-json-view": "^1.21.3",
@@ -97,7 +98,7 @@
     "@typescript-eslint/eslint-plugin": "^8.12.2",
     "@typescript-eslint/parser": "^8.8.1",
     "eslint": "^8",
-    "eslint-config-next": "14.2.7",
+    "eslint-config-next": "14.2.10",
     "postcss": "^8",
     "prettier": "^3.3.3",
     "tailwind-scrollbar": "^3.1.0",

src/agent/open-canvas/index.ts

@@ -1,11 +1,11 @@
 import { END, Send, START, StateGraph } from "@langchain/langgraph";
 import { DEFAULT_INPUTS } from "../../constants";
 import { customAction } from "./nodes/customAction";
-import { generateArtifact } from "./nodes/generateArtifact";
+import { generateArtifact } from "./nodes/generate-artifact";
 import { generateFollowup } from "./nodes/generateFollowup";
 import { generatePath } from "./nodes/generatePath";
 import { reflectNode } from "./nodes/reflect";
-import { rewriteArtifact } from "./nodes/rewriteArtifact";
+import { rewriteArtifact } from "./nodes/rewrite-artifact";
 import { rewriteArtifactTheme } from "./nodes/rewriteArtifactTheme";
 import { updateArtifact } from "./nodes/updateArtifact";
 import { replyToGeneralInput } from "./nodes/replyToGeneralInput";

src/agent/open-canvas/nodes/customAction.ts

@@ -1,6 +1,6 @@
 import { BaseMessage } from "@langchain/core/messages";
 import { LangGraphRunnableConfig } from "@langchain/langgraph";
-import { initChatModel } from "langchain/chat_models/universal";
+import { getModelFromConfig } from "../../utils";
 import { getArtifactContent } from "../../../contexts/utils";
 import { isArtifactMarkdownContent } from "../../../lib/artifact_content_types";
 import {
@@ -10,11 +10,7 @@ import {
   CustomQuickAction,
   Reflections,
 } from "../../../types";
-import {
-  ensureStoreInConfig,
-  formatReflections,
-  getModelNameAndProviderFromConfig,
-} from "../../utils";
+import { ensureStoreInConfig, formatReflections } from "../../utils";
 import {
   CUSTOM_QUICK_ACTION_ARTIFACT_CONTENT_PROMPT,
   CUSTOM_QUICK_ACTION_ARTIFACT_PROMPT_PREFIX,
@@ -39,11 +35,8 @@ export const customAction = async (
     throw new Error("No custom quick action ID found.");
   }
 
-  const { modelName, modelProvider } =
-    getModelNameAndProviderFromConfig(config);
-  const smallModel = await initChatModel(modelName, {
+  const smallModel = await getModelFromConfig(config, {
     temperature: 0.5,
-    modelProvider,
   });
 
   const store = ensureStoreInConfig(config);

src/agent/open-canvas/nodes/generate-artifact/index.ts

@@ -0,0 +1,63 @@
+import {
+  OpenCanvasGraphAnnotation,
+  OpenCanvasGraphReturnType,
+} from "../../state";
+import { LangGraphRunnableConfig } from "@langchain/langgraph";
+import {
+  getFormattedReflections,
+  getModelFromConfig,
+  getModelConfig,
+  optionallyGetSystemPromptFromConfig,
+} from "@/agent/utils";
+import { ARTIFACT_TOOL_SCHEMA } from "./schemas";
+import { ArtifactV3 } from "@/types";
+import { createArtifactContent, formatNewArtifactPrompt } from "./utils";
+
+/**
+ * Generate a new artifact based on the user's query.
+ */
+export const generateArtifact = async (
+  state: typeof OpenCanvasGraphAnnotation.State,
+  config: LangGraphRunnableConfig
+): Promise<OpenCanvasGraphReturnType> => {
+  const { modelName } = getModelConfig(config);
+  const smallModel = await getModelFromConfig(config, {
+    temperature: 0.5,
+  });
+
+  const modelWithArtifactTool = smallModel.bindTools(
+    [
+      {
+        name: "generate_artifact",
+        schema: ARTIFACT_TOOL_SCHEMA,
+      },
+    ],
+    { tool_choice: "generate_artifact" }
+  );
+
+  const memoriesAsString = await getFormattedReflections(config);
+  const formattedNewArtifactPrompt = formatNewArtifactPrompt(
+    memoriesAsString,
+    modelName
+  );
+
+  const userSystemPrompt = optionallyGetSystemPromptFromConfig(config);
+  const fullSystemPrompt = userSystemPrompt
+    ? `${userSystemPrompt}\n${formattedNewArtifactPrompt}`
+    : formattedNewArtifactPrompt;
+
+  const response = await modelWithArtifactTool.invoke(
+    [{ role: "system", content: fullSystemPrompt }, ...state.messages],
+    { runName: "generate_artifact" }
+  );
+
+  const newArtifactContent = createArtifactContent(response.tool_calls?.[0]);
+  const newArtifact: ArtifactV3 = {
+    currentIndex: 1,
+    contents: [newArtifactContent],
+  };
+
+  return {
+    artifact: newArtifact,
+  };
+};

src/agent/open-canvas/nodes/generate-artifact/schemas.ts

@@ -0,0 +1,33 @@
+import { PROGRAMMING_LANGUAGES } from "@/types";
+import { z } from "zod";
+
+export const ARTIFACT_TOOL_SCHEMA = z.object({
+  type: z
+    .enum(["code", "text"])
+    .describe("The content type of the artifact generated."),
+  language: z
+    .enum(
+      PROGRAMMING_LANGUAGES.map((lang) => lang.language) as [
+        string,
+        ...string[],
+      ]
+    )
+    .optional()
+    .describe(
+      "The language/programming language of the artifact generated.\n" +
+        "If generating code, it should be one of the options, or 'other'.\n" +
+        "If not generating code, the language should ALWAYS be 'other'."
+    ),
+  isValidReact: z
+    .boolean()
+    .optional()
+    .describe(
+      "Whether or not the generated code is valid React code. Only populate this field if generating code."
+    ),
+  artifact: z.string().describe("The content of the artifact to generate."),
+  title: z
+    .string()
+    .describe(
+      "A short title to give to the artifact. Should be less than 5 words."
+    ),
+});

src/agent/open-canvas/nodes/generate-artifact/utils.ts

@@ -0,0 +1,39 @@
+import { NEW_ARTIFACT_PROMPT } from "../../prompts";
+import { ArtifactCodeV3, ArtifactMarkdownV3 } from "@/types";
+import { ToolCall } from "@langchain/core/messages/tool";
+
+export const formatNewArtifactPrompt = (
+  memoriesAsString: string,
+  modelName: string
+): string => {
+  return NEW_ARTIFACT_PROMPT.replace("{reflections}", memoriesAsString).replace(
+    "{disableChainOfThought}",
+    modelName.includes("claude")
+      ? "\n\nIMPORTANT: Do NOT preform chain of thought beforehand. Instead, go STRAIGHT to generating the tool response. This is VERY important."
+      : ""
+  );
+};
+
+export const createArtifactContent = (
+  toolCall: ToolCall | undefined
+): ArtifactCodeV3 | ArtifactMarkdownV3 => {
+  const toolArgs = toolCall?.args;
+  const artifactType = toolArgs?.type;
+
+  if (artifactType === "code") {
+    return {
+      index: 1,
+      type: "code",
+      title: toolArgs?.title,
+      code: toolArgs?.artifact,
+      language: toolArgs?.language,
+    };
+  }
+
+  return {
+    index: 1,
+    type: "text",
+    title: toolArgs?.title,
+    fullMarkdown: toolArgs?.artifact,
+  };
+};

src/agent/open-canvas/nodes/generateFollowup.ts

@@ -1,13 +1,9 @@
 import { LangGraphRunnableConfig } from "@langchain/langgraph";
-import { initChatModel } from "langchain/chat_models/universal";
+import { getModelFromConfig } from "../../utils";
 import { getArtifactContent } from "../../../contexts/utils";
 import { isArtifactMarkdownContent } from "../../../lib/artifact_content_types";
 import { Reflections } from "../../../types";
-import {
-  ensureStoreInConfig,
-  formatReflections,
-  getModelNameAndProviderFromConfig,
-} from "../../utils";
+import { ensureStoreInConfig, formatReflections } from "../../utils";
 import { FOLLOWUP_ARTIFACT_PROMPT } from "../prompts";
 import { OpenCanvasGraphAnnotation, OpenCanvasGraphReturnType } from "../state";
 
@@ -18,12 +14,8 @@ export const generateFollowup = async (
   state: typeof OpenCanvasGraphAnnotation.State,
   config: LangGraphRunnableConfig
 ): Promise<OpenCanvasGraphReturnType> => {
-  const { modelName, modelProvider } =
-    getModelNameAndProviderFromConfig(config);
-  const smallModel = await initChatModel(modelName, {
-    temperature: 0.5,
+  const smallModel = await getModelFromConfig(config, {
     maxTokens: 250,
-    modelProvider,
   });
 
   const store = ensureStoreInConfig(config);