chore: move perf tests from langgraph folder to sdk-validation

.changeset/config.json

@@ -13,8 +13,12 @@
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": ["@langchain/langgraph-sdk-validation", "examples"],
+  "ignore": [
+    "@langchain/langgraph-sdk-validation",
+    "@langchain/langgraph-benchmark",
+    "examples"
+  ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }
-}
+}

internal/bench/README.md

@@ -0,0 +1,9 @@
+# LangGraph Benchmarks
+
+This package contains benchmarks for the LangGraph.js.
+
+## Running the tests
+
+```bash
+yarn test
+```

internal/bench/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@langchain/langgraph-benchmark",
+  "version": "0.0.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "format": "prettier --write src",
+    "lint": "prettier --check src",
+    "bench": "vitest bench"
+  },
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "@langchain/langgraph": "workspace:*",
+    "@tsconfig/recommended": "^1.0.2",
+    "@types/node": "^18.15.11",
+    "prettier": "^2.8.3",
+    "typescript": "^4.9.5 || ^5.4.5",
+    "vitest": "^3.1.2"
+  }
+}

libs/langgraph/src/tests/bench/index.bench.test.ts → internal/bench/src/index.bench.test.ts

@@ -3,7 +3,7 @@ import { HumanMessage } from "@langchain/core/messages";
 import { MemorySaver } from "@langchain/langgraph-checkpoint";
 import { createSequential } from "./sequential.js";
 import { reactAgent } from "./react_agent.js";
-import { runGraph, runFirstEventLatency } from "./bench.utils.js";
+import { runGraph, runFirstEventLatency } from "./utils.js";
 
 // Sequential benchmarks
 describe("sequential_10", () => {

libs/langgraph/src/tests/bench/react_agent.ts → internal/bench/src/react_agent.ts

@@ -1,13 +1,11 @@
-/* eslint-disable no-promise-executor-return */
-/* eslint-disable import/order */
-/* eslint-disable import/first */
+/* eslint-disable no-promise-executor-return, import/order, import/first */
 import { v4 as uuid } from "uuid";
 import { AIMessage, HumanMessage } from "@langchain/core/messages";
 import { tool } from "@langchain/core/tools";
 import { z } from "zod/v3";
 import { MemorySaver } from "@langchain/langgraph-checkpoint";
-import { createReactAgent } from "../../prebuilt/index.js";
-import { FakeToolCallingChatModel } from "../utils.models.js";
+import { createReactAgent } from "@langchain/langgraph/prebuilt";
+import { FakeToolCallingChatModel } from "./utils.js";
 
 /**
  * Creates a React agent with specified number of tools for benchmarking.
@@ -78,7 +76,7 @@ async function main() {
   console.timeEnd("stream");
 
   if (inspector.url()) {
-    await new Promise((resolve) => setTimeout(resolve, 360_000));
+    await new Promise((resolve) => setTimeout(resolve, 3_600_000));
   }
 
   return result.length;

libs/langgraph/src/tests/bench/sequential.ts → internal/bench/src/sequential.ts

@@ -1,12 +1,10 @@
-/* eslint-disable no-promise-executor-return */
-/* eslint-disable import/order */
-/* eslint-disable import/first */
+/* eslint-disable no-promise-executor-return, import/order, import/first */
 import {
   MessagesAnnotation,
   StateGraph,
   StateType,
   UpdateType,
-} from "../../index.js";
+} from "@langchain/langgraph";
 
 /**
  * Create a sequential no-op graph consisting of many nodes.
@@ -53,7 +51,7 @@ async function main() {
   console.timeEnd("stream");
 
   if (inspector.url()) {
-    await new Promise((resolve) => setTimeout(resolve, 360_000));
+    await new Promise((resolve) => setTimeout(resolve, 3_600_000));
   }
 
   return result.length;

internal/bench/src/utils.ts

@@ -0,0 +1,186 @@
+/* eslint-disable no-promise-executor-return, import/no-extraneous-dependencies */
+import { BaseMessage } from "@langchain/core/messages";
+import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
+
+import {
+  BaseChatModelParams,
+  BaseChatModel,
+  BindToolsInput,
+  BaseChatModelCallOptions,
+} from "@langchain/core/language_models/chat_models";
+
+import { ChatResult } from "@langchain/core/outputs";
+import { RunnableLambda } from "@langchain/core/runnables";
+
+import type { CompiledStateGraph } from "@langchain/langgraph";
+import { randomUUID } from "crypto";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type AnyStateGraph = CompiledStateGraph<any, any, any, any, any, any>;
+
+// Helper functions
+export const runGraph = async (
+  graph: AnyStateGraph,
+  input: Record<string, unknown>
+) => {
+  const results = await gatherIterator(
+    graph.stream(input, {
+      configurable: { thread_id: randomUUID() },
+      recursionLimit: 1000000000,
+    })
+  );
+  return results.length;
+};
+
+export const runFirstEventLatency = async (
+  graph: AnyStateGraph,
+  input: Record<string, unknown>
+) => {
+  const iterator = await graph.stream(input, {
+    configurable: { thread_id: randomUUID() },
+    recursionLimit: 1000000000,
+  });
+  await iterator.next();
+};
+
+export class FakeToolCallingChatModel extends BaseChatModel {
+  sleep?: number = 50;
+
+  responses?: BaseMessage[];
+
+  thrownErrorString?: string;
+
+  idx: number;
+
+  toolStyle: "openai" | "anthropic" | "bedrock" | "google" = "openai";
+
+  structuredResponse?: Record<string, unknown>;
+
+  // Track messages passed to structured output calls
+  structuredOutputMessages: BaseMessage[][] = [];
+
+  constructor(
+    fields: {
+      sleep?: number;
+      responses?: BaseMessage[];
+      thrownErrorString?: string;
+      toolStyle?: "openai" | "anthropic" | "bedrock" | "google";
+      structuredResponse?: Record<string, unknown>;
+    } & BaseChatModelParams
+  ) {
+    super(fields);
+    this.sleep = fields.sleep ?? this.sleep;
+    this.responses = fields.responses;
+    this.thrownErrorString = fields.thrownErrorString;
+    this.idx = 0;
+    this.toolStyle = fields.toolStyle ?? this.toolStyle;
+    this.structuredResponse = fields.structuredResponse;
+    this.structuredOutputMessages = [];
+  }
+
+  _llmType() {
+    return "fake";
+  }
+
+  async _generate(
+    messages: BaseMessage[],
+    _options: this["ParsedCallOptions"],
+    runManager?: CallbackManagerForLLMRun
+  ): Promise<ChatResult> {
+    if (this.thrownErrorString) {
+      throw new Error(this.thrownErrorString);
+    }
+    if (this.sleep !== undefined) {
+      await new Promise((resolve) => setTimeout(resolve, this.sleep));
+    }
+    const responses = this.responses?.length ? this.responses : messages;
+    const msg = responses[this.idx % responses.length];
+    const generation: ChatResult = {
+      generations: [
+        {
+          text: "",
+          message: msg,
+        },
+      ],
+    };
+    this.idx += 1;
+
+    if (typeof msg.content === "string") {
+      await runManager?.handleLLMNewToken(msg.content);
+    }
+    return generation;
+  }
+
+  bindTools(tools: BindToolsInput[]) {
+    const toolDicts = [];
+    const serverTools = [];
+    for (const tool of tools) {
+      if (!("name" in tool)) {
+        serverTools.push(tool);
+        continue;
+      }
+
+      // NOTE: this is a simplified tool spec for testing purposes only
+      if (this.toolStyle === "openai") {
+        toolDicts.push({
+          type: "function",
+          function: {
+            name: tool.name,
+          },
+        });
+      } else if (["anthropic", "google"].includes(this.toolStyle)) {
+        toolDicts.push({
+          name: tool.name,
+        });
+      } else if (this.toolStyle === "bedrock") {
+        toolDicts.push({
+          toolSpec: {
+            name: tool.name,
+          },
+        });
+      }
+    }
+    let toolsToBind: BindToolsInput[] = toolDicts;
+    if (this.toolStyle === "google") {
+      toolsToBind = [{ functionDeclarations: toolDicts }];
+    }
+    return this.bind({
+      tools: [...toolsToBind, ...serverTools],
+    } as BaseChatModelCallOptions);
+  }
+
+  withStructuredOutput<
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    RunOutput extends Record<string, any> = Record<string, any>
+  >(_: unknown) {
+    if (!this.structuredResponse) {
+      throw new Error("No structured response provided");
+    }
+    // Create a runnable that returns the proper structured format
+    return RunnableLambda.from(async (messages: BaseMessage[]) => {
+      if (this.sleep) {
+        await new Promise((resolve) => setTimeout(resolve, this.sleep));
+      }
+
+      // Store the messages that were sent to generate structured output
+      this.structuredOutputMessages.push([...messages]);
+
+      // Return in the format expected: { raw: BaseMessage, parsed: RunOutput }
+      return this.structuredResponse as RunOutput;
+    });
+  }
+}
+
+export async function gatherIterator<T>(
+  i:
+    | AsyncIterable<T>
+    | Promise<AsyncIterable<T>>
+    | Iterable<T>
+    | Promise<Iterable<T>>
+): Promise<Array<T>> {
+  const out: T[] = [];
+  for await (const item of await i) {
+    out.push(item);
+  }
+  return out;
+}

internal/bench/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "extends": "@tsconfig/recommended",
+  "compilerOptions": {
+    "target": "ES2021",
+    "lib": ["ES2021", "ES2022.Object", "ES2022.Error", "DOM"],
+    "module": "NodeNext",
+    "moduleResolution": "nodenext",
+    "esModuleInterop": true,
+    "declaration": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "useDefineForClassFields": true,
+    "strictPropertyInitialization": false,
+    "allowJs": true,
+    "strict": true,
+    "jsx": "react-jsx",
+    "outDir": "dist"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "coverage"],
+  "includeVersion": true
+}

internal/bench/vitest.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    globals: true,
+    fileParallelism: false,
+    maxConcurrency: 1,
+    include: ["**/*.test.ts"],
+    testTimeout: 300_000, // 5 minutes for benchmarks
+    benchmark: {
+      include: ["**/*.test.ts"],
+    },
+  },
+});

libs/checkpoint-redis/package.json

@@ -103,4 +103,4 @@
   "files": [
     "dist/"
   ]
-}
+}

yarn.lock

@@ -1948,6 +1948,19 @@ __metadata:
   languageName: unknown
   linkType: soft
 
+"@langchain/langgraph-benchmark@workspace:internal/bench":
+  version: 0.0.0-use.local
+  resolution: "@langchain/langgraph-benchmark@workspace:internal/bench"
+  dependencies:
+    "@langchain/langgraph": "workspace:*"
+    "@tsconfig/recommended": "npm:^1.0.2"
+    "@types/node": "npm:^18.15.11"
+    prettier: "npm:^2.8.3"
+    typescript: "npm:^4.9.5 || ^5.4.5"
+    vitest: "npm:^3.1.2"
+  languageName: unknown
+  linkType: soft
+
 "@langchain/langgraph-checkpoint-mongodb@workspace:*, @langchain/langgraph-checkpoint-mongodb@workspace:libs/checkpoint-mongodb":
   version: 0.0.0-use.local
   resolution: "@langchain/langgraph-checkpoint-mongodb@workspace:libs/checkpoint-mongodb"