Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/common/enum.ts
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,17 @@ export enum Constants {
/** Anthropic server tool ID prefix (web_search, code_execution, etc.) */
ANTHROPIC_SERVER_TOOL_PREFIX = 'srvtoolu_',
SKILL_TOOL = 'skill',
/**
* Callback-metadata keys stamped by `attemptInvoke` /
* `tryFallbackProviders` carrying the provider (SDK `Providers` enum
* value) and configured model that actually served a model invocation.
* Unlike `ls_provider` — which derived providers inherit from their base
* class (e.g. DeepSeek/OpenRouter report `'openai'`) — these reflect the
* SDK's own routing, including fallback-provider calls. Consumed by the
* subagent usage-capture handler to tag billing events.
*/
INVOKED_PROVIDER = '__invoked_provider',
INVOKED_MODEL = '__invoked_model',
READ_FILE = 'read_file',
BASH_TOOL = 'bash_tool',
BASH_PROGRAMMATIC_TOOL_CALLING = 'run_tools_with_bash',
Expand Down
10 changes: 10 additions & 0 deletions src/graphs/Graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -825,6 +825,13 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
agentContexts: Map<string, AgentContext> = new Map();
/** Default agent ID to use */
defaultAgentId: string;
/**
* Host sink for model usage emitted inside subagent child runs. Threaded
* into each `SubagentExecutor` this graph creates (and from there into
* child graphs, so nested subagents report too). See
* {@link t.StandardGraphInput.subagentUsageSink}.
*/
subagentUsageSink?: t.SubagentUsageSink;

constructor({
runId,
Expand All @@ -834,11 +841,13 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
tokenCounter,
indexTokenCountMap,
calibrationRatio,
subagentUsageSink,
}: t.StandardGraphInput) {
super();
this.runId = runId;
this.signal = signal;
this.langfuse = langfuse;
this.subagentUsageSink = subagentUsageSink;

if (agents.length === 0) {
throw new Error('At least one agent configuration is required');
Expand Down Expand Up @@ -2063,6 +2072,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
parentAgentId: agentContext.agentId,
langfuse: this.langfuse,
tokenCounter: agentContext.tokenCounter,
usageSink: this.subagentUsageSink,
maxDepth: effectiveSubagentDepth,
createChildGraph: (input): StandardGraph => {
const childGraph = new StandardGraph(input);
Expand Down
80 changes: 79 additions & 1 deletion src/llm/invoke.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import type { BaseMessage } from '@langchain/core/messages';
import type * as t from '@/types';
import { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
import { Constants, Providers } from '@/common';
import { ToolNode } from '@/tools/ToolNode';
import { Providers } from '@/common';

/**
* Minimal stub model shape `attemptInvoke` reads. Either `invoke` or
Expand Down Expand Up @@ -341,6 +341,84 @@ describe('tryFallbackProviders applies the same lazy annotation transform', () =
});
});

describe('invocation attribution metadata', () => {
it('stamps INVOKED_PROVIDER on the config passed to the model', async () => {
const capturedConfigs: unknown[] = [];
const model: StubModel = {
invoke: jest.fn(
async (_m: BaseMessage[], config?: unknown): Promise<AIMessage> => {
capturedConfigs.push(config);
return new AIMessage({ content: 'ok' });
}
),
};

await attemptInvoke(
{
model: model as t.ChatModel,
messages: [new HumanMessage('hi')],
/** A ChatOpenAI-derived provider — `ls_provider` would lie here. */
provider: Providers.DEEPSEEK,
},
{ configurable: { run_id: 'run-attr' }, metadata: { existing: true } }
);

const config = capturedConfigs[0] as {
metadata?: Record<string, unknown>;
};
expect(config.metadata?.[Constants.INVOKED_PROVIDER]).toBe(
Providers.DEEPSEEK
);
/** Pre-existing metadata is preserved, not replaced. */
expect(config.metadata?.existing).toBe(true);
});

it('stamps INVOKED_MODEL from the fallback clientOptions in tryFallbackProviders', async () => {
const capturedConfigs: unknown[] = [];
const model: StubModel = {
invoke: jest.fn(
async (_m: BaseMessage[], config?: unknown): Promise<AIMessage> => {
capturedConfigs.push(config);
return new AIMessage({ content: 'ok' });
}
),
};

jest.doMock('@/llm/init', () => ({
initializeModel: (): unknown => model,
}));
jest.resetModules();
const { tryFallbackProviders: freshTry } = (await import(
'@/llm/invoke'
)) as { tryFallbackProviders: typeof tryFallbackProviders };

await freshTry({
fallbacks: [
{
provider: Providers.ANTHROPIC,
clientOptions: { model: 'claude-fallback-1' },
},
],
messages: [new HumanMessage('hi')],
primaryError: new Error('primary failed'),
config: { configurable: { run_id: 'run-attr-fb' } },
});

const config = capturedConfigs[0] as {
metadata?: Record<string, unknown>;
};
expect(config.metadata?.[Constants.INVOKED_MODEL]).toBe(
'claude-fallback-1'
);
expect(config.metadata?.[Constants.INVOKED_PROVIDER]).toBe(
Providers.ANTHROPIC
);

jest.dontMock('@/llm/init');
jest.resetModules();
});
});

describe('cross-run hydration through ToolNode + attemptInvoke', () => {
it('annotates run 2 refs but leaves hydrated run 1 ToolMessages untouched', async () => {
/**
Expand Down
62 changes: 58 additions & 4 deletions src/llm/invoke.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import type { BaseMessage } from '@langchain/core/messages';
import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
import type * as t from '@/types';
import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
import { Constants, GraphEvents, Providers } from '@/common';
import { manualToolStreamProviders } from '@/llm/providers';
import { modifyDeltaProperties } from '@/messages';
import { ChatModelStreamHandler } from '@/stream';
import { GraphEvents, Providers } from '@/common';
import { initializeModel } from '@/llm/init';

/**
Expand Down Expand Up @@ -208,6 +208,23 @@ export async function attemptInvoke(
const runId = config?.configurable?.run_id as string | undefined;
const messagesForProvider = annotateMessagesForLLM(messages, registry, runId);

/**
* Stamp the provider that is ACTUALLY serving this invocation onto the
* callback metadata. `attemptInvoke` is the single funnel for primary,
* fallback, and summarization model calls, so consumers that need
* provider attribution per call (the subagent usage-capture handler)
* read this key instead of trusting static agent config — which is
* wrong for fallback-served calls — or `ls_provider` — which derived
* providers inherit from their base class.
*/
config = {
...config,
metadata: {
...(config?.metadata ?? {}),
[Constants.INVOKED_PROVIDER]: provider,
},
};

if (model.stream) {
const stream = await model.stream(messagesForProvider, config);
let finalChunk: AIMessageChunk | undefined;
Expand All @@ -224,7 +241,7 @@ export async function attemptInvoke(
});
}
} else if (registeredStreamHandler == null) {
const metadata = config?.metadata as Record<string, unknown> | undefined;
const metadata = config.metadata as Record<string, unknown> | undefined;
const streamHandler = new ChatModelStreamHandler();
for await (const chunk of stream) {
const handlingChunk = getStreamHandlingChunk({
Expand All @@ -247,7 +264,7 @@ export async function attemptInvoke(
});
}
} else {
const metadata = config?.metadata as Record<string, unknown> | undefined;
const metadata = config.metadata as Record<string, unknown> | undefined;
for await (const chunk of stream) {
const handlingChunk = getStreamHandlingChunk({
current: finalChunk,
Expand Down Expand Up @@ -292,6 +309,25 @@ export async function attemptInvoke(
return { messages: [finalMessage] };
}

/**
* Best-effort read of the configured model name from client options.
* Providers disagree on the key (`model` vs `modelName`).
*/
function extractClientOptionsModel(
clientOptions: t.ClientOptions | undefined
): string | undefined {
const options = clientOptions as
| { model?: unknown; modelName?: unknown }
| undefined;
if (typeof options?.model === 'string' && options.model !== '') {
return options.model;
}
if (typeof options?.modelName === 'string' && options.modelName !== '') {
return options.modelName;
}
return undefined;
}

/**
* Attempts each fallback provider in order until one succeeds.
* Throws the last error if all fallbacks fail.
Expand Down Expand Up @@ -321,6 +357,24 @@ export async function tryFallbackProviders({
clientOptions: fb.clientOptions,
tools,
});
/**
* Stamp the fallback's configured model onto callback metadata so
* per-call attribution (subagent usage capture) doesn't fall back to
* the PRIMARY config's model when the provider reports no
* `ls_model_name`. The serving provider is stamped uniformly by
* `attemptInvoke` (`INVOKED_PROVIDER`).
*/
const fbModelName = extractClientOptionsModel(fb.clientOptions);
const fbConfig: RunnableConfig | undefined =
fbModelName == null
? config
: {
...config,
metadata: {
...(config?.metadata ?? {}),
[Constants.INVOKED_MODEL]: fbModelName,
},
};
const result = await attemptInvoke(
{
model: fbModel as t.ChatModel,
Expand All @@ -329,7 +383,7 @@ export async function tryFallbackProviders({
context,
onChunk,
},
config
fbConfig
);
return result;
} catch (e) {
Expand Down
4 changes: 4 additions & 0 deletions src/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ export class Run<_T extends t.BaseGraphState> {
private toolOutputReferences?: t.ToolOutputReferencesConfig;
private eagerEventToolExecution?: t.EagerEventToolExecutionConfig;
private toolExecution?: t.ToolExecutionConfig;
private subagentUsageSink?: t.SubagentUsageSink;
private indexTokenCountMap?: Record<string, number>;
calibrationRatio: number = 1;
graphRunnable?: t.CompiledStateWorkflow;
Expand Down Expand Up @@ -176,6 +177,7 @@ export class Run<_T extends t.BaseGraphState> {
this.toolOutputReferences = config.toolOutputReferences;
this.eagerEventToolExecution = config.eagerEventToolExecution;
this.toolExecution = config.toolExecution;
this.subagentUsageSink = config.subagentUsageSink;

if (!config.graphConfig) {
throw new Error('Graph config not provided');
Expand Down Expand Up @@ -249,6 +251,7 @@ export class Run<_T extends t.BaseGraphState> {
tokenCounter: this.tokenCounter,
indexTokenCountMap: this.indexTokenCountMap,
calibrationRatio: this.calibrationRatio,
subagentUsageSink: this.subagentUsageSink,
});
/** Propagate compile options from graph config */
standardGraph.compileOptions = this.applyHITLCheckpointerFallback(
Expand Down Expand Up @@ -276,6 +279,7 @@ export class Run<_T extends t.BaseGraphState> {
tokenCounter: this.tokenCounter,
indexTokenCountMap: this.indexTokenCountMap,
calibrationRatio: this.calibrationRatio,
subagentUsageSink: this.subagentUsageSink,
});

multiAgentGraph.compileOptions =
Expand Down
Loading
Loading