Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/ToolHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import type {DataFormat} from './McpResponse.js';
import {McpResponse} from './McpResponse.js';
import {SlimMcpResponse} from './SlimMcpResponse.js';
import {ClearcutLogger} from './telemetry/ClearcutLogger.js';
import {bucketizeLatency, buildContext} from './telemetry/transformation.js';
import type {CallToolResult} from './third_party/index.js';
import {zod} from './third_party/index.js';
import type {ToolCategory} from './tools/categories.js';
Expand Down Expand Up @@ -383,14 +382,14 @@ export class ToolHandler {
isError: true,
};
} finally {
const context = buildContext(devToolsData, pageUrl);
void ClearcutLogger.get()?.logToolInvocation({
toolName: this.tool.name,
params,
schema: this.inputSchema,
success,
latencyMs: bucketizeLatency(Date.now() - startTime),
context,
latencyMs: Date.now() - startTime,
devToolsData,
pageUrl,
});
guard[Symbol.dispose]();
}
Expand Down
19 changes: 13 additions & 6 deletions src/telemetry/ClearcutLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,21 @@ import {logger} from '../utils/logger.js';

import type {ErrorCode} from './errors.js';
import type {LocalState, Persistence} from './persistence.js';
import {sanitizeParams, stripUnderscoreBeforeNumber} from './transformation.js';
import {
bucketizeLatency,
buildContext,
sanitizeParams,
stripUnderscoreBeforeNumber,
} from './transformation.js';
import {
McpClient,
type FlagUsage,
WatchdogMessageType,
OsType,
type ToolInvocation,
type ToolInvocationContext,
} from './types.js';
import {WatchdogClient} from './WatchdogClient.js';
import type {DevToolsData} from '../tools/ToolDefinition.js';

const MS_PER_DAY = 24 * 60 * 60 * 1000;

Expand Down Expand Up @@ -121,16 +126,18 @@ export class ClearcutLogger {
schema: zod.ZodRawShape;
success: boolean;
latencyMs: number;
context: ToolInvocationContext;
devToolsData?: DevToolsData;
pageUrl?: string;
}): Promise<void> {
const context = buildContext(args.devToolsData, args.pageUrl);
const sanitizedToolName = stripUnderscoreBeforeNumber(args.toolName);
const tool_invocation: ToolInvocation = {
tool_name: sanitizedToolName,
success: args.success,
latency_ms: args.latencyMs,
latency_ms: bucketizeLatency(args.latencyMs),
};
if (Object.keys(args.context).length > 0) {
tool_invocation.context = args.context;
if (Object.keys(context).length > 0) {
tool_invocation.context = context;
}
if (Object.keys(args.params).length > 0) {
tool_invocation.tool_params = {
Expand Down
21 changes: 6 additions & 15 deletions tests/ToolHandler.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {ToolHandler} from '../src/ToolHandler.js';
import {ToolCategory} from '../src/tools/categories.js';
import type {
DefinedPageTool,
DevToolsData,
ToolDefinition,
} from '../src/tools/ToolDefinition.js';
import {createTools} from '../src/tools/tools.js';
Expand Down Expand Up @@ -165,7 +166,7 @@ describe('ToolHandler', () => {
assert.strictEqual(result.isError, undefined);
});

it('appends correct context to tool call logs', async () => {
it('passes devToolsData and pageUrl to logger', async () => {
const baseTool: ToolDefinition = {
name: 'test_tool',
description: 'A test tool',
Expand All @@ -183,9 +184,8 @@ describe('ToolHandler', () => {

const testCases: Array<{
tool: ToolDefinition | DefinedPageTool;
devToolsData: Record<string, unknown>;
devToolsData: DevToolsData;
pageUrl?: string;
expectedContext: Record<string, unknown>;
}> = [
{
tool: {
Expand All @@ -195,13 +195,6 @@ describe('ToolHandler', () => {
},
devToolsData: {cdpBackendNodeId: 1},
pageUrl: 'http://localhost:9222/',
expectedContext: {
is_devtools_open: true,
is_localhost: true,
devtools_data: {
is_dom_element_selected: true,
},
},
},
{
tool: {
Expand All @@ -210,9 +203,6 @@ describe('ToolHandler', () => {
},
devToolsData: {},
pageUrl: undefined,
expectedContext: {
is_devtools_open: false,
},
},
];

Expand Down Expand Up @@ -251,9 +241,10 @@ describe('ToolHandler', () => {

assert.strictEqual(logSpy.calledOnce, true);
assert.deepStrictEqual(
logSpy.firstCall.args[0].context,
testCase.expectedContext,
logSpy.firstCall.args[0].devToolsData,
testCase.devToolsData,
);
assert.strictEqual(logSpy.firstCall.args[0].pageUrl, testCase.pageUrl);
assert.strictEqual(handlerCalled, true);

sinon.restore();
Expand Down
13 changes: 7 additions & 6 deletions tests/telemetry/ClearcutLogger.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,14 @@ describe('ClearcutLogger', () => {
schema: {},
success: true,
latencyMs: 123,
context: {},
});

assert(mockWatchdogClient.send.calledOnce);
const msg = mockWatchdogClient.send.firstCall.args[0];
assert.strictEqual(msg.type, WatchdogMessageType.LOG_EVENT);
assert.strictEqual(msg.payload.tool_invocation?.tool_name, 'test_tool');
assert.strictEqual(msg.payload.tool_invocation?.success, true);
assert.strictEqual(msg.payload.tool_invocation?.latency_ms, 123);
assert.strictEqual(msg.payload.tool_invocation?.latency_ms, 250);
});
it('sends context when provided', async () => {
const logger = ClearcutLogger.initialize({
Expand All @@ -72,10 +71,10 @@ describe('ClearcutLogger', () => {
schema: {},
success: true,
latencyMs: 123,
context: {
is_devtools_open: true,
is_localhost: false,
devToolsData: {
cdpBackendNodeId: 1,
},
pageUrl: 'https://example.com',
});

assert(mockWatchdogClient.send.calledOnce);
Expand All @@ -84,6 +83,9 @@ describe('ClearcutLogger', () => {
assert.deepStrictEqual(msg.payload.tool_invocation?.context, {
is_devtools_open: true,
is_localhost: false,
devtools_data: {
is_dom_element_selected: true,
},
});
});
it('sends sanitized params', async () => {
Expand All @@ -109,7 +111,6 @@ describe('ClearcutLogger', () => {
schema,
success: true,
latencyMs: 123,
context: {},
});

assert(mockWatchdogClient.send.calledOnce);
Expand Down