Skip to content

Filter out empty messages from LLM prompt construction #3967

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
IanCal wants to merge 2 commits into
base: main
Choose a base branch
from
+284 −7
Open

Filter out empty messages from LLM prompt construction #3967

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
034a846
Filter out empty messages from LLM prompt construction
claude Feb 10, 2026
01b3b74
Remove over-engineered safety filter, fix test field name
claude Feb 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
274 changes: 274 additions & 0 deletions packages/ai-bot/tests/prompt-construction-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5150,6 +5150,280 @@ new
'cache_control should be set to ephemeral',
);
});

test('excludes assistant messages with empty body and no tool calls', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Hello',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
body: '',
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
isStreamingFinished: true,
},
sender: '@aibot:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '3',
origin_server_ts: 3,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Can you help me?',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '3',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const assistantMessages = result.filter(
(message) => message.role === 'assistant',
);
assert.equal(
assistantMessages.length,
0,
'Empty assistant message should not be included',
);

const userMessages = result.filter((message) => message.role === 'user');
assert.equal(
userMessages.length,
2,
'Both user messages should be included',
);
});

test('keeps assistant messages with empty body when they have tool calls', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Update my card',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
body: '',
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
isStreamingFinished: true,
[APP_BOXEL_COMMAND_REQUESTS_KEY]: [
{
id: 'call_1',
name: 'patchCardInstance',
arguments: JSON.stringify({
card_id: 'http://localhost/card/1',
attributes: { title: 'Updated' },
}),
},
],
},
sender: '@aibot:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
{
type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
event_id: '3',
origin_server_ts: 3,
content: {
msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
body: 'Command result',
commandRequestId: 'call_1',
'm.relates_to': {
rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE,
event_id: '2',
key: 'applied',
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '3',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const assistantMessages = result.filter(
(message) => message.role === 'assistant',
);
assert.equal(
assistantMessages.length,
1,
'Assistant message with tool calls should be kept even with empty body',
);
assert.ok(
assistantMessages[0].tool_calls?.length,
'Assistant message should have tool calls',
);

const toolMessages = result.filter((message) => message.role === 'tool');
assert.equal(
toolMessages.length,
1,
'Tool result message should be present alongside the assistant tool call',
);
assert.equal(
toolMessages[0].tool_call_id,
'call_1',
'Tool result should reference the correct tool call id',
);
});

test('excludes user messages with empty body', async () => {
const history: DiscreteMatrixEvent[] = [
{
type: 'm.room.message',
event_id: '1',
origin_server_ts: 1,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: '',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '1',
},
status: EventStatus.SENT,
},
{
type: 'm.room.message',
event_id: '2',
origin_server_ts: 2,
content: {
msgtype: APP_BOXEL_MESSAGE_MSGTYPE,
format: 'org.matrix.custom.html',
body: 'Hello',
isStreamingFinished: true,
data: {
context: {
submode: 'interact',
},
},
},
sender: '@user:localhost',
room_id: 'room1',
unsigned: {
age: 1000,
transaction_id: '2',
},
status: EventStatus.SENT,
},
];

const result = await buildPromptForModel(
history,
'@aibot:localhost',
undefined,
undefined,
[],
fakeMatrixClient,
);

const userMessages = result.filter((message) => message.role === 'user');
assert.equal(
userMessages.length,
1,
'Only the non-empty user message should be included',
);
assert.equal(userMessages[0].content, 'Hello');
});
});

module('set model in prompt', (hooks) => {
Expand Down
17 changes: 10 additions & 7 deletions packages/runtime-common/ai/prompt.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1208,15 +1208,18 @@ export async function buildPromptForModel(
event as CardMessageEvent,
history,
);
let historicalMessage: OpenAIPromptMessage = {
role: 'assistant',
content: elideCodeBlocks(body, codePatchResults),
};
let content = elideCodeBlocks(body, codePatchResults);
let toolCalls = toToolCalls(event as CardMessageEvent);
if (toolCalls.length) {
historicalMessage.tool_calls = toolCalls;
if (content || toolCalls.length) {
let historicalMessage: OpenAIPromptMessage = {
role: 'assistant',
content,
};
if (toolCalls.length) {
historicalMessage.tool_calls = toolCalls;
}
historicalMessages.push(historicalMessage);
}
historicalMessages.push(historicalMessage);
let commandResults = getCommandResults(
event as CardMessageEvent,
history,
Expand Down
Loading