fix(google): keep unspoken model text out of the realtime transcript

.changeset/google-realtime-unspoken-text.md

@@ -0,0 +1,5 @@
+---
+'@livekit/agents-plugin-google': patch
+---
+
+Gemini realtime: keep unspoken model text parts out of the transcript when the session runs with audio output and output transcription enabled.

plugins/google/src/realtime/realtime_api.test.ts

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2026 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+import type { LiveServerContent } from '@google/genai';
 import { Behavior, FunctionResponseScheduling } from '@google/genai';
 import { llm } from '@livekit/agents';
 import { describe, expect, it, vi } from 'vitest';
@@ -182,6 +183,75 @@ describe('Google Realtime non-blocking tool scheduling', () => {
   });
 });
 
+type ServerContentSessionInternals = {
+  _realtimeModel: { capabilities: { audioOutput: boolean } };
+  options: { outputAudioTranscription?: Record<string, never> };
+  earlyCompletionPending: boolean;
+  currentGeneration: {
+    outputText: string;
+    textChannel: { write: ReturnType<typeof vi.fn> };
+  };
+  handleServerContent(serverContent: LiveServerContent): void;
+};
+
+function createServerContentSession({
+  audioOutput,
+  outputAudioTranscription,
+}: {
+  audioOutput: boolean;
+  outputAudioTranscription?: Record<string, never>;
+}): ServerContentSessionInternals {
+  const session = Object.create(RealtimeSession.prototype) as ServerContentSessionInternals;
+  session._realtimeModel = { capabilities: { audioOutput } };
+  session.options = { outputAudioTranscription };
+  session.earlyCompletionPending = false;
+  session.currentGeneration = {
+    outputText: '',
+    textChannel: { write: vi.fn() },
+  };
+  return session;
+}
+
+describe('Google Realtime model text parts', () => {
+  const modelTextTurn: LiveServerContent = {
+    modelTurn: { parts: [{ text: 'call:getWeather{location:Seattle' }] },
+    outputTranscription: { text: 'Let me check.' },
+  };
+
+  it('keeps unspoken model text out of the transcript in an audio session', () => {
+    const session = createServerContentSession({
+      audioOutput: true,
+      outputAudioTranscription: {},
+    });
+
+    session.handleServerContent(modelTextTurn);
+
+    expect(session.currentGeneration.textChannel.write.mock.calls).toEqual([['Let me check.']]);
+    expect(session.currentGeneration.outputText).toBe('Let me check.');
+  });
+
+  it('forwards model text when the session runs in text modality', () => {
+    const session = createServerContentSession({
+      audioOutput: false,
+      outputAudioTranscription: {},
+    });
+
+    session.handleServerContent({ modelTurn: { parts: [{ text: 'Hello there.' }] } });
+
+    expect(session.currentGeneration.textChannel.write.mock.calls).toEqual([['Hello there.']]);
+    expect(session.currentGeneration.outputText).toBe('Hello there.');
+  });
+
+  it('forwards model text when output transcription is disabled', () => {
+    const session = createServerContentSession({ audioOutput: true });
+
+    session.handleServerContent({ modelTurn: { parts: [{ text: 'Hello there.' }] } });
+
+    expect(session.currentGeneration.textChannel.write.mock.calls).toEqual([['Hello there.']]);
+    expect(session.currentGeneration.outputText).toBe('Hello there.');
+  });
+});
+
 describe('Google Realtime initial history seeding', () => {
   function historyConfigFor(model: string) {
     const { capabilities } = new RealtimeModel({ model, apiKey: 'test-key' });

plugins/google/src/realtime/realtime_api.ts

@@ -1651,6 +1651,14 @@ export class RealtimeSession extends llm.RealtimeSession {
 
     const discardOutput = this.earlyCompletionPending;
 
+    // With audio output and output transcription on, the spoken words arrive as
+    // outputTranscription; a text part on the model turn is never spoken (an unflagged
+    // thought, or a function call the model wrote out as text) and would leak unspoken
+    // text into the transcript.
+    const forwardModelText =
+      !this._realtimeModel.capabilities.audioOutput ||
+      this.options.outputAudioTranscription === undefined;
+
     if (serverContent.modelTurn && !discardOutput) {
       const turn = serverContent.modelTurn;
 
@@ -1660,7 +1668,7 @@ export class RealtimeSession extends llm.RealtimeSession {
           continue;
         }
 
-        if (part.text) {
+        if (part.text && forwardModelText) {
           gen.outputText += part.text;
           gen.textChannel.write(part.text);
         }

plugins/google/src/realtime/realtime_transcript.test.ts

@@ -0,0 +1,168 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type * as genai from '@google/genai';
+import { Modality } from '@google/genai';
+import type { llm } from '@livekit/agents';
+import { describe, expect, it, vi } from 'vitest';
+import { RealtimeModel } from './realtime_api.js';
+
+/** The fields of a server message the session reads — `LiveServerMessage` itself is a class. */
+type ServerFrame = Pick<genai.LiveServerMessage, 'serverContent' | 'toolCall'>;
+
+type ServerCallbacks = {
+  onopen: () => void;
+  onmessage: (message: ServerFrame) => void;
+};
+
+/**
+ * Captures the callbacks the plugin hands to `live.connect()` so a test can
+ * play server frames into the real `RealtimeSession`. Hoisted because `vi.mock` is.
+ */
+const { live } = vi.hoisted(() => ({
+  live: { callbacks: undefined as ServerCallbacks | undefined },
+}));
+
+vi.mock('@google/genai', async (importOriginal) => {
+  const actual = await importOriginal<typeof genai>();
+  return {
+    ...actual,
+    GoogleGenAI: class {
+      live = {
+        connect: async ({ callbacks }: { callbacks: ServerCallbacks }) => {
+          live.callbacks = callbacks;
+          callbacks.onopen();
+          return {
+            sendClientContent: () => {},
+            sendRealtimeInput: () => {},
+            sendToolResponse: () => {},
+            close: () => {},
+          };
+        },
+      };
+    },
+  };
+});
+
+type SessionOptions = Partial<ConstructorParameters<typeof RealtimeModel>[0]>;
+
+/** Two PCM16 samples — enough for the plugin to build one AudioFrame. */
+const AUDIO_PART: genai.Part = {
+  inlineData: { data: Buffer.from([0, 0, 0, 0]).toString('base64'), mimeType: 'audio/pcm' },
+};
+
+async function openSession(options: SessionOptions = {}) {
+  live.callbacks = undefined;
+  const session = new RealtimeModel({
+    model: 'gemini-2.0-flash-live-001',
+    apiKey: 'test-key',
+    ...options,
+  }).session();
+
+  const generations: llm.GenerationCreatedEvent[] = [];
+  session.on('generation_created', (ev) => generations.push(ev));
+
+  // Frames are only accepted once the main task holds the connected session.
+  const internals = session as unknown as { activeSession?: unknown };
+  await vi.waitFor(() => expect(internals.activeSession).toBeDefined());
+
+  return {
+    generations,
+    serverSends: (message: ServerFrame) => live.callbacks!.onmessage(message),
+  };
+}
+
+/**
+ * Frames are handled asynchronously (the plugin serialises them behind its
+ * session lock), so wait for the generation they open, then drain it the way
+ * AgentActivity would: every message's text and audio, then the tool calls.
+ */
+async function drainFirst(generations: llm.GenerationCreatedEvent[]) {
+  await vi.waitFor(() => expect(generations).toHaveLength(1));
+  const ev = generations[0]!;
+
+  let text = '';
+  let audioFrames = 0;
+  for await (const message of ev.messageStream) {
+    for await (const chunk of message.textStream) {
+      text += typeof chunk === 'string' ? chunk : chunk.text;
+    }
+    for await (const _frame of message.audioStream) {
+      audioFrames += 1;
+    }
+  }
+
+  const functionCalls: string[] = [];
+  for await (const call of ev.functionStream) {
+    functionCalls.push(call.name);
+  }
+
+  return { text, audioFrames, functionCalls };
+}
+
+describe('Gemini realtime transcript', () => {
+  it('carries only the output transcription when audio and transcription are on', async () => {
+    const { generations, serverSends } = await openSession();
+
+    // The exact shape seen in production: the model writes a function call out as
+    // text, then speaks. Only the spoken words belong in the transcript.
+    serverSends({
+      serverContent: { modelTurn: { parts: [{ text: 'call:assetGenerator{context:' }] } },
+    });
+    serverSends({ serverContent: { modelTurn: { parts: [AUDIO_PART] } } });
+    serverSends({ serverContent: { outputTranscription: { text: 'Tako je!' } } });
+    serverSends({ serverContent: { generationComplete: true } });
+    serverSends({ serverContent: { turnComplete: true } });
+
+    await expect(drainFirst(generations)).resolves.toEqual({
+      text: 'Tako je!',
+      audioFrames: 1,
+      functionCalls: [],
+    });
+  });
+
+  it('still delivers the tool call the model makes after writing one out as text', async () => {
+    const { generations, serverSends } = await openSession();
+
+    serverSends({
+      serverContent: { modelTurn: { parts: [{ text: 'call:getWeather{location:' }] } },
+    });
+    serverSends({
+      toolCall: {
+        functionCalls: [{ id: 'fc-1', name: 'getWeather', args: { location: 'Seattle' } }],
+      },
+    });
+
+    await expect(drainFirst(generations)).resolves.toEqual({
+      text: '',
+      audioFrames: 0,
+      functionCalls: ['getWeather'],
+    });
+  });
+
+  it('keeps forwarding model text in text modality', async () => {
+    const { generations, serverSends } = await openSession({ modalities: [Modality.TEXT] });
+
+    serverSends({ serverContent: { modelTurn: { parts: [{ text: 'Hello there.' }] } } });
+    serverSends({ serverContent: { turnComplete: true } });
+
+    await expect(drainFirst(generations)).resolves.toEqual({
+      text: 'Hello there.',
+      audioFrames: 0,
+      functionCalls: [],
+    });
+  });
+
+  it('keeps forwarding model text when output transcription is disabled', async () => {
+    const { generations, serverSends } = await openSession({ outputAudioTranscription: null });
+
+    serverSends({ serverContent: { modelTurn: { parts: [{ text: 'Hello there.' }] } } });
+    serverSends({ serverContent: { turnComplete: true } });
+
+    await expect(drainFirst(generations)).resolves.toEqual({
+      text: 'Hello there.',
+      audioFrames: 0,
+      functionCalls: [],
+    });
+  });
+});