feat(voice): release a reply's opening chunk at a clause or word boundary, and drop indented code from speech (#13103, #17953)

autobot-frontend/src/components/chat/ChatInterface.vue

@@ -330,7 +330,7 @@ import VoiceConversationOverlay from './VoiceConversationOverlay.vue'
 import VoiceConversationPanel from './VoiceConversationPanel.vue'
 import ChatSettingsModal from './ChatSettingsModal.vue'
 import { fetchWithAuth } from '@/utils/fetchWithAuth'
-import { createSpeechShaper, extractCompleteSentences } from '@/utils/ttsSentences'
+import { createSpeechShaper, createTtsStream } from '@/utils/ttsSentences'
 // Issue #3232: chain-of-thought reasoning trace
 import ReasoningTrace from './ReasoningTrace.vue'
 import { useReasoningTrace } from '@/composables/useReasoningTrace'
@@ -1316,6 +1316,7 @@ let _lastStreamingMsgId: string | null = null
 // Shapes each reply's spoken slices; fence state spans slices (#13102)
 const _newSpeechShaper = () => createSpeechShaper({ url: t('voice.speech.url'), path: t('voice.speech.path') })
 let _speechShaper = _newSpeechShaper()
+let _ttsStream = createTtsStream()
 
 /** Prime TTS cursor to end of current last message. Used on voice-enable and session switch. */
 function _primeTtsCursor(): void {
@@ -1330,6 +1331,7 @@ function _primeTtsCursor(): void {
     // Read the skipped text unspoken so an open code fence is known (#13102)
     _speechShaper = _newSpeechShaper()
     _speechShaper.push(m.content)
+    _ttsStream = createTtsStream(true)
     break
   }
 }
@@ -1371,6 +1373,7 @@ watch(
       _lastStreamingMsgId = current.id
       _lastSpokenIdx = store.isTyping ? 0 : current.content.length
       _speechShaper = _newSpeechShaper()
+      _ttsStream = createTtsStream(!store.isTyping)
     }
 
     if (store.isTyping && current.content) {
@@ -1379,10 +1382,7 @@ watch(
       // by the summed sentence lengths, which drift over multi-paragraph replies
       // and drop/duplicate slices (#12502).
       const newText = current.content.slice(_lastSpokenIdx)
-      const { spans, consumed } = extractCompleteSentences(
-        newText,
-        _MIN_TTS_SENTENCE_CHARS,
-      )
+      const { spans, consumed } = _ttsStream.next(newText)
       for (const span of spans) speakStreaming(_speechShaper.push(span))
       _lastSpokenIdx += consumed
     } else if (!store.isTyping && current.content) {
@@ -1403,11 +1403,6 @@ watch(
   }
 )
 
-// Minimum chars a candidate sentence must have before it is dispatched to TTS (#1485).
-// Short fragments like "Hello there! " (13 chars) sound choppy when spoken alone —
-// buffer them until they combine with the next sentence or flush as a remainder.
-const _MIN_TTS_SENTENCE_CHARS = 20
-
 </script>
 
 <style scoped>

autobot-frontend/src/utils/__tests__/ttsFastStart.test.ts

@@ -0,0 +1,104 @@
+// Copyright 2025-2026 mrveiss
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Opening-chunk fast start (#13103): the first chunk of a reply goes to TTS at
+ * a clause or word boundary, everything after it keeps the terminated-sentence
+ * rule, and the cursor advances by exactly the consumed span.
+ */
+import { describe, it, expect } from 'vitest'
+import {
+  createTtsStream,
+  extractOpeningChunk,
+  MIN_TTS_SENTENCE_CHARS,
+  OPENING_CLAUSE_MIN_CHARS,
+  OPENING_WORD_MIN_CHARS,
+} from '../ttsSentences'
+
+/** Feed `full` in `step`-char deltas the way ChatInterface's watcher does. */
+function stream(full: string, step: number, opened = false) {
+  const tts = createTtsStream(opened)
+  const spoken: string[] = []
+  let cursor = 0
+  for (let end = step; end < full.length + step; end += step) {
+    const { spans, consumed } = tts.next(full.slice(0, Math.min(end, full.length)).slice(cursor))
+    spoken.push(...spans)
+    cursor += consumed
+  }
+  return { spoken, remainder: full.slice(cursor) }
+}
+
+describe('extractOpeningChunk (#13103)', () => {
+  it('releases the opening at a clause boundary below the standard bar', () => {
+    const text = 'Looking at your logs, the failure starts'
+    expect(text.indexOf(',') + 1).toBeGreaterThanOrEqual(OPENING_CLAUSE_MIN_CHARS)
+    expect(extractOpeningChunk(text)).toEqual({ spans: ['Looking at your logs, '], consumed: 22 })
+  })
+
+  it('skips a clause boundary that comes too early', () => {
+    expect(extractOpeningChunk('Well, that is')).toBeNull()
+  })
+
+  it('falls back to a word boundary for a long opening with no early pause', () => {
+    const text = 'This opening sentence runs on for quite a while without any pause in it'
+    expect(text.length).toBeGreaterThanOrEqual(OPENING_WORD_MIN_CHARS)
+    const chunk = extractOpeningChunk(text)!
+    expect(chunk.spans[0]).toBe('This opening sentence runs on for quite a while without any pause in ')
+    expect(chunk.consumed).toBe(chunk.spans[0].length)
+  })
+
+  it('never cuts the opening inside a markdown link', () => {
+    // The comma and every word gap sit inside the still-open link text.
+    expect(extractOpeningChunk('[see the docs for the parser, especially the long section on')).toBeNull()
+    // Once the link has closed, a later gap is a fair cut.
+    expect(extractOpeningChunk('[the docs](http://x.y/z) explain the parser in a long section on')!.spans[0]).toBe(
+      '[the docs](http://x.y/z) explain the parser in a long section ',
+    )
+  })
+
+  it('waits while the opening is short and has no clause boundary', () => {
+    expect(extractOpeningChunk('Short opening with no pause')).toBeNull()
+  })
+
+  it('stays out of the way once any sentence has terminated', () => {
+    // "Hi there." is shorter than the standard bar, so it is HELD for merging;
+    // an early opening must not jump ahead of it.
+    expect('Hi there.'.length).toBeLessThan(MIN_TTS_SENTENCE_CHARS)
+    expect(extractOpeningChunk('Hi there. And then, after a long while it goes on')).toBeNull()
+  })
+})
+
+describe('createTtsStream (#13103)', () => {
+  it('speaks the opening early, then only terminated sentences', () => {
+    const tts = createTtsStream()
+    expect(tts.next('Looking at your logs, the failure')).toEqual({ spans: ['Looking at your logs, '], consumed: 22 })
+    // After the opening, a clause boundary alone releases nothing.
+    expect(tts.next('the failure starts in the parser, then')).toEqual({ spans: [], consumed: 0 })
+    expect(tts.next('the failure starts in the parser. Then')).toEqual({
+      spans: ['the failure starts in the parser. '],
+      consumed: 34,
+    })
+  })
+
+  it('gives no fast start to a reply already under way', () => {
+    const tts = createTtsStream(true)
+    expect(tts.next('Looking at your logs, the failure')).toEqual({ spans: [], consumed: 0 })
+  })
+
+  it('treats a standard sentence as the opening when one arrives first', () => {
+    const tts = createTtsStream()
+    expect(tts.next('The build failed in the parser. Next')).toEqual({
+      spans: ['The build failed in the parser. '],
+      consumed: 32,
+    })
+    expect(tts.next('Next, we look at the tokenizer')).toEqual({ spans: [], consumed: 0 })
+  })
+
+  it.each([1, 3, 7, 50])('advances the cursor by the exact consumed span — no drift (step %i)', (step) => {
+    const full =
+      'Looking at your logs, the failure starts in the parser. It reads a stray token.\n\nThe fix is small: skip it.'
+    const { spoken, remainder } = stream(full, step)
+
+    expect(spoken.join('') + remainder).toBe(full)
+    expect(spoken.length).toBeGreaterThan(0)
+  })
+})

autobot-frontend/src/utils/__tests__/ttsSpeechShaping.test.ts

@@ -103,6 +103,36 @@ describe('createSpeechShaper — fenced code (#13102)', () => {
     expect(shaper.push('\nrm -rf build\n```\nDone.')).toBe('Done.')
   })
 
+  it('drops an indented code block that follows a blank line (#17953)', () => {
+    expect(shape('Intro:\n\n    secret()\n    more()\n\nOutro.')).toBe('Intro: Outro.')
+  })
+
+  it('keeps an indented line that continues a paragraph (#17953)', () => {
+    expect(shape('Intro line\n    continues here.')).toBe('Intro line continues here.')
+  })
+
+  it('keeps indented list content, nested items included (#17953)', () => {
+    expect(shape('- item\n\n    - nested item\n    second para')).toBe('item nested item second para')
+  })
+
+  it('drops an indented block that arrives over several streamed slices (#17953)', () => {
+    const shaper = createSpeechShaper(PH)
+    const spoken = [shaper.push('Example below.\n\n'), shaper.push('    run()\n'), shaper.push('\nDone.')]
+    expect(spoken).toEqual(['Example below.', '', 'Done.'])
+  })
+
+  it('keeps a paragraph continuation that arrives after a slice ending in a newline (#17953)', () => {
+    const shaper = createSpeechShaper(PH)
+    expect([shaper.push('Intro.\n'), shaper.push('    continued here.')]).toEqual(['Intro.', 'continued here.'])
+  })
+
+  it('classifies a line only once its indent and list marker are complete (#17953)', () => {
+    const indent = createSpeechShaper(PH)
+    expect([indent.push('Example.\n\n  '), indent.push('  run()\n\nDone.')]).toEqual(['Example.', 'Done.'])
+    const marker = createSpeechShaper(PH)
+    expect([marker.push('Steps:\n-'), marker.push(' item\n\n    more about it')]).toEqual(['Steps:', 'item more about it'])
+  })
+
   it('treats a split single backtick as inline code', () => {
     const shaper = createSpeechShaper(PH)
     expect(shaper.push('Run `')).toBe('Run')

autobot-frontend/src/utils/ttsSentences.ts

@@ -114,9 +114,10 @@ interface OpenFence {
 // A fence line per CommonMark: up to 3 spaces of indent, then 3+ backticks or
 // tildes. A backtick fence's info string may not itself contain a backtick.
 const FENCE_LINE_RE = /^[ \t]{0,3}(`{3,}|~{3,})(.*)$/
-// A line that so far is only a 1-2 character marker run: it may become a
-// fence once the next slice arrives, so it is held rather than shaped.
-const PARTIAL_FENCE_RE = /^[ \t]{0,3}(?:`{1,2}|~{1,2})$/
+// A line start whose kind is not yet known: only indentation, a lone list
+// marker, or a 1-2 character fence run. It may become a fence, indented code
+// or a list item once the next slice arrives, so it is held, not classified.
+const UNDECIDED_LINE_RE = /^[ \t]*(?:[-*+]|\d+[.)]?|`{1,2}|~{1,2})?$/
 
 /** Opens a fence? Returns the fence, or null for an ordinary line. */
 function openingFence(line: string): OpenFence | null {
@@ -131,6 +132,39 @@ function closesFence(line: string, fence: OpenFence): boolean {
   return !!m && m[1][0] === fence.char && m[1].length >= fence.len && m[2].trim() === ''
 }
 
+const INDENTED_RE = /^(?: {4}| {0,3}\t)/
+const LIST_ITEM_RE = /^[ \t]*(?:[-*+]|\d+[.)])[ \t]/
+
+/**
+ * Indented code blocks (#17953), conservatively: a run of lines indented 4+
+ * spaces (or a tab) that follows a blank line and is not list content. An
+ * indented line straight after prose continues the paragraph, and indented
+ * lines under a list item are nested list content -- both stay spoken.
+ */
+function createIndentedCodeTracker() {
+  let prevBlank = true
+  let inList = false
+  let inCode = false
+  return {
+    inCode: () => inCode,
+    /** Records a complete line start; returns whether it is speakable as far as indented code goes. */
+    recordLine(line: string, eligible: boolean): boolean {
+      const blank = line.trim() === ''
+      const indented = INDENTED_RE.test(line)
+      if (inCode && !blank && !indented) inCode = false
+      if (!inCode && eligible && prevBlank && indented && !blank && !inList) inCode = true
+      if (!blank) inList = LIST_ITEM_RE.test(line) || (inList && indented)
+      prevBlank = blank
+      return !inCode
+    },
+    reset() {
+      prevBlank = true
+      inList = false
+      inCode = false
+    },
+  }
+}
+
 /**
  * Stateful speech shaper for one reply. `push()` takes successive streamed
  * slices and returns the speakable text. Fenced code blocks (``` or ~~~) are
@@ -142,34 +176,42 @@ export function createSpeechShaper(ph: SpeechPlaceholders) {
   let fence: OpenFence | null = null
   let atLineStart = true
   let held = ''
+  const indentedCode = createIndentedCodeTracker()
 
   const keepLine = (line: string, lineStart: boolean): boolean => {
+    if (!lineStart) return !fence && !indentedCode.inCode()
     if (fence) {
-      if (lineStart && closesFence(line, fence)) fence = null
+      if (closesFence(line, fence)) fence = null
+      indentedCode.recordLine(line, false)
       return false
     }
-    const opened = lineStart ? openingFence(line) : null
+    const opened = indentedCode.inCode() ? null : openingFence(line)
     if (opened) fence = opened
-    return !opened
+    return indentedCode.recordLine(line, !opened) && !opened
   }
 
   return {
     push(slice: string): string {
-      let text = held + slice
-      const lastBreak = text.lastIndexOf('\n')
-      const tailStart = lastBreak + 1
-      const tailAtLineStart = lastBreak >= 0 || atLineStart
-      held = tailAtLineStart && PARTIAL_FENCE_RE.test(text.slice(tailStart)) ? text.slice(tailStart) : ''
-      if (held) text = text.slice(0, tailStart)
-      const kept = text.split('\n').filter((line, i) => keepLine(line, i > 0 || atLineStart))
-      atLineStart = held !== '' || text.endsWith('\n')
+      const text = held + slice
+      const tailStart = text.lastIndexOf('\n') + 1
+      const tail = text.slice(tailStart)
+      // An undecided line start is held whole -- never recorded twice, never
+      // mistaken for a blank line just because a slice ended after a newline.
+      const holdTail = (tailStart > 0 || atLineStart) && UNDECIDED_LINE_RE.test(tail)
+      held = holdTail ? tail : ''
+      const body = holdTail ? text.slice(0, tailStart) : text
+      const lines = body === '' ? [] : body.split('\n')
+      if (body.endsWith('\n')) lines.pop()
+      const kept = lines.filter((line, i) => keepLine(line, i > 0 || atLineStart))
+      atLineStart = holdTail || body.endsWith('\n')
       return shapeProse(kept.join('\n'), ph).replace(/\s+/g, ' ').trim()
     },
-    /** End of reply: a held partial marker is only backticks or tildes -- never spoken. */
+    /** End of reply: a held undecided line start is markup or indent -- never spoken. */
     flush(): string {
       held = ''
       fence = null
       atLineStart = true
+      indentedCode.reset()
       return ''
     },
   }
@@ -179,3 +221,77 @@ export function createSpeechShaper(ph: SpeechPlaceholders) {
 export function shapeForSpeech(text: string, ph: SpeechPlaceholders): string {
   return createSpeechShaper(ph).push(text)
 }
+
+// ─── Opening-chunk fast start (#13103) ───────────────────────────────────
+
+/**
+ * Minimum chars a terminated sentence needs before it is dispatched (#1485).
+ * Short fragments like "Hello there! " (13 chars) sound choppy spoken alone, so
+ * they wait to merge with the next sentence or flush as the remainder.
+ */
+export const MIN_TTS_SENTENCE_CHARS = 20
+/**
+ * The opening chunk of a reply may go out at a clause boundary (`,` `;` `:`
+ * `—` `–`) once it is this long. MODEL-SPECIFIC: too short an opening renders in
+ * a different timbre on some TTS models, so this is set by a listening test
+ * against our TTS worker (#13103), not by a unit test.
+ */
+export const OPENING_CLAUSE_MIN_CHARS = 16
+/** With no early clause boundary, the opening goes out at a word boundary once this long. */
+export const OPENING_WORD_MIN_CHARS = 48
+
+interface SpeechChunk {
+  spans: string[]
+  consumed: number
+}
+
+/** A cut here would split a markdown link, so the shaper could not rejoin it. */
+const insideLink = (span: string): boolean => /\[[^\]]*$|\]\([^)]*$/.test(span)
+
+/**
+ * The opening chunk of `text`, released before any sentence terminates, or
+ * null. Null whenever a terminator is present: the standard rule owns that
+ * text, and a short terminated sentence there is HELD for merging -- an early
+ * opening must never jump ahead of it.
+ */
+export function extractOpeningChunk(text: string): SpeechChunk | null {
+  if (/[.!?]\s/.test(text)) return null
+  const clause = /[,;:—–]\s+/g
+  let m: RegExpExecArray | null
+  while ((m = clause.exec(text)) !== null) {
+    const end = m.index + m[0].length
+    if (m.index + 1 < OPENING_CLAUSE_MIN_CHARS || insideLink(text.slice(0, end))) continue
+    return { spans: [text.slice(0, end)], consumed: end }
+  }
+  if (text.length < OPENING_WORD_MIN_CHARS) return null
+  const lastGap = /\s+(?=\S*$)/.exec(text)
+  if (!lastGap || lastGap.index === 0) return null
+  const end = lastGap.index + lastGap[0].length
+  if (insideLink(text.slice(0, end))) return null
+  return { spans: [text.slice(0, end)], consumed: end }
+}
+
+/**
+ * Per-reply TTS chunking: the opening chunk on a weaker signal, then the
+ * standard terminated-sentence rule for everything after it. `next()` takes
+ * the unspoken tail and returns the spans to speak plus the exact span
+ * consumed, so the caller's cursor never drifts (#12502).
+ *
+ * @param alreadyOpened  true for a reply already under way (e.g. voice enabled
+ *                       mid-reply), which must not get a second "opening".
+ */
+export function createTtsStream(alreadyOpened = false) {
+  let opened = alreadyOpened
+  return {
+    next(text: string): SpeechChunk {
+      const standard = extractCompleteSentences(text, MIN_TTS_SENTENCE_CHARS)
+      if (opened || standard.spans.length > 0) {
+        opened = true
+        return { spans: standard.spans, consumed: standard.consumed }
+      }
+      const opening = extractOpeningChunk(text)
+      if (opening) opened = true
+      return opening ?? { spans: [], consumed: 0 }
+    },
+  }
+}