Add POST /redrob/speech so a client can play speech without holding the key

packages/redrob/src/server/routes/instance/httpapi/groups/control.ts

@@ -1,7 +1,8 @@
 import { Auth } from "@/auth"
 
 import { Schema } from "effect"
-import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, OpenApi } from "effect/unstable/httpapi"
+import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi"
+import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors"
 import { described } from "./metadata"
 import { ProviderV2 } from "@redrob-code/core/provider"
 
@@ -28,9 +29,18 @@ export const LogInput = Schema.Struct({
   }),
 })
 
+export const SpeechInput = Schema.Struct({
+  text: Schema.String.annotate({ description: "The words to speak, at most 4096 characters" }),
+  voice: Schema.optional(Schema.String).annotate({ description: "A voice of the chosen model" }),
+  model: Schema.optional(Schema.String).annotate({
+    description: "A speech model id served by the Redrob gateway. Defaults to elevenlabs/eleven-flash-v2.5, for playback.",
+  }),
+})
+
 export const ControlPaths = {
   auth: "/auth/:providerID",
   log: "/log",
+  speech: "/redrob/speech",
 } as const
 
 export const ControlApi = HttpApi.make("control").add(
@@ -59,6 +69,21 @@ export const ControlApi = HttpApi.make("control").add(
           description: "Remove authentication credentials",
         }),
       ),
+      /*
+       * Speech on the engine's own Redrob credential, for a client that has to play audio and must
+       * not hold the key: Redrob Cowork's read-aloud. The key never leaves this process.
+       */
+      HttpApiEndpoint.post("speech", ControlPaths.speech, {
+        payload: SpeechInput,
+        success: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array({ contentType: "audio/mpeg" })),
+        error: [InvalidRequestError, ServiceUnavailableError, UpstreamError, TimeoutError],
+      }).annotateMerge(
+        OpenApi.annotations({
+          identifier: "redrob.speech",
+          summary: "Speak text",
+          description: "Turn text into mp3 speech through the Redrob gateway, billed to the connected Redrob account.",
+        }),
+      ),
       HttpApiEndpoint.post("log", ControlPaths.log, {
         query: LogQuery,
         payload: LogInput,

packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts

@@ -3,12 +3,37 @@ import { Auth } from "@/auth"
 import { Effect } from "effect"
 import { HttpApiBuilder } from "effect/unstable/httpapi"
 import { RootHttpApi } from "../api"
-import { LogInput } from "../groups/control"
+import { LogInput, SpeechInput } from "../groups/control"
+import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors"
+import { PLAYBACK_DEFAULTS, prepareSpeech, requestSpeech, speechKey, type SpeechError } from "@/tool/speech_generate"
+import { HttpClient } from "effect/unstable/http"
 import { ProviderV2 } from "@redrob-code/core/provider"
 
 export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (handlers) =>
   Effect.gen(function* () {
     const auth = yield* Auth.Service
+    const http = yield* HttpClient.HttpClient
+
+    /** Each failure as the endpoint declares it; no_key is 503 because the engine, not the caller, lacks it. */
+    const speechFailure = (error: SpeechError) =>
+      error.reason === "invalid"
+        ? new InvalidRequestError({ message: error.message })
+        : error.reason === "no_key"
+          ? new ServiceUnavailableError({ message: error.message, service: "redrob" })
+          : error.reason === "timeout"
+            ? new TimeoutError({ message: error.message, operation: "speech" })
+            : new UpstreamError({
+                message: error.message,
+                service: "redrob",
+                ...(error.status ? { status: error.status } : {}),
+              })
+
+    const speech = Effect.fn("ControlHttpApi.speech")(function* (ctx: { payload: typeof SpeechInput.Type }) {
+      // This route is for audio that is played at once, so it defaults to the fast playback model.
+      const prepared = yield* prepareSpeech(ctx.payload, PLAYBACK_DEFAULTS)
+      const key = yield* speechKey(auth)
+      return (yield* requestSpeech(http, key, prepared)).audio
+    }, Effect.mapError(speechFailure))
 
     const authSet = Effect.fn("ControlHttpApi.authSet")(function* (ctx: {
       params: { providerID: ProviderV2.ID }
@@ -38,6 +63,10 @@ export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (han
       return true
     })
 
-    return handlers.handle("authSet", authSet).handle("authRemove", authRemove).handle("log", log)
+    return handlers
+      .handle("authSet", authSet)
+      .handle("authRemove", authRemove)
+      .handle("speech", speech)
+      .handle("log", log)
   }),
 )

packages/redrob/src/tool/speech_generate.ts

@@ -16,21 +16,141 @@ import { gatewayMessage, gatewayUrl, redrobKey, slug } from "./image_generate"
  * Speech through the Redrob gateway's `POST /v1/audio/speech`, on the engine's own Redrob credential,
  * for the reason image_generate gives: the key is the engine's and stays here.
  *
- * The audio is saved and NOT returned as an attachment. An attachment goes back to the model with the
- * next turn, and audio input would steer `auto` to an audio-input model and bill the clip as input on
- * every turn after, to tell the model nothing it does not already know: it wrote the words.
+ * Two callers share `prepareSpeech` and `requestSpeech`: the `speech_generate` tool, which saves an
+ * mp3, and the engine's `POST /redrob/speech` route, which hands the audio to Redrob Cowork to play
+ * a reply aloud without the app ever holding the key.
+ *
+ * The tool saves its audio and does NOT return it as an attachment. An attachment goes back to the
+ * model with the next turn, and audio input would steer `auto` to an audio-input model and bill the
+ * clip as input on every turn after, to tell the model nothing it does not already know: it wrote
+ * the words.
  */
 
 /** Multilingual, so a Korean script is spoken as Korean. Any speech model the gateway serves can be named. */
 export const DEFAULT_SPEECH_MODEL = "elevenlabs/eleven-multilingual-v2"
 export const DEFAULT_SPEECH_VOICE = "sarah"
 
+/**
+ * For speech that is played as it arrives, a reply read aloud or a voice conversation's answer, where
+ * the wait before the first word matters more than the last bit of quality. Same voice, so a saved file
+ * and a reply read aloud sound like the same speaker. Multilingual, so Korean is spoken as Korean.
+ */
+export const PLAYBACK_SPEECH_MODEL = "elevenlabs/eleven-flash-v2.5"
+
+export type SpeechDefaults = { model: string; voice: string }
+const FILE_DEFAULTS: SpeechDefaults = { model: DEFAULT_SPEECH_MODEL, voice: DEFAULT_SPEECH_VOICE }
+export const PLAYBACK_DEFAULTS: SpeechDefaults = { model: PLAYBACK_SPEECH_MODEL, voice: DEFAULT_SPEECH_VOICE }
+
 /** The gateway's limit for one request, checked here so an over-long request costs nothing. */
 export const MAX_SPEECH_CHARACTERS = 4_096
 
 const TIMEOUT_MS = 180_000
 const MAX_AUDIO_BYTES = 50 * 1024 * 1024
 
+/**
+ * Why speech could not be made. `invalid` and `no_key` are found before anything is sent and cost
+ * nothing; `upstream` and `timeout` are the gateway's.
+ */
+export class SpeechError extends Schema.TaggedErrorClass<SpeechError>()("SpeechError", {
+  reason: Schema.Literals(["invalid", "no_key", "upstream", "timeout"]),
+  message: Schema.String,
+  status: Schema.optional(Schema.Number),
+}) {}
+
+export type PreparedSpeech = { text: string; model: string; voice?: string; characters: number }
+
+/** Characters as the gateway bills them: code points, so the limit here agrees with its limit there. */
+function characters(text: string): number {
+  let count = 0
+  for (const _ of text) count += 1
+  return count
+}
+
+/** The request as it will be sent, or why it would be refused, with nothing sent either way. */
+export const prepareSpeech = Effect.fn("Speech.prepare")(function* (
+  input: { text: string; model?: string; voice?: string },
+  defaults: SpeechDefaults = FILE_DEFAULTS,
+) {
+  const text = input.text.trim()
+  if (!text) return yield* new SpeechError({ reason: "invalid", message: "Give the words to speak." })
+  const length = characters(text)
+  if (length > MAX_SPEECH_CHARACTERS) {
+    return yield* new SpeechError({
+      reason: "invalid",
+      message: `The text is ${length} characters; speak at most ${MAX_SPEECH_CHARACTERS} per call and save longer material in parts.`,
+    })
+  }
+  const model = input.model?.trim() || defaults.model
+  /* Voices belong to a model, so the default voice only goes with the default model. */
+  const voice = input.voice?.trim() || (model === defaults.model ? defaults.voice : undefined)
+  const prepared: PreparedSpeech = { text, model, ...(voice ? { voice } : {}), characters: length }
+  return prepared
+})
+
+/** The engine's Redrob key, or a `no_key` failure, which is found before anything is sent. */
+export const speechKey = Effect.fn("Speech.key")(function* (auth: Auth.Interface) {
+  const key = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined)))
+  if (!key) {
+    return yield* new SpeechError({
+      reason: "no_key",
+      message: "Connect Redrob to generate speech: no Redrob API key is available.",
+    })
+  }
+  return key
+})
+
+/** One call to the gateway. The audio is checked for being audio before anyone keeps or plays it. */
+export const requestSpeech = Effect.fn("Speech.request")(function* (
+  http: HttpClient.HttpClient,
+  key: string,
+  speech: PreparedSpeech,
+) {
+  const request = yield* HttpClientRequest.post(`${gatewayUrl()}/audio/speech`).pipe(
+    HttpClientRequest.bearerToken(key),
+    HttpClientRequest.bodyJson({
+      model: speech.model,
+      input: speech.text,
+      response_format: "mp3",
+      ...(speech.voice ? { voice: speech.voice } : {}),
+    }),
+    Effect.orDie,
+  )
+  const response = yield* http.execute(request).pipe(
+    Effect.mapError(() => new SpeechError({ reason: "upstream", message: "The Redrob gateway could not be reached." })),
+    Effect.timeoutOrElse({
+      duration: TIMEOUT_MS,
+      orElse: () => Effect.fail(new SpeechError({ reason: "timeout", message: "Speech generation timed out." })),
+    }),
+  )
+  if (response.status < 200 || response.status >= 300) {
+    const body = yield* response.text.pipe(Effect.orElseSucceed(() => ""))
+    return yield* new SpeechError({
+      reason: "upstream",
+      status: response.status,
+      message: `The Redrob gateway answered ${response.status}: ${gatewayMessage(body)}`,
+    })
+  }
+  const contentType = response.headers["content-type"]?.split(";")[0]?.trim().toLowerCase() ?? ""
+  if (!contentType.startsWith("audio/")) {
+    return yield* new SpeechError({
+      reason: "upstream",
+      message: `The Redrob gateway answered with ${contentType || "no content type"}, not audio.`,
+    })
+  }
+  const buffer = yield* response.arrayBuffer.pipe(
+    Effect.mapError(() => new SpeechError({ reason: "upstream", message: "The Redrob gateway's audio was cut off." })),
+  )
+  const audio = new Uint8Array(buffer)
+  if (audio.byteLength === 0) {
+    return yield* new SpeechError({ reason: "upstream", message: "The Redrob gateway returned no audio." })
+  }
+  if (audio.byteLength > MAX_AUDIO_BYTES) {
+    return yield* new SpeechError({ reason: "upstream", message: "The audio is over 50 MB." })
+  }
+  const cost = Number(response.headers["x-redrob-cost-usd"])
+  return { audio, contentType, ...(Number.isFinite(cost) ? { costUsd: cost } : {}) }
+})
+
 export const Parameters = Schema.Struct({
   text: Schema.String.annotate({ description: `The words to speak, at most ${MAX_SPEECH_CHARACTERS} characters` }),
   voice: Schema.optional(Schema.String).annotate({
@@ -44,13 +164,6 @@ export const Parameters = Schema.Struct({
   }),
 })
 
-/** Characters as the gateway bills them: code points, so the limit here agrees with its limit there. */
-function characters(text: string): number {
-  let count = 0
-  for (const _ of text) count += 1
-  return count
-}
-
 export const SpeechGenerateTool = Tool.define(
   "speech_generate",
   Effect.gen(function* () {
@@ -64,22 +177,12 @@ export const SpeechGenerateTool = Tool.define(
       parameters: Parameters,
       execute: (params: Schema.Schema.Type<typeof Parameters>, ctx: Tool.Context) =>
         Effect.gen(function* () {
-          const text = params.text.trim()
-          if (!text) throw new Error("Give the words to speak.")
-          const length = characters(text)
-          if (length > MAX_SPEECH_CHARACTERS) {
-            throw new Error(
-              `The text is ${length} characters; speak at most ${MAX_SPEECH_CHARACTERS} per call and save longer material in parts.`,
-            )
-          }
-          const key = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined)))
-          if (!key) throw new Error("Connect Redrob to generate speech: no Redrob API key is available.")
-          const model = params.model?.trim() || DEFAULT_SPEECH_MODEL
-          const voice = params.voice?.trim() || (model === DEFAULT_SPEECH_MODEL ? DEFAULT_SPEECH_VOICE : undefined)
+          const speech = yield* prepareSpeech(params)
+          const key = yield* speechKey(auth)
 
           const instance = yield* InstanceState.context
           const directory = path.join(instance.directory, "artifacts")
-          const stem = slug(params.filename?.trim() || text)
+          const stem = slug(params.filename?.trim() || speech.text)
           let filepath = path.join(directory, `${stem}.mp3`)
           for (let n = 2; yield* fs.existsSafe(filepath); n += 1) {
             filepath = path.join(directory, `${stem}-${n}.mp3`)
@@ -89,53 +192,30 @@ export const SpeechGenerateTool = Tool.define(
             permission: "edit",
             patterns: [path.relative(instance.worktree, filepath)],
             always: ["*"],
-            metadata: { filepath, model, voice },
+            metadata: { filepath, model: speech.model, voice: speech.voice },
           })
 
-          const request = yield* HttpClientRequest.post(`${gatewayUrl()}/audio/speech`).pipe(
-            HttpClientRequest.bearerToken(key),
-            HttpClientRequest.bodyJson({
-              model,
-              input: text,
-              response_format: "mp3",
-              ...(voice ? { voice } : {}),
-            }),
-          )
-          const response = yield* http.execute(request).pipe(
-            Effect.timeoutOrElse({
-              duration: TIMEOUT_MS,
-              orElse: () => Effect.die(new Error("Speech generation timed out")),
-            }),
-          )
-          if (response.status < 200 || response.status >= 300) {
-            throw new Error(`The Redrob gateway answered ${response.status}: ${gatewayMessage(yield* response.text)}`)
-          }
-          const contentType = response.headers["content-type"]?.split(";")[0]?.trim().toLowerCase() ?? ""
-          if (!contentType.startsWith("audio/")) {
-            throw new Error(`The Redrob gateway answered with ${contentType || "no content type"}, not audio.`)
-          }
-          const audio = new Uint8Array(yield* response.arrayBuffer)
-          if (audio.byteLength === 0) throw new Error("The Redrob gateway returned no audio.")
-          if (audio.byteLength > MAX_AUDIO_BYTES) throw new Error("The audio is over 50 MB.")
-
-          yield* fs.writeWithDirs(filepath, audio)
+          const spoken = yield* requestSpeech(http, key, speech)
+          yield* fs.writeWithDirs(filepath, spoken.audio)
           yield* events.publish(FileSystem.Event.Edited, { file: filepath })
           yield* events.publish(Watcher.Event.Updated, { file: filepath, event: "add" })
 
           const relative = path.relative(instance.worktree, filepath)
-          const cost = Number(response.headers["x-redrob-cost-usd"])
           return {
             title: relative,
-            output: `Saved ${relative}: ${length} characters spoken with ${model}${voice ? ` (${voice})` : ""}.`,
+            output: `Saved ${relative}: ${speech.characters} characters spoken with ${speech.model}${speech.voice ? ` (${speech.voice})` : ""}.`,
             metadata: {
               filepath,
-              model,
-              ...(voice ? { voice } : {}),
-              characters: length,
-              ...(Number.isFinite(cost) ? { costUsd: cost } : {}),
+              model: speech.model,
+              ...(speech.voice ? { voice: speech.voice } : {}),
+              characters: speech.characters,
+              ...(spoken.costUsd === undefined ? {} : { costUsd: spoken.costUsd }),
             },
           }
-        }).pipe(Effect.orDie),
+        }).pipe(
+          Effect.catchTag("SpeechError", (error) => Effect.die(new Error(error.message))),
+          Effect.orDie,
+        ),
     }
   }),
 )