feat(voice): GPT Live voice agent with fast commands, existing-thread follow-ups, and integrated voice UX

apps/server/src/environment/ServerEnvironment.test.ts

@@ -19,6 +19,7 @@ import {
   RELAY_URL_SECRET,
 } from "../cloud/config.ts";
 import * as ServerConfig from "../config.ts";
+import { OPENAI_API_KEY_SECRET_NAME } from "../voice/broker.ts";
 import * as ServerEnvironment from "./ServerEnvironment.ts";
 
 const isServerEnvironmentIdPersistenceError = Schema.is(
@@ -229,6 +230,49 @@ it.layer(NodeServices.layer)("ServerEnvironmentLive", (it) => {
     }),
   );
 
+  it.effect("advertises voiceLive only while the OpenAI key secret is configured", () =>
+    Effect.gen(function* () {
+      const fileSystem = yield* FileSystem.FileSystem;
+      const baseDir = yield* fileSystem.makeTempDirectoryScoped({
+        prefix: "t3-server-environment-voicelive-test-",
+      });
+      const testLayer = Layer.mergeAll(
+        ServerEnvironment.layer.pipe(Layer.provide(ServerSecretStore.layer)),
+        ServerSecretStore.layer,
+      ).pipe(Layer.provide(ServerConfig.layerTest(process.cwd(), baseDir)));
+
+      yield* Effect.gen(function* () {
+        const secrets = yield* ServerSecretStore.ServerSecretStore;
+        const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;
+        const encode = (value: string) => new TextEncoder().encode(value);
+
+        // Without the environment-owned OpenAI key the environment cannot
+        // broker a Live session, so the capability stays absent (absent means
+        // unsupported to clients) instead of advertising an entry point that
+        // can only fail.
+        const withoutKey = yield* serverEnvironment.getDescriptor;
+        expect(withoutKey.capabilities.voiceLive).toBeUndefined();
+
+        for (const emptyKey of ["", "   \n\t"]) {
+          yield* secrets.set(OPENAI_API_KEY_SECRET_NAME, encode(emptyKey));
+          const empty = yield* serverEnvironment.getDescriptor;
+          expect(empty.capabilities.voiceLive).toBeUndefined();
+        }
+
+        // Configuring the key secret publishes the capability...
+        yield* secrets.set(OPENAI_API_KEY_SECRET_NAME, encode("test-key"));
+        const withKey = yield* serverEnvironment.getDescriptor;
+        expect(withKey.capabilities.voiceLive).toBe(true);
+
+        // ...and removing it withdraws the capability without a restart,
+        // matching the broker's per-mint key read.
+        yield* secrets.remove(OPENAI_API_KEY_SECRET_NAME);
+        const afterRemoval = yield* serverEnvironment.getDescriptor;
+        expect(afterRemoval.capabilities.voiceLive).toBeUndefined();
+      }).pipe(Effect.provide(testLayer));
+    }),
+  );
+
   it.effect("advertises desktopAppUpdate only with desktop mode and the control fd", () =>
     Effect.gen(function* () {
       const fileSystem = yield* FileSystem.FileSystem;

apps/server/src/environment/ServerEnvironment.ts

@@ -10,6 +10,7 @@ import * as Crypto from "effect/Crypto";
 import * as Effect from "effect/Effect";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
 import * as Path from "effect/Path";
 import * as Schema from "effect/Schema";
 
@@ -20,6 +21,7 @@ import { resolveServerSelfUpdateCapability } from "../cloud/selfUpdate.ts";
 import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts";
 import * as ServerConfig from "../config.ts";
 import * as ProcessRunner from "../processRunner.ts";
+import { OPENAI_API_KEY_SECRET_NAME } from "../voice/broker.ts";
 import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts";
 import { detectServerEnvironmentMachineKind } from "./ServerEnvironmentMachine.ts";
 
@@ -255,15 +257,32 @@ export const make = Effect.gen(function* () {
 
   return ServerEnvironment.of({
     getEnvironmentId: Effect.succeed(environmentId),
-    // The publish opt-in and relay link change at runtime (`t3 connect
-    // publish`, the client settings toggle), so the capability is read per
+    // The publish opt-in, relay link, and voice brokering eligibility change
+    // at runtime (`t3 connect publish`, the client settings toggle, and the
+    // environment-owned OpenAI key secret), so the capabilities are read per
     // descriptor request rather than baked in at startup.
-    getDescriptor: readAgentActivityPublishingActive(secrets).pipe(
-      Effect.map((agentActivityPublishing) => ({
+    getDescriptor: Effect.gen(function* () {
+      const agentActivityPublishing = yield* readAgentActivityPublishingActive(secrets);
+      const openAiKey = yield* secrets
+        .get(OPENAI_API_KEY_SECRET_NAME)
+        .pipe(
+          Effect.catch((cause) =>
+            Effect.logWarning("failed to read the OpenAI API key secret", { cause }).pipe(
+              Effect.as(Option.none<Uint8Array>()),
+            ),
+          ),
+        );
+      return {
         ...descriptor,
-        capabilities: { ...descriptor.capabilities, agentActivityPublishing },
-      })),
-    ),
+        capabilities: {
+          ...descriptor.capabilities,
+          agentActivityPublishing,
+          ...(Option.isSome(openAiKey) && new TextDecoder().decode(openAiKey.value).trim()
+            ? { voiceLive: true }
+            : {}),
+        },
+      };
+    }),
   });
 });
 

apps/server/src/server.ts

@@ -38,6 +38,7 @@ import {
 import { guardHttpResponseWriteErrors } from "./httpResponseErrorGuard.ts";
 import { fixPath } from "./os-jank.ts";
 import { websocketRpcRouteLayer } from "./ws.ts";
+import * as VoiceBroker from "./voice/broker.ts";
 import * as ExternalLauncher from "./process/externalLauncher.ts";
 import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts";
 import { pullRequestHttpApiLayer } from "./pullRequest/http.ts";
@@ -602,6 +603,7 @@ export const makeRoutesLayer = Layer.mergeAll(
     assetRouteLayer,
     attachmentUploadRouteLayer,
     deviceHubProxyRouteLayer,
+    VoiceBroker.voiceBrokerRouteLayer,
     staticAndDevRouteLayer,
     websocketRpcRouteLayer,
   ),