perf(mobile): defer audio recorder creation until dictation (#15248)

Author: juliusmarminge

apps/mobile/src/features/voice-input/VoiceInputProvider.tsx

@@ -1,9 +1,10 @@
 import {
+  AudioModule,
   RecordingPresets,
   requestRecordingPermissionsAsync,
   setAudioModeAsync,
   setIsAudioActiveAsync,
-  useAudioRecorder,
+  type RecorderState,
   type RecordingStatus,
 } from "expo-audio";
 import { File } from "expo-file-system";
@@ -18,7 +19,7 @@ import {
   useState,
   type ReactNode,
 } from "react";
-import { AppState } from "react-native";
+import { AppState, Platform } from "react-native";
 import { useSharedValue } from "react-native-reanimated";
 
 import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
@@ -28,14 +29,23 @@ import {
   voiceInputBlocksSubmission,
   type VoiceInputState,
 } from "@t3tools/client-runtime/voice-input";
+import { createLazyVoiceRecorder, type LazyVoiceRecorder } from "./lazyVoiceRecorder";
 import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
 import { VoiceInputSession } from "./voiceInputSession";
 
 const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
 const VOICE_METERING_INTERVAL_MS = 80;
+// The native constructor takes platform-flattened options, as `useAudioRecorder`
+// builds them with expo-audio's internal `createRecordingOptions`.
+const { ios: IOS_RECORDING_OPTIONS, android: ANDROID_RECORDING_OPTIONS } =
+  RecordingPresets.HIGH_QUALITY;
 const VOICE_RECORDING_OPTIONS = {
-  ...RecordingPresets.HIGH_QUALITY,
+  extension: RecordingPresets.HIGH_QUALITY.extension,
+  sampleRate: RecordingPresets.HIGH_QUALITY.sampleRate,
+  numberOfChannels: RecordingPresets.HIGH_QUALITY.numberOfChannels,
+  bitRate: RecordingPresets.HIGH_QUALITY.bitRate,
   isMeteringEnabled: true,
+  ...(Platform.OS === "ios" ? IOS_RECORDING_OPTIONS : ANDROID_RECORDING_OPTIONS),
 };
 
 async function releaseVoiceRecordingAudio(): Promise<void> {
@@ -102,17 +112,22 @@ function useVoiceInputRuntime() {
   const audioLevelsRef = useRef(Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
   const audioLevels = useSharedValue(audioLevelsRef.current);
   const sessionRef = useRef<VoiceInputSession | null>(null);
-  const handleRecorderStatus = useCallback((status: RecordingStatus) => {
-    sessionRef.current?.controller.handleRecorderStatus({
-      isFinished: status.isFinished,
-      hasError: status.hasError || status.mediaServicesDidReset === true,
-      error: status.error,
-      url: status.url,
+  const recorderRef = useRef<LazyVoiceRecorder<RecorderState> | null>(null);
+
+  if (!sessionRef.current || !recorderRef.current) {
+    // The native recorder is created when dictation starts, not on app launch.
+    const recorder = createLazyVoiceRecorder({
+      create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
+      onStatus: (status: RecordingStatus) => {
+        sessionRef.current?.controller.handleRecorderStatus({
+          isFinished: status.isFinished,
+          hasError: status.hasError || status.mediaServicesDidReset === true,
+          error: status.error,
+          url: status.url,
+        });
+      },
     });
-  }, []);
-  const recorder = useAudioRecorder(VOICE_RECORDING_OPTIONS, handleRecorderStatus);
-
-  if (!sessionRef.current) {
+    recorderRef.current = recorder;
     sessionRef.current = new VoiceInputSession({
       recorder,
       getTranscriber: getLocalVoiceTranscriber,
@@ -134,6 +149,7 @@ function useVoiceInputRuntime() {
 
   const session = sessionRef.current;
   const controller = session.controller;
+  const recorder = recorderRef.current;
 
   useEffect(() => {
     const subscription = AppState.addEventListener("change", (nextState) => {
@@ -145,7 +161,14 @@ function useVoiceInputRuntime() {
     return () => subscription.remove();
   }, [controller]);
 
-  useEffect(() => () => controller.dispose(), [controller]);
+  useEffect(
+    () => () => {
+      // Dispose first so an active recording is stopped before the recorder is released.
+      controller.dispose();
+      recorder.release();
+    },
+    [controller, recorder],
+  );
 
   useEffect(() => {
     if (state.phase !== "recording") return;
@@ -175,7 +198,7 @@ function useVoiceInputRuntime() {
     const sampleRecording = () => {
       if (controller.currentState.phase !== "recording") return;
       const status = recorder.getStatus();
-      if (!status.isRecording) return;
+      if (!status?.isRecording) return;
 
       const level = normalizeVoiceInputDecibels(status.metering);
       const history = audioLevelsRef.current;

apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts

@@ -0,0 +1,181 @@
+import { describe, expect, it, vi } from "vite-plus/test";
+
+import { VoiceInputController } from "@t3tools/client-runtime/voice-input";
+
+import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
+
+type Status = { readonly isFinished: boolean };
+type State = { readonly isRecording: boolean };
+
+function deferred() {
+  let resolve!: () => void;
+  const promise = new Promise<void>((resolvePromise) => {
+    resolve = resolvePromise;
+  });
+  return { promise, resolve };
+}
+
+function createNativeRecorder(uri: string) {
+  const listeners = new Set<(status: Status) => void>();
+  return {
+    uri: null as string | null,
+    prepared: uri,
+    prepareToRecordAsync: vi.fn(async function (this: { uri: string | null; prepared: string }) {
+      this.uri = this.prepared;
+    }),
+    record: vi.fn(),
+    stop: vi.fn(async (): Promise<void> => undefined),
+    getStatus: vi.fn((): State => ({ isRecording: true })),
+    addListener: vi.fn((_event: "recordingStatusUpdate", listener: (status: Status) => void) => {
+      listeners.add(listener);
+      return { remove: () => listeners.delete(listener) };
+    }),
+    release: vi.fn(),
+    emit: (status: Status) => listeners.forEach((listener) => listener(status)),
+  };
+}
+
+function createHarness() {
+  const created: Array<ReturnType<typeof createNativeRecorder>> = [];
+  const statuses: Status[] = [];
+  const recorder = createLazyVoiceRecorder({
+    create: () => {
+      const native = createNativeRecorder(`file:///voice-${created.length}.m4a`);
+      created.push(native);
+      return native;
+    },
+    onStatus: (status) => statuses.push(status),
+  });
+  return { recorder, created, statuses };
+}
+
+describe("createLazyVoiceRecorder", () => {
+  it("does not create a native recorder until a recording prepares", async () => {
+    const { recorder, created } = createHarness();
+
+    expect(recorder.uri).toBeNull();
+    expect(recorder.getStatus()).toBeNull();
+    await recorder.stop();
+    recorder.release();
+    expect(created).toHaveLength(0);
+
+    await recorder.prepareToRecordAsync();
+    recorder.record({ forDuration: 300 });
+
+    expect(created).toHaveLength(1);
+    expect(created[0]!.record).toHaveBeenCalledWith({ forDuration: 300 });
+    expect(recorder.uri).toBe("file:///voice-0.m4a");
+    expect(recorder.getStatus()).toEqual({ isRecording: true });
+  });
+
+  it("reuses one native recorder across recordings", async () => {
+    const { recorder, created } = createHarness();
+
+    await recorder.prepareToRecordAsync();
+    await recorder.stop();
+    await recorder.prepareToRecordAsync();
+
+    expect(created).toHaveLength(1);
+    expect(created[0]!.prepareToRecordAsync).toHaveBeenCalledTimes(2);
+  });
+
+  it("forwards status events until the recorder is released", async () => {
+    const { recorder, created, statuses } = createHarness();
+    await recorder.prepareToRecordAsync();
+    const native = created[0]!;
+
+    native.emit({ isFinished: true });
+    recorder.release();
+    native.emit({ isFinished: false });
+
+    expect(statuses).toEqual([{ isFinished: true }]);
+  });
+
+  it("releases only after a pending stop settles and keeps the final uri", async () => {
+    const { recorder, created } = createHarness();
+    await recorder.prepareToRecordAsync();
+    const native = created[0]!;
+    const stopped = deferred();
+    native.stop.mockReturnValueOnce(stopped.promise);
+
+    const stopping = recorder.stop();
+    recorder.release();
+    await Promise.resolve();
+    expect(native.release).not.toHaveBeenCalled();
+    expect(recorder.uri).toBe("file:///voice-0.m4a");
+
+    stopped.resolve();
+    await stopping;
+    await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1));
+    expect(recorder.uri).toBe("file:///voice-0.m4a");
+    expect(recorder.getStatus()).toBeNull();
+  });
+
+  it("releases after a failed stop and creates a fresh recorder for the next start", async () => {
+    const { recorder, created } = createHarness();
+    await recorder.prepareToRecordAsync();
+    const first = created[0]!;
+    first.stop.mockRejectedValueOnce(new Error("not recording"));
+
+    await expect(recorder.stop()).rejects.toThrow("not recording");
+    recorder.release();
+    await vi.waitFor(() => expect(first.release).toHaveBeenCalledTimes(1));
+    expect(() => recorder.record({ forDuration: 1 })).toThrow("not prepared");
+
+    await recorder.prepareToRecordAsync();
+    expect(created).toHaveLength(2);
+    expect(recorder.uri).toBe("file:///voice-1.m4a");
+  });
+
+  describe("with the voice input controller", () => {
+    function createController(granted: boolean) {
+      const harness = createHarness();
+      const deleted: string[] = [];
+      const controller = new VoiceInputController({
+        recorder: harness.recorder,
+        getTranscriber: () => ({
+          prepare: async () => ({ locale: "en-US", transcribe: async () => "text" }),
+        }),
+        requestPermission: async () => ({ granted, canAskAgain: true }),
+        configureRecording: async () => undefined,
+        releaseRecording: async () => undefined,
+        deleteRecording: (uri) => deleted.push(uri),
+        readDraft: () => ({
+          ownerKey: "thread",
+          text: "",
+          selection: { start: 0, end: 0 },
+          revision: 0,
+        }),
+        commitDraft: () => undefined,
+        onStateChange: () => undefined,
+      });
+      return { ...harness, controller, deleted };
+    }
+
+    it("does not create a native recorder when microphone permission is denied", async () => {
+      const { controller, created } = createController(false);
+
+      await controller.start();
+
+      expect(controller.currentState.phase).toBe("error");
+      expect(created).toHaveLength(0);
+    });
+
+    it("stops, deletes, and releases the recorder created by a cancelled recording", async () => {
+      const { controller, recorder, created, deleted } = createController(true);
+
+      await controller.start();
+      expect(controller.currentState.phase).toBe("recording");
+      expect(created).toHaveLength(1);
+      const native = created[0]!;
+
+      controller.cancel();
+      recorder.release();
+      await vi.waitFor(() => expect(native.release).toHaveBeenCalledTimes(1));
+
+      expect(native.stop).toHaveBeenCalledTimes(1);
+      expect(deleted).toEqual(["file:///voice-0.m4a"]);
+      expect(controller.currentState.phase).toBe("idle");
+    });
+  });
+});

apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts

@@ -0,0 +1,88 @@
+import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input";
+
+type Subscription = { remove(): void };
+
+// The subset of expo-audio's native `AudioRecorder` the voice input hook uses.
+export interface NativeVoiceRecorder<TStatus, TRecorderState> {
+  readonly uri: string | null;
+  prepareToRecordAsync(): Promise<void>;
+  record(options: { readonly forDuration: number }): void;
+  stop(): Promise<void>;
+  getStatus(): TRecorderState;
+  addListener(
+    eventName: "recordingStatusUpdate",
+    listener: (status: TStatus) => void,
+  ): Subscription;
+  release(): void;
+}
+
+export interface LazyVoiceRecorder<TRecorderState> extends VoiceRecorder {
+  /** Status of the native recorder, or null before a recording has prepared one. */
+  getStatus(): TRecorderState | null;
+  /**
+   * Releases the native recorder after its pending prepare and stop calls settle.
+   * A later recording creates a new one, so replayed effects keep working.
+   */
+  release(): void;
+}
+
+/**
+ * Creates expo-audio's native `AudioRecorder` when a recording first prepares,
+ * instead of on composer render. Construction blocks the JS thread for tens of
+ * milliseconds on a cold composer, and most composer mounts never record.
+ */
+export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
+  readonly create: () => NativeVoiceRecorder<TStatus, TRecorderState>;
+  readonly onStatus: (status: TStatus) => void;
+}): LazyVoiceRecorder<TRecorderState> {
+  let recorder: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
+  let subscription: Subscription | null = null;
+  // A released recorder still answers `uri` so the controller can delete its file.
+  let retired: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
+  let retiredUri: string | null = null;
+  let pending: Promise<unknown> = Promise.resolve();
+
+  const track = <T>(promise: Promise<T>): Promise<T> => {
+    pending = Promise.allSettled([pending, promise]).then(() => undefined);
+    return promise;
+  };
+
+  return {
+    get uri() {
+      if (recorder) return recorder.uri;
+      return retired ? retired.uri : retiredUri;
+    },
+    prepareToRecordAsync() {
+      if (!recorder) {
+        recorder = input.create();
+        subscription = recorder.addListener("recordingStatusUpdate", input.onStatus);
+      }
+      return track(recorder.prepareToRecordAsync());
+    },
+    record(options) {
+      if (!recorder) throw new Error("Voice recorder is not prepared.");
+      recorder.record(options);
+    },
+    stop() {
+      return recorder ? track(recorder.stop()) : Promise.resolve();
+    },
+    getStatus() {
+      return recorder ? recorder.getStatus() : null;
+    },
+    release() {
+      const released = recorder;
+      if (!released) return;
+      recorder = null;
+      subscription?.remove();
+      subscription = null;
+      retired = released;
+      void pending.then(() => {
+        if (retired === released) {
+          retiredUri = released.uri;
+          retired = null;
+        }
+        released.release();
+      });
+    },
+  };
+}