Update TranscriptionResult API

chen-argmax · web-flow · commit 5626e1dc024a · 2025-11-07T11:53:38.000-08:00
* Make `TranscriptionResult` an open class and ensure thread safety

* Add `AudioProcessing.startStreamingRecordingLive` to record with an `AsyncStream`
diff --git a/.github/workflows/development-tests.yml b/.github/workflows/development-tests.yml
@@ -14,11 +14,12 @@ jobs:
       cancel-in-progress: true
     uses: ./.github/workflows/unit-tests.yml
     with:
-      ios-version: "18.6"
-      ios-device: "iPhone 16"
-      watchos-version: "11.5"
-      visionos-version: "2.5"
-      macos-runner: "macos-15"
+      ios-version: "26.1"
+      ios-device: "iPhone 17"
+      watchos-version: "26.1"
+      visionos-version: "26.1"
+      macos-runner: "macos-26"
+      xcode-version: "latest-stable"
 
   check-approvals:
     runs-on: ubuntu-latest
@@ -50,18 +51,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: macos-13-xlarge
-            ios-version: "17.2"
-            ios-device: "iPhone 14"
-            watchos-version: "10.2"
-            visionos-version: "1.0"
-            xcode-version: "15.2"
           - os: macos-14
             ios-version: "17.2"
             ios-device: "iPhone 15"
             watchos-version: "10.2"
             visionos-version: "1.0"
-            xcode-version: "15.2"
+            xcode-version: "16.1"
+          - os: macos-15
+            ios-version: "18.5"
+            ios-device: "iPhone 16"
+            watchos-version: "11.5"
+            visionos-version: "2.5"
+            xcode-version: "16.4"
     uses: ./.github/workflows/unit-tests.yml
     with:
       macos-runner: ${{ matrix.os }}
diff --git a/.github/workflows/release-tests.yml b/.github/workflows/release-tests.yml
@@ -11,23 +11,24 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: macos-13-xlarge
-            ios-version: "17.2" # TODO: Download older simulators for macOS 13
-            ios-device: "iPhone 14"
-            watchos-version: "10.2"
-            visionos-version: "1.0"
-            xcode-version: "15.2"
           - os: macos-14
             ios-version: "17.5"
             ios-device: "iPhone 15"
             watchos-version: "10.2"
             visionos-version: "1.0"
-            xcode-version: "15.4"
+            xcode-version: "16.1"
           - os: macos-15
-            ios-version: "18.6" # Latest available version
+            ios-version: "18.5"
             ios-device: "iPhone 16"
             watchos-version: "11.5"
             visionos-version: "2.5"
+            xcode-version: "16.4"
+          - os: macos-26
+            ios-version: "26.1"
+            ios-device: "iPhone 17"
+            watchos-version: "26.1"
+            visionos-version: "26.1"
+            macos-runner: "macos-26"
             xcode-version: "latest-stable"
     uses: ./.github/workflows/unit-tests.yml
     with:
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -43,13 +43,13 @@ jobs:
             }
           - {
               name: "watchOS",
-              condition: "${{ inputs.macos-runner == 'macos-15' }}",
+              condition: "${{ inputs.macos-runner == 'macos-26' }}",
               clean-destination: "generic/platform=watchOS",
-              test-destination: "platform=watchOS Simulator,OS=${{ inputs.watchos-version }},name=Apple Watch Ultra 2 (49mm)",
+              test-destination: "platform=watchOS Simulator,OS=${{ inputs.watchos-version }},name=Apple Watch Ultra 3 (49mm)",
             }
           - {
               name: "visionOS",
-              condition: "${{ inputs.macos-runner == 'macos-15' }}",
+              condition: "${{ inputs.macos-runner == 'macos-26' }}",
               clean-destination: "generic/platform=visionOS",
               test-destination: "platform=visionOS Simulator,OS=${{ inputs.visionos-version }},name=Apple Vision Pro",
             }
@@ -83,7 +83,7 @@ jobs:
           echo "Destinations for testing:"
           xcodebuild test-without-building -testPlan UnitTestsPlan -scheme whisperkit-Package -showdestinations
       - name: Boot Simulator and Wait
-        if: ${{ matrix.run-config['condition'] == true }} && ${{ matrix.run-config['name'] != 'macOS' }} && ${{ inputs.macos-runner == 'macos-15' }}
+        if: ${{ matrix.run-config['condition'] == true }} && ${{ matrix.run-config['name'] != 'macOS' }} && ${{ inputs.macos-runner == 'macos-26' }}
         # Slower runners require some time to fully boot the simulator
         # Parse the simulator name from the destination string, boot it, and wait
         run: |
@@ -104,4 +104,4 @@ jobs:
           name: test-results-${{ matrix.run-config['name']}}-on-${{ inputs.macos-runner }}
           path: |
             ~/Library/Developer/Xcode/DerivedData/**/Logs/Test/*.xcresult
-          retention-days: 5
+          retention-days: 5
diff --git a/Sources/WhisperKit/Core/Audio/AudioProcessor.swift b/Sources/WhisperKit/Core/Audio/AudioProcessor.swift
@@ -96,6 +96,13 @@ public protocol AudioProcessing {
     /// Starts recording audio from the specified input device, resetting the previous state
     func startRecordingLive(inputDeviceID: DeviceID?, callback: (([Float]) -> Void)?) throws
 
+    /// Starts live audio recording and provides an async stream of audio samples.
+    ///
+    /// - Parameter inputDeviceID: The device ID of the input audio device to use for recording.
+    ///   This parameter is only valid on macOS; iOS always uses the default input device.
+    /// - Returns: A tuple containing the async stream of audio sample buffers and the stream's continuation.
+    func startStreamingRecordingLive(inputDeviceID: DeviceID?) -> (AsyncThrowingStream<[Float], Error>, AsyncThrowingStream<[Float], Error>.Continuation)
+
     /// Pause recording
     func pauseRecording()
 
@@ -128,10 +135,14 @@ public extension AudioProcessing {
             try AudioProcessor.loadAudio(fromPath: audioFilePath)
         }.value
     }
-
+    
     func startRecordingLive(inputDeviceID: DeviceID? = nil, callback: (([Float]) -> Void)?) throws {
         try startRecordingLive(inputDeviceID: inputDeviceID, callback: callback)
     }
+    
+    func startStreamingRecordingLive(inputDeviceID: DeviceID? = nil) -> (AsyncThrowingStream<[Float], Error>, AsyncThrowingStream<[Float], Error>.Continuation) {
+        return startStreamingRecordingLive(inputDeviceID: inputDeviceID)
+    }
 
     func resumeRecordingLive(inputDeviceID: DeviceID? = nil, callback: (([Float]) -> Void)?) throws {
         try resumeRecordingLive(inputDeviceID: inputDeviceID, callback: callback)
@@ -1023,6 +1034,28 @@ public extension AudioProcessor {
         lastInputDevice = inputDeviceID
     }
 
+    /// Starts live audio recording and returns an async stream that yields sample buffers.
+    /// Recording stops automatically when the stream terminates.
+    func startStreamingRecordingLive(inputDeviceID: DeviceID? = nil) -> (AsyncThrowingStream<[Float], Error>, AsyncThrowingStream<[Float], Error>.Continuation) {
+        let (stream, continuation) = AsyncThrowingStream<[Float], Error>.makeStream(bufferingPolicy: .unbounded)
+        
+        continuation.onTermination = { [weak self] _ in
+            guard let self = self else { return }
+            self.audioBufferCallback = nil
+            self.stopRecording()
+        }
+        
+        do {
+            try self.startRecordingLive(inputDeviceID: inputDeviceID) { @Sendable floats in
+                continuation.yield(floats)
+            }
+        } catch {
+            continuation.finish(throwing: error)
+        }
+
+        return (stream, continuation)
+    }
+
     func resumeRecordingLive(inputDeviceID: DeviceID? = nil, callback: (([Float]) -> Void)? = nil) throws {
         try? setupAudioSessionForDevice()
 
diff --git a/Sources/WhisperKit/Core/Models.swift b/Sources/WhisperKit/Core/Models.swift
@@ -501,14 +501,17 @@ public struct DecodingResult {
     }
 }
 
-// Structs
-
-public struct TranscriptionResult: Codable, Sendable {
-    public var text: String
-    public var segments: [TranscriptionSegment]
-    public var language: String
-    public var timings: TranscriptionTimings
-    public var seekTime: Float?
+/// Reference-type container for transcription output.
+/// The stored properties stay thread-safe because each one uses
+/// `TranscriptionPropertyLock`, so reads/writes hop through a private `NSLock`
+/// before the value is accessed, making this shared `@unchecked Sendable` class
+/// safe to hand across concurrent contexts.
+open class TranscriptionResult: Codable, @unchecked Sendable {
+    @TranscriptionPropertyLock public var text: String
+    @TranscriptionPropertyLock public var segments: [TranscriptionSegment]
+    @TranscriptionPropertyLock public var language: String
+    @TranscriptionPropertyLock public var timings: TranscriptionTimings
+    @TranscriptionPropertyLock public var seekTime: Float?
 
     public init(
         text: String,
@@ -600,12 +603,38 @@ public struct TranscriptionResult: Codable, Sendable {
     }
 }
 
+/// Value-type equivalent of `TranscriptionResult` without property locking.
+public struct TranscriptionResultStruct: Codable, Sendable {
+    public var text: String
+    public var segments: [TranscriptionSegment]
+    public var language: String
+    public var timings: TranscriptionTimings
+    public var seekTime: Float?
+
+    public init(
+        text: String,
+        segments: [TranscriptionSegment],
+        language: String,
+        timings: TranscriptionTimings,
+        seekTime: Float? = nil
+    ) {
+        self.text = text
+        self.segments = segments
+        self.language = language
+        self.timings = timings
+        self.seekTime = seekTime
+    }
+}
+
+
 public extension TranscriptionResult {
     var allWords: [WordTiming] {
         return segments.compactMap { $0.words }.flatMap { $0 }
     }
 }
 
+// Structs
+
 public struct TranscriptionSegment: Hashable, Codable, Sendable {
     public var id: Int
     public var seek: Int
diff --git a/Sources/WhisperKit/Utilities/Concurrency.swift b/Sources/WhisperKit/Utilities/Concurrency.swift
@@ -2,6 +2,7 @@
 //  Copyright © 2024 Argmax, Inc. All rights reserved.
 
 import Foundation
+import os.lock
 
 /// An actor that provides thread-safe early stopping functionality using UUIDs as keys
 public actor EarlyStopActor {
@@ -31,3 +32,58 @@ public actor EarlyStopActor {
         return shouldStop.removeValue(forKey: uuid)
     }
 }
+
+/// Serializes access to a value with an `os_unfair_lock` so mutation stays
+/// thread-safe. The wrapper is used by `TranscriptionResult`, which is marked
+/// `@unchecked Sendable`; guarding each property with this lock helps keep the
+/// result instance safe when shared across concurrent contexts.
+@propertyWrapper
+public struct TranscriptionPropertyLock<Value: Codable & Sendable>: Sendable, Codable {
+    private let lock: UnfairLock
+    private var value: Value
+    
+    public init(wrappedValue: Value) {
+        self.lock = UnfairLock()
+        self.value = wrappedValue
+    }
+    public init(from decoder: Swift.Decoder) throws {
+        self.lock = UnfairLock()
+        self.value = try Value(from: decoder)
+    }
+    
+    public func encode(to encoder: Encoder) throws {
+        try lock.withLock {
+            try value.encode(to: encoder)
+        }
+        
+    }
+    
+    public var wrappedValue: Value {
+        get {
+            lock.withLock {
+                return value
+            }
+        }
+        set {
+            lock.withLock {
+                value = newValue
+            }
+        }
+    }
+}
+
+/// Thin wrapper around `os_unfair_lock` that exposes a Swift-friendly
+/// `withLock` helper. This lock is non-reentrant and optimized for low
+/// contention, matching the semantics of Core Foundation’s unfair lock.
+@usableFromInline
+final class UnfairLock: @unchecked Sendable {
+    @usableFromInline
+    var lock = os_unfair_lock()
+
+    @inlinable
+    func withLock<T>(_ body: () throws -> T) rethrows -> T {
+        os_unfair_lock_lock(&lock)
+        defer { os_unfair_lock_unlock(&lock) }
+        return try body()
+    }
+}
