[camera_avfoundation] Fix crash when streaming while recording (flutter#9691)

In `captureOutput`, we receive both pixel and audio buffers (audio only during recording with audio enabled, when streaming alone is enabled, everything works correctly). Currently, the streaming part of this method doesn't handle non-pixel buffers properly, causing a crash because of force unwrapping of nil.

This PR changes the logic to ignore non-pixel buffers for purposes of streaming.

Resolves flutter#172894.

## Pre-Review Checklist

**Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed.

[^1]: Regular contributors who have demonstrated familiarity with the repository guidelines only need to comment if the PR is not auto-exempted by repo tooling.

Author: RobertOdrowaz

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.21
+
+* Fixes crash when streaming is enabled during recording.
+
 ## 0.9.20+7
 
 * Updates kotlin version to 2.2.0 to enable gradle 8.11 support.

packages/camera/camera_avfoundation/example/ios/RunnerTests/StreamingTests.swift

@@ -36,6 +36,7 @@ final class StreamingTests: XCTestCase {
     DefaultCamera,
     AVCaptureOutput,
     CMSampleBuffer,
+    CMSampleBuffer,
     AVCaptureConnection
   ) {
     let captureSessionQueue = DispatchQueue(label: "testing")
@@ -45,13 +46,14 @@ final class StreamingTests: XCTestCase {
     let camera = CameraTestUtils.createTestCamera(configuration)
     let testAudioOutput = CameraTestUtils.createTestAudioOutput()
     let sampleBuffer = CameraTestUtils.createTestSampleBuffer()
+    let audioSampleBuffer = CameraTestUtils.createTestAudioSampleBuffer()
     let testAudioConnection = CameraTestUtils.createTestConnection(testAudioOutput)
 
-    return (camera, testAudioOutput, sampleBuffer, testAudioConnection)
+    return (camera, testAudioOutput, sampleBuffer, audioSampleBuffer, testAudioConnection)
   }
 
   func testExceedMaxStreamingPendingFramesCount() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -87,7 +89,7 @@ final class StreamingTests: XCTestCase {
   }
 
   func testReceivedImageStreamData() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -124,8 +126,34 @@ final class StreamingTests: XCTestCase {
     waitForExpectations(timeout: 30, handler: nil)
   }
 
+  func testIgnoresNonImageBuffers() {
+    let (camera, testAudioOutput, _, audioSampleBuffer, testAudioConnection) = createCamera()
+    let handlerMock = MockImageStreamHandler()
+    handlerMock.eventSinkStub = { event in
+      XCTFail()
+    }
+
+    let finishStartStreamExpectation = expectation(
+      description: "Finish startStream")
+
+    let messenger = MockFlutterBinaryMessenger()
+    camera.startImageStream(
+      with: messenger, imageStreamHandler: handlerMock,
+      completion: {
+        _ in
+        finishStartStreamExpectation.fulfill()
+      })
+
+    waitForExpectations(timeout: 30, handler: nil)
+    XCTAssertEqual(camera.isStreamingImages, true)
+
+    camera.captureOutput(testAudioOutput, didOutput: audioSampleBuffer, from: testAudioConnection)
+
+    waitForQueueRoundTrip(with: DispatchQueue.main)
+  }
+
   func testImageStreamEventFormat() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
 
     let expectation = expectation(description: "Received a valid event")
 

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/DefaultCamera.swift

@@ -758,73 +758,7 @@ final class DefaultCamera: FLTCam, Camera {
       return
     }
 
-    if isStreamingImages {
-      if let eventSink = imageStreamHandler?.eventSink,
-        streamingPendingFramesCount < maxStreamingPendingFramesCount
-      {
-        streamingPendingFramesCount += 1
-
-        let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
-        // Must lock base address before accessing the pixel data
-        CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
-        let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
-
-        var planes: [[String: Any]] = []
-
-        let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
-        let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
-
-        for i in 0..<planeCount {
-          let planeAddress: UnsafeMutableRawPointer?
-          let bytesPerRow: Int
-          let height: Int
-          let width: Int
-
-          if isPlanar {
-            planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
-            bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
-            height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
-            width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
-          } else {
-            planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
-            bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
-            height = CVPixelBufferGetHeight(pixelBuffer)
-            width = CVPixelBufferGetWidth(pixelBuffer)
-          }
-
-          let length = bytesPerRow * height
-          let bytes = Data(bytes: planeAddress!, count: length)
-
-          let planeBuffer: [String: Any] = [
-            "bytesPerRow": bytesPerRow,
-            "width": width,
-            "height": height,
-            "bytes": FlutterStandardTypedData(bytes: bytes),
-          ]
-          planes.append(planeBuffer)
-        }
-
-        // Lock the base address before accessing pixel data, and unlock it afterwards.
-        // Done accessing the `pixelBuffer` at this point.
-        CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageBuffer: [String: Any] = [
-          "width": imageWidth,
-          "height": imageHeight,
-          "format": videoFormat,
-          "planes": planes,
-          "lensAperture": Double(captureDevice.lensAperture()),
-          "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
-          "sensorSensitivity": Double(captureDevice.iso()),
-        ]
-
-        DispatchQueue.main.async {
-          eventSink(imageBuffer)
-        }
-      }
-    }
+    handleSampleBufferStreaming(sampleBuffer)
 
     if isRecording && !isRecordingPaused {
       if videoWriter?.status == .failed, let error = videoWriter?.error {
@@ -905,6 +839,81 @@ final class DefaultCamera: FLTCam, Camera {
     }
   }
 
+  private func handleSampleBufferStreaming(_ sampleBuffer: CMSampleBuffer) {
+    guard isStreamingImages,
+      let eventSink = imageStreamHandler?.eventSink,
+      streamingPendingFramesCount < maxStreamingPendingFramesCount
+    else {
+      return
+    }
+
+    // Non-pixel buffer samples, such as audio samples, are ignored for streaming
+    guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
+      return
+    }
+
+    streamingPendingFramesCount += 1
+
+    // Must lock base address before accessing the pixel data
+    CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
+
+    let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
+    let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
+
+    var planes: [[String: Any]] = []
+
+    let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
+    let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
+
+    for i in 0..<planeCount {
+      let planeAddress: UnsafeMutableRawPointer?
+      let bytesPerRow: Int
+      let height: Int
+      let width: Int
+
+      if isPlanar {
+        planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
+        bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
+        height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
+        width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
+      } else {
+        planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
+        bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
+        height = CVPixelBufferGetHeight(pixelBuffer)
+        width = CVPixelBufferGetWidth(pixelBuffer)
+      }
+
+      let length = bytesPerRow * height
+      let bytes = Data(bytes: planeAddress!, count: length)
+
+      let planeBuffer: [String: Any] = [
+        "bytesPerRow": bytesPerRow,
+        "width": width,
+        "height": height,
+        "bytes": FlutterStandardTypedData(bytes: bytes),
+      ]
+      planes.append(planeBuffer)
+    }
+
+    // Lock the base address before accessing pixel data, and unlock it afterwards.
+    // Done accessing the `pixelBuffer` at this point.
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
+
+    let imageBuffer: [String: Any] = [
+      "width": imageWidth,
+      "height": imageHeight,
+      "format": videoFormat,
+      "planes": planes,
+      "lensAperture": Double(captureDevice.lensAperture()),
+      "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
+      "sensorSensitivity": Double(captureDevice.iso()),
+    ]
+
+    DispatchQueue.main.async {
+      eventSink(imageBuffer)
+    }
+  }
+
   private func copySampleBufferWithAdjustedTime(_ sample: CMSampleBuffer, by offset: CMTime)
     -> CMSampleBuffer?
   {

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.20+7
+version: 0.9.21
 
 environment:
   sdk: ^3.6.0