Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
a978667
fix race condition when starting image stream on iOS
js2702 Feb 27, 2025
494fe89
update pubspec
js2702 Feb 27, 2025
e902881
update changelog
js2702 Feb 27, 2025
c4aa926
Test not working
js2702 Mar 1, 2025
e6dd1a6
fix test
js2702 Mar 1, 2025
423c038
Merge branch 'main' into fix_startstream_race_condition
js2702 Mar 2, 2025
0e16acb
Lint + add async/await
js2702 Mar 2, 2025
364d2d6
Merge tag 'camera_avfoundation-v0.9.18+9' into fix_startstream_race_c…
js2702 Mar 5, 2025
8502b67
remove async from tests
js2702 Mar 5, 2025
7c462d9
fix lint + missing parameter
js2702 Mar 6, 2025
5dc4fb0
Fix lint again
js2702 Mar 6, 2025
680820f
lint
js2702 Mar 6, 2025
d46b835
lint once again
js2702 Mar 6, 2025
2ed081e
Merge remote-tracking branch 'origin/main' into fix_startstream_race_…
js2702 Mar 21, 2025
80b733b
fixes
js2702 Mar 21, 2025
0add70e
empty callback
js2702 Mar 21, 2025
7bd57e5
Merge branch 'main' into fix_startstream_race_condition
js2702 Apr 8, 2025
b50bab8
call completion in all paths
js2702 Apr 8, 2025
42be013
formatting
js2702 Apr 8, 2025
2e78c19
call completion in all paths
js2702 Apr 11, 2025
870f7f5
lint
js2702 Apr 15, 2025
fba0373
Merge branch 'main' into fix_startstream_race_condition
js2702 Apr 29, 2025
a2b3269
separate in function
js2702 Jun 13, 2025
488bf75
Merge remote-tracking branch 'origin/main' into fix_startstream_race_…
js2702 Jun 13, 2025
c5075a0
adapt after merging
js2702 Jun 13, 2025
5dfac20
formatting
js2702 Jun 13, 2025
c0f8e90
rename setup method
davidmartos96 Jun 13, 2025
987f067
update error as param
davidmartos96 Jun 13, 2025
48354d9
lint
js2702 Jun 13, 2025
ce94fad
trigger tests
js2702 Jun 16, 2025
4661cc8
trigger tests
js2702 Jun 16, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions packages/camera/camera_avfoundation/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
## 0.9.19+3

* Fixes race condition when starting image stream.


## 0.9.19+2

* Adds the `Camera` Swift protocol.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
978D90B42D5F630300CD817E /* StreamingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 978D90B32D5F630300CD817E /* StreamingTests.swift */; };
97922B0D2D6380C300A9B4CF /* SampleBufferTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97922B0C2D6380C300A9B4CF /* SampleBufferTests.swift */; };
979B3DFB2D5B6BC7009BDE1A /* ExceptionCatcher.m in Sources */ = {isa = PBXBuildFile; fileRef = 979B3DFA2D5B6BC7009BDE1A /* ExceptionCatcher.m */; };
979B3DFE2D5B985B009BDE1A /* CameraCaptureSessionQueueRaceConditionTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 979B3DFD2D5B985B009BDE1A /* CameraCaptureSessionQueueRaceConditionTests.swift */; };
979B3DFE2D5B985B009BDE1A /* CameraInitRaceConditionsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 979B3DFD2D5B985B009BDE1A /* CameraInitRaceConditionsTests.swift */; };
979B3E002D5B9E6C009BDE1A /* CameraMethodChannelTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 979B3DFF2D5B9E6C009BDE1A /* CameraMethodChannelTests.swift */; };
979B3E022D5BA48F009BDE1A /* CameraOrientationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 979B3E012D5BA48F009BDE1A /* CameraOrientationTests.swift */; };
97BD4A0E2D5CC5AE00F857D5 /* CameraSettingsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97BD4A0D2D5CC5AE00F857D5 /* CameraSettingsTests.swift */; };
Expand Down Expand Up @@ -114,7 +114,7 @@
979B3DF92D5B6BA2009BDE1A /* ExceptionCatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = ExceptionCatcher.h; sourceTree = "<group>"; };
979B3DFA2D5B6BC7009BDE1A /* ExceptionCatcher.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = ExceptionCatcher.m; sourceTree = "<group>"; };
979B3DFC2D5B985B009BDE1A /* RunnerTests-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "RunnerTests-Bridging-Header.h"; sourceTree = "<group>"; };
979B3DFD2D5B985B009BDE1A /* CameraCaptureSessionQueueRaceConditionTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraCaptureSessionQueueRaceConditionTests.swift; sourceTree = "<group>"; };
979B3DFD2D5B985B009BDE1A /* CameraInitRaceConditionsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraInitRaceConditionsTests.swift; sourceTree = "<group>"; };
979B3DFF2D5B9E6C009BDE1A /* CameraMethodChannelTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraMethodChannelTests.swift; sourceTree = "<group>"; };
979B3E012D5BA48F009BDE1A /* CameraOrientationTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraOrientationTests.swift; sourceTree = "<group>"; };
97BD4A0D2D5CC5AE00F857D5 /* CameraSettingsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraSettingsTests.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -191,7 +191,7 @@
979B3DF92D5B6BA2009BDE1A /* ExceptionCatcher.h */,
979B3DFA2D5B6BC7009BDE1A /* ExceptionCatcher.m */,
979B3DFC2D5B985B009BDE1A /* RunnerTests-Bridging-Header.h */,
979B3DFD2D5B985B009BDE1A /* CameraCaptureSessionQueueRaceConditionTests.swift */,
979B3DFD2D5B985B009BDE1A /* CameraInitRaceConditionsTests.swift */,
979B3DFF2D5B9E6C009BDE1A /* CameraMethodChannelTests.swift */,
979B3E012D5BA48F009BDE1A /* CameraOrientationTests.swift */,
97BD4A0D2D5CC5AE00F857D5 /* CameraSettingsTests.swift */,
Expand Down Expand Up @@ -552,7 +552,7 @@
E1ABED6D2D94392700AED9CC /* MockAssetWriterInput.swift in Sources */,
977A25242D5A511600931E34 /* CameraPermissionTests.swift in Sources */,
970ADABE2D6740A900EFDCD9 /* MockWritableData.swift in Sources */,
979B3DFE2D5B985B009BDE1A /* CameraCaptureSessionQueueRaceConditionTests.swift in Sources */,
979B3DFE2D5B985B009BDE1A /* CameraInitRaceConditionsTests.swift in Sources */,
E142F13A2D85940600824824 /* MockCapturePhotoOutput.swift in Sources */,
E12C4FF82D68E85500515E70 /* MockFLTCameraPermissionManager.swift in Sources */,
97922B0D2D6380C300A9B4CF /* SampleBufferTests.swift in Sources */,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import XCTest
import camera_avfoundation_objc
#endif

final class CameraCaptureSessionQueueRaceConditionTests: XCTestCase {
final class CameraInitRaceConditionsTests: XCTestCase {
private func createCameraPlugin() -> (CameraPlugin, DispatchQueue) {
let captureSessionQueue = DispatchQueue(label: "io.flutter.camera.captureSessionQueue")

Expand Down Expand Up @@ -62,4 +62,36 @@ final class CameraCaptureSessionQueueRaceConditionTests: XCTestCase {
XCTAssertNotNil(
captureSessionQueue, "captureSessionQueue must not be nil after create method.")
}

func testFlutterChannelInitializedWhenStartingImageStream() {
let (cameraPlugin, _captureSessionQueue) = createCameraPlugin()
let createExpectation = expectation(description: "create's result block must be called")

cameraPlugin.createCameraOnSessionQueue(
withName: "acamera",
settings: FCPPlatformMediaSettings.make(
with: .medium,
framesPerSecond: nil,
videoBitrate: nil,
audioBitrate: nil,
enableAudio: true
)
) { result, error in
createExpectation.fulfill()
}

waitForExpectations(timeout: 30, handler: nil)

// Start stream and wait for its completion.
let startStreamExpectation = expectation(
description: "startImageStream's result block must be called")
cameraPlugin.startImageStream(completion: {
_ in
startStreamExpectation.fulfill()
})

waitForExpectations(timeout: 30, handler: nil)
XCTAssertEqual(cameraPlugin.camera?.isStreamingImages, true)
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -261,8 +261,9 @@ final class CameraPluginDelegatingMethodTests: XCTestCase {
let expectation = expectation(description: "Call completed")

var startImageStreamCalled = false
mockCamera.startImageStreamStub = { _ in
mockCamera.startImageStreamStub = { messenger, completion in
startImageStreamCalled = true
completion(nil)
}

cameraPlugin.startImageStream { error in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ final class MockCamera: NSObject, Camera {
var pausePreviewStub: (() -> Void)?
var resumePreviewStub: (() -> Void)?
var setDescriptionWhileRecordingStub: ((String, ((FlutterError?) -> Void)?) -> Void)?
var startImageStreamStub: ((FlutterBinaryMessenger) -> Void)?
var startImageStreamStub: ((FlutterBinaryMessenger, (FlutterError?) -> Void) -> Void)?
var stopImageStreamStub: (() -> Void)?

var dartAPI: FCPCameraEventApi? {
Expand All @@ -63,6 +63,7 @@ final class MockCamera: NSObject, Camera {
var videoFormat: FourCharCode = kCVPixelFormatType_32BGRA

var isPreviewPaused: Bool = false
var isStreamingImages: Bool = false

var minimumExposureOffset: CGFloat {
return getMinimumExposureOffsetStub?() ?? 0
Expand Down Expand Up @@ -186,8 +187,11 @@ final class MockCamera: NSObject, Camera {
setDescriptionWhileRecordingStub?(cameraName, completion)
}

func startImageStream(with messenger: FlutterBinaryMessenger) {
startImageStreamStub?(messenger)
func startImageStream(
with messenger: FlutterBinaryMessenger,
completion: @escaping (FlutterError?) -> Void
) {
startImageStreamStub?(messenger, completion)
}

func stopImageStream() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,28 @@ final class StreamingTests: XCTestCase {

func testExceedMaxStreamingPendingFramesCount() {
let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
let handlerMock = MockImageStreamHandler()

let finishStartStreamExpectation = expectation(
description: "Finish startStream")

let messenger = MockFlutterBinaryMessenger()
camera.startImageStream(
with: messenger, imageStreamHandler: handlerMock,
completion: {
_ in
finishStartStreamExpectation.fulfill()
})

waitForExpectations(timeout: 30, handler: nil)

// Setup mocked event sink after the stream starts
let streamingExpectation = expectation(
description: "Must not call handler over maxStreamingPendingFramesCount")
let handlerMock = MockImageStreamHandler()

handlerMock.eventSinkStub = { event in
streamingExpectation.fulfill()
}
let messenger = MockFlutterBinaryMessenger()
camera.startImageStream(with: messenger, imageStreamHandler: handlerMock)

waitForQueueRoundTrip(with: DispatchQueue.main)
XCTAssertEqual(camera.isStreamingImages, true)
Expand All @@ -74,14 +88,27 @@ final class StreamingTests: XCTestCase {

func testReceivedImageStreamData() {
let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
let handlerMock = MockImageStreamHandler()

let finishStartStreamExpectation = expectation(
description: "Finish startStream")

let messenger = MockFlutterBinaryMessenger()
camera.startImageStream(
with: messenger, imageStreamHandler: handlerMock,
completion: {
_ in
finishStartStreamExpectation.fulfill()
})

waitForExpectations(timeout: 30, handler: nil)

// Setup mocked event sink after the stream starts
let streamingExpectation = expectation(
description: "Must be able to call the handler again when receivedImageStreamData is called")
let handlerMock = MockImageStreamHandler()
handlerMock.eventSinkStub = { event in
streamingExpectation.fulfill()
}
let messenger = MockFlutterBinaryMessenger()
camera.startImageStream(with: messenger, imageStreamHandler: handlerMock)

waitForQueueRoundTrip(with: DispatchQueue.main)
XCTAssertEqual(camera.isStreamingImages, true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ protocol Camera: FlutterTexture, AVCaptureVideoDataOutputSampleBufferDelegate,
var videoFormat: FourCharCode { get set }

var isPreviewPaused: Bool { get }
var isStreamingImages: Bool { get }

var minimumAvailableZoomFactor: CGFloat { get }
var maximumAvailableZoomFactor: CGFloat { get }
Expand Down Expand Up @@ -86,7 +87,8 @@ protocol Camera: FlutterTexture, AVCaptureVideoDataOutputSampleBufferDelegate,
withCompletion: @escaping (_ error: FlutterError?) -> Void
)

func startImageStream(with: FlutterBinaryMessenger)
func startImageStream(
with: FlutterBinaryMessenger, completion: @escaping (_ error: FlutterError?) -> Void)
func stopImageStream()

// Override to make `AVCaptureVideoDataOutputSampleBufferDelegate`/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,11 @@ extension CameraPlugin: FCPCameraApi {

public func startImageStream(completion: @escaping (FlutterError?) -> Void) {
captureSessionQueue.async { [weak self] in
guard let strongSelf = self else { return }
strongSelf.camera?.startImageStream(with: strongSelf.messenger)
completion(nil)
guard let strongSelf = self else {
completion(nil)
return
}
strongSelf.camera?.startImageStream(with: strongSelf.messenger, completion: completion)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ - (void)captureToFileWithCompletion:(void (^)(NSString *_Nullable,
NSString *path = [self getTemporaryFilePathWithExtension:extension
subfolder:@"pictures"
prefix:@"CAP_"
error:error];
error:&error];
if (error) {
completion(nil, FlutterErrorFromNSError(error));
return;
Expand Down Expand Up @@ -362,7 +362,7 @@ - (AVCaptureVideoOrientation)getVideoOrientationForDeviceOrientation:
- (NSString *)getTemporaryFilePathWithExtension:(NSString *)extension
subfolder:(NSString *)subfolder
prefix:(NSString *)prefix
error:(NSError *)error {
error:(NSError **)error {
NSString *docDir =
NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES)[0];
NSString *fileDir =
Expand All @@ -373,11 +373,11 @@ - (NSString *)getTemporaryFilePathWithExtension:(NSString *)extension

NSFileManager *fm = [NSFileManager defaultManager];
if (![fm fileExistsAtPath:fileDir]) {
[[NSFileManager defaultManager] createDirectoryAtPath:fileDir
withIntermediateDirectories:true
attributes:nil
error:&error];
if (error) {
BOOL success = [[NSFileManager defaultManager] createDirectoryAtPath:fileDir
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch on this change!

withIntermediateDirectories:true
attributes:nil
error:error];
if (!success) {
return nil;
}
}
Expand Down Expand Up @@ -498,43 +498,50 @@ - (void)dealloc {
[_motionManager stopAccelerometerUpdates];
}

/// Main logic to setup the video recording.
- (void)setUpVideoRecordingWithCompletion:(void (^)(FlutterError *_Nullable))completion {
NSError *error;
_videoRecordingPath = [self getTemporaryFilePathWithExtension:@"mp4"
subfolder:@"videos"
prefix:@"REC_"
error:&error];
if (error) {
completion(FlutterErrorFromNSError(error));
return;
}
if (![self setupWriterForPath:_videoRecordingPath]) {
completion([FlutterError errorWithCode:@"IOError" message:@"Setup Writer Failed" details:nil]);
return;
}
// startWriting should not be called in didOutputSampleBuffer where it can cause state
// in which _isRecording is YES but _videoWriter.status is AVAssetWriterStatusUnknown
// in stopVideoRecording if it is called after startVideoRecording but before
// didOutputSampleBuffer had chance to call startWriting and lag at start of video
// https://github.com/flutter/flutter/issues/132016
// https://github.com/flutter/flutter/issues/151319
[_videoWriter startWriting];
_isFirstVideoSample = YES;
_isRecording = YES;
_isRecordingPaused = NO;
_videoTimeOffset = CMTimeMake(0, 1);
_audioTimeOffset = CMTimeMake(0, 1);
_videoIsDisconnected = NO;
_audioIsDisconnected = NO;
completion(nil);
}

- (void)startVideoRecordingWithCompletion:(void (^)(FlutterError *_Nullable))completion
messengerForStreaming:(nullable NSObject<FlutterBinaryMessenger> *)messenger {
if (!_isRecording) {
if (messenger != nil) {
[self startImageStreamWithMessenger:messenger];
}

NSError *error;
_videoRecordingPath = [self getTemporaryFilePathWithExtension:@"mp4"
subfolder:@"videos"
prefix:@"REC_"
error:error];
if (error) {
completion(FlutterErrorFromNSError(error));
return;
}
if (![self setupWriterForPath:_videoRecordingPath]) {
completion([FlutterError errorWithCode:@"IOError"
message:@"Setup Writer Failed"
details:nil]);
[self startImageStreamWithMessenger:messenger
completion:^(FlutterError *_Nullable error) {
[self setUpVideoRecordingWithCompletion:completion];
}];
return;
}
// startWriting should not be called in didOutputSampleBuffer where it can cause state
// in which _isRecording is YES but _videoWriter.status is AVAssetWriterStatusUnknown
// in stopVideoRecording if it is called after startVideoRecording but before
// didOutputSampleBuffer had chance to call startWriting and lag at start of video
// https://github.com/flutter/flutter/issues/132016
// https://github.com/flutter/flutter/issues/151319
[_videoWriter startWriting];
_isFirstVideoSample = YES;
_isRecording = YES;
_isRecordingPaused = NO;
_videoTimeOffset = CMTimeMake(0, 1);
_audioTimeOffset = CMTimeMake(0, 1);
_videoIsDisconnected = NO;
_audioIsDisconnected = NO;
completion(nil);

[self setUpVideoRecordingWithCompletion:completion];
} else {
completion([FlutterError errorWithCode:@"Error"
message:@"Video is already recording"
Expand Down Expand Up @@ -831,14 +838,17 @@ - (void)setExposureOffset:(double)offset {
[_captureDevice unlockForConfiguration];
}

- (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger {
- (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger
completion:(void (^)(FlutterError *))completion {
[self startImageStreamWithMessenger:messenger
imageStreamHandler:[[FLTImageStreamHandler alloc]
initWithCaptureSessionQueue:_captureSessionQueue]];
initWithCaptureSessionQueue:_captureSessionQueue]
completion:completion];
}

- (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messenger
imageStreamHandler:(FLTImageStreamHandler *)imageStreamHandler {
imageStreamHandler:(FLTImageStreamHandler *)imageStreamHandler
completion:(void (^)(FlutterError *))completion {
if (!_isStreamingImages) {
id<FLTEventChannel> eventChannel = [FlutterEventChannel
eventChannelWithName:@"plugins.flutter.io/camera_avfoundation/imageStream"
Expand All @@ -851,19 +861,27 @@ - (void)startImageStreamWithMessenger:(NSObject<FlutterBinaryMessenger> *)messen
[threadSafeEventChannel setStreamHandler:_imageStreamHandler
completion:^{
typeof(self) strongSelf = weakSelf;
if (!strongSelf) return;
if (!strongSelf) {
completion(nil);
return;
}

dispatch_async(strongSelf.captureSessionQueue, ^{
// cannot use the outter strongSelf
typeof(self) strongSelf = weakSelf;
if (!strongSelf) return;
if (!strongSelf) {
completion(nil);
return;
}

strongSelf.isStreamingImages = YES;
strongSelf.streamingPendingFramesCount = 0;
completion(nil);
});
}];
} else {
[self reportErrorMessage:@"Images from camera are already streaming!"];
completion(nil);
}
}

Expand Down
Loading