Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Use canvaskit toByteData for unsupported videoFrame formats #38361

Merged
merged 2 commits into from
Dec 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion lib/web_ui/lib/src/engine/canvaskit/image.dart
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,9 @@ class CkImage implements ui.Image, StackTraceDebugger {
ui.ImageByteFormat format = ui.ImageByteFormat.rawRgba,
}) {
assert(_debugCheckIsNotDisposed());
if (videoFrame != null) {
// readPixelsFromVideoFrame currently does not convert I420, I444, I422
// videoFrame formats to RGBA
if (videoFrame != null && videoFrame!.format != 'I420' && videoFrame!.format != 'I444' && videoFrame!.format != 'I422') {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we still leaving readPixelsFromVideoFrame because of the shared memory bug in _readPixelsFromSkImage?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is one reason. Another is that using wasm codecs toByteData with png requested format fails when CkBrowserImageDecoder is used. There is potentially an issue when the videoFrame format is I420, I444, I422 and a toByteData call is made with requested format png.

return readPixelsFromVideoFrame(videoFrame!, format);
} else {
return _readPixelsFromSkImage(format);
Expand Down
51 changes: 51 additions & 0 deletions lib/web_ui/test/canvaskit/image_golden_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,57 @@ void _testForImageCodecs({required bool useBrowserImageDecoder}) {
testCollector.collectNow();
});

test('toByteData with decodeImageFromPixels on videoFrame formats', () async {
// This test ensures that toByteData() returns pixels that can be used by decodeImageFromPixels
// for the following videoFrame formats:
// [BGRX, I422, I420, I444, BGRA]
final DomResponse listingResponse = await httpFetch('/test_images/');
final List<String> testFiles = (await listingResponse.json() as List<dynamic>).cast<String>();

Future<ui.Image> testDecodeFromPixels(Uint8List pixels, int width, int height) async {
final Completer<ui.Image> completer = Completer<ui.Image>();
ui.decodeImageFromPixels(
pixels,
width,
height,
ui.PixelFormat.rgba8888,
(ui.Image image) {
completer.complete(image);
},
);
return completer.future;
}

// Sanity-check the test file list. If suddenly test files are moved or
// deleted, and the test server returns an empty list, or is missing some
// important test files, we want to know.
expect(testFiles, isNotEmpty);
expect(testFiles, contains(matches(RegExp(r'.*\.jpg'))));
expect(testFiles, contains(matches(RegExp(r'.*\.png'))));
expect(testFiles, contains(matches(RegExp(r'.*\.gif'))));
expect(testFiles, contains(matches(RegExp(r'.*\.webp'))));
expect(testFiles, contains(matches(RegExp(r'.*\.bmp'))));

for (final String testFile in testFiles) {
final DomResponse imageResponse = await httpFetch('/test_images/$testFile');
final Uint8List imageData = (await imageResponse.arrayBuffer() as ByteBuffer).asUint8List();
final ui.Codec codec = await skiaInstantiateImageCodec(imageData);
expect(codec.frameCount, greaterThan(0));
expect(codec.repetitionCount, isNotNull);

final ui.FrameInfo frame = await codec.getNextFrame();
final CkImage ckImage = frame.image as CkImage;
final ByteData imageBytes = await ckImage.toByteData();
expect(imageBytes.lengthInBytes, greaterThan(0));

final Uint8List pixels = imageBytes.buffer.asUint8List();
final ui.Image testImage = await testDecodeFromPixels(pixels, ckImage.width, ckImage.height);
expect(testImage, isNotNull);
codec.dispose();
}
// TODO(hterkelsen): Firefox and Safari do not currently support ImageDecoder.
}, skip: isFirefox || isSafari);

test('CkImage.clone also clones the VideoFrame', () async {
final CkBrowserImageDecoder image = await CkBrowserImageDecoder.create(
data: kAnimatedGif,
Expand Down