support fallback codecs on Windows, macOS, iOS (flutter#19989)

Author: dnfield

lib/ui/painting.dart

@@ -20,7 +20,10 @@ part of dart.ui;
 
 // Update this list when changing the list of supported codecs.
 /// {@template flutter.dart:ui.imageFormats}
-/// JPEG, PNG, GIF, Animated GIF, WebP, Animated WebP, BMP, and WBMP
+/// JPEG, PNG, GIF, Animated GIF, WebP, Animated WebP, BMP, and WBMP. Additional
+/// formats may be supported by the underlying platform. Flutter will
+/// attempt to call platform API to decode unrecognized formats, and if the
+/// platform API supports decoding the image Flutter will be able to render it.
 /// {@endtemplate}
 
 bool _rectIsValid(Rect rect) {
@@ -1557,14 +1560,17 @@ enum PixelFormat {
 
 /// Opaque handle to raw decoded image data (pixels).
 ///
-/// To obtain an [Image] object, use [instantiateImageCodec].
+/// To obtain an [Image] object, use the [ImageDescriptor] API.
 ///
 /// To draw an [Image], use one of the methods on the [Canvas] class, such as
 /// [Canvas.drawImage].
 ///
 /// See also:
 ///
 ///  * [Image](https://api.flutter.dev/flutter/widgets/Image-class.html), the class in the [widgets] library.
+///  * [ImageDescriptor], which allows reading information about the image and
+///    creating a codec to decode it.
+///  * [instantiateImageCodec], a utility method that wraps [ImageDescriptor].
 ///
 @pragma('vm:entry-point')
 class Image extends NativeFieldWrapperClass2 {
@@ -1678,6 +1684,11 @@ class Codec extends NativeFieldWrapperClass2 {
 
 /// Instantiates an image [Codec].
 ///
+/// This method is a convenience wrapper around the [ImageDescriptor] API, and
+/// using [ImageDescriptor] directly is preferred since it allows the caller to
+/// make better determinations about how and whether to use the `targetWidth`
+/// and `targetHeight` parameters.
+///
 /// The `list` parameter is the binary image data (e.g a PNG or GIF binary data).
 /// The data can be for either static or animated images. The following image
 /// formats are supported: {@macro flutter.dart:ui.imageFormats}

lib/ui/painting/image_decoder.cc

@@ -105,7 +105,7 @@ sk_sp<SkImage> ImageFromCompressedData(fml::RefPtr<ImageDescriptor> descriptor,
 
   if (!descriptor->should_resize(target_width, target_height)) {
     // No resizing requested. Just decode & rasterize the image.
-    return SkImage::MakeFromEncoded(descriptor->data())->makeRasterImage();
+    return descriptor->image()->makeRasterImage();
   }
 
   const SkISize source_dimensions = descriptor->image_info().dimensions();
@@ -149,7 +149,7 @@ sk_sp<SkImage> ImageFromCompressedData(fml::RefPtr<ImageDescriptor> descriptor,
     }
   }
 
-  auto image = SkImage::MakeFromEncoded(descriptor->data());
+  auto image = descriptor->image();
   if (!image) {
     return nullptr;
   }

lib/ui/painting/image_decoder_unittests.cc

@@ -165,7 +165,8 @@ TEST_F(ImageDecoderFixtureTest, InvalidImageResultsError) {
     ASSERT_FALSE(data);
 
     fml::RefPtr<ImageDescriptor> image_descriptor =
-        fml::MakeRefCounted<ImageDescriptor>(std::move(data), nullptr);
+        fml::MakeRefCounted<ImageDescriptor>(std::move(data),
+                                             std::unique_ptr<SkCodec>(nullptr));
 
     ImageDecoder::ImageResult callback = [&](SkiaGPUObject<SkImage> image) {
       ASSERT_TRUE(runners.GetUITaskRunner()->RunsTasksOnCurrentThread());

lib/ui/painting/image_descriptor.cc

@@ -4,17 +4,30 @@
 
 #include "flutter/lib/ui/painting/image_descriptor.h"
 
+#include "flutter/fml/build_config.h"
 #include "flutter/fml/logging.h"
 #include "flutter/fml/trace_event.h"
 #include "flutter/lib/ui/painting/codec.h"
 #include "flutter/lib/ui/painting/image_decoder.h"
 #include "flutter/lib/ui/painting/multi_frame_codec.h"
 #include "flutter/lib/ui/painting/single_frame_codec.h"
 #include "flutter/lib/ui/ui_dart_state.h"
-#include "third_party/skia/src/codec/SkCodecImageGenerator.h"
 #include "third_party/tonic/dart_binding_macros.h"
 #include "third_party/tonic/logging/dart_invoke.h"
 
+#ifdef OS_MACOSX
+#include "third_party/skia/include/ports/SkImageGeneratorCG.h"
+#define PLATFORM_IMAGE_GENERATOR(data) \
+  SkImageGeneratorCG::MakeFromEncodedCG(data)
+#elif OS_WIN
+#include "third_party/skia/include/ports/SkImageGeneratorWIC.h"
+#define PLATFORM_IMAGE_GENERATOR(data) \
+  SkImageGeneratorWIC::MakeFromEncodedWIC(data)
+#else
+#define PLATFORM_IMAGE_GENERATOR(data) \
+  std::unique_ptr<SkImageGenerator>(nullptr)
+#endif
+
 namespace flutter {
 
 IMPLEMENT_WRAPPERTYPEINFO(ui, ImageDescriptor);
@@ -35,17 +48,21 @@ void ImageDescriptor::RegisterNatives(tonic::DartLibraryNatives* natives) {
 }
 
 const SkImageInfo ImageDescriptor::CreateImageInfo() const {
-  if (!generator_) {
-    return SkImageInfo::MakeUnknown();
+  if (generator_) {
+    return generator_->getInfo();
+  }
+  if (platform_image_generator_) {
+    return platform_image_generator_->getInfo();
   }
-  return generator_->getInfo();
+  return SkImageInfo::MakeUnknown();
 }
 
 ImageDescriptor::ImageDescriptor(sk_sp<SkData> buffer,
                                  const SkImageInfo& image_info,
                                  std::optional<size_t> row_bytes)
     : buffer_(std::move(buffer)),
       generator_(nullptr),
+      platform_image_generator_(nullptr),
       image_info_(std::move(image_info)),
       row_bytes_(row_bytes) {}
 
@@ -56,6 +73,15 @@ ImageDescriptor::ImageDescriptor(sk_sp<SkData> buffer,
           static_cast<SkCodecImageGenerator*>(
               SkCodecImageGenerator::MakeFromCodec(std::move(codec))
                   .release()))),
+      platform_image_generator_(nullptr),
+      image_info_(CreateImageInfo()),
+      row_bytes_(std::nullopt) {}
+
+ImageDescriptor::ImageDescriptor(sk_sp<SkData> buffer,
+                                 std::unique_ptr<SkImageGenerator> generator)
+    : buffer_(std::move(buffer)),
+      generator_(nullptr),
+      platform_image_generator_(std::move(generator)),
       image_info_(CreateImageInfo()),
       row_bytes_(std::nullopt) {}
 
@@ -77,15 +103,28 @@ void ImageDescriptor::initEncoded(Dart_NativeArguments args) {
     return;
   }
 
+  // This call will succeed if Skia has a built-in codec for this.
+  // If it fails, we will check if the platform knows how to decode this image.
   std::unique_ptr<SkCodec> codec =
       SkCodec::MakeFromData(immutable_buffer->data());
+  fml::RefPtr<ImageDescriptor> descriptor;
   if (!codec) {
-    Dart_SetReturnValue(args, tonic::ToDart("Invalid image data"));
-    return;
+    std::unique_ptr<SkImageGenerator> generator =
+        PLATFORM_IMAGE_GENERATOR(immutable_buffer->data());
+    if (!generator) {
+      // We don't have a Skia codec for this image, and the platform doesn't
+      // know how to decode it.
+      Dart_SetReturnValue(args, tonic::ToDart("Invalid image data"));
+      return;
+    }
+    descriptor = fml::MakeRefCounted<ImageDescriptor>(immutable_buffer->data(),
+                                                      std::move(generator));
+  } else {
+    descriptor = fml::MakeRefCounted<ImageDescriptor>(immutable_buffer->data(),
+                                                      std::move(codec));
   }
 
-  auto descriptor = fml::MakeRefCounted<ImageDescriptor>(
-      immutable_buffer->data(), std::move(codec));
+  FML_DCHECK(descriptor);
 
   descriptor->AssociateWithDartWrapper(descriptor_handle);
   tonic::DartInvoke(callback_handle, {Dart_TypeVoid()});
@@ -128,4 +167,31 @@ void ImageDescriptor::instantiateCodec(Dart_Handle codec_handle,
   }
   ui_codec->AssociateWithDartWrapper(codec_handle);
 }
+
+sk_sp<SkImage> ImageDescriptor::image() const {
+  SkBitmap bitmap;
+  if (!bitmap.tryAllocPixels(image_info_)) {
+    FML_LOG(ERROR) << "Failed to allocate memory for bitmap of size "
+                   << image_info_.computeMinByteSize() << "B";
+    return nullptr;
+  }
+
+  const auto& pixmap = bitmap.pixmap();
+  if (!get_pixels(pixmap)) {
+    FML_LOG(ERROR) << "Failed to get pixels for image.";
+    return nullptr;
+  }
+  bitmap.setImmutable();
+  return SkImage::MakeFromBitmap(bitmap);
+}
+
+bool ImageDescriptor::get_pixels(const SkPixmap& pixmap) const {
+  if (generator_) {
+    return generator_->getPixels(pixmap.info(), pixmap.writable_addr(),
+                                 pixmap.rowBytes());
+  }
+  FML_DCHECK(platform_image_generator_);
+  return platform_image_generator_->getPixels(pixmap);
+}
+
 }  // namespace flutter

lib/ui/painting/image_descriptor.h

@@ -13,6 +13,7 @@
 #include "flutter/lib/ui/dart_wrapper.h"
 #include "flutter/lib/ui/painting/immutable_buffer.h"
 #include "third_party/skia/include/codec/SkCodec.h"
+#include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/core/SkImageInfo.h"
 #include "third_party/skia/src/codec/SkCodecImageGenerator.h"
 #include "third_party/tonic/dart_library_natives.h"
@@ -81,31 +82,31 @@ class ImageDescriptor : public RefCountedDartWrappable<ImageDescriptor> {
   /// The underlying buffer for this image.
   sk_sp<SkData> data() const { return buffer_; }
 
+  sk_sp<SkImage> image() const;
+
   /// Whether this descriptor represents compressed (encoded) data or not.
-  bool is_compressed() const { return !!generator_; }
+  bool is_compressed() const { return generator_ || platform_image_generator_; }
 
   /// The orientation corrected image info for this image.
   const SkImageInfo& image_info() const { return image_info_; }
 
+  /// Gets the scaled dimensions of this image, if backed by a codec that can
+  /// perform efficient subpixel scaling.
   SkISize get_scaled_dimensions(float scale) {
-    if (!generator_) {
-      FML_DCHECK(false);
-      return image_info_.dimensions();
+    if (generator_) {
+      return generator_->getScaledDimensions(scale);
     }
-    return generator_->getScaledDimensions(scale);
+    return image_info_.dimensions();
   }
 
   /// Gets pixels for this image transformed based on the EXIF orientation tag,
   /// if applicable.
-  bool get_pixels(const SkPixmap& pixmap) const {
-    FML_DCHECK(generator_);
-    return generator_->getPixels(pixmap.info(), pixmap.writable_addr(),
-                                 pixmap.rowBytes());
-  }
+  bool get_pixels(const SkPixmap& pixmap) const;
 
   void dispose() {
     ClearDartWrapper();
     generator_.reset();
+    platform_image_generator_.reset();
   }
 
   size_t GetAllocationSize() const override {
@@ -119,9 +120,12 @@ class ImageDescriptor : public RefCountedDartWrappable<ImageDescriptor> {
                   const SkImageInfo& image_info,
                   std::optional<size_t> row_bytes);
   ImageDescriptor(sk_sp<SkData> buffer, std::unique_ptr<SkCodec> codec);
+  ImageDescriptor(sk_sp<SkData> buffer,
+                  std::unique_ptr<SkImageGenerator> generator);
 
   sk_sp<SkData> buffer_;
   std::shared_ptr<SkCodecImageGenerator> generator_;
+  std::unique_ptr<SkImageGenerator> platform_image_generator_;
   const SkImageInfo image_info_;
   std::optional<size_t> row_bytes_;
 

testing/dart/image_descriptor_test.dart

@@ -70,6 +70,19 @@ void main() {
     final Codec codec = await descriptor.instantiateCodec();
     expect(codec.frameCount, 1);
   });
+
+  test('HEIC image', () async {
+    final Uint8List bytes = await readFile('grill_chicken.heic');
+    final ImmutableBuffer buffer = await ImmutableBuffer.fromUint8List(bytes);
+    final ImageDescriptor descriptor = await ImageDescriptor.encoded(buffer);
+
+    expect(descriptor.width, 300);
+    expect(descriptor.height, 400);
+    expect(descriptor.bytesPerPixel, 4);
+
+    final Codec codec = await descriptor.instantiateCodec();
+    expect(codec.frameCount, 1);
+  }, skip: !(Platform.isIOS || Platform.isMacOS || Platform.isWindows));
 }
 
 Future<Uint8List> readFile(String fileName, ) async {