[Impeller] Support YUV external textures on iOS

ci/licenses_golden/licenses_flutter

@@ -1212,6 +1212,8 @@ FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_conten
 FILE: ../../../flutter/impeller/entity/contents/filters/morphology_filter_contents.h
 FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.cc
 FILE: ../../../flutter/impeller/entity/contents/filters/srgb_to_linear_filter_contents.h
+FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.cc
+FILE: ../../../flutter/impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h
 FILE: ../../../flutter/impeller/entity/contents/gradient_generator.cc
 FILE: ../../../flutter/impeller/entity/contents/gradient_generator.h
 FILE: ../../../flutter/impeller/entity/contents/linear_gradient_contents.cc
@@ -1301,6 +1303,8 @@ FILE: ../../../flutter/impeller/entity/shaders/texture_fill.vert
 FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.frag
 FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.vert
 FILE: ../../../flutter/impeller/entity/shaders/vertices.frag
+FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.frag
+FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.vert
 FILE: ../../../flutter/impeller/geometry/color.cc
 FILE: ../../../flutter/impeller/geometry/color.h
 FILE: ../../../flutter/impeller/geometry/constants.cc

common/graphics/texture.h

@@ -16,6 +16,10 @@
 
 class GrDirectContext;
 
+namespace impeller {
+class AiksContext;
+};
+
 namespace flutter {
 
 class ContextListener {
@@ -39,6 +43,7 @@ class Texture : public ContextListener {
     SkCanvas* canvas = nullptr;
     DisplayListBuilder* builder = nullptr;
     GrDirectContext* gr_context = nullptr;
+    impeller::AiksContext* aiks_context = nullptr;
     const SkPaint* sk_paint = nullptr;
     const DlPaint* dl_paint = nullptr;
   };

flow/compositor_context.cc

@@ -83,11 +83,12 @@ std::unique_ptr<CompositorContext::ScopedFrame> CompositorContext::AcquireFrame(
     bool surface_supports_readback,
     fml::RefPtr<fml::RasterThreadMerger>
         raster_thread_merger,  // NOLINT(performance-unnecessary-value-param)
-    DisplayListBuilder* display_list_builder) {
+    DisplayListBuilder* display_list_builder,
+    impeller::AiksContext* aiks_context) {
   return std::make_unique<ScopedFrame>(
       *this, gr_context, canvas, view_embedder, root_surface_transformation,
       instrumentation_enabled, surface_supports_readback, raster_thread_merger,
-      display_list_builder);
+      display_list_builder, aiks_context);
 }
 
 CompositorContext::ScopedFrame::ScopedFrame(
@@ -99,11 +100,13 @@ CompositorContext::ScopedFrame::ScopedFrame(
     bool instrumentation_enabled,
     bool surface_supports_readback,
     fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger,
-    DisplayListBuilder* display_list_builder)
+    DisplayListBuilder* display_list_builder,
+    impeller::AiksContext* aiks_context)
     : context_(context),
       gr_context_(gr_context),
       canvas_(canvas),
       display_list_builder_(display_list_builder),
+      aiks_context_(aiks_context),
       view_embedder_(view_embedder),
       root_surface_transformation_(root_surface_transformation),
       instrumentation_enabled_(instrumentation_enabled),

flow/compositor_context.h

@@ -114,7 +114,8 @@ class CompositorContext {
                 bool instrumentation_enabled,
                 bool surface_supports_readback,
                 fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger,
-                DisplayListBuilder* display_list_builder);
+                DisplayListBuilder* display_list_builder,
+                impeller::AiksContext* aiks_context);
 
     virtual ~ScopedFrame();
 
@@ -136,6 +137,8 @@ class CompositorContext {
 
     GrDirectContext* gr_context() const { return gr_context_; }
 
+    impeller::AiksContext* aiks_context() const { return aiks_context_; }
+
     virtual RasterStatus Raster(LayerTree& layer_tree,
                                 bool ignore_raster_cache,
                                 FrameDamage* frame_damage);
@@ -145,6 +148,7 @@ class CompositorContext {
     GrDirectContext* gr_context_;
     SkCanvas* canvas_;
     DisplayListBuilder* display_list_builder_;
+    impeller::AiksContext* aiks_context_;
     ExternalViewEmbedder* view_embedder_;
     const SkMatrix& root_surface_transformation_;
     const bool instrumentation_enabled_;
@@ -168,7 +172,8 @@ class CompositorContext {
       bool instrumentation_enabled,
       bool surface_supports_readback,
       fml::RefPtr<fml::RasterThreadMerger> raster_thread_merger,
-      DisplayListBuilder* display_list_builder);
+      DisplayListBuilder* display_list_builder,
+      impeller::AiksContext* aiks_context);
 
   void OnGrContextCreated();
 

flow/layers/layer.h

@@ -154,6 +154,7 @@ struct PaintContext {
   SkScalar inherited_opacity = SK_Scalar1;
   DisplayListBuilder* leaf_nodes_builder = nullptr;
   DisplayListBuilderMultiplexer* builder_multiplexer = nullptr;
+  impeller::AiksContext* aiks_context;
 };
 
 // Represents a single composited layer. Created on the UI thread but then

flow/layers/layer_tree.cc

@@ -158,6 +158,7 @@ void LayerTree::Paint(CompositorContext::ScopedFrame& frame,
       .inherited_opacity             = SK_Scalar1,
       .leaf_nodes_builder            = builder,
       .builder_multiplexer           = builder ? &builder_multiplexer : nullptr,
+      .aiks_context                  = frame.aiks_context(),
       // clang-format on
   };
 

flow/layers/layer_tree_unittests.cc

@@ -28,6 +28,7 @@ class LayerTreeTest : public CanvasTest {
                                                        false,
                                                        true,
                                                        nullptr,
+                                                       nullptr,
                                                        nullptr)) {}
 
   LayerTree& layer_tree() { return layer_tree_; }

flow/layers/texture_layer.cc

@@ -63,6 +63,7 @@ void TextureLayer::Paint(PaintContext& context) const {
       .canvas = context.leaf_nodes_canvas,
       .builder = context.leaf_nodes_builder,
       .gr_context = context.gr_context,
+      .aiks_context = context.aiks_context,
       .sk_paint = cache_paint.sk_paint(),
       .dl_paint = cache_paint.dl_paint(),
   };

impeller/aiks/aiks_context.cc

@@ -32,6 +32,10 @@ std::shared_ptr<Context> AiksContext::GetContext() const {
   return context_;
 }
 
+const ContentContext& AiksContext::GetContentContext() const {
+  return *content_context_;
+}
+
 bool AiksContext::Render(const Picture& picture, RenderTarget& render_target) {
   if (!IsValid()) {
     return false;

impeller/aiks/aiks_context.h

@@ -26,6 +26,8 @@ class AiksContext {
 
   std::shared_ptr<Context> GetContext() const;
 
+  const ContentContext& GetContentContext() const;
+
   bool Render(const Picture& picture, RenderTarget& render_target);
 
  private:

impeller/display_list/display_list_image_impeller.cc

@@ -4,6 +4,9 @@
 
 #include "impeller/display_list/display_list_image_impeller.h"
 
+#include "impeller/aiks/aiks_context.h"
+#include "impeller/entity/contents/filters/filter_contents.h"
+
 namespace impeller {
 
 sk_sp<DlImageImpeller> DlImageImpeller::Make(std::shared_ptr<Texture> texture) {
@@ -13,6 +16,23 @@ sk_sp<DlImageImpeller> DlImageImpeller::Make(std::shared_ptr<Texture> texture) {
   return sk_sp<DlImageImpeller>(new DlImageImpeller(std::move(texture)));
 }
 
+sk_sp<DlImageImpeller> DlImageImpeller::MakeFromYUVTextures(
+    AiksContext* aiks_context,
+    std::shared_ptr<Texture> y_texture,
+    std::shared_ptr<Texture> uv_texture,
+    YUVColorSpace yuv_color_space) {
+  if (!aiks_context || !y_texture || !uv_texture) {
+    return nullptr;
+  }
+  auto yuv_to_rgb_filter_contents = FilterContents::MakeYUVToRGBFilter(
+      std::move(y_texture), std::move(uv_texture), yuv_color_space);
+  impeller::Entity entity;
+  entity.SetBlendMode(impeller::BlendMode::kSource);
+  auto snapshot = yuv_to_rgb_filter_contents->RenderToSnapshot(
+      aiks_context->GetContentContext(), entity);
+  return impeller::DlImageImpeller::Make(snapshot->texture);
+}
+
 DlImageImpeller::DlImageImpeller(std::shared_ptr<Texture> texture)
     : texture_(std::move(texture)) {}
 

impeller/display_list/display_list_image_impeller.h

@@ -10,10 +10,18 @@
 
 namespace impeller {
 
+class AiksContext;
+
 class DlImageImpeller final : public flutter::DlImage {
  public:
   static sk_sp<DlImageImpeller> Make(std::shared_ptr<Texture> texture);
 
+  static sk_sp<DlImageImpeller> MakeFromYUVTextures(
+      AiksContext* aiks_context,
+      std::shared_ptr<Texture> y_texture,
+      std::shared_ptr<Texture> uv_texture,
+      YUVColorSpace yuv_color_space);
+
   // |DlImage|
   ~DlImageImpeller() override;
 

impeller/entity/BUILD.gn

@@ -61,6 +61,8 @@ impeller_shaders("entity_shaders") {
     "shaders/tiled_texture_fill.frag",
     "shaders/tiled_texture_fill.vert",
     "shaders/vertices.frag",
+    "shaders/yuv_to_rgb_filter.frag",
+    "shaders/yuv_to_rgb_filter.vert",
   ]
 }
 
@@ -106,6 +108,8 @@ impeller_component("entity") {
     "contents/filters/morphology_filter_contents.h",
     "contents/filters/srgb_to_linear_filter_contents.cc",
     "contents/filters/srgb_to_linear_filter_contents.h",
+    "contents/filters/yuv_to_rgb_filter_contents.cc",
+    "contents/filters/yuv_to_rgb_filter_contents.h",
     "contents/gradient_generator.cc",
     "contents/gradient_generator.h",
     "contents/linear_gradient_contents.cc",

impeller/entity/contents/content_context.cc

@@ -214,6 +214,8 @@ ContentContext::ContentContext(std::shared_ptr<Context> context)
   geometry_position_pipelines_[{}] =
       CreateDefaultPipeline<GeometryPositionPipeline>(*context_);
   atlas_pipelines_[{}] = CreateDefaultPipeline<AtlasPipeline>(*context_);
+  yuv_to_rgb_filter_pipelines_[{}] =
+      CreateDefaultPipeline<YUVToRGBFilterPipeline>(*context_);
 
   // Pipelines that are variants of the base pipelines with custom descriptors.
   // TODO(98684): Rework this API to allow fetching the descriptor without

impeller/entity/contents/content_context.h

@@ -61,6 +61,8 @@
 #include "impeller/entity/tiled_texture_fill.frag.h"
 #include "impeller/entity/tiled_texture_fill.vert.h"
 #include "impeller/entity/vertices.frag.h"
+#include "impeller/entity/yuv_to_rgb_filter.frag.h"
+#include "impeller/entity/yuv_to_rgb_filter.vert.h"
 #include "impeller/renderer/formats.h"
 #include "impeller/renderer/pipeline.h"
 
@@ -160,6 +162,8 @@ using GeometryPositionPipeline =
     RenderPipelineT<PositionVertexShader, VerticesFragmentShader>;
 using GeometryColorPipeline =
     RenderPipelineT<PositionColorVertexShader, VerticesFragmentShader>;
+using YUVToRGBFilterPipeline =
+    RenderPipelineT<YuvToRgbFilterVertexShader, YuvToRgbFilterFragmentShader>;
 
 struct ContentContextOptions {
   SampleCount sample_count = SampleCount::kCount1;
@@ -299,6 +303,11 @@ class ContentContext {
     return GetPipeline(atlas_pipelines_, opts);
   }
 
+  std::shared_ptr<Pipeline<PipelineDescriptor>> GetYUVToRGBFilterPipeline(
+      ContentContextOptions opts) const {
+    return GetPipeline(yuv_to_rgb_filter_pipelines_, opts);
+  }
+
   // Advanced blends.
 
   std::shared_ptr<Pipeline<PipelineDescriptor>> GetBlendColorPipeline(
@@ -422,6 +431,7 @@ class ContentContext {
   mutable Variants<AtlasPipeline> atlas_pipelines_;
   mutable Variants<GeometryPositionPipeline> geometry_position_pipelines_;
   mutable Variants<GeometryColorPipeline> geometry_color_pipelines_;
+  mutable Variants<YUVToRGBFilterPipeline> yuv_to_rgb_filter_pipelines_;
   // Advanced blends.
   mutable Variants<BlendColorPipeline> blend_color_pipelines_;
   mutable Variants<BlendColorBurnPipeline> blend_colorburn_pipelines_;

impeller/entity/contents/filters/filter_contents.cc

@@ -20,6 +20,7 @@
 #include "impeller/entity/contents/filters/local_matrix_filter_contents.h"
 #include "impeller/entity/contents/filters/matrix_filter_contents.h"
 #include "impeller/entity/contents/filters/morphology_filter_contents.h"
+#include "impeller/entity/contents/filters/yuv_to_rgb_filter_contents.h"
 #include "impeller/entity/contents/texture_contents.h"
 #include "impeller/entity/entity.h"
 #include "impeller/geometry/path_builder.h"
@@ -129,6 +130,17 @@ std::shared_ptr<FilterContents> FilterContents::MakeLocalMatrixFilter(
   return filter;
 }
 
+std::shared_ptr<FilterContents> FilterContents::MakeYUVToRGBFilter(
+    std::shared_ptr<Texture> y_texture,
+    std::shared_ptr<Texture> uv_texture,
+    YUVColorSpace yuv_color_space) {
+  auto filter = std::make_shared<impeller::YUVToRGBFilterContents>();
+  filter->SetInputs({impeller::FilterInput::Make(y_texture),
+                     impeller::FilterInput::Make(uv_texture)});
+  filter->SetYUVColorSpace(yuv_color_space);
+  return filter;
+}
+
 FilterContents::FilterContents() = default;
 
 FilterContents::~FilterContents() = default;

impeller/entity/contents/filters/filter_contents.h

@@ -84,6 +84,11 @@ class FilterContents : public Contents {
       FilterInput::Ref input,
       const Matrix& matrix);
 
+  static std::shared_ptr<FilterContents> MakeYUVToRGBFilter(
+      std::shared_ptr<Texture> y_texture,
+      std::shared_ptr<Texture> uv_texture,
+      YUVColorSpace yuv_color_space);
+
   FilterContents();
 
   ~FilterContents() override;