[macOS] Implement unobstructed platform views

Author: knopp

shell/platform/darwin/macos/framework/Source/FlutterCompositor.mm

@@ -57,6 +57,12 @@
         info.surface = surface;
         info.offset = CGPointMake(layer->offset.x, layer->offset.y);
         info.zIndex = i;
+        FlutterBackingStorePresentInfo* present_info = layer->backing_store_present_info;
+        if (present_info != nullptr && present_info->paint_region_rects != nullptr) {
+          info.paintRegion = std::vector<FlutterRect>(
+              present_info->paint_region_rects,
+              present_info->paint_region_rects + present_info->paint_region_rects_count);
+        }
         [surfaces addObject:info];
       }
     }

shell/platform/darwin/macos/framework/Source/FlutterSurfaceManager.h

@@ -5,6 +5,8 @@
 #import <Cocoa/Cocoa.h>
 #import <QuartzCore/QuartzCore.h>
 
+#include <vector>
+
 #import "flutter/shell/platform/darwin/macos/framework/Source/FlutterSurface.h"
 
 /**
@@ -15,6 +17,7 @@
 @property(readwrite, strong, nonatomic, nonnull) FlutterSurface* surface;
 @property(readwrite, nonatomic) CGPoint offset;
 @property(readwrite, nonatomic) size_t zIndex;
+@property(readwrite, nonatomic) std::vector<FlutterRect> paintRegion;
 
 @end
 

shell/platform/darwin/macos/framework/Source/FlutterSurfaceManager.mm

@@ -29,6 +29,9 @@ @interface FlutterSurfaceManager () {
 
   // Currently visible layers.
   NSMutableArray<CALayer*>* _layers;
+
+  NSNumber* _enableSurfaceDebugInfo;
+  CATextLayer* _infoLayer;
 }
 
 /**
@@ -38,6 +41,56 @@ - (void)commit:(NSArray<FlutterSurfacePresentInfo*>*)surfaces;
 
 @end
 
+static NSColor* GetBorderColorForLayer(int layer) {
+  NSArray* colors = @[
+    [NSColor yellowColor],
+    [NSColor cyanColor],
+    [NSColor magentaColor],
+    [NSColor greenColor],
+    [NSColor purpleColor],
+    [NSColor orangeColor],
+    [NSColor blueColor],
+  ];
+  return colors[layer % colors.count];
+}
+
+static void UpdateContentSubLayers(CALayer* layer,
+                                   IOSurfaceRef surface,
+                                   CGFloat scale,
+                                   CGSize surfaceSize,
+                                   NSColor* borderColor,
+                                   const std::vector<FlutterRect>& covered_area) {
+  while (layer.sublayers.count > covered_area.size()) {
+    [layer.sublayers.lastObject removeFromSuperlayer];
+  }
+
+  while (layer.sublayers.count < covered_area.size()) {
+    CALayer* newLayer = [CALayer layer];
+    [layer addSublayer:newLayer];
+  }
+
+  for (size_t i = 0; i < covered_area.size(); i++) {
+    CALayer* subLayer = [layer.sublayers objectAtIndex:i];
+    const auto& rect = covered_area[i];
+    subLayer.frame = CGRectMake(rect.left / scale, rect.top / scale,
+                                (rect.right - rect.left) / scale, (rect.bottom - rect.top) / scale);
+
+    double w = surfaceSize.width;
+    double h = surfaceSize.height;
+
+    subLayer.contentsRect = CGRectMake(rect.left / w, rect.top / h, (rect.right - rect.left) / w,
+                                       (rect.bottom - rect.top) / h);
+
+    if (borderColor != nil) {
+      // Visualize sublayer
+      subLayer.borderColor = borderColor.CGColor;
+      subLayer.borderWidth = 1.0;
+    }
+
+    subLayer.contents = (__bridge id)surface;
+  }
+}
+
 @implementation FlutterSurfaceManager
 
 - (instancetype)initWithDevice:(id<MTLDevice>)device
@@ -77,6 +130,17 @@ - (FlutterSurface*)surfaceForSize:(CGSize)size {
   return surface;
 }
 
+- (BOOL)enableSurfaceDebugInfo {
+  if (_enableSurfaceDebugInfo == nil) {
+    _enableSurfaceDebugInfo =
+        [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTEnableSurfaceDebugInfo"];
+    if (_enableSurfaceDebugInfo == nil) {
+      _enableSurfaceDebugInfo = @NO;
+    }
+  }
+  return [_enableSurfaceDebugInfo boolValue];
+}
+
 - (void)commit:(NSArray<FlutterSurfacePresentInfo*>*)surfaces {
   FML_DCHECK([NSThread isMainThread]);
 
@@ -100,16 +164,38 @@ - (void)commit:(NSArray<FlutterSurfacePresentInfo*>*)surfaces {
     [_layers addObject:layer];
   }
 
+  bool enableSurfaceDebugInfo = self.enableSurfaceDebugInfo;
+
   // Update contents of surfaces.
   for (size_t i = 0; i < surfaces.count; ++i) {
     FlutterSurfacePresentInfo* info = surfaces[i];
     CALayer* layer = _layers[i];
     CGFloat scale = _containingLayer.contentsScale;
-    layer.frame = CGRectMake(info.offset.x / scale, info.offset.y / scale,
-                             info.surface.size.width / scale, info.surface.size.height / scale);
-    layer.contents = (__bridge id)info.surface.ioSurface;
+    if (i == 0) {
+      layer.frame = CGRectMake(info.offset.x / scale, info.offset.y / scale,
+                               info.surface.size.width / scale, info.surface.size.height / scale);
+      layer.contents = (__bridge id)info.surface.ioSurface;
+    } else {
+      layer.frame = CGRectZero;
+      NSColor* borderColor = enableSurfaceDebugInfo ? GetBorderColorForLayer(i - 1) : nil;
+      UpdateContentSubLayers(layer, info.surface.ioSurface, scale, info.surface.size, borderColor,
+                             info.paintRegion);
+    }
     layer.zPosition = info.zIndex;
   }
+
+  if (enableSurfaceDebugInfo) {
+    if (_infoLayer == nil) {
+      _infoLayer = [[CATextLayer alloc] init];
+      [_containingLayer addSublayer:_infoLayer];
+      _infoLayer.fontSize = 15;
+      _infoLayer.foregroundColor = [NSColor yellowColor].CGColor;
+      _infoLayer.frame = CGRectMake(15, 15, 300, 100);
+      _infoLayer.contentsScale = _containingLayer.contentsScale;
+      _infoLayer.zPosition = 100000;
+    }
+    _infoLayer.string = [NSString stringWithFormat:@"Surface count: %li", _layers.count];
+  }
 }
 
 static CGSize GetRequiredFrameSize(NSArray<FlutterSurfacePresentInfo*>* surfaces) {

shell/platform/darwin/macos/framework/Source/FlutterSurfaceManagerTest.mm

@@ -25,6 +25,7 @@ - (instancetype)init {
   self = [super initWithFrame:NSZeroRect];
   if (self) {
     [self setWantsLayer:YES];
+    self.layer.contentsScale = 2.0;
   }
   return self;
 }
@@ -50,11 +51,13 @@ - (void)onPresent:(CGSize)frameSize withBlock:(nonnull dispatch_block_t)block {
 
 static FlutterSurfacePresentInfo* CreatePresentInfo(FlutterSurface* surface,
                                                     CGPoint offset = CGPointZero,
-                                                    size_t index = 0) {
+                                                    size_t index = 0,
+                                                    std::vector<FlutterRect> paintRegion = {}) {
   FlutterSurfacePresentInfo* res = [[FlutterSurfacePresentInfo alloc] init];
   res.surface = surface;
   res.offset = offset;
   res.zIndex = index;
+  res.paintRegion = paintRegion;
   return res;
 }
 
@@ -160,6 +163,10 @@ - (void)onPresent:(CGSize)frameSize withBlock:(nonnull dispatch_block_t)block {
   EXPECT_EQ(surface3, surface1);
 }
 
+inline bool operator==(const CGRect& lhs, const CGRect& rhs) {
+  return CGRectEqualToRect(lhs, rhs);
+}
+
 TEST(FlutterSurfaceManager, LayerManagement) {
   TestView* testView = [[TestView alloc] init];
   FlutterSurfaceManager* surfaceManager = CreateSurfaceManager(testView);
@@ -176,15 +183,68 @@ - (void)onPresent:(CGSize)frameSize withBlock:(nonnull dispatch_block_t)block {
   auto surface2_2 = [surfaceManager surfaceForSize:CGSizeMake(20, 20)];
   [surfaceManager present:@[
     CreatePresentInfo(surface2_1, CGPointMake(20, 10), 1),
-    CreatePresentInfo(surface2_2, CGPointMake(40, 50), 2)
+    CreatePresentInfo(surface2_2, CGPointMake(40, 50), 2,
+                      {
+                          FlutterRect{0, 0, 20, 20},
+                          FlutterRect{40, 0, 60, 20},
+                      })
   ]
                    notify:nil];
 
   EXPECT_EQ(testView.layer.sublayers.count, 2ul);
-  EXPECT_EQ([testView.layer.sublayers objectAtIndex:0].zPosition, 1.0);
-  EXPECT_EQ([testView.layer.sublayers objectAtIndex:1].zPosition, 2.0);
+  EXPECT_EQ(testView.layer.sublayers[0].zPosition, 1.0);
+  EXPECT_EQ(testView.layer.sublayers[1].zPosition, 2.0);
+  CALayer* firstOverlaySublayer;
+  {
+    NSArray<CALayer*>* sublayers = testView.layer.sublayers[1].sublayers;
+    EXPECT_EQ(sublayers.count, 2ul);
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[0].frame, CGRectMake(0, 0, 10, 10)));
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[1].frame, CGRectMake(20, 0, 10, 10)));
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[0].contentsRect, CGRectMake(0, 0, 1, 1)));
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[1].contentsRect, CGRectMake(2, 0, 1, 1)));
+    EXPECT_EQ(sublayers[0].contents, sublayers[1].contents);
+    firstOverlaySublayer = sublayers[0];
+  }
   EXPECT_TRUE(CGSizeEqualToSize(testView.presentedFrameSize, CGSizeMake(70, 70)));
 
+  // Check second overlay sublayer is removed while first is reused and updated
+  [surfaceManager present:@[
+    CreatePresentInfo(surface2_1, CGPointMake(20, 10), 1),
+    CreatePresentInfo(surface2_2, CGPointMake(40, 50), 2,
+                      {
+                          FlutterRect{0, 10, 20, 20},
+                      })
+  ]
+                   notify:nil];
+  EXPECT_EQ(testView.layer.sublayers.count, 2ul);
+  {
+    NSArray<CALayer*>* sublayers = testView.layer.sublayers[1].sublayers;
+    EXPECT_EQ(sublayers.count, 1ul);
+    EXPECT_EQ(sublayers[0], firstOverlaySublayer);
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[0].frame, CGRectMake(0, 5, 10, 5)));
+  }
+
+  // Check that second overlay sublayer is added back while first is reused and updated
+  [surfaceManager present:@[
+    CreatePresentInfo(surface2_1, CGPointMake(20, 10), 1),
+    CreatePresentInfo(surface2_2, CGPointMake(40, 50), 2,
+                      {
+                          FlutterRect{0, 0, 20, 20},
+                          FlutterRect{40, 0, 60, 20},
+                      })
+  ]
+                   notify:nil];
+
+  EXPECT_EQ(testView.layer.sublayers.count, 2ul);
+  {
+    NSArray<CALayer*>* sublayers = testView.layer.sublayers[1].sublayers;
+    EXPECT_EQ(sublayers.count, 2ul);
+    EXPECT_EQ(sublayers[0], firstOverlaySublayer);
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[0].frame, CGRectMake(0, 0, 10, 10)));
+    EXPECT_TRUE(CGRectEqualToRect(sublayers[1].frame, CGRectMake(20, 0, 10, 10)));
+    EXPECT_EQ(sublayers[0].contents, sublayers[1].contents);
+  }
+
   auto surface3_1 = [surfaceManager surfaceForSize:CGSizeMake(50, 30)];
   [surfaceManager present:@[ CreatePresentInfo(surface3_1, CGPointMake(20, 10)) ] notify:nil];
 

shell/platform/embedder/embedder.h

@@ -1568,6 +1568,18 @@ typedef enum {
   kFlutterLayerContentTypePlatformView,
 } FlutterLayerContentType;
 
+typedef struct {
+  size_t struct_size;
+
+  /// The area of the backing store that contains Flutter contents. Pixels
+  /// outside of this area are transparent and the embedder may choose not
+  /// to render them. Coordinates are in physical pixels.
+  FlutterRect* paint_region_rects;
+
+  /// Number of rectangles in the paint_region_rects array.
+  size_t paint_region_rects_count;
+} FlutterBackingStorePresentInfo;
+
 typedef struct {
   /// This size of this struct. Must be sizeof(FlutterLayer).
   size_t struct_size;
@@ -1587,6 +1599,10 @@ typedef struct {
   FlutterPoint offset;
   /// The size of the layer (in physical pixels).
   FlutterSize size;
+
+  /// Extra information for the backing store that the embedder may
+  /// used during presentation.
+  FlutterBackingStorePresentInfo* backing_store_present_info;
 } FlutterLayer;
 
 typedef bool (*FlutterBackingStoreCreateCallback)(

shell/platform/embedder/embedder_external_view.cc

@@ -62,6 +62,11 @@ bool EmbedderExternalView::HasPlatformView() const {
   return view_identifier_.platform_view_id.has_value();
 }
 
+std::list<SkRect> EmbedderExternalView::GetDrawnRects(
+    const SkRect& query) const {
+  return slice_->searchNonOverlappingDrawnRects(query);
+}
+
 bool EmbedderExternalView::HasEngineRenderedContents() {
   if (has_engine_rendered_contents_.has_value()) {
     return has_engine_rendered_contents_.value();

shell/platform/embedder/embedder_external_view.h

@@ -109,6 +109,8 @@ class EmbedderExternalView {
 
   bool Render(const EmbedderRenderTarget& render_target);
 
+  std::list<SkRect> GetDrawnRects(const SkRect& query) const;
+
  private:
   // End the recording of the slice.
   // Noop if the slice's recording has already ended.

shell/platform/embedder/embedder_external_view_embedder.cc

@@ -238,8 +238,15 @@ void EmbedderExternalViewEmbedder::SubmitFrame(
       // platform view.
       if (external_view->HasEngineRenderedContents()) {
         const auto& exteral_render_target = matched_render_targets.at(view_id);
+        const auto& external_view = pending_views_.at(view_id);
+        auto rect_list = external_view->GetDrawnRects(SkRect::MakeIWH(
+            pending_frame_size_.width(), pending_frame_size_.height()));
+        std::vector<SkIRect> rects;
+        for (const auto& rect : rect_list) {
+          rects.push_back(rect.roundOut());
+        }
         presented_layers.PushBackingStoreLayer(
-            exteral_render_target->GetBackingStore());
+            exteral_render_target->GetBackingStore(), rects);
       }
     }
 

shell/platform/embedder/embedder_layers.cc

@@ -17,7 +17,9 @@ EmbedderLayers::EmbedderLayers(SkISize frame_size,
 
 EmbedderLayers::~EmbedderLayers() = default;
 
-void EmbedderLayers::PushBackingStoreLayer(const FlutterBackingStore* store) {
+void EmbedderLayers::PushBackingStoreLayer(
+    const FlutterBackingStore* store,
+    const std::vector<SkIRect>& paint_region) {
   FlutterLayer layer = {};
 
   layer.struct_size = sizeof(FlutterLayer);
@@ -35,6 +37,29 @@ void EmbedderLayers::PushBackingStoreLayer(const FlutterBackingStore* store) {
   layer.size.width = transformed_layer_bounds.width();
   layer.size.height = transformed_layer_bounds.height();
 
+  auto paint_region_rects = std::make_unique<std::vector<FlutterRect>>();
+  paint_region_rects->reserve(paint_region.size());
+
+  for (const auto& rect : paint_region) {
+    auto transformed_rect =
+        root_surface_transformation_.mapRect(SkRect::Make(rect));
+    paint_region_rects->push_back(FlutterRect{
+        transformed_rect.x(),
+        transformed_rect.y(),
+        transformed_rect.right(),
+        transformed_rect.bottom(),
+    });
+  }
+
+  auto present_info = std::make_unique<FlutterBackingStorePresentInfo>();
+  present_info->struct_size = sizeof(FlutterBackingStorePresentInfo);
+  present_info->paint_region_rects = paint_region_rects->data();
+  present_info->paint_region_rects_count = paint_region_rects->size();
+  paint_region_rects_referenced_.push_back(std::move(paint_region_rects));
+
+  layer.backing_store_present_info = present_info.get();
+  present_info_refereced_.push_back(std::move(present_info));
+
   presented_layers_.push_back(layer);
 }
 

shell/platform/embedder/embedder_layers.h

@@ -24,7 +24,8 @@ class EmbedderLayers {
 
   ~EmbedderLayers();
 
-  void PushBackingStoreLayer(const FlutterBackingStore* store);
+  void PushBackingStoreLayer(const FlutterBackingStore* store,
+                             const std::vector<SkIRect>& drawn_region);
 
   void PushPlatformViewLayer(FlutterPlatformViewIdentifier identifier,
                              const EmbeddedViewParams& params);
@@ -42,6 +43,10 @@ class EmbedderLayers {
       mutations_referenced_;
   std::vector<std::unique_ptr<std::vector<const FlutterPlatformViewMutation*>>>
       mutations_arrays_referenced_;
+  std::vector<std::unique_ptr<FlutterBackingStorePresentInfo>>
+      present_info_refereced_;
+  std::vector<std::unique_ptr<std::vector<FlutterRect>>>
+      paint_region_rects_referenced_;
   std::vector<FlutterLayer> presented_layers_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(EmbedderLayers);