[web] Fixes canvas pixelation and overallocation due to transforms. (flutter#22160)

Author: ferhatb

lib/web_ui/lib/src/engine/bitmap_canvas.dart

@@ -104,18 +104,23 @@ class BitmapCanvas extends EngineCanvas {
   /// can be constructed from contents.
   bool _preserveImageData = false;
 
+  /// Canvas pixel to screen pixel ratio. Similar to dpi but
+  /// uses global transform of canvas to compute ratio.
+  final double _density;
+
   /// Allocates a canvas with enough memory to paint a picture within the given
   /// [bounds].
   ///
   /// This canvas can be reused by pictures with different paint bounds as long
   /// as the [Rect.size] of the bounds fully fit within the size used to
   /// initialize this canvas.
-  BitmapCanvas(this._bounds)
+  BitmapCanvas(this._bounds, {double density = 1.0})
       : assert(_bounds != null), // ignore: unnecessary_null_comparison
+        _density = density,
         _widthInBitmapPixels = _widthToPhysical(_bounds.width),
         _heightInBitmapPixels = _heightToPhysical(_bounds.height),
         _canvasPool = _CanvasPool(_widthToPhysical(_bounds.width),
-            _heightToPhysical(_bounds.height)) {
+            _heightToPhysical(_bounds.height), density) {
     rootElement.style.position = 'absolute';
     // Adds one extra pixel to the requested size. This is to compensate for
     // _initializeViewport() snapping canvas position to 1 pixel, causing
@@ -179,10 +184,11 @@ class BitmapCanvas extends EngineCanvas {
   }
 
   // Used by picture to assess if canvas is large enough to reuse as is.
-  bool doesFitBounds(ui.Rect newBounds) {
+  bool doesFitBounds(ui.Rect newBounds, double newDensity) {
     assert(newBounds != null); // ignore: unnecessary_null_comparison
     return _widthInBitmapPixels >= _widthToPhysical(newBounds.width) &&
-        _heightInBitmapPixels >= _heightToPhysical(newBounds.height);
+        _heightInBitmapPixels >= _heightToPhysical(newBounds.height) &&
+        _density == newDensity;
   }
 
   @override

lib/web_ui/lib/src/engine/canvas_pool.dart

@@ -33,8 +33,10 @@ class _CanvasPool extends _SaveStackTracking {
 
   html.HtmlElement? _rootElement;
   int _saveContextCount = 0;
+  final double _density;
 
-  _CanvasPool(this._widthInBitmapPixels, this._heightInBitmapPixels);
+  _CanvasPool(this._widthInBitmapPixels, this._heightInBitmapPixels,
+      this._density);
 
   html.CanvasRenderingContext2D get context {
     html.CanvasRenderingContext2D? ctx = _context;
@@ -83,7 +85,12 @@ class _CanvasPool extends _SaveStackTracking {
   void _createCanvas() {
     bool requiresClearRect = false;
     bool reused = false;
-    html.CanvasElement canvas;
+    html.CanvasElement? canvas;
+    if (_canvas != null) {
+      _canvas!.width = 0;
+      _canvas!.height = 0;
+      _canvas = null;
+    }
     if (_reusablePool != null && _reusablePool!.isNotEmpty) {
       canvas = _canvas = _reusablePool!.removeAt(0);
       requiresClearRect = true;
@@ -99,10 +106,7 @@ class _CanvasPool extends _SaveStackTracking {
           _widthInBitmapPixels / EnginePlatformDispatcher.browserDevicePixelRatio;
       final double cssHeight =
           _heightInBitmapPixels / EnginePlatformDispatcher.browserDevicePixelRatio;
-      canvas = html.CanvasElement(
-        width: _widthInBitmapPixels,
-        height: _heightInBitmapPixels,
-      );
+      canvas = _allocCanvas(_widthInBitmapPixels, _heightInBitmapPixels);
       _canvas = canvas;
 
       // Why is this null check here, even though we just allocated a canvas element above?
@@ -113,12 +117,9 @@ class _CanvasPool extends _SaveStackTracking {
       if (_canvas == null) {
         // Evict BitmapCanvas(s) and retry.
         _reduceCanvasMemoryUsage();
-        canvas = html.CanvasElement(
-          width: _widthInBitmapPixels,
-          height: _heightInBitmapPixels,
-        );
+        canvas = _allocCanvas(_widthInBitmapPixels, _heightInBitmapPixels);
       }
-      canvas.style
+      canvas!.style
         ..position = 'absolute'
         ..width = '${cssWidth}px'
         ..height = '${cssHeight}px';
@@ -131,19 +132,55 @@ class _CanvasPool extends _SaveStackTracking {
       _rootElement!.append(canvas);
     }
 
-    if (reused) {
-      // If a canvas is the first element we set z-index = -1 in [BitmapCanvas]
-      // endOfPaint to workaround blink compositing bug. To make sure this
-      // does not leak when reused reset z-index.
-      canvas.style.removeProperty('z-index');
+    try {
+      if (reused) {
+        // If a canvas is the first element we set z-index = -1 in [BitmapCanvas]
+        // endOfPaint to workaround blink compositing bug. To make sure this
+        // does not leak when reused reset z-index.
+        canvas.style.removeProperty('z-index');
+      }
+      _context = canvas.context2D;
+    } catch (e) {
+      // Handle OOM.
     }
-
-    final html.CanvasRenderingContext2D context = _context = canvas.context2D;
-    _contextHandle = ContextStateHandle(this, context);
+    if (_context == null) {
+      _reduceCanvasMemoryUsage();
+      _context = canvas.context2D;
+    }
+    if (_context == null) {
+      /// Browser ran out of memory, try to recover current allocation
+      /// and bail.
+      _canvas?.width = 0;
+      _canvas?.height = 0;
+      _canvas = null;
+      return;
+    }
+    _contextHandle = ContextStateHandle(this, _context!, this._density);
     _initializeViewport(requiresClearRect);
     _replayClipStack();
   }
 
+  html.CanvasElement? _allocCanvas(int width, int height) {
+    final dynamic canvas =
+      js_util.callMethod(html.document, 'createElement', <dynamic>['CANVAS']);
+    if (canvas != null) {
+      try {
+        canvas.width = (width * _density).ceil();
+        canvas.height = (height * _density).ceil();
+      } catch (e) {
+        return null;
+      }
+      return canvas as html.CanvasElement;
+    }
+    return null;
+    // !!! We don't use the code below since NNBD assumes it can never return
+    // null and optimizes out code.
+    // return canvas = html.CanvasElement(
+    //   width: _widthInBitmapPixels,
+    //   height: _heightInBitmapPixels,
+    // );
+  }
+
   @override
   void clear() {
     super.clear();
@@ -188,7 +225,7 @@ class _CanvasPool extends _SaveStackTracking {
             clipTimeTransform[5] != prevTransform[5] ||
             clipTimeTransform[12] != prevTransform[12] ||
             clipTimeTransform[13] != prevTransform[13]) {
-          final double ratio = EnginePlatformDispatcher.browserDevicePixelRatio;
+          final double ratio = dpi;
           ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
           ctx.transform(
               clipTimeTransform[0],
@@ -222,7 +259,7 @@ class _CanvasPool extends _SaveStackTracking {
         transform[5] != prevTransform[5] ||
         transform[12] != prevTransform[12] ||
         transform[13] != prevTransform[13]) {
-      final double ratio = EnginePlatformDispatcher.browserDevicePixelRatio;
+      final double ratio = dpi;
       ctx.setTransform(ratio, 0, 0, ratio, 0, 0);
       ctx.transform(transform[0], transform[1], transform[4], transform[5],
           transform[12], transform[13]);
@@ -300,15 +337,19 @@ class _CanvasPool extends _SaveStackTracking {
     // is applied on the DOM elements.
     ctx.setTransform(1, 0, 0, 1, 0, 0);
     if (clearCanvas) {
-      ctx.clearRect(0, 0, _widthInBitmapPixels, _heightInBitmapPixels);
+      ctx.clearRect(0, 0, _widthInBitmapPixels * _density,
+          _heightInBitmapPixels * _density);
     }
 
     // This scale makes sure that 1 CSS pixel is translated to the correct
     // number of bitmap pixels.
-    ctx.scale(EnginePlatformDispatcher.browserDevicePixelRatio,
-        EnginePlatformDispatcher.browserDevicePixelRatio);
+    ctx.scale(dpi, dpi);
   }
 
+  /// Returns effective dpi (browser DPI and pixel density due to transform).
+  double get dpi =>
+      EnginePlatformDispatcher.browserDevicePixelRatio * _density;
+
   void resetTransform() {
     final html.CanvasElement? canvas = _canvas;
     if (canvas != null) {
@@ -688,8 +729,9 @@ class _CanvasPool extends _SaveStackTracking {
 class ContextStateHandle {
   final html.CanvasRenderingContext2D context;
   final _CanvasPool _canvasPool;
+  final double density;
 
-  ContextStateHandle(this._canvasPool, this.context);
+  ContextStateHandle(this._canvasPool, this.context, this.density);
   ui.BlendMode? _currentBlendMode = ui.BlendMode.srcOver;
   ui.StrokeCap? _currentStrokeCap = ui.StrokeCap.butt;
   ui.StrokeJoin? _currentStrokeJoin = ui.StrokeJoin.miter;
@@ -778,7 +820,8 @@ class ContextStateHandle {
     if (paint.shader != null) {
       final EngineGradient engineShader = paint.shader as EngineGradient;
       final Object paintStyle =
-          engineShader.createPaintStyle(_canvasPool.context, shaderBounds);
+          engineShader.createPaintStyle(_canvasPool.context, shaderBounds,
+              density);
       fillStyle = paintStyle;
       strokeStyle = paintStyle;
     } else if (paint.color != null) {

lib/web_ui/lib/src/engine/html/picture.dart

@@ -90,6 +90,7 @@ class PersistedPicture extends PersistedLeafSurface {
   final EnginePicture picture;
   final ui.Rect? localPaintBounds;
   final int hints;
+  double _density = 1.0;
 
   /// Cache for reusing elements such as images across picture updates.
   CrossFrameCache<html.HtmlElement>? _elementCache =
@@ -107,6 +108,23 @@ class PersistedPicture extends PersistedLeafSurface {
       _transform = _transform!.clone();
       _transform!.translate(dx, dy);
     }
+    final double paintWidth = localPaintBounds!.width;
+    final double paintHeight = localPaintBounds!.height;
+    final double newDensity = localPaintBounds == null || paintWidth == 0 || paintHeight == 0
+        ? 1.0 : _computePixelDensity(_transform, paintWidth, paintHeight);
+    if (newDensity != _density) {
+      _density = newDensity;
+      if (_canvas != null) {
+        // If cull rect and density hasn't changed, this will only repaint.
+        // If density doesn't match canvas, a new canvas will be created
+        // and paint queued.
+        //
+        // Similar to preroll for transform where transform is updated, for
+        // picture this means we need to repaint so pixelation doesn't occur
+        // due to transform changing overall dpi.
+        applyPaint(_canvas);
+      }
+    }
     _computeExactCullRects();
   }
 
@@ -296,7 +314,12 @@ class PersistedPicture extends PersistedLeafSurface {
       // painting. This removes all the setup work and scaffolding objects
       // that won't be useful for anything anyway.
       _recycleCanvas(oldCanvas);
-      domRenderer.clearDom(rootElement!);
+      if (rootElement != null) {
+        domRenderer.clearDom(rootElement!);
+      }
+      if (_canvas != null) {
+        _recycleCanvas(_canvas);
+      }
       _canvas = null;
       return;
     }
@@ -339,7 +362,7 @@ class PersistedPicture extends PersistedLeafSurface {
         // We did not allocate a canvas last time. This can happen when the
         // picture is completely clipped out of the view.
         return 1.0;
-      } else if (!oldCanvas.doesFitBounds(_exactLocalCullRect!)) {
+      } else if (!oldCanvas.doesFitBounds(_exactLocalCullRect!, _density)) {
         // The canvas needs to be resized before painting.
         return 1.0;
       } else {
@@ -382,7 +405,7 @@ class PersistedPicture extends PersistedLeafSurface {
 
   void _applyBitmapPaint(EngineCanvas? oldCanvas) {
     if (oldCanvas is BitmapCanvas &&
-        oldCanvas.doesFitBounds(_optimalLocalCullRect!) &&
+        oldCanvas.doesFitBounds(_optimalLocalCullRect!, _density) &&
         oldCanvas.isReusable()) {
       if (_debugShowCanvasReuseStats) {
         DebugCanvasReuseOverlay.instance.keptCount++;
@@ -451,7 +474,7 @@ class PersistedPicture extends PersistedLeafSurface {
       final double candidatePixelCount =
           candidateSize.width * candidateSize.height;
 
-      final bool fits = candidate.doesFitBounds(bounds);
+      final bool fits = candidate.doesFitBounds(bounds, _density);
       final bool isSmaller = candidatePixelCount < lastPixelCount;
       if (fits && isSmaller) {
         // [isTooSmall] is used to make sure that a small picture doesn't
@@ -493,7 +516,7 @@ class PersistedPicture extends PersistedLeafSurface {
     if (_debugShowCanvasReuseStats) {
       DebugCanvasReuseOverlay.instance.createdCount++;
     }
-    final BitmapCanvas canvas = BitmapCanvas(bounds);
+    final BitmapCanvas canvas = BitmapCanvas(bounds, density: _density);
     canvas.setElementCache(_elementCache);
     if (_debugExplainSurfaceStats) {
       _surfaceStatsFor(this)
@@ -536,8 +559,12 @@ class PersistedPicture extends PersistedLeafSurface {
     final bool cullRectChangeRequiresRepaint =
         _computeOptimalCullRect(oldSurface);
     if (identical(picture, oldSurface.picture)) {
+      bool densityChanged =
+          (_canvas is BitmapCanvas &&
+              _density != (_canvas as BitmapCanvas)._density);
+
       // The picture is the same. Attempt to avoid repaint.
-      if (cullRectChangeRequiresRepaint) {
+      if (cullRectChangeRequiresRepaint || densityChanged) {
         // Cull rect changed such that a repaint is still necessary.
         _applyPaint(oldSurface);
       } else {
@@ -603,3 +630,72 @@ class PersistedPicture extends PersistedLeafSurface {
     }
   }
 }
+
+/// Given size of a rectangle and transform, computes pixel density
+/// (scale factor).
+double _computePixelDensity(Matrix4? transform, double width, double height) {
+  if (transform == null || transform.isIdentity()) {
+    return 1.0;
+  }
+  final Float32List m = transform.storage;
+  // Apply perspective transform to all 4 corners. Can't use left,top, bottom,
+  // right since for example rotating 45 degrees would yield inaccurate size.
+  double minX = m[12] * m[15];
+  double minY = m[13] * m[15];
+  double maxX = minX;
+  double maxY = minY;
+  double x = width;
+  double y = height;
+  double wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
+  double xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
+  double yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
+  print('$xp,$yp');
+  minX = math.min(minX, xp);
+  maxX = math.max(maxX, xp);
+  minY = math.min(minY, yp);
+  maxY = math.max(maxY, yp);
+  x = 0;
+  wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
+  xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
+  yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
+  print('$xp,$yp');
+  minX = math.min(minX, xp);
+  maxX = math.max(maxX, xp);
+  minY = math.min(minY, yp);
+  maxY = math.max(maxY, yp);
+  x = width;
+  y = 0;
+  wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
+  xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
+  yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
+  print('$xp,$yp');
+  minX = math.min(minX, xp);
+  maxX = math.max(maxX, xp);
+  minY = math.min(minY, yp);
+  maxY = math.max(maxY, yp);
+  double scaleX = (maxX - minX) / width;
+  double scaleY = (maxY - minY) / height;
+  double scale = math.min(scaleX, scaleY);
+  // kEpsilon guards against divide by zero below.
+  if (scale < kEpsilon || scale == 1) {
+    // Handle local paint bounds scaled to 0, typical when using
+    // transform animations and nothing is drawn.
+    return 1.0;
+  }
+  if (scale > 1) {
+    // Normalize scale to multiples of 2: 1x, 2x, 4x, 6x, 8x.
+    // This is to prevent frequent rescaling of canvas during animations.
+    //
+    // On a fullscreen high dpi device dpi*density*resolution will demand
+    // too much memory, so clamp at 4.
+    scale = math.min(4.0, ((scale / 2.0).ceil() * 2.0));
+    // Guard against webkit absolute limit.
+    const double kPixelLimit = 1024 * 1024 * 4;
+    if ((width * height * scale * scale) > kPixelLimit && scale > 2) {
+      scale = (kPixelLimit * 0.8) / (width * height);
+    }
+  } else {
+    scale = math.max(2.0 / (2.0 / scale).floor(), 0.0001);
+  }
+  return scale;
+}