[web:semantics] fix node positioning; expose debug tree (#24903)

Author: yjbanov

lib/web_ui/lib/src/engine/dom_renderer.dart

@@ -54,12 +54,27 @@ class DomRenderer {
   /// This element is created and inserted in the HTML DOM once. It is never
   /// removed or moved. However the [sceneElement] may be replaced inside it.
   ///
-  /// This element precedes the [glassPaneElement] so that it never receives
-  /// input events. All input events are processed by [glassPaneElement] and the
-  /// semantics tree.
+  /// This element is inserted after the [semanticsHostElement] so that
+  /// platform views take precedence in DOM event handling.
   html.Element? get sceneHostElement => _sceneHostElement;
   html.Element? _sceneHostElement;
 
+  /// The element that contains the semantics tree.
+  ///
+  /// This element is created and inserted in the HTML DOM once. It is never
+  /// removed or moved.
+  ///
+  /// We render semantics inside the glasspane for proper focus and event
+  /// handling. If semantics is behind the glasspane, the phone will disable
+  /// focusing by touch, only by tabbing around the UI. If semantics is in
+  /// front of glasspane, then DOM event won't bubble up to the glasspane so
+  /// it can forward events to the framework.
+  ///
+  /// This element is inserted before the [semanticsHostElement] so that
+  /// platform views take precedence in DOM event handling.
+  html.Element? get semanticsHostElement => _semanticsHostElement;
+  html.Element? _semanticsHostElement;
+
   /// The last scene element rendered by the [render] method.
   html.Element? get sceneElement => _sceneElement;
   html.Element? _sceneElement;
@@ -427,6 +442,14 @@ flt-glass-pane * {
 
     _sceneHostElement = createElement('flt-scene-host');
 
+    final html.Element semanticsHostElement = createElement('flt-semantics-host');
+    semanticsHostElement.style
+      ..position = 'absolute'
+      ..transformOrigin = '0 0 0';
+    _semanticsHostElement = semanticsHostElement;
+    updateSemanticsScreenProperties();
+    glassPaneElement.append(semanticsHostElement);
+
     // Don't allow the scene to receive pointer events.
     _sceneHostElement!.style.pointerEvents = 'none';
 
@@ -443,6 +466,12 @@ flt-glass-pane * {
     // by the platform view.
     glassPaneElement.insertBefore(_accesibilityPlaceholder, _sceneHostElement);
 
+    // When debugging semantics, make the scene semi-transparent so that the
+    // semantics tree is visible.
+    if (_debugShowSemanticsNodes) {
+      _sceneHostElement!.style.opacity = '0.3';
+    }
+
     PointerBinding.initInstance(glassPaneElement);
     KeyboardBinding.initInstance(glassPaneElement);
 
@@ -559,6 +588,13 @@ flt-glass-pane * {
     EnginePlatformDispatcher.instance._updateLocales();
   }
 
+  /// The framework specifies semantics in physical pixels, but CSS uses
+  /// logical pixels. To compensate, we inject an inverse scale at the root
+  /// level.
+  void updateSemanticsScreenProperties() {
+    _semanticsHostElement!.style.transform = 'scale(${1 / html.window.devicePixelRatio})';
+  }
+
   /// Called immediately after browser window metrics change.
   ///
   /// When there is a text editing going on in mobile devices, do not change
@@ -569,6 +605,7 @@ flt-glass-pane * {
   /// Note: always check for rotations for a mobile device. Update the physical
   /// size if the change is caused by a rotation.
   void _metricsDidChange(html.Event? event) {
+    updateSemanticsScreenProperties();
     if (isMobile && !window.isRotation() && textEditing.isEditing) {
       window.computeOnScreenKeyboardInsets();
       EnginePlatformDispatcher.instance.invokeOnMetricsChanged();

lib/web_ui/lib/src/engine/semantics/label_and_value.dart

@@ -96,7 +96,11 @@ class LabelAndValue extends RoleManager {
           ..width = '${semanticsObject.rect!.width}px'
           ..height = '${semanticsObject.rect!.height}px';
       }
-      _auxiliaryValueElement!.style.fontSize = '6px';
+
+      // Normally use a small font size so that text doesn't leave the scope
+      // of the semantics node. When debugging semantics, use a font size
+      // that's reasonably visible.
+      _auxiliaryValueElement!.style.fontSize = _debugShowSemanticsNodes ? '12px' : '6px';
       semanticsObject.element.append(_auxiliaryValueElement!);
     }
     _auxiliaryValueElement!.text = combinedValue.toString();

lib/web_ui/lib/src/engine/semantics/semantics.dart

@@ -6,10 +6,20 @@
 part of engine;
 
 /// Set this flag to `true` to cause the engine to visualize the semantics tree
-/// on the screen.
+/// on the screen for debugging.
 ///
-/// This is useful for debugging.
-const bool _debugShowSemanticsNodes = false;
+/// This only works in profile and release modes. Debug mode does not support
+/// passing compile-time constants.
+///
+/// Example:
+///
+/// ```
+/// flutter run -d chrome --profile --dart-define=FLUTTER_WEB_DEBUG_SHOW_SEMANTICS=true
+/// ```
+const bool _debugShowSemanticsNodes = bool.fromEnvironment(
+  'FLUTTER_WEB_DEBUG_SHOW_SEMANTICS',
+  defaultValue: false,
+);
 
 /// Contains updates for the semantics tree.
 ///
@@ -233,27 +243,29 @@ class SemanticsObject {
   /// Creates a semantics tree node with the given [id] and [owner].
   SemanticsObject(this.id, this.owner) {
     // DOM nodes created for semantics objects are positioned absolutely using
-    // transforms. We use a transparent color instead of "visibility:hidden" or
-    // "display:none" so that a screen reader does not ignore these elements.
+    // transforms.
     element.style.position = 'absolute';
 
     // The root node has some properties that other nodes do not.
-    if (id == 0) {
+    if (id == 0 && !_debugShowSemanticsNodes) {
       // Make all semantics transparent. We use `filter` instead of `opacity`
       // attribute because `filter` is stronger. `opacity` does not apply to
       // some elements, particularly on iOS, such as the slider thumb and track.
+      //
+      // We use transparency instead of "visibility:hidden" or "display:none"
+      // so that a screen reader does not ignore these elements.
       element.style.filter = 'opacity(0%)';
 
       // Make text explicitly transparent to signal to the browser that no
       // rasterization needs to be done.
       element.style.color = 'rgba(0,0,0,0)';
     }
 
+    // Make semantic elements visible for debugging by outlining them using a
+    // green border. We do not use `border` attribute because it affects layout
+    // (`outline` does not).
     if (_debugShowSemanticsNodes) {
-      element.style
-        ..filter = 'opacity(90%)'
-        ..outline = '1px solid green'
-        ..color = 'purple';
+      element.style.outline = '1px solid green';
     }
   }
 
@@ -853,9 +865,9 @@ class SemanticsObject {
         hasIdentityTransform &&
         verticalContainerAdjustment == 0.0 &&
         horizontalContainerAdjustment == 0.0) {
-      _resetElementOffsets(element);
+      _clearSemanticElementTransform(element);
       if (containerElement != null) {
-        _resetElementOffsets(containerElement);
+        _clearSemanticElementTransform(containerElement);
       }
       return;
     }
@@ -879,81 +891,48 @@ class SemanticsObject {
       effectiveTransformIsIdentity = false;
     }
 
-    if (!effectiveTransformIsIdentity || isMacOrIOS) {
-      if (effectiveTransformIsIdentity) {
-        effectiveTransform = Matrix4.identity();
-      }
-      if (isDesktop) {
-        element.style
-          ..transformOrigin = '0 0 0'
-          ..transform = (effectiveTransformIsIdentity ? 'translate(0px 0px 0px)'
-              : matrix4ToCssTransform(effectiveTransform));
-      } else {
-        // Mobile screen readers observed to have errors while calculating the
-        // semantics focus borders if css `transform` properties are used.
-        // See: https://github.com/flutter/flutter/issues/68225
-        // Therefore we are calculating a bounding rectangle for the
-        // effective transform and use that rectangle to set TLWH css style
-        // properties.
-        // Note: Identity matrix is not using this code path.
-        final ui.Rect rect =
-            computeBoundingRectangleFromMatrix(effectiveTransform, _rect!);
-        element.style
-          ..top = '${rect.top}px'
-          ..left = '${rect.left}px'
-          ..width = '${rect.width}px'
-          ..height = '${rect.height}px';
-      }
+    if (!effectiveTransformIsIdentity) {
+      element.style
+        ..transformOrigin = '0 0 0'
+        ..transform = matrix4ToCssTransform(effectiveTransform);
     } else {
-      _resetElementOffsets(element);
-      // TODO: https://github.com/flutter/flutter/issues/73347
+      _clearSemanticElementTransform(element);
     }
 
     if (containerElement != null) {
       if (!hasZeroRectOffset ||
-          isMacOrIOS ||
           verticalContainerAdjustment != 0.0 ||
           horizontalContainerAdjustment != 0.0) {
         final double translateX = -_rect!.left + horizontalContainerAdjustment;
         final double translateY = -_rect!.top + verticalContainerAdjustment;
-        if (isDesktop) {
-          containerElement.style
-            ..transformOrigin = '0 0 0'
-            ..transform = 'translate(${translateX}px, ${translateY}px)';
-        } else {
-          containerElement.style
-            ..top = '${translateY}px'
-            ..left = '${translateX}px';
-        }
+        containerElement.style
+          ..top = '${translateY}px'
+          ..left = '${translateX}px';
       } else {
-        _resetElementOffsets(containerElement);
+        _clearSemanticElementTransform(containerElement);
       }
     }
   }
 
-  // On Mac OS and iOS, VoiceOver requires left=0 top=0 value to correctly
-  // handle order. See https://github.com/flutter/flutter/issues/73347.
-  static void _resetElementOffsets(html.Element element) {
+  /// Clears the transform on a semantic element as if an identity transform is
+  /// applied.
+  ///
+  /// On macOS and iOS, VoiceOver requires `left=0; top=0` value to correctly
+  /// handle traversal order.
+  ///
+  /// See https://github.com/flutter/flutter/issues/73347.
+  static void _clearSemanticElementTransform(html.Element element) {
+    element.style
+      ..removeProperty('transform-origin')
+      ..removeProperty('transform');
     if (isMacOrIOS) {
-      if (isDesktop) {
-        element.style
-          ..transformOrigin = '0 0 0'
-          ..transform = 'translate(0px, 0px)';
-      } else {
-        element.style
-          ..top = '0px'
-          ..left = '0px';
-      }
+      element.style
+        ..top = '0px'
+        ..left = '0px';
     } else {
-      if (isDesktop) {
-        element.style
-          ..removeProperty('transform-origin')
-          ..removeProperty('transform');
-      } else {
-        element.style
-          ..removeProperty('top')
-          ..removeProperty('left');
-      }
+      element.style
+        ..removeProperty('top')
+        ..removeProperty('left');
     }
   }
 
@@ -1493,7 +1472,10 @@ class EngineSemanticsOwner {
   /// Updates the semantics tree from data in the [uiUpdate].
   void updateSemantics(ui.SemanticsUpdate uiUpdate) {
     if (!_semanticsEnabled) {
-      return;
+      // If we're receiving a semantics update from the framework, it means the
+      // developer enabled it programmatically, so we enable it in the engine
+      // too.
+      semanticsEnabled = true;
     }
 
     final SemanticsUpdate update = uiUpdate as SemanticsUpdate;
@@ -1505,19 +1487,7 @@ class EngineSemanticsOwner {
     if (_rootSemanticsElement == null) {
       final SemanticsObject root = _semanticsTree[0]!;
       _rootSemanticsElement = root.element;
-      // We render semantics inside the glasspane for proper focus and event
-      // handling. If semantics is behind the glasspane, the phone will disable
-      // focusing by touch, only by tabbing around the UI. If semantics is in
-      // front of glasspane, then DOM event won't bubble up to the glasspane so
-      // it can forward events to the framework.
-      //
-      // We insert the semantics root before the scene host. For all widgets
-      // in the scene, except for platform widgets, the scene host will pass the
-      // pointer events through to the semantics tree. However, for platform
-      // views, the pointer events will not pass through, and will be handled
-      // by the platform view.
-      domRenderer.glassPaneElement!
-          .insertBefore(_rootSemanticsElement!, domRenderer.sceneHostElement);
+      domRenderer.semanticsHostElement!.append(root.element);
     }
 
     _finalizeTree();

lib/web_ui/lib/src/engine/util.dart

@@ -595,47 +595,3 @@ int clampInt(int value, int min, int max) {
     return value;
   }
 }
-
-ui.Rect computeBoundingRectangleFromMatrix(Matrix4 transform, ui.Rect rect) {
-  final Float32List m = transform.storage;
-  // Apply perspective transform to all 4 corners. Can't use left,top, bottom,
-  // right since for example rotating 45 degrees would yield inaccurate size.
-  double x = rect.left;
-  double y = rect.top;
-  double wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
-  double xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
-  double yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
-  double minX = xp, maxX = xp;
-  double minY = yp, maxY = yp;
-  x = rect.right;
-  y = rect.bottom;
-  wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
-  xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
-  yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
-
-  minX = math.min(minX, xp);
-  maxX = math.max(maxX, xp);
-  minY = math.min(minY, yp);
-  maxY = math.max(maxY, yp);
-
-  x = rect.left;
-  y = rect.bottom;
-  wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
-  xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
-  yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
-  minX = math.min(minX, xp);
-  maxX = math.max(maxX, xp);
-  minY = math.min(minY, yp);
-  maxY = math.max(maxY, yp);
-
-  x = rect.right;
-  y = rect.top;
-  wp = 1.0 / ((m[3] * x) + (m[7] * y) + m[15]);
-  xp = ((m[0] * x) + (m[4] * y) + m[12]) * wp;
-  yp = ((m[1] * x) + (m[5] * y) + m[13]) * wp;
-  minX = math.min(minX, xp);
-  maxX = math.max(maxX, xp);
-  minY = math.min(minY, yp);
-  maxY = math.max(maxY, yp);
-  return ui.Rect.fromLTWH(minX, minY, maxX - minX, maxY - minY);
-}