Adjust stylus event to allow handwriting in nearby inputs

The computed touch action for an input element allows stylus writing
only in the editing area of input box. This makes it slightly tough to start writing in many search fields that have a lot of padding outside the editing area and inside the Input box.

Adjust the pointer event to nearest input in some fixed area around the pointer down to improve the chances of writing in input fields.
This is useful to support the upcoming split-merge ('|') stylus gesture,
which is highly likely to start outside the editing area.

Bug: 1398127
Change-Id: I957ea0e4bd1de51f95ac915a4c11882fae39d4a1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4085362
Reviewed-by: Robert Flack <flackr@chromium.org>
Commit-Queue: Mahesh Machavolu <mahesh.ma@samsung.com>
Cr-Commit-Position: refs/heads/main@{#1082918}

components/stylus_handwriting/android/java/src/org/chromium/components/stylus_handwriting/DirectWritingServiceCallback.java

@@ -139,10 +139,16 @@ private void handleDwGesture(Bundle bundle) {
         if (gestureType.equals(GESTURE_TYPE_BACKSPACE) || gestureType.equals(GESTURE_TYPE_ZIGZAG)) {
             startPoint = bundle.getFloatArray(GESTURE_BUNDLE_KEY_START_POINT);
             float[] endPoint = bundle.getFloatArray(GESTURE_BUNDLE_KEY_END_POINT);
-            // Clamp x-coordinates of gesture to Editable bounds in order to allow delete gesture
-            // even if delete strokes cross editable bounds.
-            startPoint[0] = Math.max(startPoint[0], mEditableBounds.left);
-            endPoint[0] = Math.min(endPoint[0], mEditableBounds.right);
+            // Clamp coordinates of gesture to Editable bounds in order to allow delete gesture even
+            // if delete strokes cross editable bounds.
+            startPoint[0] =
+                    Math.max(mEditableBounds.left, Math.min(startPoint[0], mEditableBounds.right));
+            startPoint[1] =
+                    Math.max(mEditableBounds.top, Math.min(startPoint[1], mEditableBounds.bottom));
+            endPoint[0] =
+                    Math.max(mEditableBounds.left, Math.min(endPoint[0], mEditableBounds.right));
+            endPoint[1] =
+                    Math.max(mEditableBounds.top, Math.min(endPoint[1], mEditableBounds.bottom));
 
             gestureData.endPoint = toMojoPoint(endPoint);
             gestureData.action = StylusWritingGestureAction.DELETE_TEXT;

third_party/blink/common/features.cc

@@ -1469,6 +1469,10 @@ BASE_FEATURE(kStylusWritingToInput,
              "StylusWritingToInput",
              base::FEATURE_ENABLED_BY_DEFAULT);
 
+BASE_FEATURE(kStylusPointerAdjustment,
+             "StylusPointerAdjustment",
+             base::FEATURE_DISABLED_BY_DEFAULT);
+
 BASE_FEATURE(kDisableArrayBufferSizeLimitsForTesting,
              "DisableArrayBufferSizeLimitsForTesting",
              base::FEATURE_DISABLED_BY_DEFAULT);

third_party/blink/public/common/features.h

@@ -742,6 +742,10 @@ BLINK_COMMON_EXPORT BASE_DECLARE_FEATURE(kEarlyExitOnNoopClassOrStyleChange);
 // Stylus handwriting recognition to text input feature.
 BLINK_COMMON_EXPORT BASE_DECLARE_FEATURE(kStylusWritingToInput);
 
+// Apply touch adjustment for stylus pointer events. This feature allows
+// enabling functions like writing into a nearby input element.
+BLINK_COMMON_EXPORT BASE_DECLARE_FEATURE(kStylusPointerAdjustment);
+
 // TODO(https://crbug.com/1201109): temporary flag to disable new ArrayBuffer
 // size limits, so that tests can be written against code receiving these
 // buffers. Remove when the bindings code instituting these limits is removed.

third_party/blink/renderer/core/input/event_handler.cc

@@ -1751,6 +1751,35 @@ bool EventHandler::BestContextMenuNodeForHitTestResult(
                                       HeapVector<Member<Node>>(nodes));
 }
 
+bool EventHandler::BestStylusWritableNodeForHitTestResult(
+    const HitTestLocation& location,
+    const HitTestResult& result,
+    gfx::Point& target_point,
+    Node*& target_node) {
+  DCHECK(location.IsRectBasedTest());
+
+  // If the touch is over a scrollbar, don't adjust the touch point since touch
+  // adjustment only takes into account DOM nodes so a touch over a scrollbar
+  // will be adjusted towards nearby nodes. This leads to things like textarea
+  // scrollbars being untouchable.
+  if (result.GetScrollbar() || result.IsOverResizer()) {
+    target_node = nullptr;
+    return false;
+  }
+
+  gfx::Point touch_center =
+      frame_->View()->ConvertToRootFrame(ToRoundedPoint(location.Point()));
+  gfx::Rect touch_rect =
+      frame_->View()->ConvertToRootFrame(location.ToEnclosingRect());
+  HeapVector<Member<Node>, 11> nodes(result.ListBasedTestResult());
+
+  // FIXME: the explicit Vector conversion copies into a temporary and is
+  // wasteful.
+  return FindBestStylusWritableCandidate(target_node, target_point,
+                                         touch_center, touch_rect,
+                                         HeapVector<Member<Node>>(nodes));
+}
+
 // Update the hover and active state across all frames.  This logic is
 // different than the mouse case because mice send MouseLeave events to frames
 // as they're exited.  With gestures or manual applications, a single event

third_party/blink/renderer/core/input/event_handler.h

@@ -207,6 +207,10 @@ class CORE_EXPORT EventHandler final : public GarbageCollected<EventHandler> {
                                            const HitTestResult&,
                                            gfx::Point& target_point,
                                            Node*& target_node);
+  bool BestStylusWritableNodeForHitTestResult(const HitTestLocation& location,
+                                              const HitTestResult&,
+                                              gfx::Point& target_point,
+                                              Node*& target_node);
   void CacheTouchAdjustmentResult(uint32_t, gfx::PointF);
 
   // Dispatch a context menu event. If |override_target_element| is provided,

third_party/blink/renderer/core/input/pointer_event_manager.cc

@@ -6,6 +6,7 @@
 
 #include "base/auto_reset.h"
 #include "base/metrics/field_trial_params.h"
+#include "third_party/blink/public/common/features.h"
 #include "third_party/blink/public/common/input/web_touch_event.h"
 #include "third_party/blink/public/mojom/frame/user_activation_notification_type.mojom-blink.h"
 #include "third_party/blink/public/mojom/input/input_handler.mojom-blink.h"
@@ -38,6 +39,7 @@
 #include "third_party/blink/renderer/platform/geometry/layout_size.h"
 #include "third_party/blink/renderer/platform/instrumentation/use_counter.h"
 #include "third_party/blink/renderer/platform/runtime_enabled_features.h"
+#include "ui/display/screen_info.h"
 
 namespace blink {
 
@@ -49,6 +51,11 @@ const char kSkipTouchEventFilterTrialProcessParamName[] =
     "skip_filtering_process";
 const char kSkipTouchEventFilterTrialTypeParamName[] = "type";
 
+// Width and height of area of rectangle to hit test for potentially important
+// input fields to write into. This improves the chances of writing into the
+// intended input if the user starts writing close to it.
+const size_t kStylusWritableAdjustmentSizeDip = 30;
+
 size_t ToPointerTypeIndex(WebPointerProperties::PointerType t) {
   return static_cast<size_t>(t);
 }
@@ -371,20 +378,49 @@ void PointerEventManager::HandlePointerInterruption(
 
 bool PointerEventManager::ShouldAdjustPointerEvent(
     const WebPointerEvent& pointer_event) const {
-  return pointer_event.pointer_type ==
-             WebPointerProperties::PointerType::kTouch &&
+  return (pointer_event.pointer_type ==
+              WebPointerProperties::PointerType::kTouch ||
+          ShouldAdjustStylusPointerEvent(pointer_event)) &&
          pointer_event.GetType() == WebInputEvent::Type::kPointerDown &&
          pointer_event_factory_.IsPrimary(pointer_event);
 }
 
-void PointerEventManager::AdjustTouchPointerEvent(
-    WebPointerEvent& pointer_event) {
-  DCHECK(pointer_event.pointer_type ==
-         WebPointerProperties::PointerType::kTouch);
+bool PointerEventManager::ShouldAdjustStylusPointerEvent(
+    const WebPointerEvent& pointer_event) const {
+  return base::FeatureList::IsEnabled(
+             blink::features::kStylusPointerAdjustment) &&
+         (pointer_event.pointer_type ==
+              WebPointerProperties::PointerType::kPen ||
+          pointer_event.pointer_type ==
+              WebPointerProperties::PointerType::kEraser);
+}
+
+void PointerEventManager::AdjustPointerEvent(WebPointerEvent& pointer_event) {
+  DCHECK(
+      pointer_event.pointer_type == WebPointerProperties::PointerType::kTouch ||
+      pointer_event.pointer_type == WebPointerProperties::PointerType::kPen ||
+      pointer_event.pointer_type == WebPointerProperties::PointerType::kEraser);
+
+  float adjustment_width = 0.0f;
+  float adjustment_height = 0.0f;
+  if (pointer_event.pointer_type == WebPointerProperties::PointerType::kTouch) {
+    adjustment_width = pointer_event.width;
+    adjustment_height = pointer_event.height;
+  } else {
+    // Calculate adjustment size for stylus tool types.
+    ChromeClient& chrome_client = frame_->GetChromeClient();
+    float device_scale_factor =
+        chrome_client.GetScreenInfo(*frame_).device_scale_factor;
+
+    float page_scale_factor = frame_->GetPage()->PageScaleFactor();
+    adjustment_width = adjustment_height =
+        kStylusWritableAdjustmentSizeDip *
+        (device_scale_factor / page_scale_factor);
+  }
 
   LayoutSize hit_rect_size = GetHitTestRectForAdjustment(
-      *frame_, LayoutSize(LayoutUnit(pointer_event.width),
-                          LayoutUnit(pointer_event.height)));
+      *frame_,
+      LayoutSize(LayoutUnit(adjustment_width), LayoutUnit(adjustment_height)));
 
   if (hit_rect_size.IsEmpty())
     return;
@@ -402,14 +438,31 @@ void PointerEventManager::AdjustTouchPointerEvent(
       root_frame.GetEventHandler().HitTestResultAtLocation(location, hit_type);
   Node* adjusted_node = nullptr;
   gfx::Point adjusted_point;
-  bool adjusted = frame_->GetEventHandler().BestClickableNodeForHitTestResult(
-      location, hit_test_result, adjusted_point, adjusted_node);
 
-  if (adjusted)
-    pointer_event.SetPositionInWidget(adjusted_point.x(), adjusted_point.y());
-
-  frame_->GetEventHandler().CacheTouchAdjustmentResult(
-      pointer_event.unique_touch_event_id, pointer_event.PositionInWidget());
+  if (pointer_event.pointer_type == WebPointerProperties::PointerType::kTouch) {
+    bool adjusted = frame_->GetEventHandler().BestClickableNodeForHitTestResult(
+        location, hit_test_result, adjusted_point, adjusted_node);
+
+    if (adjusted)
+      pointer_event.SetPositionInWidget(adjusted_point.x(), adjusted_point.y());
+
+    frame_->GetEventHandler().CacheTouchAdjustmentResult(
+        pointer_event.unique_touch_event_id, pointer_event.PositionInWidget());
+  } else if (pointer_event.pointer_type ==
+                 WebPointerProperties::PointerType::kPen ||
+             pointer_event.pointer_type ==
+                 WebPointerProperties::PointerType::kEraser) {
+    // We don't cache the adjusted point for Stylus in EventHandler to avoid
+    // taps being adjusted; this is intended only for stylus handwriting.
+    bool adjusted =
+        frame_->GetEventHandler().BestStylusWritableNodeForHitTestResult(
+            location, hit_test_result, adjusted_point, adjusted_node);
+
+    // TODO(crbug.com/1399797): Update Stylus pointer icon to indicate writing
+    // can be started when adjustment is about to be done.
+    if (adjusted)
+      pointer_event.SetPositionInWidget(adjusted_point.x(), adjusted_point.y());
+  }
 }
 
 bool PointerEventManager::ShouldFilterEvent(PointerEvent* pointer_event) {
@@ -632,7 +685,7 @@ WebInputEventResult PointerEventManager::HandlePointerEvent(
 
   WebPointerEvent pointer_event = event.WebPointerEventInRootFrame();
   if (ShouldAdjustPointerEvent(event))
-    AdjustTouchPointerEvent(pointer_event);
+    AdjustPointerEvent(pointer_event);
   event_handling_util::PointerEventTarget pointer_event_target =
       ComputePointerEventTarget(pointer_event);
 

third_party/blink/renderer/core/input/pointer_event_manager.h

@@ -240,10 +240,15 @@ class CORE_EXPORT PointerEventManager final
                               Element** pointer_capture_target,
                               Element** pending_pointer_capture_target);
 
-  // Only adjust touch type primary pointer down.
+  // Only adjust primary pointer down.
   bool ShouldAdjustPointerEvent(const WebPointerEvent&) const;
-  // Adjust coordinates so it can be used to find the best clickable target.
-  void AdjustTouchPointerEvent(WebPointerEvent&);
+
+  // Whether touch adjustment is to be applied for stylus pointer events.
+  bool ShouldAdjustStylusPointerEvent(const WebPointerEvent&) const;
+
+  // Touch agnostic method to adjust coordinates so that it can be used to find
+  // best touch clickable target or best stylus writable target.
+  void AdjustPointerEvent(WebPointerEvent&);
 
   // Check if the SkipTouchEventFilter experiment is configured to skip
   // filtering on the given event.

third_party/blink/renderer/core/page/touch_adjustment.cc

@@ -30,6 +30,7 @@
 #include "third_party/blink/renderer/core/frame/local_frame.h"
 #include "third_party/blink/renderer/core/frame/local_frame_view.h"
 #include "third_party/blink/renderer/core/html/html_frame_owner_element.h"
+#include "third_party/blink/renderer/core/input/touch_action_util.h"
 #include "third_party/blink/renderer/core/layout/layout_box.h"
 #include "third_party/blink/renderer/core/layout/layout_object.h"
 #include "third_party/blink/renderer/core/layout/layout_text.h"
@@ -158,6 +159,29 @@ bool ProvidesContextMenuItems(Node* node) {
   return false;
 }
 
+bool NodeRespondsToTapOrMove(Node* node) {
+  // This method considers nodes from NodeRespondsToTapGesture, those where pan
+  // touch action is disabled, and ones that are stylus writable. We do this to
+  // avoid adjusting the pointer position on drawable area or slidable control
+  // to the nearby writable input node.
+  node->GetDocument().UpdateStyleAndLayoutTree();
+
+  if (NodeRespondsToTapGesture(node))
+    return true;
+
+  TouchAction effective_touch_action =
+      touch_action_util::ComputeEffectiveTouchAction(*node);
+
+  if ((effective_touch_action & TouchAction::kPan) != TouchAction::kPan)
+    return true;
+
+  if ((effective_touch_action & TouchAction::kInternalNotWritable) !=
+      TouchAction::kInternalNotWritable) {
+    return true;
+  }
+  return false;
+}
+
 static inline void AppendQuadsToSubtargetList(
     Vector<gfx::QuadF>& quads,
     Node* node,
@@ -506,36 +530,52 @@ bool FindNodeWithLowestDistanceMetric(Node*& target_node,
   return (target_node);
 }
 
+bool FindBestCandidate(Node*& target_node,
+                       gfx::Point& target_point,
+                       const gfx::Point& touch_hotspot,
+                       const gfx::Rect& touch_area,
+                       const HeapVector<Member<Node>>& nodes,
+                       NodeFilter node_filter,
+                       AppendSubtargetsForNode append_subtargets_for_node) {
+  gfx::Rect target_area;
+  touch_adjustment::SubtargetGeometryList subtargets;
+  touch_adjustment::CompileSubtargetList(nodes, subtargets, node_filter,
+                                         append_subtargets_for_node);
+  return touch_adjustment::FindNodeWithLowestDistanceMetric(
+      target_node, target_point, target_area, touch_hotspot, touch_area,
+      subtargets, touch_adjustment::HybridDistanceFunction);
+}
+
 }  // namespace touch_adjustment
 
 bool FindBestClickableCandidate(Node*& target_node,
                                 gfx::Point& target_point,
                                 const gfx::Point& touch_hotspot,
                                 const gfx::Rect& touch_area,
                                 const HeapVector<Member<Node>>& nodes) {
-  gfx::Rect target_area;
-  touch_adjustment::SubtargetGeometryList subtargets;
-  touch_adjustment::CompileSubtargetList(
-      nodes, subtargets, touch_adjustment::NodeRespondsToTapGesture,
-      touch_adjustment::AppendBasicSubtargetsForNode);
-  return touch_adjustment::FindNodeWithLowestDistanceMetric(
-      target_node, target_point, target_area, touch_hotspot, touch_area,
-      subtargets, touch_adjustment::HybridDistanceFunction);
+  return FindBestCandidate(target_node, target_point, touch_hotspot, touch_area,
+                           nodes, touch_adjustment::NodeRespondsToTapGesture,
+                           touch_adjustment::AppendBasicSubtargetsForNode);
 }
 
 bool FindBestContextMenuCandidate(Node*& target_node,
                                   gfx::Point& target_point,
                                   const gfx::Point& touch_hotspot,
                                   const gfx::Rect& touch_area,
                                   const HeapVector<Member<Node>>& nodes) {
-  gfx::Rect target_area;
-  touch_adjustment::SubtargetGeometryList subtargets;
-  touch_adjustment::CompileSubtargetList(
-      nodes, subtargets, touch_adjustment::ProvidesContextMenuItems,
-      touch_adjustment::AppendContextSubtargetsForNode);
-  return touch_adjustment::FindNodeWithLowestDistanceMetric(
-      target_node, target_point, target_area, touch_hotspot, touch_area,
-      subtargets, touch_adjustment::HybridDistanceFunction);
+  return FindBestCandidate(target_node, target_point, touch_hotspot, touch_area,
+                           nodes, touch_adjustment::ProvidesContextMenuItems,
+                           touch_adjustment::AppendContextSubtargetsForNode);
+}
+
+bool FindBestStylusWritableCandidate(Node*& target_node,
+                                     gfx::Point& target_point,
+                                     const gfx::Point& touch_hotspot,
+                                     const gfx::Rect& touch_area,
+                                     const HeapVector<Member<Node>>& nodes) {
+  return FindBestCandidate(target_node, target_point, touch_hotspot, touch_area,
+                           nodes, touch_adjustment::NodeRespondsToTapOrMove,
+                           touch_adjustment::AppendBasicSubtargetsForNode);
 }
 
 LayoutSize GetHitTestRectForAdjustment(LocalFrame& frame,

third_party/blink/renderer/core/page/touch_adjustment.h

@@ -43,6 +43,11 @@ bool FindBestContextMenuCandidate(Node*& target_node,
                                   const gfx::Point& touch_hotspot,
                                   const gfx::Rect& touch_area,
                                   const HeapVector<Member<Node>>&);
+bool FindBestStylusWritableCandidate(Node*& target_node,
+                                     gfx::Point& target_point,
+                                     const gfx::Point& touch_hotspot,
+                                     const gfx::Rect& touch_area,
+                                     const HeapVector<Member<Node>>&);
 
 // Applies an upper bound to the touch area as the adjustment rect. The
 // touch_area is in root frame coordinates, which is in physical pixel when

third_party/blink/renderer/core/testing/internals.cc

@@ -2018,6 +2018,32 @@ Node* Internals::touchNodeAdjustedToBestContextMenuNode(
   return target_node;
 }
 
+Node* Internals::touchNodeAdjustedToBestStylusWritableNode(
+    int x,
+    int y,
+    int width,
+    int height,
+    Document* document,
+    ExceptionState& exception_state) {
+  DCHECK(document);
+  if (!document->GetFrame()) {
+    exception_state.ThrowDOMException(DOMExceptionCode::kInvalidAccessError,
+                                      "The document provided is invalid.");
+    return nullptr;
+  }
+
+  HitTestLocation location;
+  HitTestResult result;
+  HitTestRect(location, result, x, y, width, height, document);
+  Node* target_node = nullptr;
+  gfx::Point adjusted_point;
+  document->GetFrame()
+      ->GetEventHandler()
+      .BestStylusWritableNodeForHitTestResult(location, result, adjusted_point,
+                                              target_node);
+  return target_node;
+}
+
 int Internals::lastSpellCheckRequestSequence(Document* document,
                                              ExceptionState& exception_state) {
   SpellCheckRequester* requester = GetSpellCheckRequester(document);