From af5540a7b5cb96401f427d8c11d7d6eea684432d Mon Sep 17 00:00:00 2001 From: Chris Harrelson Date: Mon, 26 Sep 2022 23:25:50 +0000 Subject: [PATCH] [a11y-during-render] Move dirty_object list into Blink Bug: 1068668, 1342801 Change-Id: Ie88eee26ba09431364758dd4509a476575befa93 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3911618 Commit-Queue: Chris Harrelson Reviewed-by: Stefan Zager Reviewed-by: Alex Moshchuk Cr-Commit-Position: refs/heads/main@{#1051477} --- .../render_accessibility_impl.cc | 107 +++--------------- .../accessibility/render_accessibility_impl.h | 27 +---- content/renderer/render_frame_impl.cc | 9 +- content/renderer/render_frame_impl.h | 5 +- .../web_test/renderer/web_frame_test_proxy.cc | 10 +- .../web_test/renderer/web_frame_test_proxy.h | 5 +- third_party/blink/public/web/DEPS | 1 + third_party/blink/public/web/web_ax_context.h | 14 +++ third_party/blink/public/web/web_ax_object.h | 11 +- .../blink/public/web/web_local_frame_client.h | 13 +-- .../core/accessibility/ax_object_cache.h | 21 ++++ .../accessibility/ax_object_cache_impl.cc | 101 ++++++++++++++++- .../accessibility/ax_object_cache_impl.h | 45 ++++++++ .../modules/exported/web_ax_context.cc | 22 ++++ .../modules/exported/web_ax_object.cc | 11 ++ 15 files changed, 247 insertions(+), 155 deletions(-) diff --git a/content/renderer/accessibility/render_accessibility_impl.cc b/content/renderer/accessibility/render_accessibility_impl.cc index 44858e4ffe5b60..651e7dcc83456d 100644 --- a/content/renderer/accessibility/render_accessibility_impl.cc +++ b/content/renderer/accessibility/render_accessibility_impl.cc @@ -491,7 +491,7 @@ void RenderAccessibilityImpl::Reset(int32_t reset_token) { if (ax_context_) ax_context_->ResetSerializer(); pending_events_.clear(); - dirty_objects_.clear(); + ax_context_->ClearDirtyObjects(); const WebDocument& document = GetMainDocument(); if (!document.IsNull()) { @@ -514,11 +514,15 @@ void RenderAccessibilityImpl::MarkWebAXObjectDirty( ax::mojom::Event event_type) { DCHECK(obj.AccessibilityIsIncludedInTree()) << "Cannot serialize unincluded object: " << obj.ToString(true).Utf8(); - EnqueueDirtyObject(obj, event_from, event_from_action, event_intents, - dirty_objects_.end()); - if (subtree) - obj.InvalidateSerializerSubtree(); + obj.MarkDirty(subtree, event_from, event_from_action, event_intents); + + NotifyWebAXObjectMarkedDirty(obj, event_type); +} + +void RenderAccessibilityImpl::NotifyWebAXObjectMarkedDirty( + const blink::WebAXObject& obj, + ax::mojom::Event event_type) { // If the event occurred on the focused object, process immediately. // kLayoutComplete is an exception because it always fires on the root @@ -648,23 +652,6 @@ bool RenderAccessibilityImpl::IsImmediateProcessingRequiredForEvent( } } -std::list>::iterator -RenderAccessibilityImpl::EnqueueDirtyObject( - const blink::WebAXObject& obj, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action, - std::vector event_intents, - std::list>::iterator insertion_point) { - DCHECK(!obj.IsDetached()); - AXDirtyObject* dirty_object = new AXDirtyObject(); - dirty_object->obj = obj; - dirty_object->event_from = event_from; - dirty_object->event_from_action = event_from_action; - dirty_object->event_intents = event_intents; - return std::next(dirty_objects_.insert( - insertion_point, base::WrapUnique(dirty_object))); -} - int RenderAccessibilityImpl::GetDeferredEventsDelay() { // The amount of time, in milliseconds, to wait before sending non-interactive // events that are deferred before the initial page load. @@ -1052,72 +1039,16 @@ bool RenderAccessibilityImpl::SerializeUpdatesAndEvents( // time to inject a stylesheet for image annotation debugging. bool had_load_complete_messages = false; - // Dirty objects can be added as a result of serialization. For example, - // as children are iterated during depth first traversal in the serializer, - // the children sometimes need to be created. The initialization of these - // new children can lead to the discovery of parenting changes via - // aria-owns, or name changes on an ancestor that collects its name its from - // contents. In some cases this has led to an infinite loop, as the - // serialization of new dirty objects keeps adding new dirty objects to - // consider. The infinite loop is avoided by tracking the number of dirty - // objects that can be serialized from the loop, which is the initial - // number of dirty objects + kMaxExtraDirtyObjectsToSerialize. - // Allowing kMaxExtraDirtyObjectsToSerialize ensures that most important - // additional related changes occur at the same time, and that dump event - // tests have consistent results (the results change when dirty objects are - // processed in separate batches). - constexpr int kMaxExtraDirtyObjectsToSerialize = 100; - size_t num_remaining_objects_to_serialize = - dirty_objects_.size() + kMaxExtraDirtyObjectsToSerialize; - // Keep track of IDs serialized so we don't serialize the same node twice. std::set already_serialized_ids; // Serialize all dirty objects in the list at this point in time, stopping // either when the queue is empty, or the number of remaining objects to // serialize has been reached. - while (!dirty_objects_.empty() && --num_remaining_objects_to_serialize > 0) { - std::unique_ptr current_dirty_object = - std::move(dirty_objects_.front()); - dirty_objects_.pop_front(); - auto obj = current_dirty_object->obj; - - // Cannot serialize detached or unincluded object. - // Dirty objects can be added using MarkWebAXObjectDirty(obj) from other - // parts of the code as well, so we need to ensure the object still exists - // and is still included in the tree. Only included objects are marked - // dirty, but this can happen if the object becomes unincluded after it was - // originally marked dirty, in which case a children changed will also be - // fired on the included ancestor. The children changed event on the - // ancestor means that attempting to serialize this unincluded object is not - // necessary. - if (!obj.AccessibilityIsIncludedInTree()) - continue; - - DCHECK(obj.AxID() != ui::kInvalidAXNodeID); - - // Further down this loop, we update |already_serialized_ids| with all IDs - // actually serialized. However, add this object's ID first because there's - // a chance that we try to serialize this object but the serializer ends up - // skipping it. That's probably a Blink bug if that happens, but still we - // need to make sure we don't keep trying the same object over again. - if (!already_serialized_ids.insert(obj.AxID()).second) - continue; // No insertion, was already present. - - ui::AXTreeUpdate update; - update.event_from = current_dirty_object->event_from; - update.event_from_action = current_dirty_object->event_from_action; - update.event_intents = current_dirty_object->event_intents; - // If there's a plugin, force the tree data to be generated in every - // message so the plugin can merge its own tree data changes. - if (plugin_tree_source_) - update.has_tree_data = true; - - if (!obj.SerializeChanges(&update)) { - VLOG(1) << "Failed to serialize one accessibility event."; - continue; - } + ax_context_->SerializeDirtyObjects(updates, already_serialized_ids, + !!plugin_tree_source_); + for (auto& update : updates) { if (update.node_id_to_clear > 0) invalidate_plugin_subtree = true; @@ -1125,14 +1056,6 @@ bool RenderAccessibilityImpl::SerializeUpdatesAndEvents( AddPluginTreeToUpdate(&update, invalidate_plugin_subtree); AddImageAnnotations(document, update.nodes); - - DCHECK_GT(update.nodes.size(), 0U); - for (auto& node : update.nodes) { - DCHECK(node.id != ui::kInvalidAXNodeID); - already_serialized_ids.insert(node.id); - } - - updates.push_back(update); } // Loop over each event and generate an updated event message. @@ -1237,7 +1160,7 @@ void RenderAccessibilityImpl::SendPendingAccessibilityEvents() { } } - if (pending_events_.empty() && dirty_objects_.empty()) { + if (pending_events_.empty() && !ax_context_->HasDirtyObjects()) { // By default, assume the next batch does not have interactive events, and // defer so that the batch of events is larger. If any interactive events // come in, the batch will be processed immediately. @@ -1622,8 +1545,4 @@ void RenderAccessibilityImpl::ResetUKMData() { last_ukm_source_id_ = ukm::kInvalidSourceId; } -AXDirtyObject::AXDirtyObject() = default; -AXDirtyObject::AXDirtyObject(const AXDirtyObject& other) = default; -AXDirtyObject::~AXDirtyObject() = default; - } // namespace content diff --git a/content/renderer/accessibility/render_accessibility_impl.h b/content/renderer/accessibility/render_accessibility_impl.h index ca51ff1a1a7962..971f1cd6e779f5 100644 --- a/content/renderer/accessibility/render_accessibility_impl.h +++ b/content/renderer/accessibility/render_accessibility_impl.h @@ -54,16 +54,6 @@ class RenderAccessibilityManager; using BlinkAXTreeSerializer = ui::AXTreeSerializer; -struct AXDirtyObject { - AXDirtyObject(); - AXDirtyObject(const AXDirtyObject& other); - ~AXDirtyObject(); - blink::WebAXObject obj; - ax::mojom::EventFrom event_from; - ax::mojom::Action event_from_action; - std::vector event_intents; -}; - // The browser process implements native accessibility APIs, allowing assistive // technology (e.g., screen readers, magnifiers) to access and control the web // contents with high-level APIs. These APIs are also used by automation tools, @@ -129,6 +119,9 @@ class CONTENT_EXPORT RenderAccessibilityImpl : public RenderAccessibility, std::vector event_intents = {}, ax::mojom::Event event_type = ax::mojom::Event::kNone); + void NotifyWebAXObjectMarkedDirty(const blink::WebAXObject& obj, + ax::mojom::Event event_type = ax::mojom::Event::kNone); + // Returns the main top-level document for this page, or NULL if there's // no view or frame. blink::WebDocument GetMainDocument(); @@ -175,15 +168,6 @@ class CONTENT_EXPORT RenderAccessibilityImpl : public RenderAccessibility, kNotWaiting }; - // Add an AXDirtyObject to the dirty_objects_ queue. - // Returns an iterator pointing just after the newly inserted object. - std::list>::iterator EnqueueDirtyObject( - const blink::WebAXObject& obj, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action, - std::vector event_intents, - std::list>::iterator insertion_point); - // Callback that will be called from the browser upon handling the message // previously sent to it via SendPendingAccessibilityEvents(). void OnAccessibilityEventsHandled(); @@ -279,11 +263,6 @@ class CONTENT_EXPORT RenderAccessibilityImpl : public RenderAccessibility, // sent to the browser. std::vector pending_events_; - // Objects that need to be re-serialized, the next time - // we send an event bundle to the browser - but don't specifically need - // an event fired. - std::list> dirty_objects_; - using PluginAXTreeSerializer = ui::AXTreeSerializer; std::unique_ptr plugin_serializer_; PluginAXTreeSource* plugin_tree_source_; diff --git a/content/renderer/render_frame_impl.cc b/content/renderer/render_frame_impl.cc index 45cbdf6fe36115..226a09b87f7d71 100644 --- a/content/renderer/render_frame_impl.cc +++ b/content/renderer/render_frame_impl.cc @@ -4442,16 +4442,13 @@ void RenderFrameImpl::PostAccessibilityEvent(const ui::AXEvent& event) { event); } -void RenderFrameImpl::MarkWebAXObjectDirty( - const blink::WebAXObject& obj, - bool subtree, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action) { +void RenderFrameImpl::NotifyWebAXObjectMarkedDirty( + const blink::WebAXObject& obj) { if (!IsAccessibilityEnabled()) return; render_accessibility_manager_->GetRenderAccessibilityImpl() - ->MarkWebAXObjectDirty(obj, subtree, event_from, event_from_action); + ->NotifyWebAXObjectMarkedDirty(obj); } void RenderFrameImpl::AddObserver(RenderFrameObserver* observer) { diff --git a/content/renderer/render_frame_impl.h b/content/renderer/render_frame_impl.h index 4f78a13f4f230e..0e1296d023867b 100644 --- a/content/renderer/render_frame_impl.h +++ b/content/renderer/render_frame_impl.h @@ -604,10 +604,7 @@ class CONTENT_EXPORT RenderFrameImpl bool AllowContentInitiatedDataUrlNavigations( const blink::WebURL& url) override; void PostAccessibilityEvent(const ui::AXEvent& event) override; - void MarkWebAXObjectDirty(const blink::WebAXObject& obj, - bool subtree, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action) override; + void NotifyWebAXObjectMarkedDirty(const blink::WebAXObject& object) override; void CheckIfAudioSinkExistsAndIsAuthorized( const blink::WebString& sink_id, blink::WebSetSinkIdCompleteCallback callback) override; diff --git a/content/web_test/renderer/web_frame_test_proxy.cc b/content/web_test/renderer/web_frame_test_proxy.cc index f5843d2e66a73b..371366e48c5713 100644 --- a/content/web_test/renderer/web_frame_test_proxy.cc +++ b/content/web_test/renderer/web_frame_test_proxy.cc @@ -676,11 +676,8 @@ void WebFrameTestProxy::PostAccessibilityEvent(const ui::AXEvent& event) { RenderFrameImpl::PostAccessibilityEvent(event); } -void WebFrameTestProxy::MarkWebAXObjectDirty( - const blink::WebAXObject& object, - bool subtree, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action) { +void WebFrameTestProxy::NotifyWebAXObjectMarkedDirty( + const blink::WebAXObject& object) { HandleWebAccessibilityEvent(object, "MarkDirty", std::vector()); @@ -690,8 +687,7 @@ void WebFrameTestProxy::MarkWebAXObjectDirty( if (object.IsDetached()) return; // |this| is invalid. - RenderFrameImpl::MarkWebAXObjectDirty(object, subtree, event_from, - event_from_action); + RenderFrameImpl::NotifyWebAXObjectMarkedDirty(object); } void WebFrameTestProxy::HandleWebAccessibilityEvent( diff --git a/content/web_test/renderer/web_frame_test_proxy.h b/content/web_test/renderer/web_frame_test_proxy.h index 9d505697d36105..a8fff8f49dd87f 100644 --- a/content/web_test/renderer/web_frame_test_proxy.h +++ b/content/web_test/renderer/web_frame_test_proxy.h @@ -79,10 +79,7 @@ class WebFrameTestProxy : public RenderFrameImpl, ForRedirect for_redirect) override; void BeginNavigation(std::unique_ptr info) override; void PostAccessibilityEvent(const ui::AXEvent& event) override; - void MarkWebAXObjectDirty(const blink::WebAXObject& object, - bool subtree, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action) override; + void NotifyWebAXObjectMarkedDirty(const blink::WebAXObject& object) override; void CheckIfAudioSinkExistsAndIsAuthorized( const blink::WebString& sink_id, blink::WebSetSinkIdCompleteCallback completion_callback) override; diff --git a/third_party/blink/public/web/DEPS b/third_party/blink/public/web/DEPS index f56c016afb399f..24f170faf166cc 100644 --- a/third_party/blink/public/web/DEPS +++ b/third_party/blink/public/web/DEPS @@ -59,6 +59,7 @@ include_rules = [ "+third_party/blink/public/web", "+ui/accessibility/ax_enums.mojom-shared.h", "+ui/accessibility/ax_event.h", + "+ui/accessibility/ax_event_intent.h", "+ui/accessibility/ax_mode.h", "+ui/accessibility/ax_tree_id.h", "+ui/base/ime/mojom/ime_types.mojom-shared.h", diff --git a/third_party/blink/public/web/web_ax_context.h b/third_party/blink/public/web/web_ax_context.h index 1f4a60e31e80c0..0f7206ef94adbc 100644 --- a/third_party/blink/public/web/web_ax_context.h +++ b/third_party/blink/public/web/web_ax_context.h @@ -56,6 +56,20 @@ class BLINK_EXPORT WebAXContext { void MarkAllImageAXObjectsDirty( ax::mojom::Action event_from_action); + // Serialize all AXObjects that are dirty (have changed their state since + // the last serialization) into |updates|. (Heuristically) skips + // serializing dirty nodes whose AX id is in |already_serialized_ids|, and + // adds serialized dirty objects into |already_serialized_ids|. + void SerializeDirtyObjects(std::vector& updates, + std::set& already_serialized_ids, + bool has_plugin_tree_source); + + // Clears out the list of dirty AXObjects. + void ClearDirtyObjects(); + + // Returns true if any AXObject is dirty. + bool HasDirtyObjects(); + private: std::unique_ptr private_; }; diff --git a/third_party/blink/public/web/web_ax_object.h b/third_party/blink/public/web/web_ax_object.h index c15858909bf1ce..93935d422f118b 100644 --- a/third_party/blink/public/web/web_ax_object.h +++ b/third_party/blink/public/web/web_ax_object.h @@ -36,6 +36,7 @@ #include "third_party/blink/public/platform/web_vector.h" #include "third_party/blink/public/web/web_ax_enums.h" #include "ui/accessibility/ax_enums.mojom-shared.h" +#include "ui/accessibility/ax_event_intent.h" #include "ui/accessibility/ax_mode.h" namespace gfx { @@ -332,10 +333,12 @@ class BLINK_EXPORT WebAXObject { gfx::Transform& container_transform, bool* clips_children = nullptr) const; - // Retrieves a vector of all WebAXObjects in this document whose - // bounding boxes may have changed since the last query. Sends that vector - // via mojo to the browser process. - void SerializeLocationChanges() const; + // Marks ths object as dirty (needing serialization). If subtree is true, + // the entire AX subtree should be invalidated as well. + void MarkDirty(bool subtree, + ax::mojom::EventFrom event_from, + ax::mojom::Action event_from_action, + std::vector event_intents) const; // Exchanges a WebAXObject with another. void Swap(WebAXObject& other); diff --git a/third_party/blink/public/web/web_local_frame_client.h b/third_party/blink/public/web/web_local_frame_client.h index 5630ec39b70753..149ca596036a4a 100644 --- a/third_party/blink/public/web/web_local_frame_client.h +++ b/third_party/blink/public/web/web_local_frame_client.h @@ -623,16 +623,9 @@ class BLINK_EXPORT WebLocalFrameClient { // Notifies the embedder about an accessibility event on a WebAXObject. virtual void PostAccessibilityEvent(const ui::AXEvent& event) {} - // Notifies the embedder that a WebAXObject is dirty and its state needs - // to be serialized again. If |subtree| is true, the entire subtree is - // dirty. - // |event_from| and |event_from_action| annotate this node change with info - // about the event which caused the change. For example, an event from a user - // or an event from a focus action. - virtual void MarkWebAXObjectDirty(const WebAXObject&, - bool subtree, - ax::mojom::EventFrom event_from, - ax::mojom::Action event_from_action) {} + // Notifies tests that a WebAXObject is dirty and its state needs + // to be serialized again. + virtual void NotifyWebAXObjectMarkedDirty(const WebAXObject&) {} // Audio Output Devices API -------------------------------------------- diff --git a/third_party/blink/renderer/core/accessibility/ax_object_cache.h b/third_party/blink/renderer/core/accessibility/ax_object_cache.h index 19843388469a17..54fc34b1e1ea0c 100644 --- a/third_party/blink/renderer/core/accessibility/ax_object_cache.h +++ b/third_party/blink/renderer/core/accessibility/ax_object_cache.h @@ -212,6 +212,27 @@ class CORE_EXPORT AXObjectCache : public GarbageCollected { virtual void MarkAllImageAXObjectsDirty( ax::mojom::blink::Action event_from_action) = 0; + // Notifies that an AXObject is dirty and its state needs + // to be serialized again. If |subtree| is true, the entire subtree is + // dirty. + // |event_from| and |event_from_action| annotate this node change with info + // about the event which caused the change. For example, an event from a user + // or an event from a focus action. + virtual void MarkAXObjectDirty( + AXObject* obj, + bool subtree, + ax::mojom::blink::EventFrom event_from, + ax::mojom::blink::Action event_from_action, + const std::vector& event_intents) = 0; + + virtual void SerializeDirtyObjects(std::vector& updates, + std::set& already_serialized_ids, + bool has_plugin_tree_source) = 0; + + virtual void ClearDirtyObjects() = 0; + + virtual bool HasDirtyObjects() = 0; + // Ensure that a call to ProcessDeferredAccessibilityEvents() will occur soon. virtual void ScheduleVisualUpdate(Document& document) = 0; diff --git a/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.cc b/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.cc index 4886c22fe6497a..b3d8f99b65a451 100644 --- a/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.cc +++ b/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.cc @@ -3659,10 +3659,12 @@ void AXObjectCacheImpl::MarkAXObjectDirtyWithCleanLayoutHelper( WebLocalFrameImpl* webframe = WebLocalFrameImpl::FromFrame( obj->GetDocument()->AXObjectCacheOwner().GetFrame()); if (webframe && webframe->Client()) { - webframe->Client()->MarkWebAXObjectDirty(WebAXObject(obj), subtree, - event_from, event_from_action); + webframe->Client()->NotifyWebAXObjectMarkedDirty(WebAXObject(obj)); } + std::vector event_intents; + MarkAXObjectDirty(obj, subtree, event_from, event_from_action, event_intents); + obj->UpdateCachedAttributeValuesIfNeeded(true); for (auto agent : agents_) agent->AXObjectModified(obj, subtree); @@ -3913,6 +3915,100 @@ bool AXObjectCacheImpl::SerializeEntireTree(bool exclude_offscreen, return result; } +void AXObjectCacheImpl::MarkAXObjectDirty( + AXObject* obj, + bool subtree, + ax::mojom::blink::EventFrom event_from, + ax::mojom::blink::Action event_from_action, + const std::vector& event_intents) { + dirty_objects_.push_back( + AXDirtyObject::Create(obj, event_from, event_from_action, event_intents)); + + if (subtree) + InvalidateSerializerSubtree(*obj); +} + +void AXObjectCacheImpl::SerializeDirtyObjects( + std::vector& updates, + std::set& already_serialized_ids, + bool has_plugin_tree_source) { + // Dirty objects can be added as a result of serialization. For example, + // as children are iterated during depth first traversal in the serializer, + // the children sometimes need to be created. The initialization of these + // new children can lead to the discovery of parenting changes via + // aria-owns, or name changes on an ancestor that collects its name its from + // contents. In some cases this has led to an infinite loop, as the + // serialization of new dirty objects keeps adding new dirty objects to + // consider. The infinite loop is avoided by tracking the number of dirty + // objects that can be serialized from the loop, which is the initial + // number of dirty objects + kMaxExtraDirtyObjectsToSerialize. + // Allowing kMaxExtraDirtyObjectsToSerialize ensures that most important + // additional related changes occur at the same time, and that dump event + // tests have consistent results (the results change when dirty objects are + // processed in separate batches). + constexpr int kMaxExtraDirtyObjectsToSerialize = 100; + + size_t num_remaining_objects_to_serialize = + dirty_objects_.size() + kMaxExtraDirtyObjectsToSerialize; + + UpdateLifecycleIfNeeded(); + + while (!dirty_objects_.empty() && --num_remaining_objects_to_serialize > 0) { + AXDirtyObject* current_dirty_object = std::move(dirty_objects_.front()); + dirty_objects_.pop_front(); + AXObject* obj = current_dirty_object->obj; + + // Dirty objects can be added using MarkWebAXObjectDirty(obj) from other + // parts of the code as well, so we need to ensure the object still + // exists. + if (!obj || obj->IsDetached()) + continue; + + // Cannot serialize unincluded object. + // Only included objects are marked dirty, but this can happen if the + // object becomes unincluded after it was originally marked dirty, in which + // cas a children changed will also be fired on the included ancestor. The + // children changed event on the ancestor means that attempting to + // serialize this unincluded object is not necessary. + if (!obj->AccessibilityIsIncludedInTree()) + continue; + + DCHECK(obj->AXObjectID()); + + // Further down this loop, we update |already_serialized_ids| with all + // IDs actually serialized. However, add this object's ID first because + // there's a chance that we try to serialize this object but the serializer + // ends up skipping it. That's probably a Blink bug if that happens, but + // still we need to make sure we don't keep trying the same object over + // again. + if (!already_serialized_ids.insert(obj->AXObjectID()).second) + continue; // No insertion, was already present. + + ui::AXTreeUpdate update; + update.event_from = current_dirty_object->event_from; + update.event_from_action = current_dirty_object->event_from_action; + update.event_intents = current_dirty_object->event_intents; + + // If there's a plugin, force the tree data to be generated in every + // message so the plugin can merge its own tree data changes. + if (has_plugin_tree_source) + update.has_tree_data = true; + + if (!SerializeChanges(*obj, &update)) { + VLOG(1) << "Failed to serialize one accessibility event."; + continue; + } + + DCHECK_GT(update.nodes.size(), 0U); + for (auto& node : update.nodes) { + DCHECK(node.id); + already_serialized_ids.insert(node.id); + } + + updates.push_back(update); + } +} + mojo::Remote& AXObjectCacheImpl::GetOrCreateRemoteRenderAccessibilityHost() { if (!render_accessibility_host_) { @@ -4309,6 +4405,7 @@ void AXObjectCacheImpl::Trace(Visitor* visitor) const { visitor->Trace(nodes_with_pending_children_changed_); visitor->Trace(nodes_with_spelling_or_grammar_markers_); visitor->Trace(ax_tree_source_); + visitor->Trace(dirty_objects_); AXObjectCache::Trace(visitor); } diff --git a/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.h b/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.h index efd92945e54afc..d764ead36afa1e 100644 --- a/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.h +++ b/third_party/blink/renderer/modules/accessibility/ax_object_cache_impl.h @@ -47,6 +47,7 @@ #include "third_party/blink/renderer/modules/accessibility/blink_ax_tree_source.h" #include "third_party/blink/renderer/modules/accessibility/inspector_accessibility_agent.h" #include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/platform/heap/collection_support/heap_deque.h" #include "third_party/blink/renderer/platform/heap/collection_support/heap_hash_map.h" #include "third_party/blink/renderer/platform/heap/collection_support/heap_hash_set.h" #include "third_party/blink/renderer/platform/heap/garbage_collected.h" @@ -422,6 +423,21 @@ class MODULES_EXPORT AXObjectCacheImpl void ResetSerializer() override { ax_tree_serializer_->Reset(); } + void MarkAXObjectDirty( + AXObject* obj, + bool subtree, + ax::mojom::blink::EventFrom event_from, + ax::mojom::blink::Action event_from_action, + const std::vector& event_intents) override; + + void SerializeDirtyObjects(std::vector& updates, + std::set& already_serialized_ids, + bool has_plugin_tree_source) override; + + void ClearDirtyObjects() override { dirty_objects_.clear(); } + + bool HasDirtyObjects() override { return !dirty_objects_.empty(); } + void InvalidateSerializerSubtree(AXObject& obj) { ax_tree_serializer_->InvalidateSubtree(&obj); } @@ -491,6 +507,33 @@ class MODULES_EXPORT AXObjectCacheImpl void Remove(AXID); private: + + struct AXDirtyObject : public GarbageCollected { + AXDirtyObject(AXObject* obj_arg, + ax::mojom::blink::EventFrom event_from_arg, + ax::mojom::blink::Action event_from_action_arg, + std::vector event_intents_arg) + : obj(obj_arg), + event_from(event_from_arg), + event_from_action(event_from_action_arg), + event_intents(event_intents_arg) {} + + static AXDirtyObject* Create(AXObject* obj, + ax::mojom::blink::EventFrom event_from, + ax::mojom::blink::Action event_from_action, + std::vector event_intents) { + return MakeGarbageCollected( + obj, event_from, event_from_action, event_intents); + } + + void Trace(Visitor* visitor) const { visitor->Trace(obj); } + + Member obj; + ax::mojom::blink::EventFrom event_from; + ax::mojom::blink::Action event_from_action; + std::vector event_intents; + }; + mojo::Remote& GetOrCreateRemoteRenderAccessibilityHost(); void ProcessDeferredAccessibilityEventsImpl(Document&); @@ -805,6 +848,8 @@ class MODULES_EXPORT AXObjectCacheImpl Member ax_tree_source_; std::unique_ptr> ax_tree_serializer_; + HeapDeque> dirty_objects_; + FRIEND_TEST_ALL_PREFIXES(AccessibilityTest, PauseUpdatesAfterMaxNumberQueued); }; diff --git a/third_party/blink/renderer/modules/exported/web_ax_context.cc b/third_party/blink/renderer/modules/exported/web_ax_context.cc index 70a46c20366d81..d0f68df3898729 100644 --- a/third_party/blink/renderer/modules/exported/web_ax_context.cc +++ b/third_party/blink/renderer/modules/exported/web_ax_context.cc @@ -79,4 +79,26 @@ void WebAXContext::MarkAllImageAXObjectsDirty( private_->GetAXObjectCache().MarkAllImageAXObjectsDirty(event_from_action); } +void WebAXContext::SerializeDirtyObjects( + std::vector& updates, + std::set& already_serialized_ids, + bool has_plugin_tree_source) { + if (!private_->HasActiveDocument()) + return; + private_->GetAXObjectCache().SerializeDirtyObjects( + updates, already_serialized_ids, has_plugin_tree_source); +} + +void WebAXContext::ClearDirtyObjects() { + if (!private_->HasActiveDocument()) + return; + private_->GetAXObjectCache().ClearDirtyObjects(); +} + +bool WebAXContext::HasDirtyObjects() { + if (!private_->HasActiveDocument()) + return true; + return private_->GetAXObjectCache().HasDirtyObjects(); +} + } // namespace blink diff --git a/third_party/blink/renderer/modules/exported/web_ax_object.cc b/third_party/blink/renderer/modules/exported/web_ax_object.cc index eead0875ad38fc..b2ee23fbb6f4da 100644 --- a/third_party/blink/renderer/modules/exported/web_ax_object.cc +++ b/third_party/blink/renderer/modules/exported/web_ax_object.cc @@ -223,6 +223,17 @@ bool WebAXObject::SerializeChanges(ui::AXTreeUpdate* update) { return private_->AXObjectCache().SerializeChanges(*private_, update); } +void WebAXObject::MarkDirty( + bool subtree, + ax::mojom::blink::EventFrom event_from, + ax::mojom::blink::Action event_from_action, + std::vector event_intents) const { + if (IsDetached()) + return; + private_->AXObjectCache().MarkAXObjectDirty( + private_.Get(), subtree, event_from, event_from_action, event_intents); +} + bool WebAXObject::IsInClientTree() { if (IsDetached()) return false;