Skip to content

Commit d41eb4b

Browse files
dnfieldNoamDev
dnfield
authored and
NoamDev
committed
[fuchsia] HitTesting for fuchsia a11y (flutter#15570)
1 parent b5d50cb commit d41eb4b

File tree

3 files changed

+195
-13
lines changed

3 files changed

+195
-13
lines changed

‎shell/platform/fuchsia/flutter/accessibility_bridge.cc‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,8 @@ std::unordered_set<int32_t> AccessibilityBridge::GetDescendants(
9999

100100
auto it = nodes_.find(id);
101101
if (it != nodes_.end()) {
102-
auto const& children = it->second;
103-
for (const auto& child : children) {
102+
const auto& node = it->second;
103+
for (const auto& child : node.children_in_hit_test_order) {
104104
if (descendents.find(child) == descendents.end()) {
105105
to_process.push_back(child);
106106
} else {
@@ -180,10 +180,18 @@ void AccessibilityBridge::AddSemanticsNodeUpdate(
180180
for (const auto& value : update) {
181181
size_t this_node_size = sizeof(fuchsia::accessibility::semantics::Node);
182182
const auto& flutter_node = value.second;
183-
nodes_[flutter_node.id] =
184-
std::vector<int32_t>(flutter_node.childrenInTraversalOrder);
183+
// Store the nodes for later hit testing.
184+
nodes_[flutter_node.id] = {
185+
.id = flutter_node.id,
186+
.flags = flutter_node.flags,
187+
.rect = flutter_node.rect,
188+
.transform = flutter_node.transform,
189+
.children_in_hit_test_order = flutter_node.childrenInHitTestOrder,
190+
};
185191
fuchsia::accessibility::semantics::Node fuchsia_node;
186192
std::vector<uint32_t> child_ids;
193+
// Send the nodes in traversal order, so the manager can figure out
194+
// traversal.
187195
for (int32_t flutter_child_id : flutter_node.childrenInTraversalOrder) {
188196
child_ids.push_back(FlutterIdToFuchsiaId(flutter_child_id));
189197
}
@@ -221,13 +229,59 @@ void AccessibilityBridge::AddSemanticsNodeUpdate(
221229
}
222230

223231
PruneUnreachableNodes();
232+
UpdateScreenRects();
224233

225234
tree_ptr_->UpdateSemanticNodes(std::move(nodes));
226235
// TODO(dnfield): Implement the callback here
227236
// https://bugs.fuchsia.dev/p/fuchsia/issues/detail?id=35718.
228237
tree_ptr_->CommitUpdates([]() {});
229238
}
230239

240+
void AccessibilityBridge::UpdateScreenRects() {
241+
std::unordered_set<int32_t> visited_nodes;
242+
UpdateScreenRects(kRootNodeId, SkMatrix44::I(), &visited_nodes);
243+
}
244+
245+
void AccessibilityBridge::UpdateScreenRects(
246+
int32_t node_id,
247+
SkMatrix44 parent_transform,
248+
std::unordered_set<int32_t>* visited_nodes) {
249+
auto it = nodes_.find(node_id);
250+
if (it == nodes_.end()) {
251+
FML_LOG(ERROR) << "UpdateScreenRects called on unknown node";
252+
return;
253+
}
254+
auto& node = it->second;
255+
const auto& current_transform = parent_transform * node.transform;
256+
257+
const auto& rect = node.rect;
258+
FML_LOG(ERROR) << "nodeid: " << node_id;
259+
SkScalar quad[] = {
260+
rect.left(), rect.top(), //
261+
rect.right(), rect.top(), //
262+
rect.right(), rect.bottom(), //
263+
rect.left(), rect.bottom(), //
264+
};
265+
SkScalar dst[4 * 4];
266+
current_transform.map2(quad, 4, dst);
267+
node.screen_rect.setLTRB(dst[0], dst[1], dst[8], dst[9]);
268+
node.screen_rect.sort();
269+
std::vector<SkVector4> points = {
270+
current_transform * SkVector4(rect.left(), rect.top(), 0, 1),
271+
current_transform * SkVector4(rect.right(), rect.top(), 0, 1),
272+
current_transform * SkVector4(rect.right(), rect.bottom(), 0, 1),
273+
current_transform * SkVector4(rect.left(), rect.bottom(), 0, 1),
274+
};
275+
276+
visited_nodes->emplace(node_id);
277+
278+
for (uint32_t child_id : node.children_in_hit_test_order) {
279+
if (visited_nodes->find(child_id) == visited_nodes->end()) {
280+
UpdateScreenRects(child_id, current_transform, visited_nodes);
281+
}
282+
}
283+
}
284+
231285
// |fuchsia::accessibility::semantics::SemanticListener|
232286
void AccessibilityBridge::OnAccessibilityActionRequested(
233287
uint32_t node_id,
@@ -239,7 +293,34 @@ void AccessibilityBridge::OnAccessibilityActionRequested(
239293
void AccessibilityBridge::HitTest(
240294
fuchsia::math::PointF local_point,
241295
fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
242-
callback) {}
296+
callback) {
297+
auto hit_node_id = GetHitNode(kRootNodeId, local_point.x, local_point.y);
298+
FML_DCHECK(hit_node_id.has_value());
299+
fuchsia::accessibility::semantics::Hit hit;
300+
hit.set_node_id(hit_node_id.value_or(kRootNodeId));
301+
callback(std::move(hit));
302+
}
303+
304+
std::optional<int32_t> AccessibilityBridge::GetHitNode(int32_t node_id,
305+
float x,
306+
float y) {
307+
auto it = nodes_.find(node_id);
308+
if (it == nodes_.end()) {
309+
FML_LOG(ERROR) << "Attempted to hit test unkonwn node id: " << node_id;
310+
return {};
311+
}
312+
auto const& node = it->second;
313+
if (node.flags &
314+
static_cast<int32_t>(flutter::SemanticsFlags::kIsHidden) || //
315+
!node.screen_rect.contains(x, y)) {
316+
return {};
317+
}
318+
auto hit = node_id;
319+
for (int32_t child_id : node.children_in_hit_test_order) {
320+
hit = GetHitNode(child_id, x, y).value_or(hit);
321+
}
322+
return hit;
323+
}
243324

244325
// |fuchsia::accessibility::semantics::SemanticListener|
245326
void AccessibilityBridge::OnSemanticsModeChanged(

‎shell/platform/fuchsia/flutter/accessibility_bridge.h‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,24 @@ class AccessibilityBridge
8585
// Notifies the bridge of a 'hover move' touch exploration event.
8686
zx_status_t OnHoverMove(double x, double y);
8787

88+
// |fuchsia::accessibility::semantics::SemanticListener|
89+
void HitTest(
90+
fuchsia::math::PointF local_point,
91+
fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
92+
callback) override;
93+
8894
private:
95+
// Holds only the fields we need for hit testing.
96+
// In particular, it adds a screen_rect field to flutter::SemanticsNode.
97+
struct SemanticsNode {
98+
int32_t id;
99+
int32_t flags;
100+
SkRect rect;
101+
SkRect screen_rect;
102+
SkMatrix44 transform;
103+
std::vector<int32_t> children_in_hit_test_order;
104+
};
105+
89106
AccessibilityBridge::Delegate& delegate_;
90107

91108
static constexpr int32_t kRootNodeId = 0;
@@ -95,8 +112,8 @@ class AccessibilityBridge
95112
fuchsia::accessibility::semantics::SemanticTreePtr tree_ptr_;
96113
bool semantics_enabled_;
97114
// This is the cache of all nodes we've sent to Fuchsia's SemanticsManager.
98-
// Assists with pruning unreachable nodes.
99-
std::unordered_map<int32_t, std::vector<int32_t>> nodes_;
115+
// Assists with pruning unreachable nodes and hit testing.
116+
std::unordered_map<int32_t, SemanticsNode> nodes_;
100117

101118
// Derives the BoundingBox of a Flutter semantics node from its
102119
// rect and elevation.
@@ -127,19 +144,33 @@ class AccessibilityBridge
127144
// May result in a call to FuchsiaAccessibility::Commit().
128145
void PruneUnreachableNodes();
129146

147+
// Updates the on-screen positions of accessibility elements,
148+
// starting from the root element with an identity matrix.
149+
//
150+
// This should be called from Update.
151+
void UpdateScreenRects();
152+
153+
// Updates the on-screen positions of accessibility elements, starting
154+
// from node_id and using the specified transform.
155+
//
156+
// Update calls this via UpdateScreenRects().
157+
void UpdateScreenRects(int32_t node_id,
158+
SkMatrix44 parent_transform,
159+
std::unordered_set<int32_t>* visited_nodes);
160+
161+
// Traverses the semantics tree to find the node_id hit by the given x,y
162+
// point.
163+
//
164+
// Assumes that SemanticsNode::screen_rect is up to date.
165+
std::optional<int32_t> GetHitNode(int32_t node_id, float x, float y);
166+
130167
// |fuchsia::accessibility::semantics::SemanticListener|
131168
void OnAccessibilityActionRequested(
132169
uint32_t node_id,
133170
fuchsia::accessibility::semantics::Action action,
134171
fuchsia::accessibility::semantics::SemanticListener::
135172
OnAccessibilityActionRequestedCallback callback) override;
136173

137-
// |fuchsia::accessibility::semantics::SemanticListener|
138-
void HitTest(
139-
fuchsia::math::PointF local_point,
140-
fuchsia::accessibility::semantics::SemanticListener::HitTestCallback
141-
callback) override;
142-
143174
// |fuchsia::accessibility::semantics::SemanticListener|
144175
void OnSemanticsModeChanged(bool enabled,
145176
OnSemanticsModeChangedCallback callback) override;

‎shell/platform/fuchsia/flutter/accessibility_bridge_unittest.cc‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,10 +90,12 @@ TEST_F(AccessibilityBridgeTest, DeletesChildrenTransitively) {
9090
flutter::SemanticsNode node1;
9191
node1.id = 1;
9292
node1.childrenInTraversalOrder = {2};
93+
node1.childrenInHitTestOrder = {2};
9394

9495
flutter::SemanticsNode node0;
9596
node0.id = 0;
9697
node0.childrenInTraversalOrder = {1};
98+
node0.childrenInHitTestOrder = {1};
9799

98100
accessibility_bridge_->AddSemanticsNodeUpdate({
99101
{0, node0},
@@ -112,6 +114,7 @@ TEST_F(AccessibilityBridgeTest, DeletesChildrenTransitively) {
112114

113115
// Remove the children
114116
node0.childrenInTraversalOrder.clear();
117+
node0.childrenInHitTestOrder.clear();
115118
accessibility_bridge_->AddSemanticsNodeUpdate({
116119
{0, node0},
117120
});
@@ -141,6 +144,7 @@ TEST_F(AccessibilityBridgeTest, TruncatesLargeLabel) {
141144
std::string(fuchsia::accessibility::semantics::MAX_LABEL_SIZE + 1, '2');
142145

143146
node0.childrenInTraversalOrder = {1, 2};
147+
node0.childrenInHitTestOrder = {1, 2};
144148

145149
accessibility_bridge_->AddSemanticsNodeUpdate({
146150
{0, node0},
@@ -194,7 +198,9 @@ TEST_F(AccessibilityBridgeTest, SplitsLargeUpdates) {
194198
std::string(fuchsia::accessibility::semantics::MAX_LABEL_SIZE, '4');
195199

196200
node0.childrenInTraversalOrder = {1, 2};
201+
node0.childrenInHitTestOrder = {1, 2};
197202
node1.childrenInTraversalOrder = {3, 4};
203+
node1.childrenInHitTestOrder = {3, 4};
198204

199205
accessibility_bridge_->AddSemanticsNodeUpdate({
200206
{0, node0},
@@ -219,6 +225,7 @@ TEST_F(AccessibilityBridgeTest, HandlesCycles) {
219225
flutter::SemanticsNode node0;
220226
node0.id = 0;
221227
node0.childrenInTraversalOrder.push_back(0);
228+
node0.childrenInHitTestOrder.push_back(0);
222229
accessibility_bridge_->AddSemanticsNodeUpdate({
223230
{0, node0},
224231
});
@@ -231,9 +238,11 @@ TEST_F(AccessibilityBridgeTest, HandlesCycles) {
231238
EXPECT_FALSE(semantics_manager_.UpdateOverflowed());
232239

233240
node0.childrenInTraversalOrder = {0, 1};
241+
node0.childrenInHitTestOrder = {0, 1};
234242
flutter::SemanticsNode node1;
235243
node1.id = 1;
236244
node1.childrenInTraversalOrder = {0};
245+
node1.childrenInHitTestOrder = {0};
237246
accessibility_bridge_->AddSemanticsNodeUpdate({
238247
{0, node0},
239248
{1, node1},
@@ -260,12 +269,14 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) {
260269
flutter::SemanticsNode node;
261270
node.id = i;
262271
node0.childrenInTraversalOrder.push_back(i);
272+
node0.childrenInHitTestOrder.push_back(i);
263273
for (int32_t j = 0; j < leaf_nodes; j++) {
264274
flutter::SemanticsNode leaf_node;
265275
int id = (i * child_nodes) + ((j + 1) * leaf_nodes);
266276
leaf_node.id = id;
267277
leaf_node.label = "A relatively simple label";
268278
node.childrenInTraversalOrder.push_back(id);
279+
node.childrenInHitTestOrder.push_back(id);
269280
update.insert(std::make_pair(id, std::move(leaf_node)));
270281
}
271282
update.insert(std::make_pair(i, std::move(node)));
@@ -283,6 +294,7 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) {
283294

284295
// Remove the children
285296
node0.childrenInTraversalOrder.clear();
297+
node0.childrenInHitTestOrder.clear();
286298
accessibility_bridge_->AddSemanticsNodeUpdate({
287299
{0, node0},
288300
});
@@ -294,4 +306,62 @@ TEST_F(AccessibilityBridgeTest, BatchesLargeMessages) {
294306
EXPECT_FALSE(semantics_manager_.DeleteOverflowed());
295307
EXPECT_FALSE(semantics_manager_.UpdateOverflowed());
296308
}
309+
310+
TEST_F(AccessibilityBridgeTest, HitTest) {
311+
flutter::SemanticsNode node0;
312+
node0.id = 0;
313+
node0.rect.setLTRB(0, 0, 100, 100);
314+
315+
flutter::SemanticsNode node1;
316+
node1.id = 1;
317+
node1.rect.setLTRB(10, 10, 20, 20);
318+
319+
flutter::SemanticsNode node2;
320+
node2.id = 2;
321+
node2.rect.setLTRB(25, 10, 45, 20);
322+
323+
flutter::SemanticsNode node3;
324+
node3.id = 3;
325+
node3.rect.setLTRB(10, 25, 20, 45);
326+
327+
flutter::SemanticsNode node4;
328+
node4.id = 4;
329+
node4.rect.setLTRB(10, 10, 20, 20);
330+
node4.transform.setTranslate(20, 20, 0);
331+
332+
node0.childrenInTraversalOrder = {1, 2, 3, 4};
333+
node0.childrenInHitTestOrder = {1, 2, 3, 4};
334+
335+
accessibility_bridge_->AddSemanticsNodeUpdate({
336+
{0, node0},
337+
{1, node1},
338+
{2, node2},
339+
{3, node3},
340+
{4, node4},
341+
});
342+
RunLoopUntilIdle();
343+
344+
uint32_t hit_node_id;
345+
auto callback = [&hit_node_id](fuchsia::accessibility::semantics::Hit hit) {
346+
EXPECT_TRUE(hit.has_node_id());
347+
hit_node_id = hit.node_id();
348+
};
349+
350+
// Nodes are:
351+
// ----------
352+
// | 1 2 |
353+
// | 3 4 |
354+
// ----------
355+
356+
accessibility_bridge_->HitTest({1, 1}, callback);
357+
EXPECT_EQ(hit_node_id, 0u);
358+
accessibility_bridge_->HitTest({15, 15}, callback);
359+
EXPECT_EQ(hit_node_id, 1u);
360+
accessibility_bridge_->HitTest({30, 15}, callback);
361+
EXPECT_EQ(hit_node_id, 2u);
362+
accessibility_bridge_->HitTest({15, 30}, callback);
363+
EXPECT_EQ(hit_node_id, 3u);
364+
accessibility_bridge_->HitTest({30, 30}, callback);
365+
EXPECT_EQ(hit_node_id, 4u);
366+
}
297367
} // namespace flutter_runner_test

0 commit comments

Comments
 (0)