iOS a11y text entry (~70% of it) (#4575)

yjbanov · web-flow · commit 6dd49f0695cf · 2018-02-05T15:14:13.000-08:00
* implement iOS text field editing in a11y mode

* address Chinmay's comments

* replace node in child list when changing type
diff --git a/content_handler/accessibility_bridge.cc b/content_handler/accessibility_bridge.cc
@@ -20,8 +20,9 @@ AccessibilityBridge::AccessibilityBridge(app::ApplicationContext* context)
     : writer_(context->ConnectToEnvironmentService<maxwell::ContextWriter>()) {}
 
 void AccessibilityBridge::UpdateSemantics(
-    const std::vector<blink::SemanticsNode>& update) {
-  for (const auto& node : update) {
+    const blink::SemanticsNodeUpdates& update) {
+  for (const auto& update : update) {
+    const auto& node = update.second;
     semantics_nodes_[node.id] = node;
   }
   std::vector<int> visited_nodes;
diff --git a/content_handler/accessibility_bridge.h b/content_handler/accessibility_bridge.h
@@ -21,7 +21,7 @@ class AccessibilityBridge {
 
   // Update the internal representation of the semantics nodes, and write the
   // semantics to Context Service.
-  void UpdateSemantics(const std::vector<blink::SemanticsNode>& update);
+  void UpdateSemantics(const blink::SemanticsNodeUpdates& update);
 
  private:
   // Walk the semantics node tree starting at |id|, and store the id of each
diff --git a/content_handler/runtime_holder.cc b/content_handler/runtime_holder.cc
@@ -370,7 +370,7 @@ void RuntimeHolder::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
   }));
 }
 
-void RuntimeHolder::UpdateSemantics(std::vector<blink::SemanticsNode> update) {
+void RuntimeHolder::UpdateSemantics(blink::SemanticsNodeUpdates update) {
   accessibility_bridge_->UpdateSemantics(update);
 }
 
diff --git a/content_handler/runtime_holder.h b/content_handler/runtime_holder.h
@@ -65,7 +65,7 @@ class RuntimeHolder : public blink::RuntimeDelegate,
   std::string DefaultRouteName() override;
   void ScheduleFrame(bool regenerate_layer_tree = true) override;
   void Render(std::unique_ptr<flow::LayerTree> layer_tree) override;
-  void UpdateSemantics(std::vector<blink::SemanticsNode> update) override;
+  void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
   void HandlePlatformMessage(
       fxl::RefPtr<blink::PlatformMessage> message) override;
   void DidCreateMainIsolate(Dart_Isolate isolate) override;
diff --git a/lib/ui/semantics/semantics_node.h b/lib/ui/semantics/semantics_node.h
@@ -8,6 +8,7 @@
 #include <stdint.h>
 
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "third_party/skia/include/core/SkMatrix44.h"
@@ -70,6 +71,12 @@ struct SemanticsNode {
   std::vector<int32_t> children;
 };
 
+// Contains semantic nodes that need to be updated.
+//
+// The keys in the map are stable node IDd, and the values contain
+// semantic information for the node corresponding to the ID.
+using SemanticsNodeUpdates = std::unordered_map<int32_t, SemanticsNode>;
+
 }  // namespace blink
 
 #endif  // FLUTTER_LIB_UI_SEMANTICS_SEMANTICS_NODE_H_
diff --git a/lib/ui/semantics/semantics_update.cc b/lib/ui/semantics/semantics_update.cc
@@ -21,16 +21,16 @@ IMPLEMENT_WRAPPERTYPEINFO(ui, SemanticsUpdate);
 DART_BIND_ALL(SemanticsUpdate, FOR_EACH_BINDING)
 
 fxl::RefPtr<SemanticsUpdate> SemanticsUpdate::create(
-    std::vector<SemanticsNode> nodes) {
+    SemanticsNodeUpdates nodes) {
   return fxl::MakeRefCounted<SemanticsUpdate>(std::move(nodes));
 }
 
-SemanticsUpdate::SemanticsUpdate(std::vector<SemanticsNode> nodes)
+SemanticsUpdate::SemanticsUpdate(SemanticsNodeUpdates nodes)
     : nodes_(std::move(nodes)) {}
 
 SemanticsUpdate::~SemanticsUpdate() = default;
 
-std::vector<SemanticsNode> SemanticsUpdate::takeNodes() {
+SemanticsNodeUpdates SemanticsUpdate::takeNodes() {
   return std::move(nodes_);
 }
 
diff --git a/lib/ui/semantics/semantics_update.h b/lib/ui/semantics/semantics_update.h
@@ -5,8 +5,6 @@
 #ifndef FLUTTER_LIB_UI_SEMANTICS_SEMANTICS_UPDATE_H_
 #define FLUTTER_LIB_UI_SEMANTICS_SEMANTICS_UPDATE_H_
 
-#include <vector>
-
 #include "flutter/lib/ui/semantics/semantics_node.h"
 #include "lib/tonic/dart_wrappable.h"
 
@@ -23,18 +21,18 @@ class SemanticsUpdate : public fxl::RefCountedThreadSafe<SemanticsUpdate>,
 
  public:
   ~SemanticsUpdate() override;
-  static fxl::RefPtr<SemanticsUpdate> create(std::vector<SemanticsNode> nodes);
+  static fxl::RefPtr<SemanticsUpdate> create(SemanticsNodeUpdates nodes);
 
-  std::vector<SemanticsNode> takeNodes();
+  SemanticsNodeUpdates takeNodes();
 
   void dispose();
 
   static void RegisterNatives(tonic::DartLibraryNatives* natives);
 
  private:
-  explicit SemanticsUpdate(std::vector<SemanticsNode> nodes);
+  explicit SemanticsUpdate(SemanticsNodeUpdates nodes);
 
-  std::vector<SemanticsNode> nodes_;
+  SemanticsNodeUpdates nodes_;
 };
 
 }  // namespace blink
diff --git a/lib/ui/semantics/semantics_update_builder.cc b/lib/ui/semantics/semantics_update_builder.cc
@@ -69,7 +69,7 @@ void SemanticsUpdateBuilder::updateNode(int id,
   node.transform.setColMajord(transform.data());
   node.children = std::vector<int32_t>(
       children.data(), children.data() + children.num_elements());
-  nodes_.push_back(node);
+  nodes_[id] = node;
 }
 
 fxl::RefPtr<SemanticsUpdate> SemanticsUpdateBuilder::build() {
diff --git a/lib/ui/semantics/semantics_update_builder.h b/lib/ui/semantics/semantics_update_builder.h
@@ -51,7 +51,7 @@ class SemanticsUpdateBuilder
  private:
   explicit SemanticsUpdateBuilder();
 
-  std::vector<SemanticsNode> nodes_;
+  SemanticsNodeUpdates nodes_;
 };
 
 }  // namespace blink
diff --git a/runtime/runtime_delegate.h b/runtime/runtime_delegate.h
@@ -20,7 +20,7 @@ class RuntimeDelegate {
   virtual std::string DefaultRouteName() = 0;
   virtual void ScheduleFrame(bool regenerate_layer_tree = true) = 0;
   virtual void Render(std::unique_ptr<flow::LayerTree> layer_tree) = 0;
-  virtual void UpdateSemantics(std::vector<SemanticsNode> update) = 0;
+  virtual void UpdateSemantics(blink::SemanticsNodeUpdates update) = 0;
   virtual void HandlePlatformMessage(fxl::RefPtr<PlatformMessage> message) = 0;
 
   virtual void DidCreateMainIsolate(Dart_Isolate isolate);
diff --git a/shell/common/engine.cc b/shell/common/engine.cc
@@ -662,7 +662,7 @@ void Engine::Render(std::unique_ptr<flow::LayerTree> layer_tree) {
   animator_->Render(std::move(layer_tree));
 }
 
-void Engine::UpdateSemantics(std::vector<blink::SemanticsNode> update) {
+void Engine::UpdateSemantics(blink::SemanticsNodeUpdates update) {
   blink::Threads::Platform()->PostTask(fxl::MakeCopyable([
     platform_view = platform_view_.lock(), update = std::move(update)
   ]() mutable {
diff --git a/shell/common/engine.h b/shell/common/engine.h
@@ -85,7 +85,7 @@ class Engine : public blink::RuntimeDelegate {
   // RuntimeDelegate methods:
   std::string DefaultRouteName() override;
   void Render(std::unique_ptr<flow::LayerTree> layer_tree) override;
-  void UpdateSemantics(std::vector<blink::SemanticsNode> update) override;
+  void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
   void HandlePlatformMessage(
       fxl::RefPtr<blink::PlatformMessage> message) override;
   void DidCreateMainIsolate(Dart_Isolate isolate) override;
diff --git a/shell/common/platform_view.cc b/shell/common/platform_view.cc
@@ -130,7 +130,7 @@ VsyncWaiter* PlatformView::GetVsyncWaiter() {
   return vsync_waiter_.get();
 }
 
-void PlatformView::UpdateSemantics(std::vector<blink::SemanticsNode> update) {}
+void PlatformView::UpdateSemantics(blink::SemanticsNodeUpdates update) {}
 
 void PlatformView::HandlePlatformMessage(
     fxl::RefPtr<blink::PlatformMessage> message) {
diff --git a/shell/common/platform_view.h b/shell/common/platform_view.h
@@ -60,7 +60,7 @@ class PlatformView : public std::enable_shared_from_this<PlatformView> {
 
   virtual bool ResourceContextMakeCurrent() = 0;
 
-  virtual void UpdateSemantics(std::vector<blink::SemanticsNode> update);
+  virtual void UpdateSemantics(blink::SemanticsNodeUpdates update);
   virtual void HandlePlatformMessage(
       fxl::RefPtr<blink::PlatformMessage> message);
 
diff --git a/shell/platform/android/platform_view_android.cc b/shell/platform/android/platform_view_android.cc
@@ -460,7 +460,7 @@ bool PlatformViewAndroid::ResourceContextMakeCurrent() {
 }
 
 void PlatformViewAndroid::UpdateSemantics(
-    std::vector<blink::SemanticsNode> update) {
+    blink::SemanticsNodeUpdates update) {
   constexpr size_t kBytesPerNode = 33 * sizeof(int32_t);
   constexpr size_t kBytesPerChild = sizeof(int32_t);
 
diff --git a/shell/platform/android/platform_view_android.h b/shell/platform/android/platform_view_android.h
@@ -93,7 +93,7 @@ class PlatformViewAndroid : public PlatformView {
 
   bool ResourceContextMakeCurrent() override;
 
-  void UpdateSemantics(std::vector<blink::SemanticsNode> update) override;
+  void UpdateSemantics(blink::SemanticsNodeUpdates update) override;
 
   void HandlePlatformMessage(
       fxl::RefPtr<blink::PlatformMessage> message) override;
diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn
@@ -55,6 +55,8 @@ shared_library("create_flutter_framework_dylib") {
     "framework/Source/FlutterViewController.mm",
     "framework/Source/accessibility_bridge.h",
     "framework/Source/accessibility_bridge.mm",
+    "framework/Source/accessibility_text_entry.h",
+    "framework/Source/accessibility_text_entry.mm",
     "framework/Source/flutter_main_ios.h",
     "framework/Source/flutter_main_ios.mm",
     "framework/Source/flutter_touch_mapper.h",
diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h
@@ -5,6 +5,8 @@
 #ifndef SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTPLUGIN_H_
 #define SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTPLUGIN_H_
 
+#import <UIKit/UIKit.h>
+
 #include "flutter/shell/platform/darwin/ios/framework/Headers/FlutterChannels.h"
 #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputDelegate.h"
 
@@ -13,6 +15,33 @@
 @property(nonatomic, assign) id<FlutterTextInputDelegate> textInputDelegate;
 - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result;
 
+/**
+ * The `UITextInput` implementation used to control text entry.
+ *
+ * This is used by `AccessibilityBridge` to forward interactions with iOS'
+ * accessibility system.
+ */
+- (UIView<UITextInput>*)textInputView;
+
+@end
+
+/** An indexed position in the buffer of a Flutter text editing widget. */
+@interface FlutterTextPosition : UITextPosition
+
+@property(nonatomic, readonly) NSUInteger index;
+
++ (instancetype)positionWithIndex:(NSUInteger)index;
+- (instancetype)initWithIndex:(NSUInteger)index;
+
+@end
+
+/** A range of text in the buffer of a Flutter text editing widget. */
+@interface FlutterTextRange : UITextRange<NSCopying>
+
+@property(nonatomic, readonly) NSRange range;
+
++ (instancetype)rangeWithNSRange:(NSRange)range;
+
 @end
 
 #endif  // SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTPLUGIN_H_
diff --git a/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm b/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm
@@ -41,16 +41,6 @@ static UITextAutocapitalizationType ToUITextAutocapitalizationType(NSString* inp
 
 #pragma mark - FlutterTextPosition
 
-/** An indexed position in the buffer of a Flutter text editing widget. */
-@interface FlutterTextPosition : UITextPosition
-
-@property(nonatomic, readonly) NSUInteger index;
-
-+ (instancetype)positionWithIndex:(NSUInteger)index;
-- (instancetype)initWithIndex:(NSUInteger)index;
-
-@end
-
 @implementation FlutterTextPosition
 
 + (instancetype)positionWithIndex:(NSUInteger)index {
@@ -69,15 +59,6 @@ - (instancetype)initWithIndex:(NSUInteger)index {
 
 #pragma mark - FlutterTextRange
 
-/** A range of text in the buffer of a Flutter text editing widget. */
-@interface FlutterTextRange : UITextRange<NSCopying>
-
-@property(nonatomic, readonly) NSRange range;
-
-+ (instancetype)rangeWithNSRange:(NSRange)range;
-
-@end
-
 @implementation FlutterTextRange
 
 + (instancetype)rangeWithNSRange:(NSRange)range {
@@ -536,8 +517,28 @@ - (void)deleteBackward {
 
 @end
 
+/**
+ * Hides `FlutterTextInputView` from iOS accessibility system so it
+ * does not show up twice, once where it is in the `UIView` hierarchy,
+ * and a second time as part of the `SemanticsObject` hierarchy.
+ */
+@interface FlutterTextInputViewAccessibilityHider : UIView {
+}
+
+@end
+
+@implementation FlutterTextInputViewAccessibilityHider {
+}
+
+- (BOOL)accessibilityElementsHidden {
+  return YES;
+}
+
+@end
+
 @implementation FlutterTextInputPlugin {
   FlutterTextInputView* _view;
+  FlutterTextInputViewAccessibilityHider* _inputHider;
 }
 
 @synthesize textInputDelegate = _textInputDelegate;
@@ -547,6 +548,7 @@ - (instancetype)init {
 
   if (self) {
     _view = [[FlutterTextInputView alloc] init];
+    _inputHider = [[FlutterTextInputViewAccessibilityHider alloc] init];
   }
 
   return self;
@@ -555,10 +557,15 @@ - (instancetype)init {
 - (void)dealloc {
   [self hideTextInput];
   [_view release];
+  [_inputHider release];
 
   [super dealloc];
 }
 
+- (UIView<UITextInput>*)textInputView {
+  return _view;
+}
+
 - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result {
   NSString* method = call.method;
   id args = call.arguments;
@@ -587,13 +594,15 @@ - (void)showTextInput {
            @"The application must have a key window since the keyboard client "
            @"must be part of the responder chain to function");
   _view.textInputDelegate = _textInputDelegate;
-  [[UIApplication sharedApplication].keyWindow addSubview:_view];
+  [_inputHider addSubview:_view];
+  [[UIApplication sharedApplication].keyWindow addSubview:_inputHider];
   [_view becomeFirstResponder];
 }
 
 - (void)hideTextInput {
   [_view resignFirstResponder];
   [_view removeFromSuperview];
+  [_inputHider removeFromSuperview];
 }
 
 - (void)setTextInputClient:(int)client withConfiguration:(NSDictionary*)configuration {
diff --git a/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm b/shell/platform/darwin/ios/framework/Source/FlutterViewController.mm
@@ -197,6 +197,7 @@ - (void)performCommonViewControllerInitialization {
   [_textInputChannel.get() setMethodCallHandler:^(FlutterMethodCall* call, FlutterResult result) {
     [_textInputPlugin.get() handleMethodCall:call result:result];
   }];
+  _platformView->SetTextInputPlugin(_textInputPlugin);
 
   [self setupNotificationCenterObservers];
 }
diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.h
diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm b/shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm
diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_text_entry.h b/shell/platform/darwin/ios/framework/Source/accessibility_text_entry.h
diff --git a/shell/platform/darwin/ios/framework/Source/accessibility_text_entry.mm b/shell/platform/darwin/ios/framework/Source/accessibility_text_entry.mm
diff --git a/shell/platform/darwin/ios/platform_view_ios.h b/shell/platform/darwin/ios/platform_view_ios.h
diff --git a/shell/platform/darwin/ios/platform_view_ios.mm b/shell/platform/darwin/ios/platform_view_ios.mm
diff --git a/tools/licenses/pubspec.lock b/tools/licenses/pubspec.lock
diff --git a/travis/licenses_golden/licenses_flutter b/travis/licenses_golden/licenses_flutter

Original file line number	Diff line number	Diff line change
`@@ -370,7 +370,7 @@ void RuntimeHolder::Render(std::unique_ptr<flow::LayerTree> layer_tree) {`
`370`	`370`	`}));`
`371`	`371`	`}`
`372`	`372`
`373`		`-void RuntimeHolder::UpdateSemantics(std::vector<blink::SemanticsNode> update) {`
	`373`	`+void RuntimeHolder::UpdateSemantics(blink::SemanticsNodeUpdates update) {`
`374`	`374`	`accessibility_bridge_->UpdateSemantics(update);`
`375`	`375`	`}`
`376`	`376`
Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ void SemanticsUpdateBuilder::updateNode(int id,`
`69`	`69`	`node.transform.setColMajord(transform.data());`
`70`	`70`	`node.children = std::vector<int32_t>(`
`71`	`71`	`children.data(), children.data() + children.num_elements());`
`72`		`- nodes_.push_back(node);`
	`72`	`+ nodes_[id] = node;`
`73`	`73`	`}`
`74`	`74`
`75`	`75`	`fxl::RefPtr<SemanticsUpdate> SemanticsUpdateBuilder::build() {`