Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Flutter iOS Interactive Keyboard: Take Screenshot and Handle Pointer Movement #43972

Merged
merged 7 commits into from
Aug 4, 2023
Merged

Flutter iOS Interactive Keyboard: Take Screenshot and Handle Pointer Movement #43972

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fa51055
Keyboard Screenshot Pointer Move
Matt2D Jul 14, 2023
2d8138b
Address Feedback
Matt2D Aug 3, 2023
881e7be
Format
Matt2D Aug 3, 2023
f07d337
Rename
Matt2D Aug 3, 2023
ff6f3d5
Renamed to cached first responder
Matt2D Aug 3, 2023
09a5ef5
Update test to reflect new changes
Matt2D Aug 4, 2023
f3eadbd
Format
Matt2D Aug 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,4 +165,9 @@ FLUTTER_DARWIN_EXPORT
- (instancetype)initWithOwner:(FlutterTextInputPlugin*)textInputPlugin NS_DESIGNATED_INITIALIZER;

@end

@interface UIView (FindFirstResponder)
@property(nonatomic, readonly) id flutterFirstResponder;
@end

#endif // SHELL_PLATFORM_IOS_FRAMEWORK_SOURCE_FLUTTERTEXTINPUTPLUGIN_H_
96 changes: 95 additions & 1 deletion shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// found in the LICENSE file.

#import "flutter/shell/platform/darwin/ios/framework/Source/FlutterTextInputPlugin.h"
#import "flutter/shell/platform/darwin/ios/framework/Source/UIViewController+FlutterScreenAndSceneIfLoaded.h"

#import <Foundation/Foundation.h>
#import <UIKit/UIKit.h>
Expand Down Expand Up @@ -45,6 +46,8 @@
static NSString* const kSetSelectionRectsMethod = @"Scribble.setSelectionRects";
static NSString* const kStartLiveTextInputMethod = @"TextInput.startLiveTextInput";
static NSString* const kUpdateConfigMethod = @"TextInput.updateConfig";
static NSString* const kOnInteractiveKeyboardPointerMoveMethod =
@"TextInput.onPointerMoveForInteractiveKeyboard";

#pragma mark - TextInputConfiguration Field Names
static NSString* const kSecureTextEntry = @"obscureText";
Expand Down Expand Up @@ -761,6 +764,7 @@ @interface FlutterTextInputView ()
@property(nonatomic, assign) CGRect markedRect;
@property(nonatomic) BOOL isVisibleToAutofill;
@property(nonatomic, assign) BOOL accessibilityEnabled;
@property(nonatomic, assign) int textInputClient;
// The composed character that is temporarily removed by the keyboard API.
// This is cleared at the start of each keyboard interaction. (Enter a character, delete a character
// etc)
Expand Down Expand Up @@ -2214,6 +2218,11 @@ @interface FlutterTextInputPlugin ()
@property(nonatomic, retain) FlutterTextInputView* activeView;
@property(nonatomic, retain) FlutterTextInputViewAccessibilityHider* inputHider;
@property(nonatomic, readonly, weak) id<FlutterViewResponder> viewResponder;

@property(nonatomic, strong) UIView* keyboardViewContainer;
@property(nonatomic, strong) UIView* keyboardView;
@property(nonatomic, strong) UIView* cachedFirstResponder;
@property(nonatomic, assign) CGRect keyboardRect;
@end

@implementation FlutterTextInputPlugin {
Expand All @@ -2222,18 +2231,29 @@ @implementation FlutterTextInputPlugin {

- (instancetype)initWithDelegate:(id<FlutterTextInputDelegate>)textInputDelegate {
self = [super init];

if (self) {
// `_textInputDelegate` is a weak reference because it should retain FlutterTextInputPlugin.
_textInputDelegate = textInputDelegate;
_autofillContext = [[NSMutableDictionary alloc] init];
_inputHider = [[FlutterTextInputViewAccessibilityHider alloc] init];
_scribbleElements = [[NSMutableDictionary alloc] init];
_keyboardViewContainer = [[UIView alloc] init];

[[NSNotificationCenter defaultCenter] addObserver:self
selector:@selector(handleKeyboardWillShow:)
name:UIKeyboardWillShowNotification
object:nil];
}

return self;
}

- (void)handleKeyboardWillShow:(NSNotification*)notification {
NSDictionary* keyboardInfo = [notification userInfo];
NSValue* keyboardFrameEnd = [keyboardInfo valueForKey:UIKeyboardFrameEndUserInfoKey];
_keyboardRect = [keyboardFrameEnd CGRectValue];
}

- (void)dealloc {
[self hideTextInput];
}
Expand Down Expand Up @@ -2295,11 +2315,67 @@ - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result {
} else if ([method isEqualToString:kUpdateConfigMethod]) {
[self updateConfig:args];
result(nil);
} else if ([method isEqualToString:kOnInteractiveKeyboardPointerMoveMethod]) {
CGFloat pointerY = (CGFloat)[args[@"pointerY"] doubleValue];
[self handlePointerMove:pointerY];
result(nil);
} else {
result(FlutterMethodNotImplemented);
}
}

- (void)handlePointerMove:(CGFloat)pointerY {
// View must be loaded at this point.
UIScreen* screen = _viewController.flutterScreenIfViewLoaded;
double screenHeight = screen.bounds.size.height;
double keyboardHeight = _keyboardRect.size.height;
if (screenHeight - keyboardHeight <= pointerY) {
// If the pointer is within the bounds of the keyboard.
if (_keyboardView.superview == nil) {
// If no screenshot has been taken.
[self takeKeyboardScreenshotAndDisplay];
[self hideKeyboardWithoutAnimation];
} else {
[self setKeyboardContainerHeight:pointerY];
}
} else {
if (_keyboardView.superview != nil) {
// Keeps keyboard at proper height.
_keyboardViewContainer.frame = _keyboardRect;
}
}
}

- (void)setKeyboardContainerHeight:(CGFloat)pointerY {
CGRect frameRect = _keyboardRect;
frameRect.origin.y = pointerY;
_keyboardViewContainer.frame = frameRect;
Comment on lines +2350 to +2352
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor optional nits:

The logic of this method can also be used for the scenario where the pointer is not overtop the keyboard.
The frameRect's Y can be updated to be the max of pointerY and keyboard height.

  CGRect frameRect = _keyboardRect;
  if (screenHeight - keyboardHeight <= pointerY) {
      frameRect.origin.y = pointerY;
  }
  _keyboardViewContainer.frame = frameRect;

Then in the handlePointerMove, the logic can be simplified as

if (_keyboardView.superview != nil) {
   [self setKeyboardContainerHeight:pointerY];
} else {
   [self takeScreenshot];
   [self hideKeyboardWithoutAnimation];
}

Copy link
Contributor Author

@Matt2D Matt2D Aug 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It turns out that for the change we still need to check the height for takeScreenshot since we don't want to take the screenshot too early:

- (void)handlePointerMove:(CGFloat)pointerY {
  //View must be loaded at this point.
  UIScreen* screen = _viewController.flutterScreenIfViewLoaded;
  double screenHeight = screen.bounds.size.height;
  double keyboardHeight = _keyboardRect.size.height;

  if (_keyboardView.superview != nil) {
    // If screenshot is present.
    [self setKeyboardContainerHeight:pointerY];
  } else {
    // If no screenshot has been taken.
    if (screenHeight - keyboardHeight <= pointerY) {
      //If pointer is above keyboard.
      [self takeKeyboardScreenshotAndDisplay];
      [self hideKeyboardWithoutAnimation];
    }
  }
}

- (void)setKeyboardContainerHeight:(CGFloat)pointerY {
  //View must be loaded at this point.
  UIScreen* screen = _viewController.flutterScreenIfViewLoaded;
  double screenHeight = screen.bounds.size.height;
  double keyboardHeight = _keyboardRect.size.height;

  CGRect frameRect = _keyboardRect;
  if (screenHeight - keyboardHeight <= pointerY) {
      frameRect.origin.y = pointerY;
  }
  _keyboardViewContainer.frame = frameRect;
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah I missed that part. So the caller part can probably be

if (_keyboardView.superview != nil) {
   [self setKeyboardContainerHeight:pointerY];
} else if (screenHeight - keyboardHeight <= pointerY) {
   [self takeScreenshot];
   [self hideKeyboardWithoutAnimation];
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed offline - the duplicate logic of screenHeight - keyboardHeight <= pointerY can get error prune.

Another idea is to pass in an additional BOOL to the helper function, but that also makes the code unnecessarily complicate.

I think it makes sense to keep it as it is.

}

- (void)hideKeyboardWithoutAnimation {
[UIView setAnimationsEnabled:NO];
_cachedFirstResponder = UIApplication.sharedApplication.keyWindow.flutterFirstResponder;
[_cachedFirstResponder resignFirstResponder];
[UIView setAnimationsEnabled:YES];
}

- (void)takeKeyboardScreenshotAndDisplay {
// View must be loaded at this point
UIScreen* screen = _viewController.flutterScreenIfViewLoaded;
UIView* keyboardSnap = [screen snapshotViewAfterScreenUpdates:YES];
keyboardSnap = [keyboardSnap resizableSnapshotViewFromRect:_keyboardRect
afterScreenUpdates:YES
withCapInsets:UIEdgeInsetsZero];
_keyboardView = keyboardSnap;
[_keyboardViewContainer addSubview:_keyboardView];
if (_keyboardViewContainer.superview == nil) {
[UIApplication.sharedApplication.delegate.window.rootViewController.view
addSubview:_keyboardViewContainer];
Comment on lines +2372 to +2373
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should it be FlutterViewController instead?

In add to app scenario, the rootViewController might not be FlutterViewController, which might be covered by FlutterViewController. So the keyboard might be shown "behind" the FlutterViewController

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this is positioning a keyboard relative to the full screen -- regardless of whether the FlutterView is full screen, or a subview covering only part of the screen in an add-to-app scenario -- I think we want the root view controller don't we?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah that's true. We would need to make sure to position the keyboard screenshot on the very top. Set ZIndex to maxInt maybe?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Definitely worth adding!

}
_keyboardViewContainer.layer.zPosition = NSIntegerMax;
_keyboardViewContainer.frame = _keyboardRect;
}

- (void)setEditableSizeAndTransform:(NSDictionary*)dictionary {
[_activeView setEditableTransform:dictionary[@"transform"]];
if ([_activeView isScribbleAvailable]) {
Expand Down Expand Up @@ -2764,3 +2840,21 @@ - (BOOL)handlePress:(nonnull FlutterUIPressProxy*)press API_AVAILABLE(ios(13.4))
return NO;
}
@end

/**
* Recursively searches the UIView's subviews to locate the First Responder
*/
@implementation UIView (FindFirstResponder)
- (id)flutterFirstResponder {
if (self.isFirstResponder) {
return self;
}
for (UIView* subView in self.subviews) {
UIView* firstResponder = subView.flutterFirstResponder;
if (firstResponder) {
return firstResponder;
}
}
return nil;
}
@end
218 changes: 218 additions & 0 deletions shell/platform/darwin/ios/framework/Source/FlutterTextInputPluginTest.mm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ @interface FlutterSecureTextInputView : FlutterTextInputView

@interface FlutterTextInputPlugin ()
@property(nonatomic, assign) FlutterTextInputView* activeView;
@property(nonatomic, readonly) UIView* keyboardViewContainer;
@property(nonatomic, readonly) UIView* keyboardView;
@property(nonatomic, readonly) CGRect keyboardRect;
@property(nonatomic, readonly)
NSMutableDictionary<NSString*, FlutterTextInputView*>* autofillContext;

Expand Down Expand Up @@ -2425,4 +2428,219 @@ - (void)testSetPlatformViewClient {
XCTAssertNil(activeView.superview, @"activeView must be removed from view hierarchy.");
}

- (void)testInteractiveKeyboardAfterUserScrollWillResignFirstResponder {
FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];

[inputView setTextInputClient:123];
[inputView reloadInputViews];
[inputView becomeFirstResponder];
XCTAssert(inputView.isFirstResponder);

CGRect keyboardFrame = CGRectMake(0, 500, 500, 500);
[NSNotificationCenter.defaultCenter
postNotificationName:UIKeyboardWillShowNotification
object:nil
userInfo:@{UIKeyboardFrameEndUserInfoKey : @(keyboardFrame)}];
FlutterMethodCall* onPointerMoveCall =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(500)}];
[textInputPlugin handleMethodCall:onPointerMoveCall
result:^(id _Nullable result){
}];
XCTAssertFalse(inputView.isFirstResponder);
}

- (void)testInteractiveKeyboardAfterUserScrollToTopOfKeyboardWillTakeScreenshot {
NSSet<UIScene*>* scenes = UIApplication.sharedApplication.connectedScenes;
XCTAssertEqual(scenes.count, 1UL, @"There must only be 1 scene for test");
UIScene* scene = scenes.anyObject;
XCTAssert([scene isKindOfClass:[UIWindowScene class]], @"Must be a window scene for test");
UIWindowScene* windowScene = (UIWindowScene*)scene;
XCTAssert(windowScene.windows.count > 0, @"There must be at least 1 window for test");
UIWindow* window = windowScene.windows[0];
[window addSubview:viewController.view];

[viewController loadView];

FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];

[inputView setTextInputClient:123];
[inputView reloadInputViews];
[inputView becomeFirstResponder];

if (textInputPlugin.keyboardView.superview != nil) {
for (UIView* subView in textInputPlugin.keyboardViewContainer.subviews) {
[subView removeFromSuperview];
}
}
XCTAssert(textInputPlugin.keyboardView.superview == nil);
CGRect keyboardFrame = CGRectMake(0, 500, 500, 500);
[NSNotificationCenter.defaultCenter
postNotificationName:UIKeyboardWillShowNotification
object:nil
userInfo:@{UIKeyboardFrameEndUserInfoKey : @(keyboardFrame)}];
FlutterMethodCall* onPointerMoveCall =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(510)}];
[textInputPlugin handleMethodCall:onPointerMoveCall
result:^(id _Nullable result){
}];
XCTAssertFalse(textInputPlugin.keyboardView.superview == nil);
for (UIView* subView in textInputPlugin.keyboardViewContainer.subviews) {
[subView removeFromSuperview];
}
}

- (void)testInteractiveKeyboardScreenshotWillBeMovedDownAfterUserScroll {
NSSet<UIScene*>* scenes = UIApplication.sharedApplication.connectedScenes;
XCTAssertEqual(scenes.count, 1UL, @"There must only be 1 scene for test");
UIScene* scene = scenes.anyObject;
XCTAssert([scene isKindOfClass:[UIWindowScene class]], @"Must be a window scene for test");
UIWindowScene* windowScene = (UIWindowScene*)scene;
XCTAssert(windowScene.windows.count > 0, @"There must be at least 1 window for test");
UIWindow* window = windowScene.windows[0];
[window addSubview:viewController.view];

[viewController loadView];

FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];

[inputView setTextInputClient:123];
[inputView reloadInputViews];
[inputView becomeFirstResponder];

CGRect keyboardFrame = CGRectMake(0, 500, 500, 500);
[NSNotificationCenter.defaultCenter
postNotificationName:UIKeyboardWillShowNotification
object:nil
userInfo:@{UIKeyboardFrameEndUserInfoKey : @(keyboardFrame)}];
FlutterMethodCall* onPointerMoveCall =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(510)}];
[textInputPlugin handleMethodCall:onPointerMoveCall
result:^(id _Nullable result){
}];
XCTAssert(textInputPlugin.keyboardView.superview != nil);

XCTAssertEqual(textInputPlugin.keyboardViewContainer.frame.origin.y, keyboardFrame.origin.y);

FlutterMethodCall* onPointerMoveCallMove =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(600)}];
[textInputPlugin handleMethodCall:onPointerMoveCallMove
result:^(id _Nullable result){
}];
XCTAssert(textInputPlugin.keyboardView.superview != nil);

XCTAssertEqual(textInputPlugin.keyboardViewContainer.frame.origin.y, 600.0);

for (UIView* subView in textInputPlugin.keyboardViewContainer.subviews) {
[subView removeFromSuperview];
}
}

- (void)testInteractiveKeyboardScreenshotWillBeMovedToOrginalPositionAfterUserScroll {
NSSet<UIScene*>* scenes = UIApplication.sharedApplication.connectedScenes;
XCTAssertEqual(scenes.count, 1UL, @"There must only be 1 scene for test");
UIScene* scene = scenes.anyObject;
XCTAssert([scene isKindOfClass:[UIWindowScene class]], @"Must be a window scene for test");
UIWindowScene* windowScene = (UIWindowScene*)scene;
XCTAssert(windowScene.windows.count > 0, @"There must be at least 1 window for test");
UIWindow* window = windowScene.windows[0];
[window addSubview:viewController.view];

[viewController loadView];

FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];

[inputView setTextInputClient:123];
[inputView reloadInputViews];
[inputView becomeFirstResponder];

CGRect keyboardFrame = CGRectMake(0, 500, 500, 500);
[NSNotificationCenter.defaultCenter
postNotificationName:UIKeyboardWillShowNotification
object:nil
userInfo:@{UIKeyboardFrameEndUserInfoKey : @(keyboardFrame)}];
FlutterMethodCall* onPointerMoveCall =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(500)}];
[textInputPlugin handleMethodCall:onPointerMoveCall
result:^(id _Nullable result){
}];
XCTAssert(textInputPlugin.keyboardView.superview != nil);
XCTAssertEqual(textInputPlugin.keyboardViewContainer.frame.origin.y, keyboardFrame.origin.y);

FlutterMethodCall* onPointerMoveCallMove =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(600)}];
[textInputPlugin handleMethodCall:onPointerMoveCallMove
result:^(id _Nullable result){
}];
XCTAssert(textInputPlugin.keyboardView.superview != nil);
XCTAssertEqual(textInputPlugin.keyboardViewContainer.frame.origin.y, 600.0);

FlutterMethodCall* onPointerMoveCallBackUp =
[FlutterMethodCall methodCallWithMethodName:@"TextInput.onPointerMoveForInteractiveKeyboard"
arguments:@{@"pointerY" : @(10)}];
[textInputPlugin handleMethodCall:onPointerMoveCallBackUp
result:^(id _Nullable result){
}];
XCTAssert(textInputPlugin.keyboardView.superview != nil);
XCTAssertEqual(textInputPlugin.keyboardViewContainer.frame.origin.y, keyboardFrame.origin.y);
for (UIView* subView in textInputPlugin.keyboardViewContainer.subviews) {
[subView removeFromSuperview];
}
}

- (void)testInteractiveKeyboardFindFirstResponderRecursive {
FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];
[inputView setTextInputClient:123];
[inputView reloadInputViews];
[inputView becomeFirstResponder];

UIView* firstResponder = UIApplication.sharedApplication.keyWindow.flutterFirstResponder;
XCTAssertEqualObjects(inputView, firstResponder);
}

- (void)testInteractiveKeyboardFindFirstResponderRecursiveInMultipleSubviews {
FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
FlutterTextInputView* subInputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
FlutterTextInputView* otherSubInputView =
[[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
FlutterTextInputView* subFirstResponderInputView =
[[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[subInputView addSubview:subFirstResponderInputView];
[inputView addSubview:subInputView];
[inputView addSubview:otherSubInputView];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];
[inputView setTextInputClient:123];
[inputView reloadInputViews];
[subInputView setTextInputClient:123];
[subInputView reloadInputViews];
[otherSubInputView setTextInputClient:123];
[otherSubInputView reloadInputViews];
[subFirstResponderInputView setTextInputClient:123];
[subFirstResponderInputView reloadInputViews];
[subFirstResponderInputView becomeFirstResponder];

UIView* firstResponder = UIApplication.sharedApplication.keyWindow.flutterFirstResponder;
XCTAssertEqualObjects(subFirstResponderInputView, firstResponder);
}

- (void)testInteractiveKeyboardFindFirstResponderIsNilRecursive {
FlutterTextInputView* inputView = [[FlutterTextInputView alloc] initWithOwner:textInputPlugin];
[UIApplication.sharedApplication.keyWindow addSubview:inputView];
[inputView setTextInputClient:123];
[inputView reloadInputViews];

UIView* firstResponder = UIApplication.sharedApplication.keyWindow.flutterFirstResponder;
XCTAssertNil(firstResponder);
}

@end