Add multi-step input method support for Linux (flutter#21897)

This implements the Gtk hooks required to support multi-step input
methods on Linux. This builds on the support for composing regions
(preedit region in Gtk terminology) added to TextInputModel in
flutter/engine#21682.

Specifically, the following changes are included:

1. Add handler for TextInput.setMarkedTextRegion framework messages: On
any change to the EditableText in the framework, this message is sent
which provides an updated rect (in the local co-ordinates of the
EditableText) for the composing region. If not in composing mode, the
cursor rect is sent.

2. Add handler for TextInput.setEditableSizeAndTransform framework messages:
On any change to the RenderObject underlying the EditableText, an
updated size for the full EditableText widget, as well as an affine
transform matrix from local co-ordinates to Flutter root co-ordinates is
sent.

3. On either of the above messages, we use the transformed composing
rect to compute the cursor position in Gtk window co-ordinates and
inform Gtk, so that it can position any system IM composing window
correctly for on-the-spot composing, such as is used when inputting
Japanese text.

4. Adds handlers for preedit-start, preedit-changed, and preedit-end
signals from Gtk. These are passed on to the TextInputModel.

5. Updates the preedit-commit handler to commit the composing region to
the text or, if not composing, insert new text at the cursor.

6. Updates the handler for TextInput.setEditingState framework messages
to extract the composing range base and extent and pass these on to
TextInputModel.

7. Updates update_editing_state function to set composing base and
extent on text input state updates sent to the framework.

Author: cbracken

shell/platform/linux/fl_text_input_plugin.cc

@@ -22,6 +22,9 @@ static constexpr char kHideMethod[] = "TextInput.hide";
 static constexpr char kUpdateEditingStateMethod[] =
     "TextInputClient.updateEditingState";
 static constexpr char kPerformActionMethod[] = "TextInputClient.performAction";
+static constexpr char kSetEditableSizeAndTransform[] =
+    "TextInput.setEditableSizeAndTransform";
+static constexpr char kSetMarkedTextRect[] = "TextInput.setMarkedTextRect";
 
 static constexpr char kInputActionKey[] = "inputAction";
 static constexpr char kTextInputTypeKey[] = "inputType";
@@ -34,6 +37,8 @@ static constexpr char kSelectionIsDirectionalKey[] = "selectionIsDirectional";
 static constexpr char kComposingBaseKey[] = "composingBase";
 static constexpr char kComposingExtentKey[] = "composingExtent";
 
+static constexpr char kTransform[] = "transform";
+
 static constexpr char kTextAffinityDownstream[] = "TextAffinity.downstream";
 static constexpr char kMultilineInputType[] = "TextInputType.multiline";
 
@@ -57,6 +62,19 @@ struct _FlTextInputPlugin {
   GtkIMContext* im_context;
 
   flutter::TextInputModel* text_model;
+
+  // The owning Flutter view.
+  FlView* view;
+
+  // A 4x4 matrix that maps from `EditableText` local coordinates to the
+  // coordinate system of `PipelineOwner.rootNode`.
+  double editabletext_transform[4][4];
+
+  // The smallest rect, in local coordinates, of the text in the composing
+  // range, or of the caret in the case where there is no current composing
+  // range. This value is updated via `TextInput.setMarkedTextRect` messages
+  // over the text input channel.
+  GdkRectangle composing_rect;
 };
 
 G_DEFINE_TYPE(FlTextInputPlugin, fl_text_input_plugin, G_TYPE_OBJECT)
@@ -99,13 +117,22 @@ static void update_editing_state(FlTextInputPlugin* self) {
   fl_value_set_string_take(value, kSelectionExtentKey,
                            fl_value_new_int(selection.extent()));
 
+  int composing_base = self->text_model->composing()
+                           ? self->text_model->composing_range().base()
+                           : -1;
+  int composing_extent = self->text_model->composing()
+                             ? self->text_model->composing_range().extent()
+                             : -1;
+  fl_value_set_string_take(value, kComposingBaseKey,
+                           fl_value_new_int(composing_base));
+  fl_value_set_string_take(value, kComposingExtentKey,
+                           fl_value_new_int(composing_extent));
+
   // The following keys are not implemented and set to default values.
   fl_value_set_string_take(value, kSelectionAffinityKey,
                            fl_value_new_string(kTextAffinityDownstream));
   fl_value_set_string_take(value, kSelectionIsDirectionalKey,
                            fl_value_new_bool(FALSE));
-  fl_value_set_string_take(value, kComposingBaseKey, fl_value_new_int(-1));
-  fl_value_set_string_take(value, kComposingExtentKey, fl_value_new_int(-1));
 
   fl_value_append(args, value);
 
@@ -138,9 +165,41 @@ static void perform_action(FlTextInputPlugin* self) {
                                   nullptr, perform_action_response_cb, self);
 }
 
+// Signal handler for GtkIMContext::preedit-start
+static void im_preedit_start_cb(FlTextInputPlugin* self) {
+  self->text_model->BeginComposing();
+
+  // Set the top-level window used for system input method windows.
+  GdkWindow* window =
+      gtk_widget_get_window(gtk_widget_get_toplevel(GTK_WIDGET(self->view)));
+  gtk_im_context_set_client_window(self->im_context, window);
+}
+
+// Signal handler for GtkIMContext::preedit-changed
+static void im_preedit_changed_cb(FlTextInputPlugin* self) {
+  g_autofree gchar* buf = nullptr;
+  gint cursor_offset = 0;
+  gtk_im_context_get_preedit_string(self->im_context, &buf, nullptr,
+                                    &cursor_offset);
+  cursor_offset += self->text_model->composing_range().base();
+  self->text_model->UpdateComposingText(buf);
+  self->text_model->SetSelection(TextRange(cursor_offset, cursor_offset));
+
+  update_editing_state(self);
+}
+
 // Signal handler for GtkIMContext::commit
 static void im_commit_cb(FlTextInputPlugin* self, const gchar* text) {
   self->text_model->AddText(text);
+  if (self->text_model->composing()) {
+    self->text_model->CommitComposing();
+  }
+  update_editing_state(self);
+}
+
+// Signal handler for GtkIMContext::preedit-end
+static void im_preedit_end_cb(FlTextInputPlugin* self) {
+  self->text_model->EndComposing();
   update_editing_state(self);
 }
 
@@ -209,6 +268,8 @@ static FlMethodResponse* set_editing_state(FlTextInputPlugin* self,
                                            FlValue* args) {
   const gchar* text =
       fl_value_get_string(fl_value_lookup_string(args, kTextKey));
+  self->text_model->SetText(text);
+
   int64_t selection_base =
       fl_value_get_int(fl_value_lookup_string(args, kSelectionBaseKey));
   int64_t selection_extent =
@@ -221,6 +282,19 @@ static FlMethodResponse* set_editing_state(FlTextInputPlugin* self,
   self->text_model->SetText(text);
   self->text_model->SetSelection(TextRange(selection_base, selection_extent));
 
+  int64_t composing_base =
+      fl_value_get_int(fl_value_lookup_string(args, kComposingBaseKey));
+  int64_t composing_extent =
+      fl_value_get_int(fl_value_lookup_string(args, kComposingExtentKey));
+  if (composing_base == -1 && composing_extent == -1) {
+    self->text_model->EndComposing();
+  } else {
+    size_t composing_start = std::min(composing_base, composing_extent);
+    size_t cursor_offset = selection_base - composing_start;
+    self->text_model->SetComposingRange(
+        TextRange(composing_base, composing_extent), cursor_offset);
+  }
+
   return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
 }
 
@@ -238,6 +312,83 @@ static FlMethodResponse* hide(FlTextInputPlugin* self) {
   return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
 }
 
+// Update the IM cursor position.
+//
+// As text is input by the user, the framework sends two streams of updates
+// over the text input channel: updates to the composing rect (cursor rect when
+// not in IME composing mode) and updates to the matrix transform from local
+// coordinates to Flutter root coordinates. This function is called after each
+// of these updates. It transforms the composing rect to GTK window coordinates
+// and notifies GTK of the updated cursor position.
+static void update_im_cursor_position(FlTextInputPlugin* self) {
+  // Skip update if not composing to avoid setting to position 0.
+  if (!self->text_model->composing()) {
+    return;
+  }
+
+  // Transform the x, y positions of the cursor from local coordinates to
+  // Flutter view coordinates.
+  gint x = self->composing_rect.x * self->editabletext_transform[0][0] +
+           self->composing_rect.y * self->editabletext_transform[1][0] +
+           self->editabletext_transform[3][0] + self->composing_rect.width;
+  gint y = self->composing_rect.x * self->editabletext_transform[0][1] +
+           self->composing_rect.y * self->editabletext_transform[1][1] +
+           self->editabletext_transform[3][1] + self->composing_rect.height;
+
+  // Transform from Flutter view coordinates to GTK window coordinates.
+  GdkRectangle preedit_rect;
+  gtk_widget_translate_coordinates(
+      GTK_WIDGET(self->view), gtk_widget_get_toplevel(GTK_WIDGET(self->view)),
+      x, y, &preedit_rect.x, &preedit_rect.y);
+
+  // Set the cursor location in window coordinates so that GTK can position any
+  // system input method windows.
+  gtk_im_context_set_cursor_location(self->im_context, &preedit_rect);
+}
+
+// Handles updates to the EditableText size and position from the framework.
+//
+// On changes to the size or position of the RenderObject underlying the
+// EditableText, this update may be triggered. It provides an updated size and
+// transform from the local coordinate system of the EditableText to root
+// Flutter coordinate system.
+static FlMethodResponse* set_editable_size_and_transform(
+    FlTextInputPlugin* self,
+    FlValue* args) {
+  FlValue* transform = fl_value_lookup_string(args, kTransform);
+  size_t transform_len = fl_value_get_length(transform);
+  g_warn_if_fail(transform_len == 16);
+
+  for (size_t i = 0; i < transform_len; ++i) {
+    double val = fl_value_get_float(fl_value_get_list_value(transform, i));
+    self->editabletext_transform[i / 4][i % 4] = val;
+  }
+  update_im_cursor_position(self);
+
+  return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
+}
+
+// Handles updates to the composing rect from the framework.
+//
+// On changes to the state of the EditableText in the framework, this update
+// may be triggered. It provides an updated rect for the composing region in
+// local coordinates of the EditableText. In the case where there is no
+// composing region, the cursor rect is sent.
+static FlMethodResponse* set_marked_text_rect(FlTextInputPlugin* self,
+                                              FlValue* args) {
+  self->composing_rect.x =
+      fl_value_get_float(fl_value_lookup_string(args, "x"));
+  self->composing_rect.y =
+      fl_value_get_float(fl_value_lookup_string(args, "y"));
+  self->composing_rect.width =
+      fl_value_get_float(fl_value_lookup_string(args, "width"));
+  self->composing_rect.height =
+      fl_value_get_float(fl_value_lookup_string(args, "height"));
+  update_im_cursor_position(self);
+
+  return FL_METHOD_RESPONSE(fl_method_success_response_new(nullptr));
+}
+
 // Called when a method call is received from Flutter.
 static void method_call_cb(FlMethodChannel* channel,
                            FlMethodCall* method_call,
@@ -258,6 +409,10 @@ static void method_call_cb(FlMethodChannel* channel,
     response = clear_client(self);
   } else if (strcmp(method, kHideMethod) == 0) {
     response = hide(self);
+  } else if (strcmp(method, kSetEditableSizeAndTransform) == 0) {
+    response = set_editable_size_and_transform(self, args);
+  } else if (strcmp(method, kSetMarkedTextRect) == 0) {
+    response = set_marked_text_rect(self, args);
   } else {
     response = FL_METHOD_RESPONSE(fl_method_not_implemented_response_new());
   }
@@ -268,6 +423,11 @@ static void method_call_cb(FlMethodChannel* channel,
   }
 }
 
+static void view_weak_notify_cb(gpointer user_data, GObject* object) {
+  FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(object);
+  self->view = nullptr;
+}
+
 static void fl_text_input_plugin_dispose(GObject* object) {
   FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(object);
 
@@ -290,6 +450,15 @@ static void fl_text_input_plugin_init(FlTextInputPlugin* self) {
   self->client_id = kClientIdUnset;
   self->im_context = gtk_im_multicontext_new();
   self->input_multiline = FALSE;
+  g_signal_connect_object(self->im_context, "preedit-start",
+                          G_CALLBACK(im_preedit_start_cb), self,
+                          G_CONNECT_SWAPPED);
+  g_signal_connect_object(self->im_context, "preedit-end",
+                          G_CALLBACK(im_preedit_end_cb), self,
+                          G_CONNECT_SWAPPED);
+  g_signal_connect_object(self->im_context, "preedit-changed",
+                          G_CALLBACK(im_preedit_changed_cb), self,
+                          G_CONNECT_SWAPPED);
   g_signal_connect_object(self->im_context, "commit", G_CALLBACK(im_commit_cb),
                           self, G_CONNECT_SWAPPED);
   g_signal_connect_object(self->im_context, "retrieve-surrounding",
@@ -301,7 +470,8 @@ static void fl_text_input_plugin_init(FlTextInputPlugin* self) {
   self->text_model = new flutter::TextInputModel();
 }
 
-FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger) {
+FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger,
+                                            FlView* view) {
   g_return_val_if_fail(FL_IS_BINARY_MESSENGER(messenger), nullptr);
 
   FlTextInputPlugin* self = FL_TEXT_INPUT_PLUGIN(
@@ -312,6 +482,8 @@ FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger) {
       fl_method_channel_new(messenger, kChannelName, FL_METHOD_CODEC(codec));
   fl_method_channel_set_method_call_handler(self->channel, method_call_cb, self,
                                             nullptr);
+  self->view = view;
+  g_object_weak_ref(G_OBJECT(view), view_weak_notify_cb, self);
 
   return self;
 }

shell/platform/linux/fl_text_input_plugin.h

@@ -8,6 +8,7 @@
 #include <gdk/gdk.h>
 
 #include "flutter/shell/platform/linux/public/flutter_linux/fl_binary_messenger.h"
+#include "flutter/shell/platform/linux/public/flutter_linux/fl_view.h"
 
 G_BEGIN_DECLS
 
@@ -27,13 +28,15 @@ G_DECLARE_FINAL_TYPE(FlTextInputPlugin,
 /**
  * fl_text_input_plugin_new:
  * @messenger: an #FlBinaryMessenger.
+ * @view: the #FlView with which the text input plugin is associated.
  *
  * Creates a new plugin that implements SystemChannels.textInput from the
  * Flutter services library.
  *
  * Returns: a new #FlTextInputPlugin.
  */
-FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger);
+FlTextInputPlugin* fl_text_input_plugin_new(FlBinaryMessenger* messenger,
+                                            FlView* view);
 
 /**
  * fl_text_input_plugin_filter_keypress

shell/platform/linux/fl_view.cc

@@ -161,7 +161,7 @@ static void fl_view_constructed(GObject* object) {
   self->key_event_plugin = fl_key_event_plugin_new(messenger);
   self->mouse_cursor_plugin = fl_mouse_cursor_plugin_new(messenger, self);
   self->platform_plugin = fl_platform_plugin_new(messenger);
-  self->text_input_plugin = fl_text_input_plugin_new(messenger);
+  self->text_input_plugin = fl_text_input_plugin_new(messenger, self);
 }
 
 static void fl_view_set_property(GObject* object,