feat(ui): enable multiple keyboard layouts for "paste text" to remote host (#405)

* Enable multiple keyboard layouts for paste text from host

* Trema is the more robust method for capital umlauts

* Improve error handling and pre-loading

* Improve accent handling

* Remove obscure Alt-Gr keys, unsure if they are supported everywhere

* Add Swiss French

* Change line ordering

* Fix whitespace

* Add French (France)

* Add English (UK)

* Add Swedish

* Add Spanish

* Fix fr_FR special characters

* Add more keys to Spanish

* Remove default value shift: false

* Add Norwegian

* Operator precedence 🤦

* Add Italian

* Add Czech

* Move guard statements outside of loop

* Move language name definitions into the keyboard layout files

* Change the locale names to their native language

German->Deutsch et. al.

* Move hold key handling into Go backend analogous to https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt

* Remove trailing whitespace

* Fix

* Add Belgisch Nederlands

* Add JSONRPC handling

* Use useSettingsStore

* Revert "Move hold key handling into Go backend analogous to https://www.kernel.org/doc/Documentation/usb/gadget_hid.txt"

This reverts commit 146cee9.

* Move FeatureFlag to navigation

* Fix: flip Y/Z

* Add useEffect dependencies

* Embolden language

* Add to useCallback dependencies

---------

Co-authored-by: Marc Brooks <IDisposable@gmail.com>

Author: dlorch

config.go

@@ -87,6 +87,7 @@ type Config struct {
 	LocalAuthMode        string                 `json:"localAuthMode"` //TODO: fix it with migration
 	WakeOnLanDevices     []WakeOnLanDevice      `json:"wake_on_lan_devices"`
 	KeyboardMacros       []KeyboardMacro        `json:"keyboard_macros"`
+	KeyboardLayout       string                 `json:"keyboard_layout"`
 	EdidString           string                 `json:"hdmi_edid_string"`
 	ActiveExtension      string                 `json:"active_extension"`
 	DisplayRotation      string                 `json:"display_rotation"`
@@ -109,6 +110,7 @@ var defaultConfig = &Config{
 	ActiveExtension:      "",
 	KeyboardMacros:       []KeyboardMacro{},
 	DisplayRotation:      "270",
+	KeyboardLayout:       "en-US",
 	DisplayMaxBrightness: 64,
 	DisplayDimAfterSec:   120,  // 2 minutes
 	DisplayOffAfterSec:   1800, // 30 minutes

jsonrpc.go

@@ -901,6 +901,18 @@ func rpcSetCloudUrl(apiUrl string, appUrl string) error {
 	return nil
 }
 
+func rpcGetKeyboardLayout() (string, error) {
+	return config.KeyboardLayout, nil
+}
+
+func rpcSetKeyboardLayout(layout string) error {
+	config.KeyboardLayout = layout
+	if err := SaveConfig(); err != nil {
+		return fmt.Errorf("failed to save config: %w", err)
+	}
+	return nil
+}
+
 func getKeyboardMacros() (interface{}, error) {
 	macros := make([]KeyboardMacro, len(config.KeyboardMacros))
 	copy(macros, config.KeyboardMacros)
@@ -1066,6 +1078,8 @@ var rpcHandlers = map[string]RPCHandler{
 	"setUsbDevices":          {Func: rpcSetUsbDevices, Params: []string{"devices"}},
 	"setUsbDeviceState":      {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}},
 	"setCloudUrl":            {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}},
+	"getKeyboardLayout":      {Func: rpcGetKeyboardLayout},
+	"setKeyboardLayout":      {Func: rpcSetKeyboardLayout, Params: []string{"layout"}},
 	"getKeyboardMacros":      {Func: getKeyboardMacros},
 	"setKeyboardMacros":      {Func: setKeyboardMacros, Params: []string{"params"}},
 }

ui/src/components/popovers/PasteModal.tsx

@@ -8,14 +8,21 @@ import { GridCard } from "@components/Card";
 import { TextAreaWithLabel } from "@components/TextArea";
 import { SettingsPageHeader } from "@components/SettingsPageheader";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
-import { useHidStore, useRTCStore, useUiStore } from "@/hooks/stores";
-import { chars, keys, modifiers } from "@/keyboardMappings";
+import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
+import { keys, modifiers } from "@/keyboardMappings";
+import { layouts, chars } from "@/keyboardLayouts";
 import notifications from "@/notifications";
 
 const hidKeyboardPayload = (keys: number[], modifier: number) => {
   return { keys, modifier };
 };
 
+const modifierCode = (shift?: boolean, altRight?: boolean) => {
+  return (shift ? modifiers["ShiftLeft"] : 0)
+       | (altRight ? modifiers["AltRight"] : 0)
+}
+const noModifier = 0
+
 export default function PasteModal() {
   const TextAreaRef = useRef<HTMLTextAreaElement>(null);
   const setPasteMode = useHidStore(state => state.setPasteModeEnabled);
@@ -27,6 +34,18 @@ export default function PasteModal() {
   const [invalidChars, setInvalidChars] = useState<string[]>([]);
   const close = useClose();
 
+  const keyboardLayout = useSettingsStore(state => state.keyboardLayout);
+  const setKeyboardLayout = useSettingsStore(
+    state => state.setKeyboardLayout,
+  );
+
+  useEffect(() => {
+    send("getKeyboardLayout", {}, resp => {
+      if ("error" in resp) return;
+      setKeyboardLayout(resp.result as string);
+    });
+  }, [send, setKeyboardLayout]);
+
   const onCancelPasteMode = useCallback(() => {
     setPasteMode(false);
     setDisableVideoFocusTrap(false);
@@ -37,33 +56,49 @@ export default function PasteModal() {
     setPasteMode(false);
     setDisableVideoFocusTrap(false);
     if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return;
+    if (!keyboardLayout) return;
+    if (!chars[keyboardLayout]) return;
 
     const text = TextAreaRef.current.value;
 
     try {
       for (const char of text) {
-        const { key, shift } = chars[char] ?? {};
+        const { key, shift, altRight, deadKey, accentKey } = chars[keyboardLayout][char]
         if (!key) continue;
 
-        await new Promise<void>((resolve, reject) => {
-          send(
-            "keyboardReport",
-            hidKeyboardPayload([keys[key]], shift ? modifiers["ShiftLeft"] : 0),
-            params => {
-              if ("error" in params) return reject(params.error);
-              send("keyboardReport", hidKeyboardPayload([], 0), params => {
+	const keyz = [ keys[key] ];
+	const modz = [ modifierCode(shift, altRight) ];
+
+	if (deadKey) {
+            keyz.push(keys["Space"]);
+            modz.push(noModifier);
+	}
+	if (accentKey) {
+            keyz.unshift(keys[accentKey.key])
+            modz.unshift(modifierCode(accentKey.shift, accentKey.altRight))
+	}
+
+	for (const [index, kei] of keyz.entries()) {
+          await new Promise<void>((resolve, reject) => {
+            send(
+              "keyboardReport",
+              hidKeyboardPayload([kei], modz[index]),
+              params => {
                 if ("error" in params) return reject(params.error);
-                resolve();
-              });
-            },
-          );
-        });
+                send("keyboardReport", hidKeyboardPayload([], 0), params => {
+                  if ("error" in params) return reject(params.error);
+                  resolve();
+                });
+              },
+            );
+          });
+	}
       }
     } catch (error) {
       console.error(error);
       notifications.error("Failed to paste text");
     }
-  }, [rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteMode]);
+  }, [rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteMode, keyboardLayout]);
 
   useEffect(() => {
     if (TextAreaRef.current) {
@@ -113,7 +148,7 @@ export default function PasteModal() {
                             // @ts-expect-error TS doesn't recognize Intl.Segmenter in some environments
                             [...new Intl.Segmenter().segment(value)]
                               .map(x => x.segment)
-                              .filter(char => !chars[char]),
+                              .filter(char => !chars[keyboardLayout][char]),
                           ),
                         ];
 
@@ -132,6 +167,11 @@ export default function PasteModal() {
                     )}
                   </div>
                 </div>
+		<div className="space-y-4">
+                  <p className="text-xs text-slate-600 dark:text-slate-400">
+                    Sending text using keyboard layout: {layouts[keyboardLayout]}
+                  </p>
+		</div>
               </div>
             </div>
           </div>

ui/src/hooks/stores.ts

@@ -302,6 +302,9 @@ interface SettingsState {
 
   backlightSettings: BacklightSettings;
   setBacklightSettings: (settings: BacklightSettings) => void;
+
+  keyboardLayout: string;
+  setKeyboardLayout: (layout: string) => void;
 }
 
 export const useSettingsStore = create(
@@ -330,6 +333,9 @@ export const useSettingsStore = create(
       },
       setBacklightSettings: (settings: BacklightSettings) =>
         set({ backlightSettings: settings }),
+
+      keyboardLayout: "en-US",
+      setKeyboardLayout: layout => set({ keyboardLayout: layout }),
     }),
     {
       name: "settings",

ui/src/keyboardLayouts.ts

@@ -0,0 +1,45 @@
+import { chars as chars_fr_BE, name as name_fr_BE } from "@/keyboardLayouts/fr_BE"
+import { chars as chars_cs_CZ, name as name_cs_CZ } from "@/keyboardLayouts/cs_CZ"
+import { chars as chars_en_UK, name as name_en_UK } from "@/keyboardLayouts/en_UK"
+import { chars as chars_en_US, name as name_en_US } from "@/keyboardLayouts/en_US"
+import { chars as chars_fr_FR, name as name_fr_FR } from "@/keyboardLayouts/fr_FR"
+import { chars as chars_de_DE, name as name_de_DE } from "@/keyboardLayouts/de_DE"
+import { chars as chars_it_IT, name as name_it_IT } from "@/keyboardLayouts/it_IT"
+import { chars as chars_nb_NO, name as name_nb_NO } from "@/keyboardLayouts/nb_NO"
+import { chars as chars_es_ES, name as name_es_ES } from "@/keyboardLayouts/es_ES"
+import { chars as chars_sv_SE, name as name_sv_SE } from "@/keyboardLayouts/sv_SE"
+import { chars as chars_fr_CH, name as name_fr_CH } from "@/keyboardLayouts/fr_CH"
+import { chars as chars_de_CH, name as name_de_CH } from "@/keyboardLayouts/de_CH"
+
+type KeyInfo = { key: string | number; shift?: boolean, altRight?: boolean }
+export type KeyCombo = KeyInfo & { deadKey?: boolean, accentKey?: KeyInfo }
+
+export const layouts: Record<string, string> = {
+  be_FR: name_fr_BE,
+  cs_CZ: name_cs_CZ,
+  en_UK: name_en_UK,
+  en_US: name_en_US,
+  fr_FR: name_fr_FR,
+  de_DE: name_de_DE,
+  it_IT: name_it_IT,
+  nb_NO: name_nb_NO,
+  es_ES: name_es_ES,
+  sv_SE: name_sv_SE,
+  fr_CH: name_fr_CH,
+  de_CH: name_de_CH,
+}
+
+export const chars: Record<string, Record<string, KeyCombo>> = {
+  be_FR: chars_fr_BE,
+  cs_CZ: chars_cs_CZ,
+  en_UK: chars_en_UK,
+  en_US: chars_en_US,
+  fr_FR: chars_fr_FR,
+  de_DE: chars_de_DE,
+  it_IT: chars_it_IT,
+  nb_NO: chars_nb_NO,
+  es_ES: chars_es_ES,
+  sv_SE: chars_sv_SE,
+  fr_CH: chars_fr_CH,
+  de_CH: chars_de_CH,
+};