Windows keyboard layout handling: get the current layout from the parent terminal process (#3786)

Author: ForNeVeR

PSReadLine/Keys.cs

@@ -115,13 +115,14 @@ public override int GetHashCode()
         public static extern uint MapVirtualKey(ConsoleKey uCode, uint uMapType);
 
         [DllImport("user32.dll", CharSet = CharSet.Unicode)]
-        public static extern int ToUnicode(
+        public static extern int ToUnicodeEx(
             ConsoleKey uVirtKey,
             uint uScanCode,
             byte[] lpKeyState,
             [MarshalAs(UnmanagedType.LPArray)] [Out] char[] chars,
             int charMaxCount,
-            uint flags);
+            uint flags,
+            IntPtr dwhkl);
 
         static readonly ThreadLocal<char[]> toUnicodeBuffer = new ThreadLocal<char[]>(() => new char[2]);
         static readonly ThreadLocal<byte[]> toUnicodeStateBuffer = new ThreadLocal<byte[]>(() => new byte[256]);
@@ -147,7 +148,9 @@ internal static void TryGetCharFromConsoleKey(ConsoleKeyInfo key, ref char resul
             {
                 flags |= (1 << 2); /* If bit 2 is set, keyboard state is not changed (Windows 10, version 1607 and newer) */
             }
-            int charCount = ToUnicode(virtualKey, scanCode, state, chars, chars.Length, flags);
+
+            IntPtr layout = PlatformWindows.GetConsoleKeyboardLayout();
+            int charCount = ToUnicodeEx(virtualKey, scanCode, state, chars, chars.Length, flags, layout);
 
             if (charCount == 1)
             {

PSReadLine/PlatformWindows.cs

@@ -140,6 +140,7 @@ internal static IConsole OneTimeInit(PSConsoleReadLine singleton)
         var breakHandlerGcHandle = GCHandle.Alloc(new BreakHandler(OnBreak));
         SetConsoleCtrlHandler((BreakHandler)breakHandlerGcHandle.Target, true);
         _enableVtOutput = !Console.IsOutputRedirected && SetConsoleOutputVirtualTerminalProcessing();
+        _terminalOwnerThreadId = GetTerminalOwnerThreadId();
 
         return _enableVtOutput ? new VirtualTerminal() : new LegacyWin32Console();
     }
@@ -1015,4 +1016,105 @@ private static void TerminateStragglers()
             }
         }
     }
+
+    private static uint _terminalOwnerThreadId;
+
+    /// <remarks>
+    /// This method helps to find the owner thread of the terminal window used by this pwsh instance,
+    /// by looking for a parent process whose <see cref="Process.MainWindowHandle"/>) is visible.
+    ///
+    /// The terminal process is not always the direct parent of the current process, but may be higher
+    /// in the process tree in case this pwsh process is a child of some other console process.
+    ///
+    /// This works well in Windows Terminal (with profile), IntelliJ and VSCode.
+    /// It doesn't work when PowerShell runs in conhost, or when it gets started from Start Menu with
+    /// Windows Terminal as the default terminal application (without profile).
+    /// </remarks>
+    private static uint GetTerminalOwnerThreadId()
+    {
+        try
+        {
+            // The window handle returned by `GetConsoleWindow` is not the correct terminal/console window for us
+            // to query about the keyboard layout change. It's the window created for a console application, such
+            // as `cmd` or `pwsh`, so its owner process in those cases will be `cmd` or `pwsh`.
+            //
+            // When we are running with conhost, this window is visible, but it's not what we want and needs to be
+            // filtered out. When running with conhost, we want the window owned by the conhost. But unfortunately,
+            // there is no reliable way to get the conhost process that is associated with the current pwsh, since
+            // it's not in the parent chain of the process tree.
+            // So, this method is supposed to always fail when running with conhost.
+            IntPtr wrongHandle = GetConsoleWindow();
+
+            // Limit for parent process walk-up for not getting stuck in a loop (possible in case pid reuse).
+            const int iterationLimit = 20;
+            var process = Process.GetCurrentProcess();
+
+            for (int i = 0; i < iterationLimit; ++i)
+            {
+                if (process.ProcessName is "explorer")
+                {
+                    // We've reached the root of the process tree. This can happen when PowerShell was started
+                    // from Start Menu with Windows Terminal as the default terminal application.
+                    // The `explorer` process has a visible window, but it doesn't help for getting the layout
+                    // change. Again, we need to find the terminal window owner.
+                    break;
+                }
+
+                IntPtr mainWindowHandle = process.MainWindowHandle;
+                if (mainWindowHandle == wrongHandle)
+                {
+                    // This can only happen when we are running with conhost.
+                    // Break early because the terminal owner process is not in the parent chain in this scenario.
+                    break;
+                }
+
+                if (mainWindowHandle != IntPtr.Zero && IsWindowVisible(mainWindowHandle))
+                {
+                    // The window is visible, so it's likely the terminal window.
+                    return GetWindowThreadProcessId(process.MainWindowHandle, out _);
+                }
+
+                // When reaching here, the main window of the process:
+                //   - doesn't exist, or
+                //   - exists but invisible
+                // So, this is likely not a terminal process.
+                // Now we get its parent process and continue with the check.
+                int parentId = GetParentPid(process);
+                process = Process.GetProcessById(parentId);
+            }
+        }
+        catch (Exception)
+        {
+            // No access to the process, or the process is already dead.
+            // Either way, we cannot determine the owner thread of the terminal window.
+        }
+
+        // We could not find the owner thread/process of the terminal window in following scenarios:
+        //  1. pwsh is running with conhost.
+        //     This happens when conhost is set as the default terminal application, and a user starts pwsh
+        //     from the Start Menu, or with `win+r` (run code) and etc.
+        //
+        //  2. pwsh is running with Windows Terminal, but was not started from a Windows Terminal profile.
+        //     This happens when Windows Terminal is set as the default terminal application, and a user
+        //     starts pwsh from the Start Menu, or with `win+r` (run code) and etc.
+        //     The `WindowsTerminal` process is not in the parent process chain in this case.
+        //
+        //  3. pwsh's parent process chain is broken -- a parent was terminated so we cannot walk up the chain.
+        return 0;
+    }
+
+    internal static IntPtr GetConsoleKeyboardLayout()
+    {
+        return GetKeyboardLayout(_terminalOwnerThreadId);
+    }
+
+    [DllImport("user32.dll")]
+    [return: MarshalAs(UnmanagedType.Bool)]
+    private static extern bool IsWindowVisible(IntPtr hWnd);
+
+    [DllImport("User32.dll", SetLastError = true)]
+    private static extern IntPtr GetKeyboardLayout(uint idThread);
+
+    [DllImport("user32.dll", SetLastError = true)]
+    private static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint proccess);
 }