gh-91048: Refactor and optimize remote debugging module

Include/cpython/pystate.h

@@ -61,6 +61,8 @@ typedef struct _stack_chunk {
     PyObject * data[1]; /* Variable sized */
 } _PyStackChunk;
 
+/* Minimum size of data stack chunk */
+#define _PY_DATA_STACK_CHUNK_SIZE (16*1024)
 struct _ts {
     /* See Python/ceval.c for comments explaining most fields */
 

Include/internal/pycore_debug_offsets.h

@@ -54,11 +54,13 @@ extern "C" {
 # define _Py_Debug_Free_Threaded 1
 # define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc)
 # define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index)
+# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation)
 #else
 # define _Py_Debug_gilruntimestate_enabled 0
 # define _Py_Debug_Free_Threaded 0
 # define _Py_Debug_code_object_co_tlbc 0
 # define _Py_Debug_interpreter_frame_tlbc_index 0
+# define _Py_Debug_interpreter_state_tlbc_generation 0
 #endif
 
 
@@ -89,6 +91,8 @@ typedef struct _Py_DebugOffsets {
         uint64_t gil_runtime_state_enabled;
         uint64_t gil_runtime_state_locked;
         uint64_t gil_runtime_state_holder;
+        uint64_t code_object_generation;
+        uint64_t tlbc_generation;
     } interpreter_state;
 
     // Thread state offset;
@@ -216,6 +220,11 @@ typedef struct _Py_DebugOffsets {
         uint64_t gi_frame_state;
     } gen_object;
 
+    struct _llist_node {
+        uint64_t next;
+        uint64_t prev;
+    } llist_node;
+
     struct _debugger_support {
         uint64_t eval_breaker;
         uint64_t remote_debugger_support;
@@ -251,6 +260,8 @@ typedef struct _Py_DebugOffsets {
         .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \
         .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \
         .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \
+        .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \
+        .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \
     }, \
     .thread_state = { \
         .size = sizeof(PyThreadState), \
@@ -347,6 +358,10 @@ typedef struct _Py_DebugOffsets {
         .gi_iframe = offsetof(PyGenObject, gi_iframe), \
         .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \
     }, \
+    .llist_node = { \
+        .next = offsetof(struct llist_node, next), \
+        .prev = offsetof(struct llist_node, prev), \
+    }, \
     .debugger_support = { \
         .eval_breaker = offsetof(PyThreadState, eval_breaker), \
         .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support),  \

Include/internal/pycore_global_strings.h

@@ -286,6 +286,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(alias)
         STRUCT_FOR_ID(align)
         STRUCT_FOR_ID(all)
+        STRUCT_FOR_ID(all_threads)
         STRUCT_FOR_ID(allow_code)
         STRUCT_FOR_ID(any)
         STRUCT_FOR_ID(append)

Include/internal/pycore_interp_structs.h

@@ -726,6 +726,10 @@ typedef struct _PyIndexPool {
 
     // Next index to allocate if no free indices are available
     int32_t next_index;
+
+    // Generation counter incremented on thread creation/destruction
+    // Used for TLBC cache invalidation in remote debugging
+    uint32_t tlbc_generation;
 } _PyIndexPool;
 
 typedef union _Py_unique_id_entry {
@@ -843,6 +847,8 @@ struct _is {
     /* The per-interpreter GIL, which might not be used. */
     struct _gil_runtime_state _gil;
 
+    uint64_t _code_object_generation;
+
      /* ---------- IMPORTANT ---------------------------
      The fields above this line are declared as early as
      possible to facilitate out-of-process observability

Lib/asyncio/tools.py

@@ -1,11 +1,10 @@
 """Tools to analyze tasks running in asyncio programs."""
 
-from dataclasses import dataclass
 from collections import defaultdict
 from itertools import count
 from enum import Enum
 import sys
-from _remote_debugging import get_all_awaited_by
+from _remote_debugging import RemoteUnwinder
 
 
 class NodeType(Enum):
@@ -118,6 +117,11 @@ def dfs(v):
 
 
 # ─── PRINT TREE FUNCTION ───────────────────────────────────────
+def get_all_awaited_by(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_all_awaited_by()
+
+
 def build_async_tree(result, task_emoji="(T)", cor_emoji=""):
     """
     Build a list of strings for pretty-print an async call tree.

Lib/test/test_external_inspection.py

@@ -4,6 +4,7 @@
 import importlib
 import sys
 import socket
+import threading
 from asyncio import staggered, taskgroups
 from unittest.mock import ANY
 from test.support import os_helper, SHORT_TIMEOUT, busy_retry
@@ -16,9 +17,7 @@
 
 try:
     from _remote_debugging import PROCESS_VM_READV_SUPPORTED
-    from _remote_debugging import get_stack_trace
-    from _remote_debugging import get_async_stack_trace
-    from _remote_debugging import get_all_awaited_by
+    from _remote_debugging import RemoteUnwinder
 except ImportError:
     raise unittest.SkipTest("Test only runs when _remote_debugging is available")
 
@@ -34,7 +33,23 @@ def _make_test_script(script_dir, script_basename, source):
 )
 
 
+def get_stack_trace(pid):
+    unwinder = RemoteUnwinder(pid, all_threads=True)
+    return unwinder.get_stack_trace()
+
+
+def get_async_stack_trace(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_async_stack_trace()
+
+
+def get_all_awaited_by(pid):
+    unwinder = RemoteUnwinder(pid)
+    return unwinder.get_all_awaited_by()
+
+
 class TestGetStackTrace(unittest.TestCase):
+    maxDiff = None
 
     @skip_if_not_supported
     @unittest.skipIf(
@@ -46,7 +61,7 @@ def test_remote_stack_trace(self):
         port = find_unused_port()
         script = textwrap.dedent(
             f"""\
-            import time, sys, socket
+            import time, sys, socket, threading
             # Connect to the test process
             sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             sock.connect(('localhost', {port}))
@@ -55,13 +70,16 @@ def bar():
                 for x in range(100):
                     if x == 50:
                         baz()
+
             def baz():
                 foo()
 
             def foo():
-                sock.sendall(b"ready"); time.sleep(10_000)  # same line number
+                sock.sendall(b"ready:thread\\n"); time.sleep(10_000)  # same line number
 
-            bar()
+            t = threading.Thread(target=bar)
+            t.start()
+            sock.sendall(b"ready:main\\n"); t.join()  # same line number
             """
         )
         stack_trace = None
@@ -82,8 +100,9 @@ def foo():
                 p = subprocess.Popen([sys.executable, script_name])
                 client_socket, _ = server_socket.accept()
                 server_socket.close()
-                response = client_socket.recv(1024)
-                self.assertEqual(response, b"ready")
+                response = b""
+                while b"ready:main" not in response or b"ready:thread" not in response:
+                    response += client_socket.recv(1024)
                 stack_trace = get_stack_trace(p.pid)
             except PermissionError:
                 self.skipTest("Insufficient permissions to read the stack trace")
@@ -94,13 +113,23 @@ def foo():
                 p.terminate()
                 p.wait(timeout=SHORT_TIMEOUT)
 
-            expected_stack_trace = [
-                ("foo", script_name, 14),
-                ("baz", script_name, 11),
+            thread_expected_stack_trace = [
+                ("foo", script_name, 15),
+                ("baz", script_name, 12),
                 ("bar", script_name, 9),
-                ("<module>", script_name, 16),
+                ('Thread.run', threading.__file__, ANY)
             ]
-            self.assertEqual(stack_trace, expected_stack_trace)
+            # Is possible that there are more threads, so we check that the
+            # expected stack traces are in the result (looking at you Windows!)
+            self.assertIn((ANY, thread_expected_stack_trace), stack_trace)
+
+            # Check that the main thread stack trace is in the result
+            frame = ("<module>", script_name, 19)
+            for _, stack in stack_trace:
+                if frame in stack:
+                    break
+            else:
+                self.fail("Main thread stack trace not found in result")
 
     @skip_if_not_supported
     @unittest.skipIf(
@@ -700,13 +729,28 @@ async def main():
     )
     def test_self_trace(self):
         stack_trace = get_stack_trace(os.getpid())
+        # Is possible that there are more threads, so we check that the
+        # expected stack traces are in the result (looking at you Windows!)
+        this_tread_stack = None
+        for thread_id, stack in stack_trace:
+            if thread_id == threading.get_native_id():
+                this_tread_stack = stack
+                break
+        self.assertIsNotNone(this_tread_stack)
         self.assertEqual(
-            stack_trace[0],
-            (
-                "TestGetStackTrace.test_self_trace",
-                __file__,
-                self.test_self_trace.__code__.co_firstlineno + 6,
-            ),
+            stack[:2],
+            [
+                (
+                    "get_stack_trace",
+                    __file__,
+                    get_stack_trace.__code__.co_firstlineno + 2,
+                ),
+                (
+                    "TestGetStackTrace.test_self_trace",
+                    __file__,
+                    self.test_self_trace.__code__.co_firstlineno + 6,
+                ),
+            ]
         )
 
 

Makefile.pre.in

@@ -1206,6 +1206,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/unicodeobject.h \
 		$(srcdir)/Include/warnings.h \
 		$(srcdir)/Include/weakrefobject.h \
+		$(srcdir)/Python/remote_debug.h \
 		\
 		pyconfig.h \
 		$(PARSER_HEADERS) \