GC crash _PyObject_AssertFailed with pdb

[kumaraditya303]

Reproducer:

def func():
    print(
      42
    )

import pdb; pdb.set_trace()
func()

Crash:

@kumaraditya303 ➜ /workspaces/cpython (main ✗) $ ./python main.py 
> /workspaces/cpython/main.py(7)<module>()
-> func()
(Pdb) s
--Call--
> /workspaces/cpython/main.py(1)func()
-> def func():
(Pdb) n
> /workspaces/cpython/main.py(2)func()
-> print(
(Pdb) n
> /workspaces/cpython/main.py(3)func()
-> 42
(Pdb) j 1
> /workspaces/cpython/main.py(1)func()
-> def func():
(Pdb) exit
Traceback (most recent call last):
  File "/workspaces/cpython/main.py", line 7, in <module>
    func()
    ^^^^^^
  File "/workspaces/cpython/main.py", line 1, in func
    def func():
    
  File "/workspaces/cpython/main.py", line 1, in func
    def func():
    
  File "/workspaces/cpython/Lib/bdb.py", line 90, in trace_dispatch
    return self.dispatch_line(frame)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/workspaces/cpython/Lib/bdb.py", line 115, in dispatch_line
    if self.quitting: raise BdbQuit
                      ^^^^^^^^^^^^^
bdb.BdbQuit
Modules/gcmodule.c:113: gc_decref: Assertion "gc_get_refs(g) > 0" failed: refcount is too small
Enable tracemalloc to get the memory block allocation traceback

object address  : 0x7f18ff3b81d0
object refcount : 2
object type     : 0x55cfef2a6460
object type name: builtin_function_or_method
object repr     : <built-in function print>

Fatal Python error: _PyObject_AssertFailed: _PyObject_AssertFailed
Python runtime state: finalizing (tstate=0x000055cfef3bca48)

Current thread 0x00007f18ff707280 (most recent call first):
  Garbage-collecting
  <no Python frame>
Aborted (core dumped)

GDB stack trace (main)

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
#1  0x00007ffff7c71859 in __GI_abort () at abort.c:79
#2  0x000055555585db52 in fatal_error_exit (status=-1) at Python/pylifecycle.c:2614
#3  fatal_error (fd=2, header=header@entry=1, prefix=<optimized out>, 
    prefix@entry=0x55555594cd70 <__func__.19026> "_PyObject_AssertFailed", msg=<optimized out>, 
    msg@entry=0x55555594c575 "_PyObject_AssertFailed", status=status@entry=-1)
    at Python/pylifecycle.c:2795
#4  0x000055555585dbb4 in _Py_FatalErrorFunc (
    func=func@entry=0x55555594cd70 <__func__.19026> "_PyObject_AssertFailed", 
    msg=msg@entry=0x55555594c575 "_PyObject_AssertFailed") at Python/pylifecycle.c:2811
#5  0x00005555557431d2 in _PyObject_AssertFailed (
    obj=obj@entry=<built-in method print of module object at remote 0x7ffff78f32f0>, 
    expr=expr@entry=0x5555559ab18c "gc_get_refs(g) > 0", 
    msg=msg@entry=0x55555594c5eb "refcount is too small", 
    file=file@entry=0x5555559ab11c "Modules/gcmodule.c", line=line@entry=113, 
    function=function@entry=0x5555559ab6c8 <__func__.18373> "gc_decref") at Objects/object.c:2360
#6  0x000055555588ad50 in gc_decref (g=0x7ffff79001c0) at Modules/gcmodule.c:116
#7  visit_decref (op=<built-in method print of module object at remote 0x7ffff78f32f0>, 
    parent=0x7ffff7900ad0) at Modules/gcmodule.c:459
#8  0x00005555557249a5 in dict_traverse (op=<optimized out>, visit=0x55555588ac96 <visit_decref>, 
    arg=0x7ffff7900ad0) at Objects/dictobject.c:3547
#9  0x000055555588a741 in subtract_refs (
    containers=containers@entry=0x555555b869a8 <_PyRuntime+53896>) at Modules/gcmodule.c:478
#10 0x000055555588be95 in deduce_unreachable (unreachable=0x7fffffffc1d0, 
    base=0x555555b869a8 <_PyRuntime+53896>) at Modules/gcmodule.c:1100
#11 gc_collect_main (tstate=tstate@entry=0x555555ba0a48 <_PyRuntime+160552>, 
    generation=generation@entry=2, n_collected=n_collected@entry=0x7fffffffc258, 
    n_uncollectable=n_uncollectable@entry=0x7fffffffc260, nofail=nofail@entry=0)
    at Modules/gcmodule.c:1226

Python versions tested:

• Python 3.11.0b3+ 41e4b42 (remotes/upstream/3.11:41e4b426ce, Jun 24 2022, 11:58:10) [GCC 9.4.0] on linux
• Python 3.12.0a0 51fd4de (heads/main:51fd4de101, Jun 24 2022, 12:02:08) [GCC 9.4.0] on linux

cc @markshannon @pablogsal

[tiran]

I can reproduce the issue in a pydebug build:

../../Modules/gcmodule.c:113: gc_decref: Assertion "gc_get_refs(g) > 0" failed: refcount is too small
Enable tracemalloc to get the memory block allocation traceback

object address  : 0x7fffea698230
object refcount : 2
object type     : 0x7ccfa0
object type name: builtin_function_or_method
object repr     : <built-in function print>

Fatal Python error: _PyObject_AssertFailed: _PyObject_AssertFailed
Python runtime state: finalizing (tstate=0x00000000008e2fa8)

Current thread 0x00007ffff7cb7740 (most recent call first):
  Garbage-collecting
  <no Python frame>

Program received signal SIGABRT, Aborted.

(gdb) bt
#0  0x00007ffff7d48c4c in __pthread_kill_implementation () from /lib64/libc.so.6
#1  0x00007ffff7cf89c6 in raise () from /lib64/libc.so.6
#2  0x00007ffff7ce27f4 in abort () from /lib64/libc.so.6
#3  0x00000000005b55dc in fatal_error_exit (status=<optimized out>) at ../../Python/pylifecycle.c:2614
#4  0x00000000005b69da in fatal_error (fd=2, header=header@entry=1, prefix=prefix@entry=0x6799b0 <__func__.1> "_PyObject_AssertFailed", msg=msg@entry=0x67985d "_PyObject_AssertFailed", status=status@entry=-1)
    at ../../Python/pylifecycle.c:2795
#5  0x00000000005b6a3b in _Py_FatalErrorFunc (func=func@entry=0x6799b0 <__func__.1> "_PyObject_AssertFailed", msg=msg@entry=0x67985d "_PyObject_AssertFailed") at ../../Python/pylifecycle.c:2811
#6  0x00000000004e855f in _PyObject_AssertFailed (obj=<built-in method print of module object at remote 0x7fffea68b350>, expr=expr@entry=0x70d394 "gc_get_refs(g) > 0", 
    msg=msg@entry=0x6798a9 "refcount is too small", file=file@entry=0x70d1fc "../../Modules/gcmodule.c", line=line@entry=113, function=function@entry=0x70d7c8 <__func__.26> "gc_decref")
    at ../../Objects/object.c:2360
#7  0x00000000005d9e20 in gc_decref (g=<optimized out>) at ../../Modules/gcmodule.c:113
#8  0x00000000005da4a5 in visit_decref (op=<built-in method print of module object at remote 0x7fffea68b350>, parent=0x7fffea698b30) at ../../Modules/gcmodule.c:459
#9  0x00000000004d347e in dict_traverse (op=<optimized out>, visit=0x5da446 <visit_decref>, arg=0x7fffea698b30) at ../../Objects/dictobject.c:3547
#10 0x00000000005d8d78 in subtract_refs (containers=containers@entry=0x8c8f08 <_PyRuntime+53896>) at ../../Modules/gcmodule.c:478
#11 0x00000000005d9f16 in deduce_unreachable (base=base@entry=0x8c8f08 <_PyRuntime+53896>, unreachable=unreachable@entry=0x7fffffffd410) at ../../Modules/gcmodule.c:1100
#12 0x00000000005da956 in gc_collect_main (tstate=tstate@entry=0x8e2fa8 <_PyRuntime+160552>, generation=generation@entry=2, n_collected=n_collected@entry=0x7fffffffd468, 
    n_uncollectable=n_uncollectable@entry=0x7fffffffd460, nofail=nofail@entry=0) at ../../Modules/gcmodule.c:1226
#13 0x00000000005dae4e in gc_collect_with_callback (tstate=tstate@entry=0x8e2fa8 <_PyRuntime+160552>, generation=generation@entry=2) at ../../Modules/gcmodule.c:1400
#14 0x00000000005db3c9 in PyGC_Collect () at ../../Modules/gcmodule.c:2086
#15 0x00000000005b6601 in Py_FinalizeEx () at ../../Python/pylifecycle.c:1823
#16 0x00000000005d8c25 in Py_RunMain () at ../../Modules/main.c:691
#17 0x00000000005d8c75 in pymain_main (args=args@entry=0x7fffffffd550) at ../../Modules/main.c:719
#18 0x00000000005d8cfa in Py_BytesMain (argc=<optimized out>, argv=<optimized out>) at ../../Modules/main.c:743
#19 0x000000000041d73f in main (argc=<optimized out>, argv=<optimized out>) at ../../Programs/python.c:15

(gdb) p op
$14 = <built-in method print of module object at remote 0x7fffea68b350>
(gdb) p *((PyGC_Head *)(((char *)(op))-sizeof(PyGC_Head)))
$15 = {_gc_next = 140737126171264, _gc_prev = 2}

[markshannon]

This might be related to #94438

[tiran]

It's seems to be a ref counting issue. The jump and frame_setlineno seems to reduce the reference count by one. Please notice that the case without jump starts interpreter shutdown with gc: 2, refcnt: 3. With jump 1 shutdown starts with gc: 1, refcnt: 2.

ref counts without jump

> tc.py(11)<module>()
-> func()
(Pdb) s
--Call--
> tc.py(1)func()
-> def func():
(Pdb) n
> tc.py(2)func()
-> print(
(Pdb) n
> tc.py(3)func()
-> 42
(Pdb) exit
...
bdb.BdbQuit
gc: 2, refcnt: 3
gc: 1, refcnt: 3
gc: 2, refcnt: 3
gc: 1, refcnt: 3
gc: 1, refcnt: 1
gc: 1, refcnt: 1

ref counts with jump

> tc.py(11)<module>()
-> func()
(Pdb) s
--Call--
> tc.py(1)func()
-> def func():
(Pdb) n
> tc.py(2)func()
-> print(
(Pdb) n
> tc.py(3)func()
-> 42
(Pdb) j 1
> tc.py(1)func()
-> def func():
(Pdb) exit
...
bdb.BdbQuit
gc: 1, refcnt: 2
../../Modules/gcmodule.c:113: gc_decref: Assertion "gc_get_refs(g) > 0" failed: refcount is too small

debug hack

--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -249,6 +249,10 @@ meth_traverse(PyCFunctionObject *m, visitproc visit, void *arg)
     Py_VISIT(PyCFunction_GET_CLASS(m));
     Py_VISIT(m->m_self);
     Py_VISIT(m->m_module);
+    if (strcmp(m->m_ml->ml_name, "print") == 0) {
+        #define AS_GC(o) ((PyGC_Head *)(((char *)(o))-sizeof(PyGC_Head)))
+        fprintf(stderr, "gc: %ld, refcnt: %ld\n", (Py_ssize_t)(AS_GC(m)->_gc_prev >> _PyGC_PREV_SHIFT), Py_REFCNT(m));
+    }
     return 0;
 }

[tiran]

The decref in frame_stack_pop seems to be the culprit. If I drop the decref, then pdb session no longer crashes. refleak checks for test_frame test_code test_traceback test_pdb do not show any refleaks either.

static void
frame_stack_pop(PyFrameObject *f)
{
    PyObject *v = _PyFrame_StackPop(f->f_frame);
    Py_XDECREF(v);
}

[iritkatriel]

I'm not sure how much that function is covered by tests. Can you verify whether it is?

[tiran]

frame_stack_pop() is only used by frame_setlineno() conditionally. I guess it is not covered by any test case at all. Otherwise we would have seen crashers.

[pablogsal]

The issue may be a bit more general. Notice the jump is going back to the bytecode so we need to ensure that jumps and continues are always idempotent