bpo-9263: _PyObject_Dump() detects freed memory (GH-10061) (GH-10662) (GH-10663)

* bpo-9263: _PyObject_Dump() detects freed memory (GH-10061)

_PyObject_Dump() now uses an heuristic to check if the object memory
has been freed: log "<freed object>" in that case.

The heuristic rely on the debug hooks on Python memory allocators
which fills the memory with DEADBYTE (0xDB) when memory is
deallocated. Use PYTHONMALLOC=debug to always enable these debug
hooks.

(cherry picked from commit 82af0b6)

* bpo-9263: Fix _PyObject_Dump() for freed object (#10661)

If _PyObject_Dump() detects that the object is freed, don't try to
dump it (exit immediately).

Enhance also _PyObject_IsFreed(): it now detects if the pointer
itself looks like freed memory.

(cherry picked from commit 2cf5d32)
(cherry picked from commit 95036ea)

Author: vstinner

Include/object.h

@@ -520,6 +520,7 @@ struct _Py_Identifier;
 PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);
 PyAPI_FUNC(void) _Py_BreakPoint(void);
 PyAPI_FUNC(void) _PyObject_Dump(PyObject *);
+PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *);
 #endif
 PyAPI_FUNC(PyObject *) PyObject_Repr(PyObject *);
 PyAPI_FUNC(PyObject *) PyObject_Str(PyObject *);

Include/pymem.h

@@ -59,7 +59,9 @@ PyAPI_FUNC(int) _PyTraceMalloc_Untrack(
 PyAPI_FUNC(PyObject*) _PyTraceMalloc_GetTraceback(
     _PyTraceMalloc_domain_t domain,
     uintptr_t ptr);
-#endif   /* !Py_LIMITED_API */
+
+PyAPI_FUNC(int) _PyMem_IsFreed(void *ptr, size_t size);
+#endif   /* !defined(Py_LIMITED_API) */
 
 
 /* BEWARE:

Objects/object.c

@@ -422,40 +422,71 @@ _Py_BreakPoint(void)
 }
 
 
+/* Heuristic checking if the object memory has been deallocated.
+   Rely on the debug hooks on Python memory allocators which fills the memory
+   with DEADBYTE (0xDB) when memory is deallocated.
+
+   The function can be used to prevent segmentation fault on dereferencing
+   pointers like 0xdbdbdbdbdbdbdbdb. Such pointer is very unlikely to be mapped
+   in memory. */
+int
+_PyObject_IsFreed(PyObject *op)
+{
+    uintptr_t ptr = (uintptr_t)op;
+    if (_PyMem_IsFreed(&ptr, sizeof(ptr))) {
+        return 1;
+    }
+    int freed = _PyMem_IsFreed(&op->ob_type, sizeof(op->ob_type));
+    /* ignore op->ob_ref: the value can have be modified
+       by Py_INCREF() and Py_DECREF(). */
+#ifdef Py_TRACE_REFS
+    freed &= _PyMem_IsFreed(&op->_ob_next, sizeof(op->_ob_next));
+    freed &= _PyMem_IsFreed(&op->_ob_prev, sizeof(op->_ob_prev));
+#endif
+    return freed;
+}
+
+
 /* For debugging convenience.  See Misc/gdbinit for some useful gdb hooks */
 void
 _PyObject_Dump(PyObject* op)
 {
-    if (op == NULL)
-        fprintf(stderr, "NULL\n");
-    else {
-#ifdef WITH_THREAD
-        PyGILState_STATE gil;
-#endif
-        PyObject *error_type, *error_value, *error_traceback;
+    if (op == NULL) {
+        fprintf(stderr, "<NULL object>\n");
+        fflush(stderr);
+        return;
+    }
 
-        fprintf(stderr, "object  : ");
-#ifdef WITH_THREAD
-        gil = PyGILState_Ensure();
-#endif
+    if (_PyObject_IsFreed(op)) {
+        /* It seems like the object memory has been freed:
+           don't access it to prevent a segmentation fault. */
+        fprintf(stderr, "<freed object>\n");
+        return;
+    }
 
-        PyErr_Fetch(&error_type, &error_value, &error_traceback);
-        (void)PyObject_Print(op, stderr, 0);
-        PyErr_Restore(error_type, error_value, error_traceback);
+    PyGILState_STATE gil;
+    PyObject *error_type, *error_value, *error_traceback;
 
-#ifdef WITH_THREAD
-        PyGILState_Release(gil);
-#endif
-        /* XXX(twouters) cast refcount to long until %zd is
-           universally available */
-        fprintf(stderr, "\n"
-            "type    : %s\n"
-            "refcount: %ld\n"
-            "address : %p\n",
-            Py_TYPE(op)==NULL ? "NULL" : Py_TYPE(op)->tp_name,
-            (long)op->ob_refcnt,
-            op);
-    }
+    fprintf(stderr, "object  : ");
+    fflush(stderr);
+    gil = PyGILState_Ensure();
+
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+    (void)PyObject_Print(op, stderr, 0);
+    fflush(stderr);
+    PyErr_Restore(error_type, error_value, error_traceback);
+
+    PyGILState_Release(gil);
+    /* XXX(twouters) cast refcount to long until %zd is
+       universally available */
+    fprintf(stderr, "\n"
+        "type    : %s\n"
+        "refcount: %ld\n"
+        "address : %p\n",
+        Py_TYPE(op)==NULL ? "NULL" : Py_TYPE(op)->tp_name,
+        (long)op->ob_refcnt,
+        op);
+    fflush(stderr);
 }
 
 PyObject *

Objects/obmalloc.c

@@ -1907,6 +1907,23 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
     return _PyMem_DebugRawAlloc(1, ctx, nbytes);
 }
 
+
+/* Heuristic checking if the memory has been freed. Rely on the debug hooks on
+   Python memory allocators which fills the memory with DEADBYTE (0xDB) when
+   memory is deallocated. */
+int
+_PyMem_IsFreed(void *ptr, size_t size)
+{
+    unsigned char *bytes = ptr;
+    for (size_t i=0; i < size; i++) {
+        if (bytes[i] != DEADBYTE) {
+            return 0;
+        }
+    }
+    return 1;
+}
+
+
 /* The debug free first checks the 2*SST bytes on each end for sanity (in
    particular, that the FORBIDDENBYTEs with the api ID are still intact).
    Then fills the original bytes with DEADBYTE.