gh-118789: Add PyUnstable_Object_ClearWeakRefsNoCallbacks (#118807)

This exposes `PyUnstable_Object_ClearWeakRefsNoCallbacks` as an unstable
C-API function to provide a thread-safe mechanism for clearing weakrefs
without executing callbacks.

Some C-API extensions need to clear weakrefs without calling callbacks,
such as after running finalizers like we do in subtype_dealloc.
Previously they could use `_PyWeakref_ClearRef` on each weakref, but
that's not thread-safe in the free-threaded build.

Co-authored-by: Petr Viktorin <encukou@gmail.com>

Author: colesbury

Doc/c-api/weakref.rst

@@ -96,3 +96,19 @@ as much as it can.
    This iterates through the weak references for *object* and calls callbacks
    for those references which have one. It returns when all callbacks have
    been attempted.
+
+
+.. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object)
+
+   Clears the weakrefs for *object* without calling the callbacks.
+
+   This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler
+   for types with finalizers (i.e., :meth:`~object.__del__`).  The handler for
+   those objects first calls :c:func:`PyObject_ClearWeakRefs` to clear weakrefs
+   and call their callbacks, then the finalizer, and finally this function to
+   clear any weakrefs that may have been created by the finalizer.
+
+   In most circumstances, it's more appropriate to use
+   :c:func:`PyObject_ClearWeakRefs` to clear weakrefs instead of this function.
+
+   .. versionadded:: 3.13

Include/cpython/object.h

@@ -288,6 +288,8 @@ PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *);
 PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *);
 PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
 
+PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *);
+
 /* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes
    dict as the last parameter. */
 PyAPI_FUNC(PyObject *)

Include/internal/pycore_weakref.h

@@ -109,7 +109,7 @@ extern Py_ssize_t _PyWeakref_GetWeakrefCount(PyObject *obj);
 
 // Clear all the weak references to obj but leave their callbacks uncalled and
 // intact.
-extern void _PyWeakref_ClearWeakRefsExceptCallbacks(PyObject *obj);
+extern void _PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj);
 
 PyAPI_FUNC(int) _PyWeakref_IsDead(PyObject *weakref);
 

Lib/test/test_capi/test_object.py

@@ -103,5 +103,33 @@ def testPyObjectPrintOSError(self):
         with self.assertRaises(OSError):
             _testcapi.pyobject_print_os_error(output_filename)
 
+
+class ClearWeakRefsNoCallbacksTest(unittest.TestCase):
+    """Test PyUnstable_Object_ClearWeakRefsNoCallbacks"""
+    def test_ClearWeakRefsNoCallbacks(self):
+        """Ensure PyUnstable_Object_ClearWeakRefsNoCallbacks works"""
+        import weakref
+        import gc
+        class C:
+            pass
+        obj = C()
+        messages = []
+        ref = weakref.ref(obj, lambda: messages.append("don't add this"))
+        self.assertIs(ref(), obj)
+        self.assertFalse(messages)
+        _testcapi.pyobject_clear_weakrefs_no_callbacks(obj)
+        self.assertIsNone(ref())
+        gc.collect()
+        self.assertFalse(messages)
+
+    def test_ClearWeakRefsNoCallbacks_no_weakref_support(self):
+        """Don't fail on objects that don't support weakrefs"""
+        import weakref
+        obj = object()
+        with self.assertRaises(TypeError):
+            ref = weakref.ref(obj)
+        _testcapi.pyobject_clear_weakrefs_no_callbacks(obj)
+
+
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/C API/2024-05-08-21-57-50.gh-issue-118789.Ni4UQx.rst

@@ -0,0 +1,2 @@
+Add :c:func:`PyUnstable_Object_ClearWeakRefsNoCallbacks`, which clears
+weakrefs without calling their callbacks.

Modules/_testcapi/object.c

@@ -117,11 +117,19 @@ pyobject_print_os_error(PyObject *self, PyObject *args)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+pyobject_clear_weakrefs_no_callbacks(PyObject *self, PyObject *obj)
+{
+    PyUnstable_Object_ClearWeakRefsNoCallbacks(obj);
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef test_methods[] = {
     {"call_pyobject_print", call_pyobject_print, METH_VARARGS},
     {"pyobject_print_null", pyobject_print_null, METH_VARARGS},
     {"pyobject_print_noref_object", pyobject_print_noref_object, METH_VARARGS},
     {"pyobject_print_os_error", pyobject_print_os_error, METH_VARARGS},
+    {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O},
 
     {NULL},
 };

Objects/typeobject.c

@@ -2533,7 +2533,7 @@ subtype_dealloc(PyObject *self)
            finalizers since they might rely on part of the object
            being finalized that has already been destroyed. */
         if (type->tp_weaklistoffset && !base->tp_weaklistoffset) {
-            _PyWeakref_ClearWeakRefsExceptCallbacks(self);
+            _PyWeakref_ClearWeakRefsNoCallbacks(self);
         }
     }
 

Objects/weakrefobject.c

@@ -1016,7 +1016,7 @@ PyObject_ClearWeakRefs(PyObject *object)
     PyObject *exc = PyErr_GetRaisedException();
     PyObject *tuple = PyTuple_New(num_weakrefs * 2);
     if (tuple == NULL) {
-        _PyWeakref_ClearWeakRefsExceptCallbacks(object);
+        _PyWeakref_ClearWeakRefsNoCallbacks(object);
         PyErr_WriteUnraisable(NULL);
         PyErr_SetRaisedException(exc);
         return;
@@ -1057,6 +1057,14 @@ PyObject_ClearWeakRefs(PyObject *object)
     PyErr_SetRaisedException(exc);
 }
 
+void
+PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *obj)
+{
+    if (_PyType_SUPPORTS_WEAKREFS(Py_TYPE(obj))) {
+        _PyWeakref_ClearWeakRefsNoCallbacks(obj);
+    }
+}
+
 /* This function is called by _PyStaticType_Dealloc() to clear weak references.
  *
  * This is called at the end of runtime finalization, so we can just
@@ -1076,7 +1084,7 @@ _PyStaticType_ClearWeakRefs(PyInterpreterState *interp, PyTypeObject *type)
 }
 
 void
-_PyWeakref_ClearWeakRefsExceptCallbacks(PyObject *obj)
+_PyWeakref_ClearWeakRefsNoCallbacks(PyObject *obj)
 {
     /* Modeled after GET_WEAKREFS_LISTPTR().