bpo-20526: Fix PyThreadState_Clear(): don't decref frame (GH-19120)

PyThreadState.frame is a borrowed reference, not a strong reference:
PyThreadState_Clear() must not call Py_CLEAR(tstate->frame).

Remove test_threading.test_warnings_at_exit(): we cannot warranty
that the Python thread state of daemon threads is cleared in a
reliable way during Python shutdown.

Author: vstinner

Include/cpython/pystate.h

@@ -55,6 +55,7 @@ struct _ts {
     struct _ts *next;
     PyInterpreterState *interp;
 
+    /* Borrowed reference to the current frame (it can be NULL) */
     struct _frame *frame;
     int recursion_depth;
     char overflowed; /* The stack has overflowed. Allow 50 more calls

Lib/test/test_threading.py

@@ -776,34 +776,6 @@ def __del__(self):
         """)
         self.assertEqual(out.rstrip(), b"thread_dict.atexit = 'value'")
 
-    def test_warnings_at_exit(self):
-        # bpo-19466: try to call most destructors at Python shutdown before
-        # destroying Python thread states
-        filename = __file__
-        rc, out, err = assert_python_ok("-Wd", "-c", """if 1:
-            import time
-            import threading
-            from test import support
-
-            def open_sleep():
-                # a warning will be emitted when the open file will be
-                # destroyed (without being explicitly closed) while the daemon
-                # thread is destroyed
-                fileobj = open(%a, 'rb')
-                start_event.set()
-                time.sleep(support.LONG_TIMEOUT)
-
-            start_event = threading.Event()
-
-            thread = threading.Thread(target=open_sleep, daemon=True)
-            thread.start()
-
-            # wait until the thread started
-            start_event.wait()
-        """ % filename)
-        self.assertRegex(err.rstrip(),
-                         b"^sys:1: ResourceWarning: unclosed file ")
-
 
 class ThreadJoinOnShutdown(BaseTestCase):
 

Misc/NEWS.d/next/Core and Builtins/2020-03-23-18-08-34.bpo-20526.NHNZIv.rst

@@ -0,0 +1,3 @@
+Fix :c:func:`PyThreadState_Clear()`. ``PyThreadState.frame`` is a borrowed
+reference, not a strong reference: ``PyThreadState_Clear()`` must not call
+``Py_CLEAR(tstate->frame)``.

Python/pystate.c

@@ -765,11 +765,19 @@ PyThreadState_Clear(PyThreadState *tstate)
 {
     int verbose = tstate->interp->config.verbose;
 
-    if (verbose && tstate->frame != NULL)
+    if (verbose && tstate->frame != NULL) {
+        /* bpo-20526: After the main thread calls
+           _PyRuntimeState_SetFinalizing() in Py_FinalizeEx(), threads must
+           exit when trying to take the GIL. If a thread exit in the middle of
+           _PyEval_EvalFrameDefault(), tstate->frame is not reset to its
+           previous value. It is more likely with daemon threads, but it can
+           happen with regular threads if threading._shutdown() fails
+           (ex: interrupted by CTRL+C). */
         fprintf(stderr,
           "PyThreadState_Clear: warning: thread still has a frame\n");
+    }
 
-    Py_CLEAR(tstate->frame);
+    /* Don't clear tstate->frame: it is a borrowed reference */
 
     Py_CLEAR(tstate->dict);
     Py_CLEAR(tstate->async_exc);