bpo-32591: Add native coroutine origin tracking

Doc/library/inspect.rst

@@ -34,6 +34,9 @@ provided as convenient choices for the second argument to :func:`getmembers`.
 They also help you determine when you can expect to find the following special
 attributes:
 
+.. this function name is too big to fit in the ascii-art table below
+.. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth`
+
 +-----------+-------------------+---------------------------+
 | Type      | Attribute         | Description               |
 +===========+===================+===========================+
@@ -215,6 +218,10 @@ attributes:
 +-----------+-------------------+---------------------------+
 |           | cr_code           | code                      |
 +-----------+-------------------+---------------------------+
+|           | cr_origin         | where coroutine was       |
+|           |                   | created, or ``None``. See |
+|           |                   | |coroutine-origin-link|   |
++-----------+-------------------+---------------------------+
 | builtin   | __doc__           | documentation string      |
 +-----------+-------------------+---------------------------+
 |           | __name__          | original name of this     |
@@ -234,6 +241,9 @@ attributes:
    The ``__name__`` attribute of generators is now set from the function
    name, instead of the code name, and it can now be modified.
 
+.. versionchanged:: 3.7
+
+   Add ``cr_origin`` attribute to coroutines.
 
 .. function:: getmembers(object[, predicate])
 

Doc/library/sys.rst

@@ -675,6 +675,18 @@ always available.
       for details.)
 
 
+.. function:: get_coroutine_origin_tracking_depth()
+
+   Get the current coroutine origin tracking depth, as set by
+   func:`set_coroutine_origin_tracking_depth`.
+
+   .. versionadded:: 3.7
+
+   .. note::
+      This function has been added on a provisional basis (see :pep:`411`
+      for details.)  Use it only for debugging purposes.
+
+
 .. function:: get_coroutine_wrapper()
 
    Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`.
@@ -686,6 +698,10 @@ always available.
       This function has been added on a provisional basis (see :pep:`411`
       for details.)  Use it only for debugging purposes.
 
+   .. deprecated:: 3.7
+      The coroutine wrapper functionality has been deprecated, and
+      will be removed in 3.8. See :issue:`32591` for details.
+
 
 .. data:: hash_info
 
@@ -1212,6 +1228,26 @@ always available.
       This function has been added on a provisional basis (see :pep:`411`
       for details.)
 
+.. function:: set_coroutine_origin_tracking_depth(depth)
+
+   Allows enabling or disabling coroutine origin tracking. When
+   enabled, the ``cr_origin`` attribute on coroutine objects will
+   contain a tuple of (filename, line number, function name) tuples
+   describing the traceback where the coroutine object was created,
+   with the most recent call first. When disabled, ``cr_origin`` will
+   be None.
+
+   To enable, pass a *depth* value greater than zero; this sets the
+   number of frames whose information will be captured. To disable,
+   pass set *depth* to zero.
+
+   This setting is thread-specific.
+
+   .. versionadded:: 3.7
+
+   .. note::
+      This function has been added on a provisional basis (see :pep:`411`
+      for details.)  Use it only for debugging purposes.
 
 .. function:: set_coroutine_wrapper(wrapper)
 
@@ -1252,6 +1288,10 @@ always available.
       This function has been added on a provisional basis (see :pep:`411`
       for details.)  Use it only for debugging purposes.
 
+   .. deprecated:: 3.7
+      The coroutine wrapper functionality has been deprecated, and
+      will be removed in 3.8. See :issue:`32591` for details.
+
 .. function:: _enablelegacywindowsfsencoding()
 
    Changes the default filesystem encoding and errors mode to 'mbcs' and

Doc/whatsnew/3.7.rst

@@ -510,6 +510,9 @@ sys
 
 Added :attr:`sys.flags.dev_mode` flag for the new development mode.
 
+Deprecated :func:`sys.set_coroutine_wrapper` and
+:func:`sys.get_coroutine_wrapper`.
+
 time
 ----
 

Include/ceval.h

@@ -31,6 +31,8 @@ PyAPI_FUNC(PyObject *) PyEval_CallMethod(PyObject *obj,
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
 PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) _PyEval_SetCoroutineOriginTrackingDepth(int new_depth);
+PyAPI_FUNC(int) _PyEval_GetCoroutineOriginTrackingDepth(void);
 PyAPI_FUNC(void) _PyEval_SetCoroutineWrapper(PyObject *);
 PyAPI_FUNC(PyObject *) _PyEval_GetCoroutineWrapper(void);
 PyAPI_FUNC(void) _PyEval_SetAsyncGenFirstiter(PyObject *);

Include/genobject.h

@@ -51,6 +51,7 @@ PyAPI_FUNC(void) _PyGen_Finalize(PyObject *self);
 #ifndef Py_LIMITED_API
 typedef struct {
     _PyGenObject_HEAD(cr)
+    PyObject *cr_origin;
 } PyCoroObject;
 
 PyAPI_DATA(PyTypeObject) PyCoro_Type;

Include/pystate.h

@@ -262,6 +262,8 @@ typedef struct _ts {
     void (*on_delete)(void *);
     void *on_delete_data;
 
+    int coroutine_origin_tracking_depth;
+
     PyObject *coroutine_wrapper;
     int in_coroutine_wrapper;
 

Include/warnings.h

@@ -56,6 +56,10 @@ PyErr_WarnExplicitFormat(PyObject *category,
 #define PyErr_Warn(category, msg) PyErr_WarnEx(category, msg, 1)
 #endif
 
+#ifndef Py_LIMITED_API
+void _PyErr_WarnUnawaitedCoroutine(PyObject *coro);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Lib/asyncio/base_events.py

@@ -34,6 +34,7 @@
 except ImportError:  # pragma: no cover
     ssl = None
 
+from . import constants
 from . import coroutines
 from . import events
 from . import futures
@@ -224,7 +225,8 @@ def __init__(self):
         self.slow_callback_duration = 0.1
         self._current_handle = None
         self._task_factory = None
-        self._coroutine_wrapper_set = False
+        self._coroutine_origin_tracking_enabled = False
+        self._coroutine_origin_tracking_saved_depth = None
 
         if hasattr(sys, 'get_asyncgen_hooks'):
             # Python >= 3.6
@@ -382,7 +384,7 @@ def run_forever(self):
         if events._get_running_loop() is not None:
             raise RuntimeError(
                 'Cannot run the event loop while another loop is running')
-        self._set_coroutine_wrapper(self._debug)
+        self._set_coroutine_origin_tracking(self._debug)
         self._thread_id = threading.get_ident()
         if self._asyncgens is not None:
             old_agen_hooks = sys.get_asyncgen_hooks()
@@ -398,7 +400,7 @@ def run_forever(self):
             self._stopping = False
             self._thread_id = None
             events._set_running_loop(None)
-            self._set_coroutine_wrapper(False)
+            self._set_coroutine_origin_tracking(False)
             if self._asyncgens is not None:
                 sys.set_asyncgen_hooks(*old_agen_hooks)
 
@@ -1531,39 +1533,20 @@ def _run_once(self):
                 handle._run()
         handle = None  # Needed to break cycles when an exception occurs.
 
-    def _set_coroutine_wrapper(self, enabled):
-        try:
-            set_wrapper = sys.set_coroutine_wrapper
-            get_wrapper = sys.get_coroutine_wrapper
-        except AttributeError:
-            return
-
-        enabled = bool(enabled)
-        if self._coroutine_wrapper_set == enabled:
+    def _set_coroutine_origin_tracking(self, enabled):
+        if bool(enabled) == bool(self._coroutine_origin_tracking_enabled):
             return
 
-        wrapper = coroutines.debug_wrapper
-        current_wrapper = get_wrapper()
-
         if enabled:
-            if current_wrapper not in (None, wrapper):
-                warnings.warn(
-                    f"loop.set_debug(True): cannot set debug coroutine "
-                    f"wrapper; another wrapper is already set "
-                    f"{current_wrapper!r}",
-                    RuntimeWarning)
-            else:
-                set_wrapper(wrapper)
-                self._coroutine_wrapper_set = True
+            self._coroutine_origin_tracking_saved_depth = (
+                sys.get_coroutine_origin_tracking_depth())
+            sys.set_coroutine_origin_tracking_depth(
+                constants.DEBUG_STACK_DEPTH)
         else:
-            if current_wrapper not in (None, wrapper):
-                warnings.warn(
-                    f"loop.set_debug(False): cannot unset debug coroutine "
-                    f"wrapper; another wrapper was set {current_wrapper!r}",
-                    RuntimeWarning)
-            else:
-                set_wrapper(None)
-                self._coroutine_wrapper_set = False
+            sys.set_coroutine_origin_tracking_depth(
+                self._coroutine_origin_tracking_saved_depth)
+
+        self._coroutine_origin_tracking_enabled = enabled
 
     def get_debug(self):
         return self._debug
@@ -1572,4 +1555,4 @@ def set_debug(self, enabled):
         self._debug = enabled
 
         if self.is_running():
-            self._set_coroutine_wrapper(enabled)
+            self.call_soon_threadsafe(self._set_coroutine_origin_tracking, enabled)

Lib/asyncio/coroutines.py

@@ -32,14 +32,6 @@ def _is_debug_mode():
 _DEBUG = _is_debug_mode()
 
 
-def debug_wrapper(gen):
-    # This function is called from 'sys.set_coroutine_wrapper'.
-    # We only wrap here coroutines defined via 'async def' syntax.
-    # Generator-based coroutines are wrapped in @coroutine
-    # decorator.
-    return CoroWrapper(gen, None)
-
-
 class CoroWrapper:
     # Wrapper for coroutine object in _DEBUG mode.
 
@@ -87,39 +79,16 @@ def gi_code(self):
         return self.gen.gi_code
 
     def __await__(self):
-        cr_await = getattr(self.gen, 'cr_await', None)
-        if cr_await is not None:
-            raise RuntimeError(
-                f"Cannot await on coroutine {self.gen!r} while it's "
-                f"awaiting for {cr_await!r}")
         return self
 
     @property
     def gi_yieldfrom(self):
         return self.gen.gi_yieldfrom
 
-    @property
-    def cr_await(self):
-        return self.gen.cr_await
-
-    @property
-    def cr_running(self):
-        return self.gen.cr_running
-
-    @property
-    def cr_code(self):
-        return self.gen.cr_code
-
-    @property
-    def cr_frame(self):
-        return self.gen.cr_frame
-
     def __del__(self):
         # Be careful accessing self.gen.frame -- self.gen might not exist.
         gen = getattr(self, 'gen', None)
         frame = getattr(gen, 'gi_frame', None)
-        if frame is None:
-            frame = getattr(gen, 'cr_frame', None)
         if frame is not None and frame.f_lasti == -1:
             msg = f'{self!r} was never yielded from'
             tb = getattr(self, '_source_traceback', ())
@@ -141,8 +110,6 @@ def coroutine(func):
     if inspect.iscoroutinefunction(func):
         # In Python 3.5 that's all we need to do for coroutines
         # defined with "async def".
-        # Wrapping in CoroWrapper will happen via
-        # 'sys.set_coroutine_wrapper' function.
         return func
 
     if inspect.isgeneratorfunction(func):

Lib/test/test_asyncio/test_pep492.py

@@ -1,5 +1,6 @@
 """Tests support for new syntax introduced by PEP 492."""
 
+import sys
 import types
 import unittest
 
@@ -148,35 +149,14 @@ async def foo():
         data = self.loop.run_until_complete(foo())
         self.assertEqual(data, 'spam')
 
-    @mock.patch('asyncio.coroutines.logger')
-    def test_async_def_wrapped(self, m_log):
-        async def foo():
-            pass
+    def test_debug_mode_manages_coroutine_origin_tracking(self):
         async def start():
-            foo_coro = foo()
-            self.assertRegex(
-                repr(foo_coro),
-                r'<CoroWrapper .*\.foo\(\) running at .*pep492.*>')
-
-            with support.check_warnings((r'.*foo.*was never',
-                                         RuntimeWarning)):
-                foo_coro = None
-                support.gc_collect()
-                self.assertTrue(m_log.error.called)
-                message = m_log.error.call_args[0][0]
-                self.assertRegex(message,
-                                 r'CoroWrapper.*foo.*was never')
+            self.assertTrue(sys.get_coroutine_origin_tracking_depth() > 0)
 
+        self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0)
         self.loop.set_debug(True)
         self.loop.run_until_complete(start())
-
-        async def start():
-            foo_coro = foo()
-            task = asyncio.ensure_future(foo_coro, loop=self.loop)
-            self.assertRegex(repr(task), r'Task.*foo.*running')
-
-        self.loop.run_until_complete(start())
-
+        self.assertEqual(sys.get_coroutine_origin_tracking_depth(), 0)
 
     def test_types_coroutine(self):
         def gen():
@@ -226,9 +206,9 @@ async def runner():
                 t.cancel()
 
         self.loop.set_debug(True)
-        with self.assertRaisesRegex(
-            RuntimeError,
-            r'Cannot await.*test_double_await.*\bafunc\b.*while.*\bsleep\b'):
+        with self.assertRaises(
+                RuntimeError,
+                msg='coroutine is being awaited already'):
 
             self.loop.run_until_complete(runner())