gh-110693: Use a Linked List for Pending Calls

Include/internal/pycore_ceval.h

@@ -51,6 +51,7 @@ extern void _PyEval_SignalReceived(void);
 typedef int _Py_add_pending_call_result;
 #define _Py_ADD_PENDING_SUCCESS 0
 #define _Py_ADD_PENDING_FULL -1
+#define _Py_ADD_PENDING_NO_MEMORY -2
 
 // Export for '_testinternalcapi' shared extension
 PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall(

Include/internal/pycore_ceval_state.h

@@ -14,15 +14,18 @@ extern "C" {
 
 typedef int (*_Py_pending_call_func)(void *);
 
+struct _pending_call;
+
 struct _pending_call {
     _Py_pending_call_func func;
     void *arg;
     int flags;
+    int from_heap;
+    struct _pending_call *next;
 };
 
-#define PENDINGCALLSARRAYSIZE 32
-
-#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE
+// We effectively drop the limit for per-interpreter pending calls.
+#define MAXPENDINGCALLS INT32_MAX
 /* For interpreter-level pending calls, we want to avoid spending too
    much time on pending calls in any one thread, so we apply a limit. */
 #if MAXPENDINGCALLS > 100
@@ -31,7 +34,10 @@ struct _pending_call {
 #  define MAXPENDINGCALLSLOOP MAXPENDINGCALLS
 #endif
 
-#define MAXPENDINGCALLS_MAIN PENDINGCALLSARRAYSIZE
+#define NPENDINGCALLSARRAY 32
+/* We keep the number small to preserve as much compatibility
+   as possible with earlier versions. */
+#define MAXPENDINGCALLS_MAIN NPENDINGCALLSARRAY
 /* For the main thread, we want to make sure all pending calls are
    run at once, for the sake of prompt signal handling.  This is
    unlikely to cause any problems since there should be very few
@@ -41,7 +47,7 @@ struct _pending_call {
 struct _pending_calls {
     int busy;
     PyMutex mutex;
-    /* Request for running pending calls. */
+    /* The number of pending calls. */
     int32_t npending;
     /* The maximum allowed number of pending calls.
        If the queue fills up to this point then _PyEval_AddPendingCall()
@@ -52,9 +58,10 @@ struct _pending_calls {
        A value of 0 means there is no limit (other than the maximum
        size of the list of pending calls). */
     int32_t maxloop;
-    struct _pending_call calls[PENDINGCALLSARRAYSIZE];
-    int first;
-    int next;
+    /* The linked list of pending calls. */
+    struct _pending_call *head;
+    struct _pending_call *tail;
+    struct _pending_call *freelist;
 };
 
 
@@ -96,6 +103,11 @@ struct _ceval_runtime_state {
     // For example, we use a preallocated array
     // for the list of pending calls.
     struct _pending_calls pending_mainthread;
+    // Using a preallocated array for the first pending calls gives us
+    // some extra stability in the case of signals.
+    // We also use this number as the max and loop max for the main thread.
+#define NPENDINGCALLSARRAY 32
+    struct _pending_call _pending_preallocated[NPENDINGCALLSARRAY];
     PyMutex sys_trace_profile_mutex;
 };
 

Include/internal/pycore_runtime_init.h

@@ -114,11 +114,11 @@ extern PyTypeObject _PyExc_MemoryError;
         .autoTSSkey = Py_tss_NEEDS_INIT, \
         .parser = _parser_runtime_state_INIT, \
         .ceval = { \
+            .perf = _PyEval_RUNTIME_PERF_INIT, \
             .pending_mainthread = { \
                 .max = MAXPENDINGCALLS_MAIN, \
                 .maxloop = MAXPENDINGCALLSLOOP_MAIN, \
             }, \
-            .perf = _PyEval_RUNTIME_PERF_INIT, \
         }, \
         .gilstate = { \
             .check_enabled = 1, \

Lib/test/test_capi/test_misc.py

@@ -1570,24 +1570,18 @@ def test_max_pending(self):
             self.assertEqual(added, maxpending)
 
         with self.subTest('not main-only'):
-            # Per-interpreter pending calls has the same low limit
+            # Per-interpreter pending calls do not have a limit
             # on how many may be pending at a time.
-            maxpending = 32
 
             l = []
             added = self.pendingcalls_submit(l, 1, main=False)
             self.pendingcalls_wait(l, added)
             self.assertEqual(added, 1)
 
             l = []
-            added = self.pendingcalls_submit(l, maxpending, main=False)
-            self.pendingcalls_wait(l, added)
-            self.assertEqual(added, maxpending)
-
-            l = []
-            added = self.pendingcalls_submit(l, maxpending+1, main=False)
+            added = self.pendingcalls_submit(l, 1000, main=False)
             self.pendingcalls_wait(l, added)
-            self.assertEqual(added, maxpending)
+            self.assertEqual(added, 1000)
 
     class PendingTask(types.SimpleNamespace):
 

Misc/NEWS.d/next/Core and Builtins/2018-12-11-15-12-03.gh-issue-79647.5xu737.rst

@@ -0,0 +1 @@
+Simply the ceval pending calls list by using a linked list.

Python/ceval_gil.c

@@ -674,6 +674,48 @@ signal_active_thread(PyInterpreterState *interp, uintptr_t bit)
    threadstate.
 */
 
+static void
+_pending_calls_init(struct _pending_calls *pending,
+                    struct _pending_call *preallocated, size_t len)
+{
+    /* We shouldn't need the lock while initializing. */
+    assert(pending->head == NULL);
+    assert(pending->tail == NULL);
+    assert(pending->npending == 0);
+    assert(!pending->busy);
+    assert(pending->max > 0);
+    assert(pending->maxloop >= 0);
+
+    assert(pending->freelist == NULL);
+    if (preallocated) {
+        assert(len > 0);
+        for (size_t i = len; i > 0; i--) {
+            struct _pending_call *call = &preallocated[i-1];
+            assert(!call->from_heap);
+            call->next = pending->freelist;
+            pending->freelist = call;
+        }
+    }
+}
+
+static void
+_pending_calls_fini(struct _pending_calls *pending)
+{
+    PyMutex_Lock(&pending->mutex);
+    struct _pending_call *head = pending->freelist;
+    pending->freelist = NULL;
+    PyMutex_Unlock(&pending->mutex);
+
+    /* Deallocate all freelist entries. */
+    while (head != NULL) {
+        struct _pending_call *cur = head;
+        head = cur->next;
+        if (cur->from_heap) {
+            PyMem_RawFree(cur);
+        }
+    }
+}
+
 /* Push one item onto the queue while holding the lock. */
 static int
 _push_pending_call(struct _pending_calls *pending,
@@ -684,52 +726,69 @@ _push_pending_call(struct _pending_calls *pending,
     }
     assert(pending->npending < pending->max);
 
-    int i = pending->next;
-    assert(pending->calls[i].func == NULL);
+    // Allocate for the pending call.
+    struct _pending_call *call = pending->freelist;
+    if (call != NULL) {
+        pending->freelist = call->next;
+    }
+    else {
+        call = PyMem_RawMalloc(sizeof(struct _pending_call));
+        if (call == NULL) {
+            return _Py_ADD_PENDING_NO_MEMORY;
+        }
+        call->from_heap = 1;
+    }
 
-    pending->calls[i].func = func;
-    pending->calls[i].arg = arg;
-    pending->calls[i].flags = flags;
+    // Initialize the data.
+    *call = (struct _pending_call){
+        .func=func,
+        .arg=arg,
+        .flags=flags,
+    };
 
-    assert(pending->npending < PENDINGCALLSARRAYSIZE);
+    // Add the call to the list.
+    if (pending->head == NULL) {
+        pending->head = call;
+    }
+    else {
+        pending->tail->next = call;
+    }
+    pending->tail = call;
     _Py_atomic_add_int32(&pending->npending, 1);
 
-    pending->next = (i + 1) % PENDINGCALLSARRAYSIZE;
-    assert(pending->next != pending->first
-            || pending->npending == pending->max);
-
     return _Py_ADD_PENDING_SUCCESS;
 }
 
-static int
-_next_pending_call(struct _pending_calls *pending,
-                   int (**func)(void *), void **arg, int *flags)
-{
-    int i = pending->first;
-    if (pending->npending == 0) {
-        /* Queue empty */
-        assert(i == pending->next);
-        assert(pending->calls[i].func == NULL);
-        return -1;
-    }
-    *func = pending->calls[i].func;
-    *arg = pending->calls[i].arg;
-    *flags = pending->calls[i].flags;
-    return i;
-}
-
 /* Pop one item off the queue while holding the lock. */
 static void
 _pop_pending_call(struct _pending_calls *pending,
                   int (**func)(void *), void **arg, int *flags)
 {
-    int i = _next_pending_call(pending, func, arg, flags);
-    if (i >= 0) {
-        pending->calls[i] = (struct _pending_call){0};
-        pending->first = (i + 1) % PENDINGCALLSARRAYSIZE;
-        assert(pending->npending > 0);
-        _Py_atomic_add_int32(&pending->npending, -1);
+    struct _pending_call *call = pending->head;
+    if (call == NULL) {
+        /* Queue empty */
+        assert(pending->npending == 0);
+        assert(pending->tail == NULL);
+        return;
+    }
+    assert(pending->npending > 0);
+    assert(pending->tail != NULL);
+
+    // Remove the next one from the list.
+    pending->head = call->next;
+    if (pending->tail == call) {
+        pending->tail = NULL;
     }
+    _Py_atomic_add_int32(&pending->npending, -1);
+
+    // Copy its data.
+    *func = call->func;
+    *arg = call->arg;
+    *flags = call->flags;
+
+    // "Deallocate" the list entry.
+    call->next = pending->freelist;
+    pending->freelist = call;
 }
 
 /* This implementation is thread-safe.  It allows
@@ -807,8 +866,6 @@ _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
     int res = 0;
     int32_t npending = -1;
 
-    assert(sizeof(pending->max) <= sizeof(size_t)
-            && ((size_t)pending->max) <= Py_ARRAY_LENGTH(pending->calls));
     int32_t maxloop = pending->maxloop;
     if (maxloop == 0) {
         maxloop = pending->max;
@@ -827,7 +884,7 @@ _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
         npending = pending->npending;
         PyMutex_Unlock(&pending->mutex);
 
-        /* Check if there are any more pending calls. */
+        /* Check if there were any pending calls left. */
         if (func == NULL) {
             assert(npending == 0);
             break;
@@ -839,6 +896,7 @@ _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
             PyMem_RawFree(arg);
         }
         if (res != 0) {
+            assert(PyErr_Occurred());
             res = -1;
             goto finally;
         }
@@ -960,11 +1018,37 @@ _Py_FinishPendingCalls(PyThreadState *tstate)
     assert(PyGILState_Check());
     assert(_PyThreadState_CheckConsistency(tstate));
 
-    if (make_pending_calls(tstate) < 0) {
-        PyObject *exc = _PyErr_GetRaisedException(tstate);
-        PyErr_BadInternalCall();
-        _PyErr_ChainExceptions1(exc);
-        _PyErr_Print(tstate);
+    struct _pending_calls *pending = &tstate->interp->ceval.pending;
+    struct _pending_calls *pending_main =
+            _Py_IsMainThread() && _Py_IsMainInterpreter(tstate->interp)
+            ? &_PyRuntime.ceval.pending_mainthread
+            : NULL;
+
+    /* make_pending_calls() may return early without making all pending
+       calls, so we keep trying until we're actually done. */
+    int32_t npending = INT32_MAX;
+    int32_t npending_prev = -1;
+    do {
+        assert(npending_prev < 0 || npending_prev > npending);
+        npending_prev = npending;
+
+        if (make_pending_calls(tstate) < 0) {
+            PyObject *exc = _PyErr_GetRaisedException(tstate);
+            PyErr_BadInternalCall();
+            _PyErr_ChainExceptions1(exc);
+            _PyErr_Print(tstate);
+        }
+
+        npending = _Py_atomic_load_int32_relaxed(&pending->npending);
+        if (pending_main != NULL) {
+            npending += _Py_atomic_load_int32_relaxed(&pending_main->npending);
+        }
+    } while (npending > 0);
+
+    /* Clean up the lingering pending calls state. */
+    _pending_calls_fini(pending);
+    if (pending_main != NULL) {
+        _pending_calls_fini(pending_main);
     }
 }
 
@@ -1011,6 +1095,14 @@ Py_MakePendingCalls(void)
 void
 _PyEval_InitState(PyInterpreterState *interp)
 {
+    _pending_calls_init(&interp->ceval.pending, NULL, 0);
+    if (_Py_IsMainInterpreter(interp)) {
+        struct _ceval_runtime_state *ceval = &_PyRuntime.ceval;
+        _pending_calls_init(&ceval->pending_mainthread,
+                            ceval->_pending_preallocated,
+                            Py_ARRAY_LENGTH(ceval->_pending_preallocated));
+    }
+
     _gil_initialize(&interp->_gil);
 }