gh-111798: Use lower Py_C_RECURSION_LIMIT in debug mode (#112124)

* Run again test_ast_recursion_limit() on WASI platform.
* Add _testinternalcapi.get_c_recursion_remaining().
* Fix test_ast and test_sys_settrace: test_ast_recursion_limit() and
  test_trace_unpack_long_sequence() now adjust the maximum recursion
  depth depending on the the remaining C recursion.

Include/cpython/pystate.h

@@ -214,7 +214,11 @@ struct _ts {
 
 };
 
-#ifdef __wasi__
+#ifdef Py_DEBUG
+   // A debug build is likely built with low optimization level which implies
+   // higher stack memory usage than a release build: use a lower limit.
+#  define Py_C_RECURSION_LIMIT 500
+#elif defined(__wasi__)
    // WASI has limited call stack. Python's recursion limit depends on code
    // layout, optimization, and WASI runtime. Wasmtime can handle about 700
    // recursions, sometimes less. 500 is a more conservative limit.

Lib/test/test_ast.py

@@ -12,6 +12,10 @@
 import weakref
 from functools import partial
 from textwrap import dedent
+try:
+    import _testinternalcapi
+except ImportError:
+    _testinternalcapi = None
 
 from test import support
 from test.support.import_helper import import_fresh_module
@@ -1118,12 +1122,14 @@ def next(self):
                     return self
         enum._test_simple_enum(_Precedence, ast._Precedence)
 
-    @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     @support.cpython_only
     def test_ast_recursion_limit(self):
         fail_depth = support.EXCEEDS_RECURSION_LIMIT
         crash_depth = 100_000
         success_depth = 1200
+        if _testinternalcapi is not None:
+            remaining = _testinternalcapi.get_c_recursion_remaining()
+            success_depth = min(success_depth, remaining)
 
         def check_limit(prefix, repeated):
             expect_ok = prefix + repeated * success_depth

Lib/test/test_sys_settrace.py

@@ -14,6 +14,10 @@
 import textwrap
 import subprocess
 import warnings
+try:
+    import _testinternalcapi
+except ImportError:
+    _testinternalcapi = None
 
 support.requires_working_socket(module=True)
 
@@ -3033,16 +3037,21 @@ def test_trace_unpack_long_sequence(self):
         self.assertEqual(counts, {'call': 1, 'line': 301, 'return': 1})
 
     def test_trace_lots_of_globals(self):
+        count = 1000
+        if _testinternalcapi is not None:
+            remaining = _testinternalcapi.get_c_recursion_remaining()
+            count = min(count, remaining)
+
         code = """if 1:
             def f():
                 return (
                     {}
                 )
-        """.format("\n+\n".join(f"var{i}\n" for i in range(1000)))
-        ns = {f"var{i}": i for i in range(1000)}
+        """.format("\n+\n".join(f"var{i}\n" for i in range(count)))
+        ns = {f"var{i}": i for i in range(count)}
         exec(code, ns)
         counts = self.count_traces(ns["f"])
-        self.assertEqual(counts, {'call': 1, 'line': 2000, 'return': 1})
+        self.assertEqual(counts, {'call': 1, 'line': count * 2, 'return': 1})
 
 
 class TestEdgeCases(unittest.TestCase):

Misc/NEWS.d/next/Core and Builtins/2023-11-15-20-20-51.gh-issue-111798.cs-3t3.rst

@@ -0,0 +1,4 @@
+When Python is built in debug mode, set the C recursion limit to 500 instead
+of 1500. A debug build is likely built with low optimization level which
+implies higher stack memory usage than a release build. Patch by Victor
+Stinner.

Modules/_testinternalcapi.c

@@ -109,6 +109,14 @@ get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
 }
 
 
+static PyObject*
+get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args))
+{
+    PyThreadState *tstate = _PyThreadState_GET();
+    return PyLong_FromLong(tstate->c_recursion_remaining);
+}
+
+
 static PyObject*
 test_bswap(PyObject *self, PyObject *Py_UNUSED(args))
 {
@@ -1611,6 +1619,7 @@ perf_trampoline_set_persist_after_fork(PyObject *self, PyObject *args)
 static PyMethodDef module_functions[] = {
     {"get_configs", get_configs, METH_NOARGS},
     {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
+    {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS},
     {"test_bswap", test_bswap, METH_NOARGS},
     {"test_popcount", test_popcount, METH_NOARGS},
     {"test_bit_length", test_bit_length, METH_NOARGS},