Merge branch 'python:main' into faster_test_pegen

Author: gpshead

Doc/whatsnew/3.11.rst

@@ -759,12 +759,19 @@ Porting to Python 3.11
   which are not available in the limited C API.
   (Contributed by Victor Stinner in :issue:`46007`.)
 
-* Changes of the :c:type:`PyFrameObject` structure members:
+* Changes of the private :c:type:`PyFrameObject` structure members.
+
+  While the documentation notes that the fields of ``PyFrameObject`` are
+  subject to change at any time, they have been stable for a long time
+  and were used in several popular extensions.
+  In Python 3.11, the frame struct was reorganized to allow performance
+  optimizations. Rather than reading the fields directly, extensions should
+  use functions:
 
   * ``f_code``: removed, use :c:func:`PyFrame_GetCode` instead.
     Warning: the function returns a :term:`strong reference`, need to call
     :c:func:`Py_DECREF`.
-  * ``f_back``: changed, use :c:func:`PyFrame_GetBack`.
+  * ``f_back``: changed (see below), use :c:func:`PyFrame_GetBack`.
   * ``f_builtins``: removed,
     use ``PyObject_GetAttrString(frame, "f_builtins")``.
   * ``f_globals``: removed,
@@ -773,13 +780,17 @@ Porting to Python 3.11
     use ``PyObject_GetAttrString(frame, "f_locals")``.
   * ``f_lasti``: removed,
     use ``PyObject_GetAttrString(frame, "f_lasti")``.
-  * ``f_valuesstack``: removed.
-  * ``f_stackdepth``: removed.
-  * ``f_gen``: removed.
-  * ``f_iblock``: removed.
-  * ``f_state``: removed.
-  * ``f_blockstack``: removed.
-  * ``f_localsplus``: removed.
+
+  The following fields were removed entirely, as they were details
+  of the old implementation:
+
+  * ``f_valuesstack``
+  * ``f_stackdepth``
+  * ``f_gen``
+  * ``f_iblock``
+  * ``f_state``
+  * ``f_blockstack``
+  * ``f_localsplus``
 
   The Python frame object is now created lazily. A side effect is that the
   ``f_back`` member must not be accessed directly, since its value is now also

Include/cpython/pystate.h

@@ -187,16 +187,19 @@ struct _ts {
     /* The following fields are here to avoid allocation during init.
        The data is exposed through PyThreadState pointer fields.
        These fields should not be accessed directly outside of init.
+       This is indicated by an underscore prefix on the field names.
 
        All other PyInterpreterState pointer fields are populated when
        needed and default to NULL.
        */
+       // Note some fields do not have a leading underscore for backward
+       // compatibility.  See https://bugs.python.org/issue45953#msg412046.
 
     /* The thread's exception stack entry.  (Always the last entry.) */
-    _PyErr_StackItem _exc_state;
+    _PyErr_StackItem exc_state;
 
     /* The bottom-most frame on the stack. */
-    CFrame _root_cframe;
+    CFrame root_cframe;
 };
 
 

Include/internal/pycore_code.h

@@ -305,9 +305,19 @@ typedef struct _call_stats {
     uint64_t pyeval_calls;
 } CallStats;
 
+typedef struct _object_stats {
+    uint64_t allocations;
+    uint64_t frees;
+    uint64_t new_values;
+    uint64_t dict_materialized_on_request;
+    uint64_t dict_materialized_new_key;
+    uint64_t dict_materialized_too_big;
+} ObjectStats;
+
 typedef struct _stats {
     OpcodeStats opcode_stats[256];
     CallStats call_stats;
+    ObjectStats object_stats;
 } PyStats;
 
 extern PyStats _py_stats;
@@ -316,6 +326,7 @@ extern PyStats _py_stats;
 #define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
 #define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[opname].execution_count++
 #define CALL_STAT_INC(name) _py_stats.call_stats.name++
+#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++
 
 void _Py_PrintSpecializationStats(int to_file);
 
@@ -326,6 +337,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
 #define STAT_DEC(opname, name) ((void)0)
 #define OPCODE_EXE_INC(opname) ((void)0)
 #define CALL_STAT_INC(name) ((void)0)
+#define OBJECT_STAT_INC(name) ((void)0)
 #endif
 
 

Lib/asyncio/sslproto.py

@@ -367,6 +367,12 @@ def get_write_buffer_size(self):
         """Return the current size of the write buffer."""
         return self._ssl_protocol._transport.get_write_buffer_size()
 
+    def get_write_buffer_limits(self):
+        """Get the high and low watermarks for write flow control. 
+        Return a tuple (low, high) where low and high are 
+        positive number of bytes."""
+        return self._ssl_protocol._transport.get_write_buffer_limits()
+
     @property
     def _protocol_paused(self):
         # Required for sendfile fallback pause_writing/resume_writing logic

Lib/asyncio/transports.py

@@ -99,6 +99,12 @@ def get_write_buffer_size(self):
         """Return the current size of the write buffer."""
         raise NotImplementedError
 
+    def get_write_buffer_limits(self):
+        """Get the high and low watermarks for write flow control. 
+        Return a tuple (low, high) where low and high are 
+        positive number of bytes."""
+        raise NotImplementedError
+
     def write(self, data):
         """Write some data bytes to the transport.
 

Lib/ctypes/test/test_python_api.py

@@ -1,5 +1,5 @@
 from ctypes import *
-import unittest, sys
+import unittest
 from test import support
 
 ################################################################
@@ -10,18 +10,14 @@
 ################################################################
 
 from sys import getrefcount as grc
-if sys.version_info > (2, 4):
-    c_py_ssize_t = c_size_t
-else:
-    c_py_ssize_t = c_int
 
 class PythonAPITestCase(unittest.TestCase):
 
     def test_PyBytes_FromStringAndSize(self):
         PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize
 
         PyBytes_FromStringAndSize.restype = py_object
-        PyBytes_FromStringAndSize.argtypes = c_char_p, c_py_ssize_t
+        PyBytes_FromStringAndSize.argtypes = c_char_p, c_size_t
 
         self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
 

Lib/lib2to3/pytree.py

@@ -720,8 +720,8 @@ def generate_matches(self, nodes):
                         r[self.name] = nodes[:count]
                     yield count, r
             except RuntimeError:
-                # We fall back to the iterative pattern matching scheme if the recursive
-                # scheme hits the recursion limit.
+                # Fall back to the iterative pattern matching scheme if the
+                # recursive scheme hits the recursion limit (RecursionError).
                 for count, r in self._iterative_matches(nodes):
                     if self.name:
                         r[self.name] = nodes[:count]

Lib/lib2to3/tests/data/infinite_recursion.py

@@ -1,5 +1,5 @@
-# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching
-# scheme in the event that the faster recursive system fails due to infinite recursion.
+# Verify that 2to3 falls back from the recursive pattern matching scheme to a
+# slower, iterative scheme in the event of a RecursionError.
 from ctypes import *
 STRING = c_char_p
 

Lib/lib2to3/tests/test_all_fixers.py

@@ -6,8 +6,10 @@
 # Author: Collin Winter
 
 # Python imports
-import unittest
+import os.path
+import sys
 import test.support
+import unittest
 
 # Local imports
 from . import support
@@ -19,9 +21,22 @@ class Test_all(support.TestCase):
     def setUp(self):
         self.refactor = support.get_refactorer()
 
+    def refactor_file(self, filepath):
+        if test.support.verbose:
+            print(f"Refactor file: {filepath}")
+        if os.path.basename(filepath) == 'infinite_recursion.py':
+            # bpo-46542: Processing infinite_recursion.py can crash Python
+            # if Python is built in debug mode: lower the recursion limit
+            # to prevent a crash.
+            with test.support.infinite_recursion(150):
+                self.refactor.refactor_file(filepath)
+        else:
+            self.refactor.refactor_file(filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            self.refactor.refactor_file(filepath)
+            with self.subTest(filepath=filepath):
+                self.refactor_file(filepath)
 
 if __name__ == '__main__':
     unittest.main()

Lib/lib2to3/tests/test_parser.py

@@ -20,6 +20,7 @@
 import subprocess
 import sys
 import tempfile
+import test.support
 import unittest
 
 # Local imports
@@ -589,25 +590,31 @@ class TestParserIdempotency(support.TestCase):
 
     """A cut-down version of pytree_idempotency.py."""
 
+    def parse_file(self, filepath):
+        if test.support.verbose:
+            print(f"Parse file: {filepath}")
+        with open(filepath, "rb") as fp:
+            encoding = tokenize.detect_encoding(fp.readline)[0]
+        self.assertIsNotNone(encoding,
+                             "can't detect encoding for %s" % filepath)
+        with open(filepath, "r", encoding=encoding) as fp:
+            source = fp.read()
+        try:
+            tree = driver.parse_string(source)
+        except ParseError:
+            try:
+                tree = driver_no_print_statement.parse_string(source)
+            except ParseError as err:
+                self.fail('ParseError on file %s (%s)' % (filepath, err))
+        new = str(tree)
+        if new != source:
+            print(diff_texts(source, new, filepath))
+            self.fail("Idempotency failed: %s" % filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            with open(filepath, "rb") as fp:
-                encoding = tokenize.detect_encoding(fp.readline)[0]
-            self.assertIsNotNone(encoding,
-                                 "can't detect encoding for %s" % filepath)
-            with open(filepath, "r", encoding=encoding) as fp:
-                source = fp.read()
-            try:
-                tree = driver.parse_string(source)
-            except ParseError:
-                try:
-                    tree = driver_no_print_statement.parse_string(source)
-                except ParseError as err:
-                    self.fail('ParseError on file %s (%s)' % (filepath, err))
-            new = str(tree)
-            if new != source:
-                print(diff_texts(source, new, filepath))
-                self.fail("Idempotency failed: %s" % filepath)
+            with self.subTest(filepath=filepath):
+                self.parse_file(filepath)
 
     def test_extended_unpacking(self):
         driver.parse_string("a, *b, c = x\n")