Merge branch 'python:main' into faster_test_pegen

Author: gpshead

Doc/whatsnew/3.11.rst

@@ -759,12 +759,19 @@ Porting to Python 3.11
   which are not available in the limited C API.
   (Contributed by Victor Stinner in :issue:`46007`.)
 
-* Changes of the :c:type:`PyFrameObject` structure members:
+* Changes of the private :c:type:`PyFrameObject` structure members.
+
+  While the documentation notes that the fields of ``PyFrameObject`` are
+  subject to change at any time, they have been stable for a long time
+  and were used in several popular extensions.
+  In Python 3.11, the frame struct was reorganized to allow performance
+  optimizations. Rather than reading the fields directly, extensions should
+  use functions:
 
   * ``f_code``: removed, use :c:func:`PyFrame_GetCode` instead.
     Warning: the function returns a :term:`strong reference`, need to call
     :c:func:`Py_DECREF`.
-  * ``f_back``: changed, use :c:func:`PyFrame_GetBack`.
+  * ``f_back``: changed (see below), use :c:func:`PyFrame_GetBack`.
   * ``f_builtins``: removed,
     use ``PyObject_GetAttrString(frame, "f_builtins")``.
   * ``f_globals``: removed,
@@ -773,13 +780,17 @@ Porting to Python 3.11
     use ``PyObject_GetAttrString(frame, "f_locals")``.
   * ``f_lasti``: removed,
     use ``PyObject_GetAttrString(frame, "f_lasti")``.
-  * ``f_valuesstack``: removed.
-  * ``f_stackdepth``: removed.
-  * ``f_gen``: removed.
-  * ``f_iblock``: removed.
-  * ``f_state``: removed.
-  * ``f_blockstack``: removed.
-  * ``f_localsplus``: removed.
+
+  The following fields were removed entirely, as they were details
+  of the old implementation:
+
+  * ``f_valuesstack``
+  * ``f_stackdepth``
+  * ``f_gen``
+  * ``f_iblock``
+  * ``f_state``
+  * ``f_blockstack``
+  * ``f_localsplus``
 
   The Python frame object is now created lazily. A side effect is that the
   ``f_back`` member must not be accessed directly, since its value is now also

Include/cpython/pystate.h

@@ -187,16 +187,19 @@ struct _ts {
     /* The following fields are here to avoid allocation during init.
        The data is exposed through PyThreadState pointer fields.
        These fields should not be accessed directly outside of init.
+       This is indicated by an underscore prefix on the field names.
 
        All other PyInterpreterState pointer fields are populated when
        needed and default to NULL.
        */
+       // Note some fields do not have a leading underscore for backward
+       // compatibility.  See https://bugs.python.org/issue45953#msg412046.
 
     /* The thread's exception stack entry.  (Always the last entry.) */
-    _PyErr_StackItem _exc_state;
+    _PyErr_StackItem exc_state;
 
     /* The bottom-most frame on the stack. */
-    CFrame _root_cframe;
+    CFrame root_cframe;
 };
 
 

Include/internal/pycore_code.h

@@ -305,9 +305,19 @@ typedef struct _call_stats {
     uint64_t pyeval_calls;
 } CallStats;
 
+typedef struct _object_stats {
+    uint64_t allocations;
+    uint64_t frees;
+    uint64_t new_values;
+    uint64_t dict_materialized_on_request;
+    uint64_t dict_materialized_new_key;
+    uint64_t dict_materialized_too_big;
+} ObjectStats;
+
 typedef struct _stats {
     OpcodeStats opcode_stats[256];
     CallStats call_stats;
+    ObjectStats object_stats;
 } PyStats;
 
 extern PyStats _py_stats;
@@ -316,6 +326,7 @@ extern PyStats _py_stats;
 #define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
 #define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[opname].execution_count++
 #define CALL_STAT_INC(name) _py_stats.call_stats.name++
+#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++
 
 void _Py_PrintSpecializationStats(int to_file);
 
@@ -326,6 +337,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
 #define STAT_DEC(opname, name) ((void)0)
 #define OPCODE_EXE_INC(opname) ((void)0)
 #define CALL_STAT_INC(name) ((void)0)
+#define OBJECT_STAT_INC(name) ((void)0)
 #endif
 
 

Lib/asyncio/sslproto.py

@@ -367,6 +367,12 @@ def get_write_buffer_size(self):
         """Return the current size of the write buffer."""
         return self._ssl_protocol._transport.get_write_buffer_size()
 
+    def get_write_buffer_limits(self):
+        """Get the high and low watermarks for write flow control. 
+        Return a tuple (low, high) where low and high are 
+        positive number of bytes."""
+        return self._ssl_protocol._transport.get_write_buffer_limits()
+
     @property
     def _protocol_paused(self):
         # Required for sendfile fallback pause_writing/resume_writing logic

Lib/asyncio/transports.py

@@ -99,6 +99,12 @@ def get_write_buffer_size(self):
         """Return the current size of the write buffer."""
         raise NotImplementedError
 
+    def get_write_buffer_limits(self):
+        """Get the high and low watermarks for write flow control. 
+        Return a tuple (low, high) where low and high are 
+        positive number of bytes."""
+        raise NotImplementedError
+
     def write(self, data):
         """Write some data bytes to the transport.
 

Lib/ctypes/test/test_python_api.py

@@ -1,5 +1,5 @@
 from ctypes import *
-import unittest, sys
+import unittest
 from test import support
 
 ################################################################
@@ -10,18 +10,14 @@
 ################################################################
 
 from sys import getrefcount as grc
-if sys.version_info > (2, 4):
-    c_py_ssize_t = c_size_t
-else:
-    c_py_ssize_t = c_int
 
 class PythonAPITestCase(unittest.TestCase):
 
     def test_PyBytes_FromStringAndSize(self):
         PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize
 
         PyBytes_FromStringAndSize.restype = py_object
-        PyBytes_FromStringAndSize.argtypes = c_char_p, c_py_ssize_t
+        PyBytes_FromStringAndSize.argtypes = c_char_p, c_size_t
 
         self.assertEqual(PyBytes_FromStringAndSize(b"abcdefghi", 3), b"abc")
 

Lib/lib2to3/pytree.py

@@ -720,8 +720,8 @@ def generate_matches(self, nodes):
                         r[self.name] = nodes[:count]
                     yield count, r
             except RuntimeError:
-                # We fall back to the iterative pattern matching scheme if the recursive
-                # scheme hits the recursion limit.
+                # Fall back to the iterative pattern matching scheme if the
+                # recursive scheme hits the recursion limit (RecursionError).
                 for count, r in self._iterative_matches(nodes):
                     if self.name:
                         r[self.name] = nodes[:count]

Lib/lib2to3/tests/data/infinite_recursion.py

@@ -1,5 +1,5 @@
-# This file is used to verify that 2to3 falls back to a slower, iterative pattern matching
-# scheme in the event that the faster recursive system fails due to infinite recursion.
+# Verify that 2to3 falls back from the recursive pattern matching scheme to a
+# slower, iterative scheme in the event of a RecursionError.
 from ctypes import *
 STRING = c_char_p
 

Lib/lib2to3/tests/test_all_fixers.py

@@ -6,8 +6,10 @@
 # Author: Collin Winter
 
 # Python imports
-import unittest
+import os.path
+import sys
 import test.support
+import unittest
 
 # Local imports
 from . import support
@@ -19,9 +21,22 @@ class Test_all(support.TestCase):
     def setUp(self):
         self.refactor = support.get_refactorer()
 
+    def refactor_file(self, filepath):
+        if test.support.verbose:
+            print(f"Refactor file: {filepath}")
+        if os.path.basename(filepath) == 'infinite_recursion.py':
+            # bpo-46542: Processing infinite_recursion.py can crash Python
+            # if Python is built in debug mode: lower the recursion limit
+            # to prevent a crash.
+            with test.support.infinite_recursion(150):
+                self.refactor.refactor_file(filepath)
+        else:
+            self.refactor.refactor_file(filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            self.refactor.refactor_file(filepath)
+            with self.subTest(filepath=filepath):
+                self.refactor_file(filepath)
 
 if __name__ == '__main__':
     unittest.main()

Lib/lib2to3/tests/test_parser.py

@@ -20,6 +20,7 @@
 import subprocess
 import sys
 import tempfile
+import test.support
 import unittest
 
 # Local imports
@@ -589,25 +590,31 @@ class TestParserIdempotency(support.TestCase):
 
     """A cut-down version of pytree_idempotency.py."""
 
+    def parse_file(self, filepath):
+        if test.support.verbose:
+            print(f"Parse file: {filepath}")
+        with open(filepath, "rb") as fp:
+            encoding = tokenize.detect_encoding(fp.readline)[0]
+        self.assertIsNotNone(encoding,
+                             "can't detect encoding for %s" % filepath)
+        with open(filepath, "r", encoding=encoding) as fp:
+            source = fp.read()
+        try:
+            tree = driver.parse_string(source)
+        except ParseError:
+            try:
+                tree = driver_no_print_statement.parse_string(source)
+            except ParseError as err:
+                self.fail('ParseError on file %s (%s)' % (filepath, err))
+        new = str(tree)
+        if new != source:
+            print(diff_texts(source, new, filepath))
+            self.fail("Idempotency failed: %s" % filepath)
+
     def test_all_project_files(self):
         for filepath in support.all_project_files():
-            with open(filepath, "rb") as fp:
-                encoding = tokenize.detect_encoding(fp.readline)[0]
-            self.assertIsNotNone(encoding,
-                                 "can't detect encoding for %s" % filepath)
-            with open(filepath, "r", encoding=encoding) as fp:
-                source = fp.read()
-            try:
-                tree = driver.parse_string(source)
-            except ParseError:
-                try:
-                    tree = driver_no_print_statement.parse_string(source)
-                except ParseError as err:
-                    self.fail('ParseError on file %s (%s)' % (filepath, err))
-            new = str(tree)
-            if new != source:
-                print(diff_texts(source, new, filepath))
-                self.fail("Idempotency failed: %s" % filepath)
+            with self.subTest(filepath=filepath):
+                self.parse_file(filepath)
 
     def test_extended_unpacking(self):
         driver.parse_string("a, *b, c = x\n")

Lib/test/test_gdb.py

@@ -897,15 +897,19 @@ def test_gc(self):
     # to suppress these. See also the comment in DebuggerTests.get_stack_trace
     def test_pycfunction(self):
         'Verify that "py-bt" displays invocations of PyCFunction instances'
+        # bpo-46600: If the compiler inlines _null_to_none() in meth_varargs()
+        # (ex: clang -Og), _null_to_none() is the frame #1. Otherwise,
+        # meth_varargs() is the frame #1.
+        expected_frame = r'#(1|2)'
         # Various optimizations multiply the code paths by which these are
         # called, so test a variety of calling conventions.
-        for func_name, args, expected_frame in (
-            ('meth_varargs', '', 1),
-            ('meth_varargs_keywords', '', 1),
-            ('meth_o', '[]', 1),
-            ('meth_noargs', '', 1),
-            ('meth_fastcall', '', 1),
-            ('meth_fastcall_keywords', '', 1),
+        for func_name, args in (
+            ('meth_varargs', ''),
+            ('meth_varargs_keywords', ''),
+            ('meth_o', '[]'),
+            ('meth_noargs', ''),
+            ('meth_fastcall', ''),
+            ('meth_fastcall_keywords', ''),
         ):
             for obj in (
                 '_testcapi',
@@ -945,10 +949,9 @@ def bar():
                         # defined.' message in stderr.
                         ignore_stderr=True,
                     )
-                    self.assertIn(
-                        f'#{expected_frame} <built-in method {func_name}',
-                        gdb_output,
-                    )
+                    regex = expected_frame
+                    regex += re.escape(f' <built-in method {func_name}')
+                    self.assertRegex(gdb_output, regex)
 
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")

Lib/test/test_peepholer.py

@@ -126,7 +126,7 @@ def test_pack_unpack(self):
                 code = compile(line,'','single')
                 self.assertInBytecode(code, elem)
                 self.assertNotInBytecode(code, 'BUILD_TUPLE')
-                self.assertNotInBytecode(code, 'UNPACK_TUPLE')
+                self.assertNotInBytecode(code, 'UNPACK_SEQUENCE')
                 self.check_lnotab(code)
 
     def test_folding_of_tuples_of_constants(self):

Lib/test/test_tabnanny.py

@@ -293,8 +293,8 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False):
         _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args)
         # Note: The `splitlines()` will solve the problem of CRLF(\r) added
         # by OS Windows.
-        out = out.decode('ascii')
-        err = err.decode('ascii')
+        out = os.fsdecode(out)
+        err = os.fsdecode(err)
         if partial:
             for std, output in ((stdout, out), (stderr, err)):
                 _output = output.splitlines()

Lib/test/test_typing.py

@@ -4643,6 +4643,13 @@ class A(typing.Match):
 
 class AnnotatedTests(BaseTestCase):
 
+    def test_new(self):
+        with self.assertRaisesRegex(
+            TypeError,
+            'Type Annotated cannot be instantiated',
+        ):
+            Annotated()
+
     def test_repr(self):
         self.assertEqual(
             repr(Annotated[int, 4, 5]),

Mac/BuildScript/build-installer.py

@@ -358,9 +358,9 @@ def library_recipes():
                   ),
           ),
           dict(
-              name="SQLite 3.36.0",
-              url="https://sqlite.org/2021/sqlite-autoconf-3360000.tar.gz",
-              checksum='f5752052fc5b8e1b539af86a3671eac7',
+              name="SQLite 3.37.2",
+              url="https://sqlite.org/2022/sqlite-autoconf-3370200.tar.gz",
+              checksum='683cc5312ee74e71079c14d24b7a6d27',
               extra_cflags=('-Os '
                             '-DSQLITE_ENABLE_FTS5 '
                             '-DSQLITE_ENABLE_FTS4 '

Misc/NEWS.d/next/Build/2022-02-01-14-07-37.bpo-46600.NNLnfj.rst

@@ -0,0 +1,3 @@
+Fix the test checking if the C compiler supports ``-Og`` option in the
+``./configure`` script to also use ``-Og`` on clang which supports it. Patch
+by Victor Stinner.

Misc/NEWS.d/next/Core and Builtins/2022-02-01-10-23-21.bpo-46564.6Xc2_H.rst

@@ -0,0 +1 @@
+Do not create frame objects when creating :class:`super` object. Patch by Kumar Aditya.

Misc/NEWS.d/next/Library/2022-01-27-12-24-38.bpo-46487.UDkN2z.rst

@@ -0,0 +1 @@
+Add the ``get_write_buffer_limits`` method to :class:`asyncio.transports.WriteTransport` and to the SSL transport.

Misc/NEWS.d/next/Tests/2022-01-31-17-34-13.bpo-46542.RTMm1T.rst

@@ -0,0 +1,2 @@
+Fix a Python crash in test_lib2to3 when using Python built in debug mode:
+limit the recursion limit. Patch by Victor Stinner.

Misc/NEWS.d/next/Tests/2022-02-01-17-13-53.bpo-46600.FMCk8Z.rst

@@ -0,0 +1,2 @@
+Fix test_gdb.test_pycfunction() for Python built with ``clang -Og``.
+Tolerate inlined functions in the gdb traceback. Patch by Victor Stinner.

Misc/NEWS.d/next/macOS/2022-01-26-12-04-09.bpo-45925.yBSiYO.rst

@@ -0,0 +1 @@
+Update macOS installer to SQLite 3.37.2.