Merge branch 'main' into gh-110481-inter-thread-queue

Author: colesbury

.github/workflows/build.yml

@@ -250,7 +250,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1w, 3.0.11, 3.1.3]
+        openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -304,7 +304,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -415,7 +415,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:

.github/workflows/jit.yml

@@ -18,6 +18,7 @@ jobs:
   jit:
     name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
     runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:

.github/workflows/reusable-ubuntu.yml

@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     env:
-      OPENSSL_VER: 3.0.11
+      OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.7
+    rev: v0.2.0
     hooks:
       - id: ruff
         name: Run Ruff on Lib/test/

Doc/c-api/dict.rst

@@ -174,6 +174,26 @@ Dictionary Objects
    .. versionadded:: 3.4
 
 
+.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result)
+
+   Inserts *default_value* into the dictionary *p* with a key of *key* if the
+   key is not already present in the dictionary. If *result* is not ``NULL``,
+   then *\*result* is set to a :term:`strong reference` to either
+   *default_value*, if the key was not present, or the existing value, if *key*
+   was already present in the dictionary.
+   Returns ``1`` if the key was present and *default_value* was not inserted,
+   or ``0`` if the key was not present and *default_value* was inserted.
+   On failure, returns ``-1``, sets an exception, and sets ``*result``
+   to ``NULL``.
+
+   For clarity: if you have a strong reference to *default_value* before
+   calling this function, then after it returns, you hold a strong reference
+   to both *default_value* and *\*result* (if it's not ``NULL``).
+   These may refer to the same object: in that case you hold two separate
+   references to it.
+   .. versionadded:: 3.13
+
+
 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result)
 
    Remove *key* from dictionary *p* and optionally return the removed value.

Doc/c-api/import.rst

@@ -13,20 +13,8 @@ Importing Modules
       single: __all__ (package variable)
       single: modules (in module sys)
 
-   This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below,
-   leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set
-   to 0.  When the *name*
-   argument contains a dot (when it specifies a submodule of a package), the
-   *fromlist* argument is set to the list ``['*']`` so that the return value is the
-   named module rather than the top-level package containing it as would otherwise
-   be the case.  (Unfortunately, this has an additional side effect when *name* in
-   fact specifies a subpackage instead of a submodule: the submodules specified in
-   the package's ``__all__`` variable are  loaded.)  Return a new reference to the
-   imported module, or ``NULL`` with an exception set on failure.  A failing
-   import of a module doesn't leave the module in :data:`sys.modules`.
-
-   This function always uses absolute imports.
-
+   This is a wrapper around :c:func:`PyImport_Import()` which takes a
+   :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`.
 
 .. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name)
 

Doc/c-api/list.rst

@@ -56,13 +56,21 @@ List Objects
    Similar to :c:func:`PyList_Size`, but without error checking.
 
 
-.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index)
+.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index)
 
    Return the object at position *index* in the list pointed to by *list*.  The
    position must be non-negative; indexing from the end of the list is not
-   supported.  If *index* is out of bounds (<0 or >=len(list)),
+   supported.  If *index* is out of bounds (:code:`<0 or >=len(list)`),
    return ``NULL`` and set an :exc:`IndexError` exception.
 
+   .. versionadded:: 3.13
+
+
+.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index)
+
+   Like :c:func:`PyList_GetItemRef`, but returns a
+   :term:`borrowed reference` instead of a :term:`strong reference`.
+
 
 .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i)
 

Doc/data/refcounts.dat

@@ -1133,6 +1133,10 @@ PyList_GetItem:PyObject*::0:
 PyList_GetItem:PyObject*:list:0:
 PyList_GetItem:Py_ssize_t:index::
 
+PyList_GetItemRef:PyObject*::+1:
+PyList_GetItemRef:PyObject*:list:0:
+PyList_GetItemRef:Py_ssize_t:index::
+
 PyList_GetSlice:PyObject*::+1:
 PyList_GetSlice:PyObject*:list:0:
 PyList_GetSlice:Py_ssize_t:low::

Doc/data/stable_abi.dat

@@ -405,22 +405,37 @@ lists.  When in doubt, use a mutex!
 Can't we get rid of the Global Interpreter Lock?
 ------------------------------------------------
 
-.. XXX link to dbeazley's talk about GIL?
-
 The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's
 deployment on high-end multiprocessor server machines, because a multi-threaded
 Python program effectively only uses one CPU, due to the insistence that
 (almost) all Python code can only run while the GIL is held.
 
-Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive
+With the approval of :pep:`703` work is now underway to remove the GIL from the
+CPython implementation of Python.  Initially it will be implemented as an
+optional compiler flag when building the interpreter, and so separate
+builds will be available with and without the GIL.  Long-term, the hope is
+to settle on a single build, once the performance implications of removing the
+GIL are fully understood.  Python 3.13 is likely to be the first release
+containing this work, although it may not be completely functional in this
+release.
+
+The current work to remove the GIL is based on a
+`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_
+by Sam Gross.
+Prior to that,
+in the days of Python 1.5, Greg Stein actually implemented a comprehensive
 patch set (the "free threading" patches) that removed the GIL and replaced it
-with fine-grained locking.  Adam Olsen recently did a similar experiment
+with fine-grained locking.  Adam Olsen did a similar experiment
 in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_
-project.  Unfortunately, both experiments exhibited a sharp drop in single-thread
+project.  Unfortunately, both of these earlier experiments exhibited a sharp
+drop in single-thread
 performance (at least 30% slower), due to the amount of fine-grained locking
-necessary to compensate for the removal of the GIL.
+necessary to compensate for the removal of the GIL.  The Python 3.9 fork
+is the first attempt at removing the GIL with an acceptable performance
+impact.
 
-This doesn't mean that you can't make good use of Python on multi-CPU machines!
+The presence of the GIL in current Python releases
+doesn't mean that you can't make good use of Python on multi-CPU machines!
 You just have to be creative with dividing the work up between multiple
 *processes* rather than multiple *threads*.  The
 :class:`~concurrent.futures.ProcessPoolExecutor` class in the new
@@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work
 done.  Some standard library modules such as :mod:`zlib` and :mod:`hashlib`
 already do this.
 
-It has been suggested that the GIL should be a per-interpreter-state lock rather
-than truly global; interpreters then wouldn't be able to share objects.
-Unfortunately, this isn't likely to happen either.  It would be a tremendous
-amount of work, because many object implementations currently have global state.
-For example, small integers and short strings are cached; these caches would
-have to be moved to the interpreter state.  Other object types have their own
-free list; these free lists would have to be moved to the interpreter state.
-And so on.
-
-And I doubt that it can even be done in finite time, because the same problem
-exists for 3rd party extensions.  It is likely that 3rd party extensions are
-being written at a faster rate than you can convert them to store all their
-global state in the interpreter state.
-
-And finally, once you have multiple interpreters not sharing any state, what
-have you gained over running each interpreter in a separate process?
+An alternative approach to reducing the impact of the GIL is
+to make the GIL a per-interpreter-state lock rather than truly global.
+This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is
+available in the C API. A Python interface to it is expected in Python 3.13.
+The main limitation to it at the moment is likely to be 3rd party extension
+modules, since these must be written with multiple interpreters in mind in
+order to be usable, so many older extension modules will not be usable.
 
 
 Input and Output

Doc/faq/library.rst

@@ -341,7 +341,7 @@ Glossary
    docstring
       A string literal which appears as the first expression in a class,
       function or module.  While ignored when the suite is executed, it is
-      recognized by the compiler and put into the :attr:`__doc__` attribute
+      recognized by the compiler and put into the :attr:`!__doc__` attribute
       of the enclosing class, function or module.  Since it is available via
       introspection, it is the canonical place for documentation of the
       object.
@@ -1104,10 +1104,12 @@ Glossary
       The :class:`collections.abc.Sequence` abstract base class
       defines a much richer interface that goes beyond just
       :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding
-      :meth:`count`, :meth:`index`, :meth:`~object.__contains__`, and
+      :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and
       :meth:`~object.__reversed__`. Types that implement this expanded
       interface can be registered explicitly using
-      :func:`~abc.ABCMeta.register`.
+      :func:`~abc.ABCMeta.register`. For more documentation on sequence
+      methods generally, see
+      :ref:`Common Sequence Operations <typesseq-common>`.
 
    set comprehension
       A compact way to process all or part of the elements in an iterable and

Doc/glossary.rst

@@ -497,13 +497,30 @@ the :meth:`~Enum.__repr__` omits the inherited class' name.  For example::
     >>> Creature.DOG
     <Creature.DOG: size='medium', legs=4>
 
-Use the :func:`!dataclass` argument ``repr=False``
+Use the :func:`~dataclasses.dataclass` argument ``repr=False``
 to use the standard :func:`repr`.
 
 .. versionchanged:: 3.12
    Only the dataclass fields are shown in the value area, not the dataclass'
    name.
 
+.. note::
+
+   Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum`
+   and its subclasses is not supported. It will not raise any errors,
+   but it will produce very strange results at runtime, such as members
+   being equal to each other::
+
+      >>> @dataclass               # don't do this: it does not make any sense
+      ... class Color(Enum):
+      ...    RED = 1
+      ...    BLUE = 2
+      ...
+      >>> Color.RED is Color.BLUE
+      False
+      >>> Color.RED == Color.BLUE  # problem is here: they should not be equal
+      True
+
 
 Pickling
 --------

Doc/howto/enum.rst

@@ -1933,30 +1933,28 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration
 
     LOGGING = {
         'version': 1,
-        'disable_existing_loggers': True,
+        'disable_existing_loggers': False,
         'formatters': {
             'verbose': {
-                'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+                'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
+                'style': '{',
             },
             'simple': {
-                'format': '%(levelname)s %(message)s'
+                'format': '{levelname} {message}',
+                'style': '{',
             },
         },
         'filters': {
             'special': {
                 '()': 'project.logging.SpecialFilter',
                 'foo': 'bar',
-            }
+            },
         },
         'handlers': {
-            'null': {
-                'level':'DEBUG',
-                'class':'django.utils.log.NullHandler',
-            },
-            'console':{
-                'level':'DEBUG',
-                'class':'logging.StreamHandler',
-                'formatter': 'simple'
+            'console': {
+                'level': 'INFO',
+                'class': 'logging.StreamHandler',
+                'formatter': 'simple',
             },
             'mail_admins': {
                 'level': 'ERROR',
@@ -1966,9 +1964,8 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration
         },
         'loggers': {
             'django': {
-                'handlers':['null'],
+                'handlers': ['console'],
                 'propagate': True,
-                'level':'INFO',
             },
             'django.request': {
                 'handlers': ['mail_admins'],