Merge branch 'main' into feature/import-cache-2

hudson-trading · May 7, 2024 · 865652b · 865652b
2 parents c4afc9a + 26bab42
commit 865652b
Show file tree

Hide file tree

Showing 339 changed files with 17,841 additions and 4,545 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -208,7 +208,7 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       # macos-14 is M1, macos-13 is Intel
-      os-matrix: '["macos-14", "macos-13"]'
+      os-matrix: '["macos-14-xlarge", "macos-13-large"]'
 
   build_macos_free_threading:
     name: 'macOS (free-threading)'
@@ -218,8 +218,8 @@ jobs:
     with:
       config_hash: ${{ needs.check_source.outputs.config_hash }}
       free-threading: true
-      # macos-14 is M1
-      os-matrix: '["macos-14"]'
+      # macos-14-large is Intel with 12 cores (most parallelism)
+      os-matrix: '["macos-14-large"]'
 
   build_ubuntu:
     name: 'Ubuntu'
@@ -250,7 +250,7 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
@@ -316,7 +316,7 @@ jobs:
 
   test_hypothesis:
     name: "Hypothesis tests on Ubuntu"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
@@ -429,7 +429,7 @@ jobs:
 
   build_asan:
     name: 'Address sanitizer'
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'

diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml
@@ -23,7 +23,7 @@ jobs:
   jit:
     name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }})
     runs-on: ${{ matrix.runner }}
-    timeout-minutes: 75
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
@@ -75,14 +75,10 @@ jobs:
             architecture: aarch64
             runner: ubuntu-latest
             compiler: gcc
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
           - target: aarch64-unknown-linux-gnu/clang
             architecture: aarch64
             runner: ubuntu-latest
             compiler: clang
-            # These fail because of emulation, not because of the JIT:
-            exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection
     env:
       CC: ${{ matrix.compiler }}
     steps:
@@ -97,7 +93,7 @@ jobs:
           choco upgrade llvm -y
           choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
-          ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
@@ -115,7 +111,7 @@ jobs:
           SDKROOT="$(xcrun --show-sdk-path)" \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
-          ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
@@ -125,11 +121,12 @@ jobs:
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
           ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
+        # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
@@ -146,4 +143,4 @@ jobs:
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
             ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
-          ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -8,6 +8,7 @@ on:
   pull_request:
     paths:
       - ".github/workflows/mypy.yml"
+      - "Lib/_pyrepl/**"
       - "Lib/test/libregrtest/**"
       - "Tools/build/generate_sbom.py"
       - "Tools/cases_generator/**"
@@ -35,8 +36,9 @@ jobs:
     strategy:
       matrix:
         target: [
+          "Lib/_pyrepl",
           "Lib/test/libregrtest",
-          "Tools/build/",
+          "Tools/build",
           "Tools/cases_generator",
           "Tools/clinic",
           "Tools/jit",

diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh
@@ -15,6 +15,7 @@ apt-get -yq install \
     libgdbm-dev \
     libgdbm-compat-dev \
     liblzma-dev \
+    libmpdec-dev \
     libncurses5-dev \
     libreadline6-dev \
     libsqlite3-dev \

diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml
@@ -22,6 +22,7 @@ jobs:
       HOMEBREW_NO_INSTALL_CLEANUP: 1
       HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     strategy:
       fail-fast: false
       matrix:
@@ -49,7 +50,7 @@ jobs:
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix openssl@3.0)"
     - name: Build CPython
-      run: make -j4
+      run: make -j8
     - name: Display build info
       run: make pythoninfo
     - name: Tests

diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml
@@ -29,7 +29,14 @@ jobs:
     - name: Install Dependencies
       run: |
         sudo ./.github/workflows/posix-deps-apt.sh
-        sudo apt install -y clang
+        # Install clang-18
+        wget https://apt.llvm.org/llvm.sh
+        chmod +x llvm.sh
+        sudo ./llvm.sh 18
+        sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
+        sudo update-alternatives --set clang /usr/bin/clang-18
+        sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
+        sudo update-alternatives --set clang++ /usr/bin/clang++-18
         # Reduce ASLR to avoid TSAN crashing
         sudo sysctl -w vm.mmap_rnd_bits=28
     - name: TSAN Option Setup

diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml
@@ -12,11 +12,12 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.13
       PYTHONSTRICTEXTENSIONBUILD: 1
+      TERM: linux
     steps:
     - uses: actions/checkout@v4
     - name: Register gcc problem matcher

diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml
@@ -9,7 +9,7 @@ jobs:
   build_wasi_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       WASMTIME_VERSION: 18.0.3
       WASI_SDK_VERSION: 21

diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst
@@ -120,12 +120,19 @@ See also :ref:`Reflection <reflection>`.
 
 .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame)
 
-   Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`).
+   Get the *frame*'s :attr:`~frame.f_locals` attribute.
+   If the frame refers to a function or comprehension, this returns
+   a write-through proxy object that allows modifying the locals.
+   In all other cases (classes, modules) it returns the :class:`dict`
+   representing the frame locals directly.
 
    Return a :term:`strong reference`.
 
    .. versionadded:: 3.11
 
+   .. versionchanged:: 3.13
+      Return a proxy object for functions and comprehensions.
+
 
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 

diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst
@@ -25,3 +25,4 @@ document the API functions in detail.
    memory.rst
    objimpl.rst
    apiabiversion.rst
+   monitoring.rst
diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst
@@ -1251,8 +1251,11 @@ PyConfig
       If non-zero, initialize the perf trampoline. See :ref:`perf_profiling`
       for more information.
 
-      Set by :option:`-X perf <-X>` command line option and by the
-      :envvar:`PYTHONPERFSUPPORT` environment variable.
+      Set by :option:`-X perf <-X>` command-line option and by the
+      :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support
+      with stack pointers and :option:`-X perf_jit <-X>` command-line option
+      and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf
+      support with DWARF JIT information.
 
       Default: ``-1``.
 

diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst
@@ -390,7 +390,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    Usage example::
 
       int32_t value;
-      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1);
+      Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1);
       if (bytes < 0) {
           // Failed. A Python exception was set with the reason.
           return NULL;
@@ -418,7 +418,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    called twice: first to determine the buffer size, then to fill it::
 
       // Ask how much space we need.
-      Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1);
+      Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1);
       if (expected < 0) {
           // Failed. A Python exception was set with the reason.
           return NULL;
@@ -430,7 +430,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
           return NULL;
       }
       // Safely get the entire value.
-      Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1);
+      Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1);
       if (bytes < 0) {  // Exception has been set.
           free(bignum);
           return NULL;

diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst
@@ -634,7 +634,7 @@ state:
 
    .. versionadded:: 3.9
 
-.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil)
+.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil)
 
    Indicate that *module* does or does not support running without the global
    interpreter lock (GIL), using one of the values from