diff --git a/.cargo/config b/.cargo/config index c17c0f23439..e5cba4fd74b 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,6 +1,3 @@ -[alias] -xtask = "run --package xtask --" - [target.'cfg(feature = "cargo-clippy")'] rustflags = [ # TODO: remove these allows once msrv increased from 1.48 @@ -20,6 +17,7 @@ rustflags = [ "-Dclippy::todo", "-Dclippy::unnecessary_wraps", "-Dclippy::useless_transmute", + "-Dclippy::used_underscore_binding", "-Delided_lifetimes_in_paths", "-Dunused_lifetimes", "-Drust_2021_prelude_collisions" diff --git a/.github/bors.toml b/.github/bors.toml deleted file mode 100644 index dfe28aba8d9..00000000000 --- a/.github/bors.toml +++ /dev/null @@ -1,7 +0,0 @@ -delete_merged_branches = true -required_approvals = 0 -use_codeowners = false -status = [ - "conclusion", -] -timeout_sec = 21600 \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 4473a007e8f..02c0d5d5c90 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,8 @@ -Thank you for contributing to pyo3! +Thank you for contributing to PyO3! + +By submitting these contributions you agree for them to be licensed under PyO3's [Apache-2.0 license](https://github.com/PyO3/pyo3#license). + +PyO3 is currently undergoing a relicensing process to match Rust's dual-license under `Apache-2.0` and `MIT` licenses. While that process is ongoing, if you are a first-time contributor please add your agreement as a comment in [#3108](https://github.com/PyO3/pyo3/pull/3108). Please consider adding the following to your pull request: - an entry for this PR in newsfragments - see [https://pyo3.rs/main/contributing.html#documenting-changes] @@ -6,5 +10,5 @@ Please consider adding the following to your pull request: - tests for all new or changed functions PyO3's CI pipeline will check your pull request. To run its tests -locally, you can run ```cargo xtask ci```. See its documentation - [here](https://github.com/PyO3/pyo3/tree/main/xtask#readme). +locally, you can run ```nox```. See ```nox --list-sessions``` +for a list of supported actions. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2155b82683c..5a14617da67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ on: jobs: build: + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7"]'), inputs.python-version) }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 @@ -122,9 +123,11 @@ jobs: - uses: dorny/paths-filter@v2 # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here - if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} + if: ${{ inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' }} id: ffi-changes with: + base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' @@ -133,10 +136,10 @@ jobs: - '.github/workflows/build.yml' - name: Run pyo3-ffi-check + # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor + # is pypy 3.9 on windows + if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check - # Allow failure on PyPy for now - continue-on-error: ${{ startsWith(inputs.python-version, 'pypy') }} - if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy-3.7' && inputs.python-version != 'pypy-3.8' }} - name: Test cross compilation diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 122d2007a12..ef597ead739 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,11 +4,9 @@ on: push: branches: - main - # for bors r+ - - staging - # for bors try - - trying pull_request: + merge_group: + types: [checks_requested] workflow_dispatch: concurrency: @@ -34,6 +32,32 @@ jobs: - name: Check rust formatting (rustfmt) run: nox -s fmt-rust + check-msrv: + needs: [fmt] + runs-on: ubuntu-latest + if: github.ref != 'refs/heads/main' + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.48.0 + targets: x86_64-unknown-linux-gnu + components: clippy,rust-src + - uses: actions/setup-python@v4 + with: + architecture: "x64" + - uses: Swatinem/rust-cache@v2 + with: + key: check-msrv-1.48.0 + continue-on-error: true + - run: python -m pip install --upgrade pip && pip install nox + - name: Prepare minimal package versions + run: nox -s set-minimal-package-versions + - run: nox -s check-all + + env: + CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu + clippy: needs: [fmt] runs-on: ${{ matrix.platform.os }} @@ -80,16 +104,6 @@ jobs: rust-target: "i686-pc-windows-msvc", }, ] - include: - - rust: 1.48.0 - python-version: "3.11" - platform: - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - } - msrv: "MSRV" name: clippy/${{ matrix.platform.rust-target }}/${{ matrix.rust }} steps: - uses: actions/checkout@v3 @@ -106,15 +120,12 @@ jobs: key: clippy-${{ matrix.platform.rust-target }}-${{ matrix.platform.os }}-${{ matrix.rust }} continue-on-error: true - run: python -m pip install --upgrade pip && pip install nox - - if: matrix.msrv == 'MSRV' - name: Prepare minimal package versions (MSRV only) - run: nox -s set-minimal-package-versions - run: nox -s clippy-all env: CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} build-pr: - if: github.event_name == 'pull_request' + if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -158,7 +169,7 @@ jobs: } ] build-full: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} needs: [fmt] uses: ./.github/workflows/build.yml @@ -183,9 +194,11 @@ jobs: "3.9", "3.10", "3.11", - "pypy-3.7", - "pypy-3.8", - "pypy-3.9" + "3.12-dev", + "pypy3.7", + "pypy3.8", + "pypy3.9", + "pypy3.10", ] platform: [ @@ -241,7 +254,7 @@ jobs: extra-features: "multiple-pymethods" valgrind: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -261,7 +274,7 @@ jobs: TRYBUILD: overwrite careful: - if: ${{ github.event_name != 'pull_request' && github.ref != 'refs/heads/main' }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || (github.event_name != 'pull_request' && github.ref != 'refs/heads/main') }} needs: [fmt] runs-on: ubuntu-latest steps: @@ -353,6 +366,7 @@ jobs: conclusion: needs: - fmt + - check-msrv - clippy - build-pr - build-full diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 20d88873b43..46d43316333 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - + - uses: actions/setup-python@v4 - uses: dtolnay/rust-toolchain@stable - uses: actions/cache@v3 @@ -74,8 +74,10 @@ jobs: - name: Run benchmarks run: | - for bench in call dict gil list pyclass pyobject set tuple; do - cargo bench --features hashbrown --bench "bench_$bench" -- --output-format bencher | tee -a output.txt + python -m pip install --upgrade pip && pip install nox + for bench in pyo3-benches/benches/*.rs; do + bench_name=$(basename "$bench" .rs) + nox -s bench -- --bench "$bench_name" -- --output-format bencher | tee -a output.txt done # Download previous benchmark result from cache (if exists) diff --git a/.netlify/build.sh b/.netlify/build.sh index 74c8bca1927..6cab914b78c 100755 --- a/.netlify/build.sh +++ b/.netlify/build.sh @@ -17,21 +17,45 @@ mv pyo3-gh-pages netlify_build ## Configure netlify _redirects file # Add redirect for each documented version +set +x # these loops get very spammy and fill the deploy log + for d in netlify_build/v*; do version="${d/netlify_build\/v/}" echo "/v$version/doc/* https://docs.rs/pyo3/$version/:splat" >> netlify_build/_redirects + if [ $version != $PYO3_VERSION ]; then + # for old versions, mark the files in the latest version as the canonical URL + for file in $(find $d -type f); do + file_path="${file/$d\//}" + # remove index.html and/or .html suffix to match the page URL on the + # final netlfiy site + url_path="$file_path" + if [[ $file_path == index.html ]]; then + url_path="" + elif [[ $file_path == *.html ]]; then + url_path="${file_path%.html}" + fi + echo "/v$version/$url_path" >> netlify_build/_headers + if test -f "netlify_build/v$PYO3_VERSION/$file_path"; then + echo " Link: ; rel=\"canonical\"" >> netlify_build/_headers + else + # this file doesn't exist in the latest guide, don't index it + echo " X-Robots-Tag: noindex" >> netlify_build/_headers + fi + done + fi done # Add latest redirect -echo "/latest/* /v${PYO3_VERSION}/:splat" >> netlify_build/_redirects +echo "/latest/* /v${PYO3_VERSION}/:splat 302" >> netlify_build/_redirects ## Add landing page redirect if [ "${CONTEXT}" == "deploy-preview" ]; then echo "/ /main/" >> netlify_build/_redirects else - echo "/ /v${PYO3_VERSION}/" >> netlify_build/_redirects + echo "/ /v${PYO3_VERSION}/ 302" >> netlify_build/_redirects fi +set -x ## Generate towncrier release notes pip install towncrier @@ -53,7 +77,7 @@ mv target/guide netlify_build/main/ ## Build public docs -cargo xtask doc +nox -s docs mv target/doc netlify_build/main/doc/ echo "" > netlify_build/main/doc/index.html @@ -61,7 +85,7 @@ echo "" > netlify_build/main/doc/in ## Build internal docs echo "
⚠️ Internal Docs ⚠️ Not Public API 👉 Official Docs Here
" > netlify_build/banner.html -RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" cargo xtask doc --internal +RUSTDOCFLAGS="--html-before-content netlify_build/banner.html" nox -s docs -- nightly internal rm netlify_build/banner.html mkdir -p netlify_build/internal diff --git a/CHANGELOG.md b/CHANGELOG.md index 11c9b6bb772..31a9e023cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,61 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.19.2] - 2023-08-01 + +### Added + +- Add FFI definitions `PyState_AddModule`, `PyState_RemoveModule` and `PyState_FindModule` for PyPy 3.9 and up. [#3295](https://github.com/PyO3/pyo3/pull/3295) +- Add FFI definitions `_PyObject_CallFunction_SizeT` and `_PyObject_CallMethod_SizeT`. [#3297](https://github.com/PyO3/pyo3/pull/3297) +- Add a "performance" section to the guide collecting performance-related tricks and problems. [#3304](https://github.com/PyO3/pyo3/pull/3304) +- Add `PyErr::Display` for all Python versions, and FFI symbol `PyErr_DisplayException` for Python 3.12. [#3334](https://github.com/PyO3/pyo3/pull/3334) +- Add FFI definition `PyType_GetDict()` for Python 3.12. [#3339](https://github.com/PyO3/pyo3/pull/3339) +- Add `PyAny::downcast_exact`. [#3346](https://github.com/PyO3/pyo3/pull/3346) +- Add `PySlice::full()` to construct a full slice (`::`). [#3353](https://github.com/PyO3/pyo3/pull/3353) + +### Changed + +- Update `PyErr` for 3.12 betas to avoid deprecated ffi methods. [#3306](https://github.com/PyO3/pyo3/pull/3306) +- Update FFI definitions of `object.h` for Python 3.12.0b4. [#3335](https://github.com/PyO3/pyo3/pull/3335) +- Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4. [#3342](https://github.com/PyO3/pyo3/pull/3342) +- Optimize conversion of `float` to `f64` (and `PyFloat::value`) on non-abi3 builds. [#3345](https://github.com/PyO3/pyo3/pull/3345) + +### Fixed + +- Fix timezone conversion bug for FixedOffset datetimes that were being incorrectly converted to and from UTC. [#3269](https://github.com/PyO3/pyo3/pull/3269) +- Fix `SystemError` raised in `PyUnicodeDecodeError_Create` on PyPy 3.10. [#3297](https://github.com/PyO3/pyo3/pull/3297) +- Correct FFI definition `Py_EnterRecursiveCall` to return `c_int` (was incorrectly returning `()`). [#3300](https://github.com/PyO3/pyo3/pull/3300) +- Fix case where `PyErr::matches` and `PyErr::is_instance` returned results inconsistent with `PyErr::get_type`. [#3313](https://github.com/PyO3/pyo3/pull/3313) +- Fix loss of panic message in `PanicException` when unwinding after the exception was "normalized". [#3326](https://github.com/PyO3/pyo3/pull/3326) +- Fix `PyErr::from_value` and `PyErr::into_value` losing traceback on conversion. [#3328](https://github.com/PyO3/pyo3/pull/3328) +- Fix reference counting of immortal objects on Python 3.12.0b4. [#3335](https://github.com/PyO3/pyo3/pull/3335) + + +## [0.19.1] - 2023-07-03 + +### Packaging + +- Extend range of supported versions of `hashbrown` optional dependency to include version 0.14 [#3258](https://github.com/PyO3/pyo3/pull/3258) +- Extend range of supported versions of `indexmap` optional dependency to include version 2. [#3277](https://github.com/PyO3/pyo3/pull/3277) +- Support PyPy 3.10. [#3289](https://github.com/PyO3/pyo3/pull/3289) + +### Added + +- Add `pyo3::types::PyFrozenSetBuilder` to allow building a `PyFrozenSet` item by item. [#3156](https://github.com/PyO3/pyo3/pull/3156) +- Add support for converting to and from Python's `ipaddress.IPv4Address`/`ipaddress.IPv6Address` and `std::net::IpAddr`. [#3197](https://github.com/PyO3/pyo3/pull/3197) +- Add support for `num-bigint` feature in combination with `abi3`. [#3198](https://github.com/PyO3/pyo3/pull/3198) +- Add `PyErr_GetRaisedException()`, `PyErr_SetRaisedException()` to FFI definitions for Python 3.12 and later. [#3248](https://github.com/PyO3/pyo3/pull/3248) +- Add `Python::with_pool` which is a safer but more limited alternative to `Python::new_pool`. [#3263](https://github.com/PyO3/pyo3/pull/3263) +- Add `PyDict::get_item_with_error` on PyPy. [#3270](https://github.com/PyO3/pyo3/pull/3270) +- Allow `#[new]` methods may to return `Py` in order to return existing instances. [#3287](https://github.com/PyO3/pyo3/pull/3287) + +### Fixed + +- Fix conversion of classes implementing `__complex__` to `Complex` when using `abi3` or PyPy. [#3185](https://github.com/PyO3/pyo3/pull/3185) +- Stop suppressing unrelated exceptions in `PyAny::hasattr`. [#3271](https://github.com/PyO3/pyo3/pull/3271) +- Fix memory leak when creating `PySet` or `PyFrozenSet` or returning types converted into these internally, e.g. `HashSet` or `BTreeSet`. [#3286](https://github.com/PyO3/pyo3/pull/3286) + + ## [0.19.0] - 2023-05-31 ### Packaging @@ -1478,7 +1533,9 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.19.2...HEAD +[0.19.2]: https://github.com/pyo3/pyo3/compare/v0.19.1...v0.19.2 +[0.19.1]: https://github.com/pyo3/pyo3/compare/v0.19.0...v0.19.1 [0.19.0]: https://github.com/pyo3/pyo3/compare/v0.18.3...v0.19.0 [0.18.3]: https://github.com/pyo3/pyo3/compare/v0.18.2...v0.18.3 [0.18.2]: https://github.com/pyo3/pyo3/compare/v0.18.1...v0.18.2 diff --git a/Cargo.toml b/Cargo.toml index b0d84aae0d2..50e04bbd1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.19.0" +version = "0.19.2" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -20,10 +20,10 @@ parking_lot = ">= 0.11, < 0.13" memoffset = "0.9" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.19.2" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.19.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.19.2", optional = true } indoc = { version = "1.0.3", optional = true } unindent = { version = "0.1.4", optional = true } @@ -34,8 +34,8 @@ inventory = { version = "0.3.0", optional = true } anyhow = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false, optional = true } eyre = { version = ">= 0.4, < 0.7", optional = true } -hashbrown = { version = ">= 0.9, < 0.14", optional = true } -indexmap = { version = "1.6", optional = true } +hashbrown = { version = ">= 0.9, < 0.15", optional = true } +indexmap = { version = ">= 1.6, < 3", optional = true } num-bigint = { version = "0.4", optional = true } num-complex = { version = ">= 0.2, < 0.5", optional = true } rust_decimal = { version = "1.0.0", default-features = false, optional = true } @@ -44,7 +44,6 @@ serde = { version = "1.0", optional = true } [dev-dependencies] assert_approx_eq = "1.1.0" chrono = { version = "0.4" } -criterion = "0.3.5" # Required for "and $N others" normalization trybuild = ">=1.0.70" rustversion = "1.0" @@ -58,7 +57,7 @@ rust_decimal = { version = "1.8.0", features = ["std"] } widestring = "0.5.1" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "0.19.2", features = ["resolve-config"] } [features] default = ["macros"] @@ -114,76 +113,14 @@ full = [ "rust_decimal", ] -[[bench]] -name = "bench_any" -harness = false - -[[bench]] -name = "bench_call" -harness = false - -[[bench]] -name = "bench_err" -harness = false - -[[bench]] -name = "bench_decimal" -harness = false -required-features = ["rust_decimal"] - -[[bench]] -name = "bench_dict" -harness = false - -[[bench]] -name = "bench_frompyobject" -harness = false -required-features = ["macros"] - -[[bench]] -name = "bench_gil" -harness = false - -[[bench]] -name = "bench_list" -harness = false - -[[bench]] -name = "bench_pyclass" -harness = false -required-features = ["macros"] - -[[bench]] -name = "bench_pyobject" -harness = false - -[[bench]] -name = "bench_set" -harness = false - -[[bench]] -name = "bench_tuple" -harness = false - -[[bench]] -name = "bench_intern" -harness = false - -[[bench]] -name = "bench_extract" -harness = false - [workspace] members = [ "pyo3-ffi", - "pyo3-ffi-check", - "pyo3-ffi-check/macro", "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", "pytests", "examples", - "xtask" ] [package.metadata.docs.rs] diff --git a/Contributing.md b/Contributing.md index a1fd31a34d9..319a6397e09 100644 --- a/Contributing.md +++ b/Contributing.md @@ -51,7 +51,7 @@ There are some specific areas of focus where help is currently needed for the do You can build the docs (including all features) with ```shell -cargo xtask doc --open +nox -s docs -- open ``` #### Doctests @@ -95,8 +95,10 @@ Tests run with all supported Python versions with the latest stable Rust compile If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. You can run these tests yourself with -```cargo xtask ci``` -See [its documentation](https://github.com/PyO3/pyo3/tree/main/xtask#readme) for more commands you can run. +```nox``` +and +```nox -l``` +lists further commands you can run. ### Documenting changes @@ -109,6 +111,36 @@ To include your changes in the release notes, you should create one (or more) ne - `removed` - for features which have been removed - `fixed` - for "changed" features which were classed as a bugfix +### Style guide + +#### Generic code + +PyO3 has a lot of generic APIs to increase usability. These can come at the cost of generic code bloat. Where reasonable, try to implement a concrete sub-portion of generic functions. There are two forms of this: + +- If the concrete sub-portion doesn't benefit from re-use by other functions, name it `inner` and keep it as a local to the function. +- If the concrete sub-portion is re-used by other functions, preferably name it `_foo` and place it directly below `foo` in the source code (where `foo` is the original generic function). + +#### FFI calls + +PyO3 makes a lot of FFI calls to Python's C API using raw pointers. Where possible try to avoid using pointers-to-temporaries in expressions: + +```rust +// dangerous +pyo3::ffi::Something(name.to_object(py).as_ptr()); + +// because the following refactoring is a use-after-free error: +let name = name.to_object(py).as_ptr(); +pyo3::ffi::Something(name) +``` + +Instead, prefer to bind the safe owned `PyObject` wrapper before passing to ffi functions: + +```rust +let name: PyObject = name.to_object(py); +pyo3::ffi::Something(name.as_ptr()) +// name will automatically be freed when it falls out of scope +``` + ## Python and Rust version support policy PyO3 aims to keep sufficient compatibility to make packaging Python extensions built with PyO3 feasible on most common package managers. @@ -133,9 +165,9 @@ CI tests both the most recent stable Rust version and the minimum supported Rust PyO3 has two sets of benchmarks for evaluating some aspects of its performance. The benchmark suite is currently very small - please open PRs with new benchmarks if you're interested in helping to expand it! -First, there are Rust-based benchmarks located in the `benches` subdirectory. As long as you have a nightly rust compiler available on your system, you can run these benchmarks with: +First, there are Rust-based benchmarks located in the `pyo3-benches` subdirectory. You can run these benchmarks with: - cargo +nightly bench + nox -s bench Second, there is a Python-based benchmark contained in the `pytests` subdirectory. You can read more about it [here](pytests). @@ -145,7 +177,7 @@ You can view what code is and isn't covered by PyO3's tests. We aim to have 100% - First, generate a `lcov.info` file with ```shell -cargo xtask coverage +nox -s coverage ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. diff --git a/README.md b/README.md index 594fbc7f519..27868be0c2e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ # PyO3 -[![actions status](https://github.com/PyO3/pyo3/workflows/CI/badge.svg)](https://github.com/PyO3/pyo3/actions) -[![benchmark](https://github.com/PyO3/pyo3/actions/workflows/bench.yml/badge.svg)](https://pyo3.rs/dev/bench/) -[![codecov](https://codecov.io/gh/PyO3/pyo3/branch/main/graph/badge.svg)](https://codecov.io/gh/PyO3/pyo3) -[![crates.io](https://img.shields.io/crates/v/pyo3)](https://crates.io/crates/pyo3) +[![actions status](https://img.shields.io/github/actions/workflow/status/PyO3/pyo3/ci.yml?branch=main&logo=github&style=)](https://github.com/PyO3/pyo3/actions) +[![benchmark](https://img.shields.io/badge/benchmark-✓-Green?logo=github)](https://pyo3.rs/dev/bench/) +[![codecov](https://img.shields.io/codecov/c/gh/PyO3/pyo3?logo=codecov)](https://codecov.io/gh/PyO3/pyo3) +[![crates.io](https://img.shields.io/crates/v/pyo3?logo=rust)](https://crates.io/crates/pyo3) [![minimum rustc 1.48](https://img.shields.io/badge/rustc-1.48+-blue.svg)](https://rust-lang.github.io/rfcs/2495-min-rust-version.html) -[![dev chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/PyO3/Lobby) -[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) +[![dev chat](https://img.shields.io/gitter/room/PyO3/Lobby?logo=gitter)](https://gitter.im/PyO3/Lobby) +[![contributing notes](https://img.shields.io/badge/contribute-on%20github-Green?logo=github)](https://github.com/PyO3/pyo3/blob/main/Contributing.md) [Rust](https://www.rust-lang.org/) bindings for [Python](https://www.python.org/), including tools for creating native Python extension modules. Running and interacting with Python code from a Rust binary is also supported. @@ -68,7 +68,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.0", features = ["extension-module"] } +pyo3 = { version = "0.19.2", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -137,7 +137,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.19.0" +version = "0.19.2" features = ["auto-initialize"] ``` @@ -192,6 +192,7 @@ about this topic. - [fastuuid](https://github.com/thedrow/fastuuid/) _Python bindings to Rust's UUID library._ - [feos](https://github.com/feos-org/feos) _Lightning fast thermodynamic modeling in Rust with fully developed Python interface._ - [forust](https://github.com/jinlow/forust) _A lightweight gradient boosted decision tree library written in Rust._ +- [haem](https://github.com/BooleanCat/haem) _A Python library for working on Bioinformatics problems._ - [html-py-ever](https://github.com/PyO3/setuptools-rust/tree/main/examples/html-py-ever) _Using [html5ever](https://github.com/servo/html5ever) through [kuchiki](https://github.com/kuchiki-rs/kuchiki) to speed up html parsing and css-selecting._ - [hyperjson](https://github.com/mre/hyperjson) _A hyper-fast Python module for reading/writing JSON data using Rust's serde-json._ - [inline-python](https://github.com/fusion-engineering/inline-python) _Inline Python code directly in your Rust code._ @@ -212,11 +213,14 @@ about this topic. - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._ - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._ +- [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._ - [wasmer-python](https://github.com/wasmerio/wasmer-python) _Python library to run WebAssembly binaries._ ## Articles and other media -- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - March 28, 2023 +- [A Week of PyO3 + rust-numpy (How to Speed Up Your Data Pipeline X Times)](https://terencezl.github.io/blog/2023/06/06/a-week-of-pyo3-rust-numpy/) - Jun 6, 2023 +- [(Podcast) PyO3 with David Hewitt](https://rustacean-station.org/episode/david-hewitt/) - May 19, 2023 +- [Making Python 100x faster with less than 100 lines of Rust](https://ohadravid.github.io/posts/2023-03-rusty-python/) - Mar 28, 2023 - [How Pydantic V2 leverages Rust's Superpowers](https://fosdem.org/2023/schedule/event/rust_how_pydantic_v2_leverages_rusts_superpowers/) - Feb 4, 2023 - [How we extended the River stats module with Rust using PyO3](https://boring-guy.sh/posts/river-rust/) - Dec 23, 2022 - [Nine Rules for Writing Python Extensions in Rust](https://towardsdatascience.com/nine-rules-for-writing-python-extensions-in-rust-d35ea3a4ec29?sk=f8d808d5f414154fdb811e4137011437) - Dec 31, 2021 @@ -244,7 +248,8 @@ If you don't have time to contribute yourself but still wish to support the proj ## License -PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). +PyO3 is licensed under the [Apache-2.0 license](https://opensource.org/licenses/APACHE-2.0). (The PyO3 project is in the process of collecting permission from past contributors to additionally license under the [MIT license](https://opensource.org/license/mit/), see [#3108](https://github.com/PyO3/pyo3/pull/3108). Once complete, PyO3 will additionally be licensed under the MIT license, the same as the Rust language itself is both Apache and MIT licensed.) + Python is licensed under the [Python License](https://docs.python.org/3/license.html). Deploys by Netlify diff --git a/codecov.yml b/codecov.yml index cfef461049a..1d903951f57 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,5 +10,5 @@ coverage: ignore: - tests/*.rs + - pytests/*.rs - src/test_hygiene/*.rs - - src/impl_/ghost.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index e76132c4afe..ab627206ae8 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "examples" +name = "pyo3-examples" version = "0.0.0" publish = false edition = "2018" [dev-dependencies] -pyo3 = { version = "0.19.0", path = "..", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.19.2", path = "..", features = ["auto-initialize", "extension-module"] } [[example]] name = "decorator" diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 0368bb1f432..d3341677b1f 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/.template/pyproject.toml b/examples/decorator/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/decorator/.template/pyproject.toml +++ b/examples/decorator/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/decorator/pyproject.toml b/examples/decorator/pyproject.toml index c6196256d35..8575ca25fc2 100644 --- a/examples/decorator/pyproject.toml +++ b/examples/decorator/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 0368bb1f432..d3341677b1f 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pyproject.toml b/examples/maturin-starter/.template/pyproject.toml index cd79e887b23..537fdacc666 100644 --- a/examples/maturin-starter/.template/pyproject.toml +++ b/examples/maturin-starter/.template/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/maturin-starter/pyproject.toml b/examples/maturin-starter/pyproject.toml index 9a18a20ea8b..fb9c808f283 100644 --- a/examples/maturin-starter/pyproject.toml +++ b/examples/maturin-starter/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 0915e77badd..6ab231df25e 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/plugin/plugin_api/pyproject.toml b/examples/plugin/plugin_api/pyproject.toml index 114687eddef..6350644ca14 100644 --- a/examples/plugin/plugin_api/pyproject.toml +++ b/examples/plugin/plugin_api/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 3ed1b01d30f..b06e0f272b1 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/requirements-dev.txt b/examples/setuptools-rust-starter/requirements-dev.txt index c52fb607d12..3235670811c 100644 --- a/examples/setuptools-rust-starter/requirements-dev.txt +++ b/examples/setuptools-rust-starter/requirements-dev.txt @@ -1,3 +1,4 @@ pytest>=3.5.0 setuptools_rust~=1.0.0 pip>=21.3 +wheel diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 68988ce3f48..1349490d477 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,3 +1,3 @@ -variable::set("PYO3_VERSION", "0.19.0"); +variable::set("PYO3_VERSION", "0.19.2"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::delete(".template"); diff --git a/examples/word-count/pyproject.toml b/examples/word-count/pyproject.toml index d8ce3650277..6f88a5170f1 100644 --- a/examples/word-count/pyproject.toml +++ b/examples/word-count/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 2198c3792e9..e75095f4bef 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -24,6 +24,7 @@ - [Debugging](debugging.md) - [Features reference](features.md) - [Memory management](memory.md) +- [Performance](performance.md) - [Advanced topics](advanced.md) - [Building and distribution](building_and_distribution.md) - [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md) diff --git a/guide/src/class.md b/guide/src/class.md index dae5cbc04ec..b5a27b517cc 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -60,7 +60,7 @@ To integrate Rust types with Python, PyO3 needs to place some restrictions on th Rust lifetimes are used by the Rust compiler to reason about a program's memory safety. They are a compile-time only concept; there is no way to access Rust lifetimes at runtime from a dynamic language like Python. -As soon as Rust data is exposed to Python, there is no guarantee which the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. +As soon as Rust data is exposed to Python, there is no guarantee that the Rust compiler can make on how long the data will live. Python is a reference-counted language and those references can be held for an arbitrarily long time which is untraceable by the Rust compiler. The only possible way to express this correctly is to require that any `#[pyclass]` does not borrow data for any lifetime shorter than the `'static` lifetime, i.e. the `#[pyclass]` cannot have any lifetime parameters. When you need to share ownership of data between Python and Rust, instead of using borrowed references with lifetimes consider using reference-counted smart pointers such as [`Arc`] or [`Py`]. @@ -74,7 +74,7 @@ Because Python objects are freely shared between threads by the Python interpret ## Constructor -By default it is not possible to create an instance of a custom class from Python code. +By default, it is not possible to create an instance of a custom class from Python code. To declare a constructor, you need to define a method and annotate it with the `#[new]` attribute. Only Python's `__new__` method can be specified, `__init__` is not available. @@ -113,8 +113,11 @@ impl Nonzero { } ``` +If you want to return an existing object (for example, because your `new` +method caches the values it returns), `new` can return `pyo3::Py`. + As you can see, the Rust method name is not important here; this way you can -still use `new()` for a Rust-level constructor. +still, use `new()` for a Rust-level constructor. If no method marked with `#[new]` is declared, object instances can only be created from Rust, but not from Python. @@ -226,8 +229,10 @@ struct FrozenCounter { value: AtomicUsize, } -let py_counter: Py = Python::with_gil(|py| { - let counter = FrozenCounter { value: AtomicUsize::new(0) }; +let py_counter: Py = Python::with_gil(|py| { + let counter = FrozenCounter { + value: AtomicUsize::new(0), + }; Py::new(py, counter).unwrap() }); @@ -264,7 +269,7 @@ use the `extends` parameter for `pyclass` with the full path to the base class. For convenience, `(T, U)` implements `Into>` where `U` is the base class of `T`. -But for more deeply nested inheritance, you have to return `PyClassInitializer` +But for a more deeply nested inheritance, you have to return `PyClassInitializer` explicitly. To get a parent class from a child, use [`PyRef`] instead of `&self` for methods, @@ -647,9 +652,9 @@ impl BaseClass { #[new] #[classmethod] fn py_new<'p>(cls: &'p PyType, py: Python<'p>) -> PyResult { - // Get an abstract attribute (presumably) declared on a subclass of this class. - let subclass_attr = cls.getattr("a_class_attr")?; - Ok(Self(subclass_attr.to_object(py))) + // Get an abstract attribute (presumably) declared on a subclass of this class. + let subclass_attr = cls.getattr("a_class_attr")?; + Ok(Self(subclass_attr.to_object(py))) } } ``` @@ -716,11 +721,72 @@ impl MyClass { } ``` +## Free functions + +Free functions defined using `#[pyfunction]` interact with classes through the same mechanisms as the self parameters of instance methods, i.e. they can take GIL-bound references, GIL-bound reference wrappers or GIL-indepedent references: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +struct MyClass { + my_field: i32, +} + +// Take a GIL-bound reference when the underlying `PyCell` is irrelevant. +#[pyfunction] +fn increment_field(my_class: &mut MyClass) { + my_class.my_field += 1; +} + +// Take a GIL-bound reference wrapper when borrowing should be automatic, +// but interaction with the underlying `PyCell` is desired. +#[pyfunction] +fn print_field(my_class: PyRef<'_, MyClass>) { + println!("{}", my_class.my_field); +} + +// Take a GIL-bound reference to the underlying cell +// when borrowing needs to be managed manually. +#[pyfunction] +fn increment_then_print_field(my_class: &PyCell) { + my_class.borrow_mut().my_field += 1; + + println!("{}", my_class.borrow().my_field); +} + +// Take a GIL-indepedent reference when you want to store the reference elsewhere. +#[pyfunction] +fn print_refcnt(my_class: Py, py: Python<'_>) { + println!("{}", my_class.get_refcnt(py)); +} +``` + +Classes can also be passed by value if they can be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +#[pyclass] +#[derive(Clone)] +struct MyClass { + my_field: Box, +} + +#[pyfunction] +fn dissamble_clone(my_class: MyClass) { + let MyClass { mut my_field } = my_class; + *my_field += 1; +} +``` + +Note that `#[derive(FromPyObject)]` on a class is usually not useful as it tries to construct a new Rust value by filling in the fields by looking up attributes of any given Python value. + ## Method arguments Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be used to specify the way that `#[pymethods]` accept arguments. Consult the documentation for [`function signatures`](./function/signature.md) to see the parameters this attribute accepts. -The following example defines a class `MyClass` with a method `method`. This method has a signature which sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: +The following example defines a class `MyClass` with a method `method`. This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: ```rust # use pyo3::prelude::*; @@ -756,7 +822,7 @@ impl MyClass { } ``` -In Python this might be used like: +In Python, this might be used like: ```python >>> import mymodule diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 38ed27c1f1e..01a5c5a3cfc 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -13,30 +13,34 @@ The table below contains the Python type and the corresponding function argument | Python | Rust | Rust (Python-native) | | ------------- |:-------------------------------:|:--------------------:| | `object` | - | `&PyAny` | -| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf` | `&PyUnicode` | +| `str` | `String`, `Cow`, `&str`, `OsString`, `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `&PyBytes` | | `bool` | `bool` | `&PyBool` | -| `int` | Any integer type (`i32`, `u32`, `usize`, etc) | `&PyLong` | +| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `&PyLong` | | `float` | `f32`, `f64` | `&PyFloat` | -| `complex` | `num_complex::Complex`[^1] | `&PyComplex` | +| `complex` | `num_complex::Complex`[^2] | `&PyComplex` | | `list[T]` | `Vec` | `&PyList` | -| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^2], `indexmap::IndexMap`[^3] | `&PyDict` | +| `dict[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyDict` | | `tuple[T, U]` | `(T, U)`, `Vec` | `&PyTuple` | -| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^2] | `&PySet` | -| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^2] | `&PyFrozenSet` | +| `set[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PySet` | +| `frozenset[T]` | `HashSet`, `BTreeSet`, `hashbrown::HashSet`[^3] | `&PyFrozenSet` | | `bytearray` | `Vec`, `Cow<[u8]>` | `&PyByteArray` | | `slice` | - | `&PySlice` | | `type` | - | `&PyType` | | `module` | - | `&PyModule` | +| `collections.abc.Buffer` | - | `PyBuffer` | | `datetime.datetime` | - | `&PyDateTime` | | `datetime.date` | - | `&PyDate` | | `datetime.time` | - | `&PyTime` | | `datetime.tzinfo` | - | `&PyTzInfo` | | `datetime.timedelta` | - | `&PyDelta` | -| `collections.abc.Buffer` | - | `PyBuffer` | +| `decimal.Decimal` | `rust_decimal::Decimal`[^5] | - | +| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | +| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | +| `pathlib.Path` | `PathBuf`, `Path` | `&PyString`, `&PyUnicode` | | `typing.Optional[T]` | `Option` | - | | `typing.Sequence[T]` | `Vec` | `&PySequence` | -| `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^2], `indexmap::IndexMap`[^3] | `&PyMapping` | +| `typing.Mapping[K, V]` | `HashMap`, `BTreeMap`, `hashbrown::HashMap`[^3], `indexmap::IndexMap`[^4] | `&PyMapping` | | `typing.Iterator[Any]` | - | `&PyIterator` | | `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.html#deriving-a-hrefhttpsdocsrspyo3latestpyo3conversiontraitfrompyobjecthtmlfrompyobjecta-for-enums) | - | @@ -94,8 +98,12 @@ Finally, the following Rust types are also able to convert to Python as return v | `PyRef` | `T` | | `PyRefMut` | `T` | -[^1]: Requires the `num-complex` optional feature. +[^1]: Requires the `num-bigint` optional feature. + +[^2]: Requires the `num-complex` optional feature. + +[^3]: Requires the `hashbrown` optional feature. -[^2]: Requires the `hashbrown` optional feature. +[^4]: Requires the `indexmap` optional feature. -[^3]: Requires the `indexmap` optional feature. +[^5]: Requires the `rust_decimal` optional feature. diff --git a/guide/src/faq.md b/guide/src/faq.md index a475b2fe6e7..8308acc64de 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -187,7 +187,7 @@ This happens on Windows when linking to the python DLL fails or the wrong one is - `python3X.dll` for Python 3.X, e.g. `python310.dll` for Python 3.10 - `python3.dll` when using PyO3's `abi3` feature -The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). The two easiest ways to achieve this are: +The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps). Some ways to achieve this are: - Put the Python DLL in the same folder as your build artifacts - Add the directory containing the Python DLL to your `PATH` environment variable, for example `C:\Users\\AppData\Local\Programs\Python\Python310` - If this happens when you are *distributing* your program, consider using [PyOxidizer](https://github.com/indygreg/PyOxidizer) to package it with your binary. diff --git a/guide/src/function/error_handling.md b/guide/src/function/error_handling.md index 38cc602eb72..b0f63885cdf 100644 --- a/guide/src/function/error_handling.md +++ b/guide/src/function/error_handling.md @@ -13,7 +13,7 @@ Rust code uses the generic [`Result`] enum to propagate errors. The error PyO3 has the [`PyErr`] type which represents a Python exception. If a PyO3 API could result in a Python exception being raised, the return type of that `API` will be [`PyResult`], which is an alias for the type `Result`. In summary: -- When Python exceptions are raised and caught by PyO3, the exception will stored in the `Err` variant of the `PyResult`. +- When Python exceptions are raised and caught by PyO3, the exception will be stored in the `Err` variant of the `PyResult`. - Passing Python exceptions through Rust code then uses all the "normal" techniques such as the `?` operator, with `PyErr` as the error type. - Finally, when a `PyResult` crosses from Rust back to Python via PyO3, if the result is an `Err` variant the contained exception will be raised. diff --git a/guide/src/getting_started.md b/guide/src/getting_started.md index 2dd45a43ce7..2074590e325 100644 --- a/guide/src/getting_started.md +++ b/guide/src/getting_started.md @@ -123,7 +123,7 @@ You should also create a `pyproject.toml` with the following contents: ```toml [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [project] diff --git a/guide/src/performance.md b/guide/src/performance.md new file mode 100644 index 00000000000..23fb59c4e90 --- /dev/null +++ b/guide/src/performance.md @@ -0,0 +1,94 @@ +# Performance + +To achieve the best possible performance, it is useful to be aware of several tricks and sharp edges concerning PyO3's API. + +## `extract` versus `downcast` + +Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&PyAny` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::{exceptions::PyTypeError, types::PyList}; + +fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { + todo!() +} + +fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { + todo!() +} + +#[pyfunction] +fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { + if let Ok(list) = value.extract::<&PyList>() { + frobnicate_list(list) + } else if let Ok(vec) = value.extract::>() { + frobnicate_vec(vec) + } else { + Err(PyTypeError::new_err("Cannot frobnicate that type.")) + } +} +``` + +This suboptimal as the `FromPyObject` trait requires `extract` to have a `Result` return type. For native types like `PyList`, it faster to use `downcast` (which `extract` calls internally) when the error value is ignored. This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e. + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::{exceptions::PyTypeError, types::PyList}; +# fn frobnicate_list(list: &PyList) -> PyResult<&PyAny> { todo!() } +# fn frobnicate_vec(vec: Vec<&PyAny>) -> PyResult<&PyAny> { todo!() } +# +#[pyfunction] +fn frobnicate(value: &PyAny) -> PyResult<&PyAny> { + // Use `downcast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly. + if let Ok(list) = value.downcast::() { + frobnicate_list(list) + } else if let Ok(vec) = value.extract::>() { + frobnicate_vec(vec) + } else { + Err(PyTypeError::new_err("Cannot frobnicate that type.")) + } +} +``` + +## Access to GIL-bound reference implies access to GIL token + +Calling `Python::with_gil` is effectively a no-op when the GIL is already held, but checking that this is the case still has a cost. If an existing GIL token can not be accessed, for example when implementing a pre-existing trait, but a GIL-bound reference is available, this cost can be avoided by exploiting that access to GIL-bound reference gives zero-cost access to a GIL token via `PyAny::py`. + +For example, instead of writing + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::PyList; + +struct Foo(Py); + +struct FooRef<'a>(&'a PyList); + +impl PartialEq for FooRef<'_> { + fn eq(&self, other: &Foo) -> bool { + Python::with_gil(|py| self.0.len() == other.0.as_ref(py).len()) + } +} +``` + +use more efficient + +```rust +# #![allow(dead_code)] +# use pyo3::prelude::*; +# use pyo3::types::PyList; +# struct Foo(Py); +# struct FooRef<'a>(&'a PyList); +# +impl PartialEq for FooRef<'_> { + fn eq(&self, other: &Foo) -> bool { + // Access to `&'a PyAny` implies access to `Python<'a>`. + let py = self.0.py(); + self.0.len() == other.0.as_ref(py).len() + } +} +``` diff --git a/guide/src/python_typing_hints.md b/guide/src/python_typing_hints.md index e5ce1fe70d8..9d6c206f110 100644 --- a/guide/src/python_typing_hints.md +++ b/guide/src/python_typing_hints.md @@ -1,6 +1,6 @@ # Typing and IDE hints for you Python package -PyO3 provides an easy to use interface to code native Python libraries in Rust. The accompanying Maturin allows you to build and publish them as a package. Yet, for the better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. +PyO3 provides an easy to use interface to code native Python libraries in Rust. The accompanying Maturin allows you to build and publish them as a package. Yet, for a better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code. Currently the best solution for the problem is to manually maintain `*.pyi` files and ship them along with the package. @@ -12,7 +12,7 @@ There is a sketch of a roadmap towards completing [the `experimental-inspect` fe > A stubs file only contains a description of the public interface of the module without any implementations. -There is also [extensive documentation on type stubs on the offical Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html). +There is also [extensive documentation on type stubs on the official Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html). Most Python developers probably already encountered them when trying to use their IDE's "Go to Definition" function on any builtin type. For example, the definitions of a few standard exceptions look like this: @@ -37,7 +37,7 @@ class StopIteration(Exception): value: Any ``` -As we can see, those are not full definitions containing implementation, but just a description of interface. It is usually all that the user of the library needs. +As we can see, those are not full definitions containing implementation, but just a description of the interface. It is usually all that the user of the library needs. ### What do the PEPs say? @@ -75,7 +75,7 @@ When source files are in the same package as stub files, they should be placed n #### If you do not have other Python files -If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. As documented in the [Maturin Guide](https://github.com/PyO3/maturin/blob/084cfaced651b28616aeea1f818bdc933a536bfe/guide/src/project_layout.md#adding-python-type-information), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. +If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you. As documented in the [Maturin Guide](https://github.com/PyO3/maturin/#mixed-rustpython-projects), the only thing you need to do is to create a stub file for your module named `.pyi` in your project root and Maturin will do the rest. ```text my-rust-project/ @@ -108,7 +108,7 @@ my-project └── lib.rs ``` -Let's go a little bit more into details regarding the files inside the package folder. +Let's go a little bit more into detail regarding the files inside the package folder. ##### `__init__.py` content diff --git a/noxfile.py b/noxfile.py index d264cdc0467..8ab8d9ec545 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,20 +4,19 @@ import subprocess import sys import tempfile -import time from functools import lru_cache from glob import glob from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple import nox -nox.options.sessions = ["test", "clippy", "fmt"] +nox.options.sessions = ["test", "clippy", "fmt", "docs"] PYO3_DIR = Path(__file__).parent -PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11") -PYPY_VERSIONS = ("3.7", "3.8", "3.9") +PY_VERSIONS = ("3.7", "3.8", "3.9", "3.10", "3.11", "3.12") +PYPY_VERSIONS = ("3.7", "3.8", "3.9", "3.10") @nox.session(venv_backend="none") @@ -35,7 +34,7 @@ def test_rust(session: nox.Session): _run_cargo_test(session) _run_cargo_test(session, features="abi3") - if not "skip-full" in session.posargs: + if "skip-full" not in session.posargs: _run_cargo_test(session, features="full") _run_cargo_test(session, features="abi3 full") @@ -50,11 +49,10 @@ def test_py(session: nox.Session) -> None: @nox.session(venv_backend="none") def coverage(session: nox.Session) -> None: session.env.update(_get_coverage_env()) - _run(session, "cargo", "llvm-cov", "clean", "--workspace", external=True) + _run_cargo(session, "llvm-cov", "clean", "--workspace") test(session) - _run( + _run_cargo( session, - "cargo", "llvm-cov", "--package=pyo3", "--package=pyo3-build-config", @@ -65,7 +63,6 @@ def coverage(session: nox.Session) -> None: "--codecov", "--output-path", "coverage.json", - external=True, ) @@ -77,7 +74,8 @@ def fmt(session: nox.Session): @nox.session(name="fmt-rust", venv_backend="none") def fmt_rust(session: nox.Session): - _run(session, "cargo", "fmt", "--all", "--check", external=True) + _run_cargo(session, "fmt", "--all", "--check") + _run_cargo(session, "fmt", _FFI_CHECK, "--all", "--check") @nox.session(name="fmt-py") @@ -88,7 +86,7 @@ def fmt_py(session: nox.Session): @nox.session(name="clippy", venv_backend="none") def clippy(session: nox.Session) -> bool: - if not _clippy(session): + if not _clippy(session) and _clippy_additional_workspaces(session): session.error("one or more jobs failed") @@ -96,26 +94,15 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: success = True env = env or os.environ for feature_set in _get_feature_sets(): - command = "clippy" - extra = ("--", "--deny=warnings") - if _get_rust_version()[:2] == (1, 48): - # 1.48 crashes during clippy because of lints requested - # in .cargo/config - command = "check" - extra = () try: - _run( + _run_cargo( session, - "cargo", - command, + "clippy", *feature_set, "--all-targets", "--workspace", - # linting pyo3-ffi-check requires docs to have been built or - # the macros will error; doesn't seem worth it on CI - "--exclude=pyo3-ffi-check", - *extra, - external=True, + "--", + "--deny=warnings", env=env, ) except Exception: @@ -123,34 +110,68 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: return success +def _clippy_additional_workspaces(session: nox.Session) -> bool: + # pyo3-benches and pyo3-ffi-check are in isolated workspaces so that their + # dependencies do not interact with MSRV + + success = True + try: + _run_cargo(session, "clippy", _BENCHES) + except Exception: + success = False + + # Run pyo3-ffi-check only on when not cross-compiling, because it needs to + # have Python headers to feed to bindgen which gets messy when cross-compiling. + target = os.environ.get("CARGO_BUILD_TARGET") + if target is None or _get_rust_default_target() == target: + try: + _build_docs_for_ffi_check(session) + _run_cargo(session, "clippy", _FFI_CHECK, "--workspace", "--all-targets") + except Exception: + success = False + return success + + +@nox.session(venv_backend="none") +def bench(session: nox.Session) -> bool: + _run_cargo(session, "bench", _BENCHES, *session.posargs) + + @nox.session(name="clippy-all", venv_backend="none") def clippy_all(session: nox.Session) -> None: success = True - with tempfile.NamedTemporaryFile("r+") as config: - env = os.environ.copy() - env["PYO3_CONFIG_FILE"] = config.name - env["PYO3_CI"] = "1" - def _clippy_with_config(implementation, version) -> bool: - config.seek(0) - config.truncate(0) - config.write( - f"""\ -implementation={implementation} -version={version} -suppress_build_script_link_lines=true -""" - ) - config.flush() + def _clippy_with_config(env: Dict[str, str]) -> None: + nonlocal success + success &= _clippy(session, env=env) - session.log(f"{implementation} {version}") - return _clippy(session, env=env) + _for_all_version_configs(session, _clippy_with_config) + success &= _clippy_additional_workspaces(session) - for version in PY_VERSIONS: - success &= _clippy_with_config("CPython", version) + if not success: + session.error("one or more jobs failed") - for version in PYPY_VERSIONS: - success &= _clippy_with_config("PyPy", version) + +@nox.session(name="check-all", venv_backend="none") +def check_all(session: nox.Session) -> None: + success = True + + def _check(env: Dict[str, str]) -> None: + nonlocal success + for feature_set in _get_feature_sets(): + try: + _run_cargo( + session, + "check", + *feature_set, + "--all-targets", + "--workspace", + env=env, + ) + except Exception: + success = False + + _for_all_version_configs(session, _check) if not success: session.error("one or more jobs failed") @@ -159,13 +180,9 @@ def _clippy_with_config(implementation, version) -> bool: @nox.session(venv_backend="none") def publish(session: nox.Session) -> None: _run_cargo_publish(session, package="pyo3-build-config") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros-backend") - time.sleep(10) _run_cargo_publish(session, package="pyo3-macros") - time.sleep(10) _run_cargo_publish(session, package="pyo3-ffi") - time.sleep(10) _run_cargo_publish(session, package="pyo3") @@ -287,6 +304,43 @@ def test_emscripten(session: nox.Session): ) +@nox.session(venv_backend="none") +def docs(session: nox.Session) -> None: + rustdoc_flags = ["-Dwarnings"] + toolchain_flags = [] + cargo_flags = [] + + if "open" in session.posargs: + cargo_flags.append("--open") + + if "nightly" in session.posargs: + rustdoc_flags.append("--cfg docsrs") + toolchain_flags.append("+nightly") + cargo_flags.extend(["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"]) + + if "nightly" in session.posargs and "internal" in session.posargs: + rustdoc_flags.append("--Z unstable-options") + rustdoc_flags.append("--document-hidden-items") + cargo_flags.append("--document-private-items") + else: + cargo_flags.extend(["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"]) + + rustdoc_flags.append(session.env.get("RUSTDOCFLAGS", "")) + session.env["RUSTDOCFLAGS"] = " ".join(rustdoc_flags) + + _run_cargo( + session, + *toolchain_flags, + "doc", + "--lib", + "--no-default-features", + "--features=full", + "--no-deps", + "--workspace", + *cargo_flags, + ) + + @nox.session(name="build-guide", venv_backend="none") def build_guide(session: nox.Session): _run(session, "mdbook", "build", "-d", "../target/guide", "guide", *session.posargs) @@ -298,8 +352,6 @@ def format_guide(session: nox.Session): for path in Path("guide").glob("**/*.md"): session.log("Working on %s", path) - content = path.read_text() - lines = iter(path.read_text().splitlines(True)) new_lines = [] @@ -346,14 +398,13 @@ def format_guide(session: nox.Session): @nox.session(name="address-sanitizer", venv_backend="none") def address_sanitizer(session: nox.Session): - _run( + _run_cargo( session, - "cargo", "+nightly", "test", "--release", "-Zbuild-std", - f"--target={_get_rust_target()}", + f"--target={_get_rust_default_target()}", "--", "--test-threads=1", env={ @@ -361,7 +412,6 @@ def address_sanitizer(session: nox.Session): "RUSTDOCFLAGS": "-Zsanitizer=address", "ASAN_OPTIONS": "detect_leaks=0", }, - external=True, ) @@ -449,6 +499,7 @@ def set_minimal_package_versions(session: nox.Session): "trybuild": "1.0.76", # pins to avoid syn 2.0 (which requires Rust 1.56) "ghost": "0.1.8", + "serde_json": "1.0.99", "serde": "1.0.156", "serde_derive": "1.0.156", "cxx": "1.0.92", @@ -458,21 +509,17 @@ def set_minimal_package_versions(session: nox.Session): "js-sys": "0.3.61", "wasm-bindgen": "0.2.84", "syn": "1.0.109", + # proc-macro2 1.0.66+ is edition 2021 + "quote": "1.0.30", + "proc-macro2": "1.0.65", } - # run cargo update first to ensure that everything is at highest # possible version, so that this matches what CI will resolve to. for project in projects: if project is None: - _run(session, "cargo", "update", external=True) + _run_cargo(session, "update") else: - _run( - session, - "cargo", - "update", - f"--manifest-path={project}/Cargo.toml", - external=True, - ) + _run_cargo(session, "update", f"--manifest-path={project}/Cargo.toml") for project in projects: lock_file = Path(project or "") / "Cargo.lock" @@ -506,22 +553,25 @@ def load_pkg_versions(): # supported on MSRV for project in projects: if project is None: - _run(session, "cargo", "metadata", silent=True, external=True) + _run_cargo(session, "metadata", silent=True) else: - _run( + _run_cargo( session, - "cargo", "metadata", f"--manifest-path={project}/Cargo.toml", silent=True, - external=True, ) @nox.session(name="ffi-check") def ffi_check(session: nox.Session): - session.run("cargo", "doc", "-p", "pyo3-ffi", "--no-deps", external=True) - _run(session, "cargo", "run", "-p", "pyo3-ffi-check", external=True) + _build_docs_for_ffi_check(session) + _run_cargo(session, "run", _FFI_CHECK) + + +def _build_docs_for_ffi_check(session: nox.Session) -> None: + # pyo3-ffi-check needs to scrape docs of pyo3-ffi + _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps") @lru_cache() @@ -540,7 +590,7 @@ def _get_rust_version() -> Tuple[int, int, int, List[str]]: return (*map(int, version_number.split(".")), extra) -def _get_rust_target() -> str: +def _get_rust_default_target() -> str: for line in _get_rust_info(): if line.startswith(_HOST_LINE_START): return line[len(_HOST_LINE_START) :].strip() @@ -603,6 +653,10 @@ def _run(session: nox.Session, *args: str, **kwargs: Any) -> None: print("::endgroup::", file=sys.stderr) +def _run_cargo(session: nox.Session, *args: str, **kwargs: Any) -> None: + _run(session, "cargo", *args, **kwargs, external=True) + + def _run_cargo_test( session: nox.Session, *, @@ -624,7 +678,7 @@ def _run_cargo_test( def _run_cargo_publish(session: nox.Session, *, package: str) -> None: - _run(session, "cargo", "publish", f"--package={package}", external=True) + _run_cargo(session, "publish", f"--package={package}") def _run_cargo_set_package_version( @@ -642,3 +696,37 @@ def _run_cargo_set_package_version( def _get_output(*args: str) -> str: return subprocess.run(args, capture_output=True, text=True, check=True).stdout + + +def _for_all_version_configs( + session: nox.Session, job: Callable[[Dict[str, str]], None] +) -> None: + with tempfile.NamedTemporaryFile("r+") as config: + env = os.environ.copy() + env["PYO3_CONFIG_FILE"] = config.name + env["PYO3_CI"] = "1" + + def _job_with_config(implementation, version) -> bool: + config.seek(0) + config.truncate(0) + config.write( + f"""\ +implementation={implementation} +version={version} +suppress_build_script_link_lines=true +""" + ) + config.flush() + + session.log(f"{implementation} {version}") + return job(env) + + for version in PY_VERSIONS: + _job_with_config("CPython", version) + + for version in PYPY_VERSIONS: + _job_with_config("PyPy", version) + + +_BENCHES = "--manifest-path=pyo3-benches/Cargo.toml" +_FFI_CHECK = "--manifest-path=pyo3-ffi-check/Cargo.toml" diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml new file mode 100644 index 00000000000..f04e5429a06 --- /dev/null +++ b/pyo3-benches/Cargo.toml @@ -0,0 +1,80 @@ +[package] +name = "pyo3-benches" +version = "0.1.0" +description = "In-tree benchmarks for the PyO3 project" +authors = ["PyO3 Project and Contributors "] +edition = "2021" +publish = false + +[dependencies] +pyo3 = { path = "../", features = ["auto-initialize"] } + +[dev-dependencies] +criterion = "0.5.1" + +[[bench]] +name = "bench_any" +harness = false + +[[bench]] +name = "bench_call" +harness = false + +[[bench]] +name = "bench_comparisons" +harness = false + +[[bench]] +name = "bench_err" +harness = false + +[[bench]] +name = "bench_decimal" +harness = false +required-features = ["pyo3/rust_decimal"] + +[[bench]] +name = "bench_dict" +harness = false +required-features = ["pyo3/hashbrown"] + +[[bench]] +name = "bench_frompyobject" +harness = false +required-features = ["pyo3/macros"] + +[[bench]] +name = "bench_gil" +harness = false + +[[bench]] +name = "bench_list" +harness = false + +[[bench]] +name = "bench_pyclass" +harness = false +required-features = ["pyo3/macros"] + +[[bench]] +name = "bench_pyobject" +harness = false + +[[bench]] +name = "bench_set" +harness = false +required-features = ["pyo3/hashbrown"] + +[[bench]] +name = "bench_tuple" +harness = false + +[[bench]] +name = "bench_intern" +harness = false + +[[bench]] +name = "bench_extract" +harness = false + +[workspace] diff --git a/benches/bench_any.rs b/pyo3-benches/benches/bench_any.rs similarity index 81% rename from benches/bench_any.rs rename to pyo3-benches/benches/bench_any.rs index ec23cedc1c0..765497fa079 100644 --- a/benches/bench_any.rs +++ b/pyo3-benches/benches/bench_any.rs @@ -5,7 +5,7 @@ use pyo3::{ PyBool, PyByteArray, PyBytes, PyDict, PyFloat, PyFrozenSet, PyInt, PyList, PyMapping, PySequence, PySet, PyString, PyTuple, }, - PyAny, Python, + PyAny, PyResult, Python, }; #[derive(PartialEq, Eq, Debug)] @@ -71,8 +71,23 @@ fn bench_identify_object_type(b: &mut Bencher<'_>) { }); } +fn bench_collect_generic_iterator(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let collection = py.eval("list(range(1 << 20))", None, None).unwrap(); + + b.iter(|| { + collection + .iter() + .unwrap() + .collect::>>() + .unwrap() + }); + }); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("identify_object_type", bench_identify_object_type); + c.bench_function("collect_generic_iterator", bench_collect_generic_iterator); } criterion_group!(benches, criterion_benchmark); diff --git a/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs similarity index 100% rename from benches/bench_call.rs rename to pyo3-benches/benches/bench_call.rs diff --git a/pyo3-benches/benches/bench_comparisons.rs b/pyo3-benches/benches/bench_comparisons.rs new file mode 100644 index 00000000000..bfa4ac63fa4 --- /dev/null +++ b/pyo3-benches/benches/bench_comparisons.rs @@ -0,0 +1,70 @@ +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; + +use pyo3::{prelude::*, pyclass::CompareOp, Python}; + +#[pyclass] +struct OrderedDunderMethods(i64); + +#[pymethods] +impl OrderedDunderMethods { + fn __lt__(&self, other: &Self) -> bool { + self.0 < other.0 + } + + fn __le__(&self, other: &Self) -> bool { + self.0 <= other.0 + } + + fn __eq__(&self, other: &Self) -> bool { + self.0 == other.0 + } + + fn __ne__(&self, other: &Self) -> bool { + self.0 != other.0 + } + + fn __gt__(&self, other: &Self) -> bool { + self.0 > other.0 + } + + fn __ge__(&self, other: &Self) -> bool { + self.0 >= other.0 + } +} + +#[pyclass] +#[derive(PartialEq, Eq, PartialOrd, Ord)] +struct OrderedRichcmp(i64); + +#[pymethods] +impl OrderedRichcmp { + fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool { + op.matches(self.cmp(other)) + } +} + +fn bench_ordered_dunder_methods(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj1 = Py::new(py, OrderedDunderMethods(0)).unwrap().into_ref(py); + let obj2 = Py::new(py, OrderedDunderMethods(1)).unwrap().into_ref(py); + + b.iter(|| obj2.gt(obj1).unwrap()); + }); +} + +fn bench_ordered_richcmp(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj1 = Py::new(py, OrderedRichcmp(0)).unwrap().into_ref(py); + let obj2 = Py::new(py, OrderedRichcmp(1)).unwrap().into_ref(py); + + b.iter(|| obj2.gt(obj1).unwrap()); + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("ordered_dunder_methods", bench_ordered_dunder_methods); + c.bench_function("ordered_richcmp", bench_ordered_richcmp); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/bench_decimal.rs b/pyo3-benches/benches/bench_decimal.rs similarity index 100% rename from benches/bench_decimal.rs rename to pyo3-benches/benches/bench_decimal.rs diff --git a/benches/bench_dict.rs b/pyo3-benches/benches/bench_dict.rs similarity index 98% rename from benches/bench_dict.rs rename to pyo3-benches/benches/bench_dict.rs index 2b92159d10a..64398a65e39 100644 --- a/benches/bench_dict.rs +++ b/pyo3-benches/benches/bench_dict.rs @@ -10,7 +10,7 @@ fn iter_dict(b: &mut Bencher<'_>) { let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); let mut sum = 0; b.iter(|| { - for (k, _v) in dict.iter() { + for (k, _v) in dict { let i: u64 = k.extract().unwrap(); sum += i; } diff --git a/benches/bench_err.rs b/pyo3-benches/benches/bench_err.rs similarity index 100% rename from benches/bench_err.rs rename to pyo3-benches/benches/bench_err.rs diff --git a/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs similarity index 100% rename from benches/bench_extract.rs rename to pyo3-benches/benches/bench_extract.rs diff --git a/benches/bench_frompyobject.rs b/pyo3-benches/benches/bench_frompyobject.rs similarity index 88% rename from benches/bench_frompyobject.rs rename to pyo3-benches/benches/bench_frompyobject.rs index 587a89bcdf3..c2dfbc0ea85 100644 --- a/benches/bench_frompyobject.rs +++ b/pyo3-benches/benches/bench_frompyobject.rs @@ -2,7 +2,7 @@ use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; use pyo3::{ prelude::*, - types::{PyList, PyString}, + types::{PyFloat, PyList, PyString}, }; #[derive(FromPyObject)] @@ -79,6 +79,15 @@ fn not_a_list_via_extract_enum(b: &mut Bencher<'_>) { }) } +fn f64_from_pyobject(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + let obj = PyFloat::new(py, 1.234); + b.iter(|| { + let _: f64 = obj.extract().unwrap(); + }); + }) +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("enum_from_pyobject", enum_from_pyobject); c.bench_function("list_via_downcast", list_via_downcast); @@ -86,6 +95,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("not_a_list_via_downcast", not_a_list_via_downcast); c.bench_function("not_a_list_via_extract", not_a_list_via_extract); c.bench_function("not_a_list_via_extract_enum", not_a_list_via_extract_enum); + c.bench_function("f64_from_pyobject", f64_from_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/benches/bench_gil.rs b/pyo3-benches/benches/bench_gil.rs similarity index 100% rename from benches/bench_gil.rs rename to pyo3-benches/benches/bench_gil.rs diff --git a/benches/bench_intern.rs b/pyo3-benches/benches/bench_intern.rs similarity index 100% rename from benches/bench_intern.rs rename to pyo3-benches/benches/bench_intern.rs diff --git a/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs similarity index 98% rename from benches/bench_list.rs rename to pyo3-benches/benches/bench_list.rs index dd305db727b..dd2e3db12ab 100644 --- a/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -9,7 +9,7 @@ fn iter_list(b: &mut Bencher<'_>) { let list = PyList::new(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in list.iter() { + for x in list { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/benches/bench_pyclass.rs b/pyo3-benches/benches/bench_pyclass.rs similarity index 100% rename from benches/bench_pyclass.rs rename to pyo3-benches/benches/bench_pyclass.rs diff --git a/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs similarity index 100% rename from benches/bench_pyobject.rs rename to pyo3-benches/benches/bench_pyobject.rs diff --git a/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs similarity index 98% rename from benches/bench_set.rs rename to pyo3-benches/benches/bench_set.rs index 58abc956337..1bc815997b9 100644 --- a/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -24,7 +24,7 @@ fn iter_set(b: &mut Bencher<'_>) { let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); let mut sum = 0; b.iter(|| { - for x in set.iter() { + for x in set { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs similarity index 98% rename from benches/bench_tuple.rs rename to pyo3-benches/benches/bench_tuple.rs index 07d7fe2dbda..e26c1700338 100644 --- a/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -9,7 +9,7 @@ fn iter_tuple(b: &mut Bencher<'_>) { let tuple = PyTuple::new(py, 0..LEN); let mut sum = 0; b.iter(|| { - for x in tuple.iter() { + for x in tuple { let i: u64 = x.extract().unwrap(); sum += i; } diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index ac8d451fada..4ba6b113880 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.19.0" +version = "0.19.2" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 6f57e0e6eed..8670b355bac 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -87,8 +87,8 @@ impl From<&'_ str> for Error { } impl From for Error { - fn from(_: std::convert::Infallible) -> Self { - unreachable!() + fn from(value: std::convert::Infallible) -> Self { + match value {} } } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index f870daa7ea0..07c5bcb6d37 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -161,6 +161,11 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 59 { println!("cargo:rustc-cfg=thread_local_const_init"); } + + // Enable use of `#[cfg(panic = "...")]` on Rust 1.60 and greater + if rustc_minor_version >= 60 { + println!("cargo:rustc-cfg=panic_unwind"); + } } /// Private exports used in PyO3's build.rs diff --git a/pyo3-ffi-check/Cargo.toml b/pyo3-ffi-check/Cargo.toml index e1663eb8cdc..06829298ea3 100644 --- a/pyo3-ffi-check/Cargo.toml +++ b/pyo3-ffi-check/Cargo.toml @@ -13,5 +13,10 @@ path = "../pyo3-ffi" features = ["extension-module"] # A lazy way of skipping linking in most cases (as we don't use any runtime symbols) [build-dependencies] -bindgen = "0.63.0" +bindgen = "0.66.1" pyo3-build-config = { path = "../pyo3-build-config" } + +[workspace] +members = [ + "macro" +] diff --git a/pyo3-ffi-check/macro/Cargo.toml b/pyo3-ffi-check/macro/Cargo.toml index a9f51f654c6..206e2f44620 100644 --- a/pyo3-ffi-check/macro/Cargo.toml +++ b/pyo3-ffi-check/macro/Cargo.toml @@ -11,4 +11,5 @@ proc-macro = true glob = "0.3" quote = "1" proc-macro2 = "1" -scraper = "0.12" +scraper = "0.17" +pyo3-build-config = { path = "../../pyo3-build-config" } diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 5d03606276e..e572a9ad76c 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -1,8 +1,14 @@ use std::{env, fs, path::PathBuf}; use proc_macro2::{Ident, Span, TokenStream, TokenTree}; +use pyo3_build_config::PythonVersion; use quote::quote; +const PY_3_12: PythonVersion = PythonVersion { + major: 3, + minor: 12, +}; + /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -130,16 +136,27 @@ pub fn for_all_fields(input: proc_macro::TokenStream) -> proc_macro::TokenStream let mut output = TokenStream::new(); for el in html.select(&selector) { - let id = el + let field_name = el .value() .id() .unwrap() .strip_prefix("structfield.") .unwrap(); - let field_ident = Ident::new(id, Span::call_site()); + let field_ident = Ident::new(field_name, Span::call_site()); + + let bindgen_field_ident = if (pyo3_build_config::get().version >= PY_3_12) + && struct_name == "PyObject" + && field_name == "ob_refcnt" + { + // PyObject since 3.12 implements ob_refcnt as a union; bindgen creates + // an anonymous name for the field + Ident::new("__bindgen_anon_1", Span::call_site()) + } else { + field_ident.clone() + }; - output.extend(quote!(#macro_name!(#struct_name, #field_ident);)); + output.extend(quote!(#macro_name!(#struct_name, #field_ident, #bindgen_field_ident);)); } output.into() diff --git a/pyo3-ffi-check/src/main.rs b/pyo3-ffi-check/src/main.rs index c537362530b..99713524702 100644 --- a/pyo3-ffi-check/src/main.rs +++ b/pyo3-ffi-check/src/main.rs @@ -40,16 +40,18 @@ fn main() { pyo3_ffi_align, bindgen_align ); - } else { - pyo3_ffi_check_macro::for_all_fields!($name, check_field); } + + pyo3_ffi_check_macro::for_all_fields!($name, check_field); }}; } macro_rules! check_field { - ($struct_name:ident, $field:ident) => {{ + ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{ + #[allow(clippy::used_underscore_binding)] let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field); - let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $field); + #[allow(clippy::used_underscore_binding)] + let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field); if pyo3_ffi_offset != bindgen_offset { failed = true; @@ -78,7 +80,10 @@ fn main() { non_camel_case_types, non_upper_case_globals, dead_code, - improper_ctypes + improper_ctypes, + clippy::all, + // clippy fails with lots of errors if this is not set specifically + clippy::used_underscore_binding )] mod bindings { include!(concat!(env!("OUT_DIR"), "/bindings.rs")); diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index bda76ab6b55..16d1e2dffe3 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.19.0" +version = "0.19.2" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,4 +38,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "0.19.2", features = ["resolve-config"] } diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 3d167360b99..0b3b7dbb3c2 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -52,8 +52,21 @@ extern "C" { ... ) -> *mut PyObject; - // skipped _PyObject_CallFunction_SizeT - // skipped _PyObject_CallMethod_SizeT + #[cfg(not(Py_3_13))] + #[cfg_attr(PyPy, link_name = "_PyPyObject_CallFunction_SizeT")] + pub fn _PyObject_CallFunction_SizeT( + callable_object: *mut PyObject, + format: *const c_char, + ... + ) -> *mut PyObject; + #[cfg(not(Py_3_13))] + #[cfg_attr(PyPy, link_name = "_PyPyObject_CallMethod_SizeT")] + pub fn _PyObject_CallMethod_SizeT( + o: *mut PyObject, + method: *const c_char, + format: *const c_char, + ... + ) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyObject_CallFunctionObjArgs")] pub fn PyObject_CallFunctionObjArgs(callable: *mut PyObject, ...) -> *mut PyObject; diff --git a/pyo3-ffi/src/boolobject.rs b/pyo3-ffi/src/boolobject.rs index 17af974b04a..0e4958c8fba 100644 --- a/pyo3-ffi/src/boolobject.rs +++ b/pyo3-ffi/src/boolobject.rs @@ -23,12 +23,12 @@ extern "C" { #[inline] pub unsafe fn Py_False() -> *mut PyObject { - addr_of_mut_shim!(_Py_FalseStruct) as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_FalseStruct) as *mut PyObject } #[inline] pub unsafe fn Py_True() -> *mut PyObject { - addr_of_mut_shim!(_Py_TrueStruct) as *mut PyLongObject as *mut PyObject + addr_of_mut_shim!(_Py_TrueStruct) as *mut PyObject } #[inline] diff --git a/pyo3-ffi/src/ceval.rs b/pyo3-ffi/src/ceval.rs index 1eb59b03423..7aae25f8c3e 100644 --- a/pyo3-ffi/src/ceval.rs +++ b/pyo3-ffi/src/ceval.rs @@ -76,7 +76,7 @@ extern "C" { extern "C" { #[cfg(Py_3_9)] #[cfg_attr(PyPy, link_name = "PyPy_EnterRecursiveCall")] - pub fn Py_EnterRecursiveCall(arg1: *const c_char); + pub fn Py_EnterRecursiveCall(arg1: *const c_char) -> c_int; #[cfg(Py_3_9)] #[cfg_attr(PyPy, link_name = "PyPy_LeaveRecursiveCall")] pub fn Py_LeaveRecursiveCall(); diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index 7d97abc4f6f..d2e3ca9d67a 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -41,11 +41,11 @@ extern "C" { ) -> *mut PyObject; } -#[cfg(all(Py_3_8))] +#[cfg(Py_3_8)] const PY_VECTORCALL_ARGUMENTS_OFFSET: Py_ssize_t = 1 << (8 * std::mem::size_of::() as Py_ssize_t - 1); -#[cfg(all(Py_3_8))] +#[cfg(Py_3_8)] #[inline(always)] pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { assert!(n <= (PY_SSIZE_T_MAX as size_t)); @@ -113,7 +113,7 @@ extern "C" { kwnames: *mut PyObject, ) -> *mut PyObject; - #[cfg(all(Py_3_8))] + #[cfg(Py_3_8)] #[cfg_attr(all(not(PyPy), not(Py_3_9)), link_name = "_PyObject_VectorcallDict")] #[cfg_attr(all(PyPy, not(Py_3_9)), link_name = "_PyPyObject_VectorcallDict")] #[cfg_attr(all(PyPy, Py_3_9), link_name = "PyPyObject_VectorcallDict")] @@ -124,7 +124,7 @@ extern "C" { kwdict: *mut PyObject, ) -> *mut PyObject; - #[cfg(all(Py_3_8))] + #[cfg(Py_3_8)] #[cfg_attr(not(any(Py_3_9, PyPy)), link_name = "_PyVectorcall_Call")] #[cfg_attr(PyPy, link_name = "PyPyVectorcall_Call")] pub fn PyVectorcall_Call( @@ -177,8 +177,8 @@ extern "C" { #[inline(always)] pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { assert!(!arg.is_null()); - let _args = [std::ptr::null_mut(), arg]; - let args = _args.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET + let args_array = [std::ptr::null_mut(), arg]; + let args = args_array.as_ptr().offset(1); // For PY_VECTORCALL_ARGUMENTS_OFFSET let tstate = PyThreadState_GET(); let nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; _PyObject_VectorcallTstate(tstate, func, args, nargsf as size_t, std::ptr::null_mut()) diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index b77b2f988d0..67a952b9b9b 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -2,14 +2,63 @@ use crate::object::*; use crate::pyport::Py_ssize_t; #[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_uchar, c_void}; +use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; + +#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] +opaque_struct!(_PyOpcache); + +pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 14; +pub const _PY_MONITORING_EVENTS: usize = 16; + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _Py_Monitors { + pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], +} // skipped _Py_CODEUNIT + // skipped _Py_OPCODE // skipped _Py_OPARG -#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped _py_make_codeunit + +// skipped _py_set_opcode + +// skipped _Py_MAKE_CODEUNIT +// skipped _Py_SET_OPCODE + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoCached { + pub _co_code: *mut PyObject, + pub _co_varnames: *mut PyObject, + pub _co_cellvars: *mut PyObject, + pub _co_freevars: *mut PyObject, +} + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoLineInstrumentationData { + pub original_opcode: u8, + pub line_delta: i8, +} + +#[cfg(Py_3_12)] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCoMonitoringData { + pub local_monitors: _Py_Monitors, + pub active_monitors: _Py_Monitors, + pub tools: *mut u8, + pub lines: *mut _PyCoLineInstrumentationData, + pub line_tools: *mut u8, + pub per_instruction_opcodes: *mut u8, + pub per_instruction_tools: *mut u8, +} #[cfg(all(not(PyPy), not(Py_3_7)))] opaque_struct!(PyCodeObject); @@ -83,17 +132,26 @@ pub struct PyCodeObject { pub co_names: *mut PyObject, pub co_exceptiontable: *mut PyObject, pub co_flags: c_int, + #[cfg(not(Py_3_12))] pub co_warmup: c_int, + pub co_argcount: c_int, pub co_posonlyargcount: c_int, pub co_kwonlyargcount: c_int, pub co_stacksize: c_int, pub co_firstlineno: c_int, + pub co_nlocalsplus: c_int, + #[cfg(Py_3_12)] + pub co_framesize: c_int, pub co_nlocals: c_int, + #[cfg(not(Py_3_12))] pub co_nplaincellvars: c_int, pub co_ncellvars: c_int, pub co_nfreevars: c_int, + #[cfg(Py_3_12)] + pub co_version: u32, + pub co_localsplusnames: *mut PyObject, pub co_localspluskinds: *mut PyObject, pub co_filename: *mut PyObject, @@ -101,8 +159,16 @@ pub struct PyCodeObject { pub co_qualname: *mut PyObject, pub co_linetable: *mut PyObject, pub co_weakreflist: *mut PyObject, + #[cfg(not(Py_3_12))] pub _co_code: *mut PyObject, + #[cfg(not(Py_3_12))] pub _co_linearray: *mut c_char, + #[cfg(Py_3_12)] + pub _co_cached: *mut _PyCoCached, + #[cfg(Py_3_12)] + pub _co_instrumentation_version: u64, + #[cfg(Py_3_12)] + pub _co_monitoring: *mut _PyCoMonitoringData, pub _co_firsttraceable: c_int, pub co_extra: *mut c_void, pub co_code_adaptive: [c_char; 1], diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 9a2afdb93e3..71af81e83e5 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -30,12 +30,27 @@ pub struct PyCompilerFlags { // skipped non-limited _PyCompilerFlags_INIT +#[cfg(all(Py_3_12, not(PyPy)))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct _PyCompilerSrcLocation { + pub lineno: c_int, + pub end_lineno: c_int, + pub col_offset: c_int, + pub end_col_offset: c_int, +} + +// skipped SRC_LOCATION_FROM_AST + #[cfg(not(PyPy))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, + #[cfg(not(Py_3_12))] pub ff_lineno: c_int, + #[cfg(Py_3_12)] + pub ff_location: _PyCompilerSrcLocation, } pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs new file mode 100644 index 00000000000..e33da0b91b9 --- /dev/null +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -0,0 +1,27 @@ +use crate::{PyFloat_Check, PyObject}; +use std::os::raw::c_double; + +#[repr(C)] +pub struct PyFloatObject { + pub ob_base: PyObject, + pub ob_fval: c_double, +} + +#[inline] +pub unsafe fn _PyFloat_CAST(op: *mut PyObject) -> *mut PyFloatObject { + debug_assert_eq!(PyFloat_Check(op), 1); + op.cast() +} + +#[inline] +pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { + (*_PyFloat_CAST(op)).ob_fval +} + +// skipped PyFloat_Pack2 +// skipped PyFloat_Pack4 +// skipped PyFloat_Pack8 + +// skipped PyFloat_Unpack2 +// skipped PyFloat_Unpack4 +// skipped PyFloat_Unpack8 diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 443988ddbd9..f5e2ae8a929 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -39,6 +39,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_12)] + pub func_typeparams: *mut PyObject, pub vectorcall: Option, #[cfg(Py_3_11)] pub func_version: u32, diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index 6b522bf8123..e409337ba79 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -15,6 +15,7 @@ pub struct PyGenObject { pub gi_frame: *mut PyFrameObject, #[cfg(not(Py_3_10))] pub gi_running: c_int, + #[cfg(not(Py_3_12))] pub gi_code: *mut PyObject, pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 7d40ba08d38..17fe7559e1b 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -91,6 +91,8 @@ pub struct PyConfig { #[cfg(all(Py_3_9, not(Py_3_10)))] pub _use_peg_parser: c_int, pub tracemalloc: c_int, + #[cfg(Py_3_12)] + pub perf_profiling: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -137,6 +139,8 @@ pub struct PyConfig { pub use_frozen_modules: c_int, #[cfg(Py_3_11)] pub safe_path: c_int, + #[cfg(Py_3_12)] + pub int_max_str_digits: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, @@ -163,7 +167,7 @@ pub struct PyConfig { pub run_filename: *mut wchar_t, pub _install_importlib: c_int, pub _init_main: c_int, - #[cfg(Py_3_9)] + #[cfg(all(Py_3_9, not(Py_3_12)))] pub _isolated_interpreter: c_int, #[cfg(Py_3_11)] pub _is_python_build: c_int, diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 7c39a9e4dd6..30c5d3b9a65 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -29,6 +29,7 @@ pub(crate) mod pymem; pub(crate) mod pystate; pub(crate) mod pythonrun; // skipped sysmodule.h +pub(crate) mod floatobject; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; pub(crate) mod weakrefobject; @@ -42,6 +43,7 @@ pub use self::compile::*; pub use self::descrobject::*; #[cfg(not(PyPy))] pub use self::dictobject::*; +pub use self::floatobject::*; pub use self::frameobject::*; pub use self::funcobject::*; pub use self::genobject::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 870473b46da..abf8f1dc61c 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -1,4 +1,6 @@ use crate::object; +#[cfg(Py_3_8)] +use crate::vectorcallfunc; use crate::{PyObject, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -112,14 +114,6 @@ mod bufferinfo { #[cfg(not(Py_3_11))] pub use self::bufferinfo::*; -#[cfg(Py_3_8)] -pub type vectorcallfunc = unsafe extern "C" fn( - callable: *mut PyObject, - args: *const *mut PyObject, - nargsf: libc::size_t, - kwnames: *mut PyObject, -) -> *mut PyObject; - #[repr(C)] #[derive(Copy, Clone)] pub struct PyNumberMethods { @@ -275,10 +269,12 @@ pub struct PyTypeObject { pub tp_version_tag: c_uint, pub tp_finalize: Option, #[cfg(Py_3_8)] - pub tp_vectorcall: Option, - #[cfg(any(all(PyPy, Py_3_8), all(not(PyPy), Py_3_8, not(Py_3_9))))] + pub tp_vectorcall: Option, + #[cfg(Py_3_12)] + pub tp_watched: c_char, + #[cfg(any(all(PyPy, Py_3_8, not(Py_3_10)), all(not(PyPy), Py_3_8, not(Py_3_9))))] pub tp_print: Option, - #[cfg(PyPy)] + #[cfg(all(PyPy, not(Py_3_10)))] pub tp_pypy_flags: std::os::raw::c_long, #[cfg(py_sys_config = "COUNT_ALLOCS")] pub tp_allocs: Py_ssize_t, @@ -297,6 +293,8 @@ pub struct PyTypeObject { #[derive(Clone)] pub struct _specialization_cache { pub getitem: *mut PyObject, + #[cfg(Py_3_12)] + pub getitem_version: u32, } #[repr(C)] @@ -347,6 +345,9 @@ pub unsafe fn PyHeapType_GET_MEMBERS( // skipped _PyType_GetModuleByDef extern "C" { + #[cfg(Py_3_12)] + pub fn PyType_GetDict(o: *mut PyTypeObject) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyObject_Print")] pub fn PyObject_Print(o: *mut PyObject, fp: *mut ::libc::FILE, flags: c_int) -> c_int; diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index abbffc0b8c2..fe7b4d4b045 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -65,6 +65,8 @@ pub struct PyImportErrorObject { pub msg: *mut PyObject, pub name: *mut PyObject, pub path: *mut PyObject, + #[cfg(Py_3_12)] + pub name_from: *mut PyObject, } #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 7cd3bfbc8fa..c1bebc4f1bc 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -118,10 +118,7 @@ where } const STATE_INTERNED_INDEX: usize = 0; -#[cfg(not(Py_3_12))] const STATE_INTERNED_WIDTH: u8 = 2; -#[cfg(Py_3_12)] -const STATE_INTERNED_WIDTH: u8 = 1; const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; const STATE_KIND_WIDTH: u8 = 3; @@ -151,8 +148,8 @@ const STATE_READY_WIDTH: u8 = 1; #[repr(C)] #[repr(align(4))] struct PyASCIIObjectState { - _bitfield_align: [u8; 0], - _bitfield: BitfieldUnit<[u8; 4usize]>, + bitfield_align: [u8; 0], + bitfield: BitfieldUnit<[u8; 4usize]>, } // c_uint and u32 are not necessarily the same type on all targets / architectures @@ -161,7 +158,7 @@ impl PyASCIIObjectState { #[inline] unsafe fn interned(&self) -> c_uint { std::mem::transmute( - self._bitfield + self.bitfield .get(STATE_INTERNED_INDEX, STATE_INTERNED_WIDTH) as u32, ) } @@ -169,57 +166,57 @@ impl PyASCIIObjectState { #[inline] unsafe fn set_interned(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_INTERNED_INDEX, STATE_INTERNED_WIDTH, val as u64) } #[inline] unsafe fn kind(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_KIND_INDEX, STATE_KIND_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_KIND_INDEX, STATE_KIND_WIDTH) as u32) } #[inline] unsafe fn set_kind(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_KIND_INDEX, STATE_KIND_WIDTH, val as u64) } #[inline] unsafe fn compact(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH) as u32) } #[inline] unsafe fn set_compact(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_COMPACT_INDEX, STATE_COMPACT_WIDTH, val as u64) } #[inline] unsafe fn ascii(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_ASCII_INDEX, STATE_ASCII_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_ASCII_INDEX, STATE_ASCII_WIDTH) as u32) } #[inline] unsafe fn set_ascii(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_ASCII_INDEX, STATE_ASCII_WIDTH, val as u64) } #[cfg(not(Py_3_12))] #[inline] unsafe fn ready(&self) -> c_uint { - std::mem::transmute(self._bitfield.get(STATE_READY_INDEX, STATE_READY_WIDTH) as u32) + std::mem::transmute(self.bitfield.get(STATE_READY_INDEX, STATE_READY_WIDTH) as u32) } #[cfg(not(Py_3_12))] #[inline] unsafe fn set_ready(&mut self, val: c_uint) { let val: u32 = std::mem::transmute(val); - self._bitfield + self.bitfield .set(STATE_READY_INDEX, STATE_READY_WIDTH, val as u64) } } @@ -228,8 +225,8 @@ impl From for PyASCIIObjectState { #[inline] fn from(value: u32) -> Self { PyASCIIObjectState { - _bitfield_align: [], - _bitfield: BitfieldUnit::new(value.to_ne_bytes()), + bitfield_align: [], + bitfield: BitfieldUnit::new(value.to_ne_bytes()), } } } @@ -237,7 +234,7 @@ impl From for PyASCIIObjectState { impl From for u32 { #[inline] fn from(value: PyASCIIObjectState) -> Self { - u32::from_ne_bytes(value._bitfield.storage) + u32::from_ne_bytes(value.bitfield.storage) } } @@ -268,7 +265,7 @@ impl PyASCIIObject { /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Returns one of: [`SSTATE_NOT_INTERNED`], [`SSTATE_INTERNED_MORTAL`], - /// or on CPython earlier than 3.12, [`SSTATE_INTERNED_IMMORTAL`] + /// or [`SSTATE_INTERNED_IMMORTAL`]. #[inline] pub unsafe fn interned(&self) -> c_uint { PyASCIIObjectState::from(self.state).interned() @@ -277,8 +274,7 @@ impl PyASCIIObject { /// Set the `interned` field of the [`PyASCIIObject`] state bitfield. /// /// Calling this function with an argument that is not [`SSTATE_NOT_INTERNED`], - /// [`SSTATE_INTERNED_MORTAL`], or on CPython earlier than 3.12, - /// [`SSTATE_INTERNED_IMMORTAL`] is invalid. + /// [`SSTATE_INTERNED_MORTAL`], or [`SSTATE_INTERNED_IMMORTAL`] is invalid. #[inline] pub unsafe fn set_interned(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); @@ -288,8 +284,9 @@ impl PyASCIIObject { /// Get the `kind` field of the [`PyASCIIObject`] state bitfield. /// - /// Returns one of: [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], - /// [`PyUnicode_4BYTE_KIND`] + /// Returns one of: + #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")] + /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`]. #[inline] pub unsafe fn kind(&self) -> c_uint { PyASCIIObjectState::from(self.state).kind() @@ -297,8 +294,9 @@ impl PyASCIIObject { /// Set the `kind` field of the [`PyASCIIObject`] state bitfield. /// - /// Calling this function with an argument that is not [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`], - /// [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid. + /// Calling this function with an argument that is not + #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")] + /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid. #[inline] pub unsafe fn set_kind(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); @@ -398,12 +396,14 @@ extern "C" { pub const SSTATE_NOT_INTERNED: c_uint = 0; pub const SSTATE_INTERNED_MORTAL: c_uint = 1; -#[cfg(not(Py_3_12))] pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; +#[cfg(Py_3_12)] +pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).ascii() @@ -420,7 +420,7 @@ pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { } #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] +#[deprecated(note = "Removed in Python 3.12")] pub const PyUnicode_WCHAR_KIND: c_uint = 0; pub const PyUnicode_1BYTE_KIND: c_uint = 1; @@ -445,6 +445,7 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).kind() @@ -484,6 +485,7 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { #[inline] pub unsafe fn PyUnicode_GET_LENGTH(op: *mut PyObject) -> Py_ssize_t { debug_assert!(crate::PyUnicode_Check(op) != 0); + #[cfg(not(Py_3_12))] debug_assert!(PyUnicode_IS_READY(op) != 0); (*(op as *mut PyASCIIObject)).length @@ -502,8 +504,13 @@ pub unsafe fn PyUnicode_IS_READY(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ready() } +#[cfg(Py_3_12)] +#[inline] +pub unsafe fn PyUnicode_READY(_op: *mut PyObject) -> c_int { + 0 +} + #[cfg(not(Py_3_12))] -#[cfg_attr(Py_3_10, deprecated(note = "Python 3.10"))] #[inline] pub unsafe fn PyUnicode_READY(op: *mut PyObject) -> c_int { debug_assert!(crate::PyUnicode_Check(op) != 0); diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index b03fbb303a8..aa9435f6170 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -23,6 +23,7 @@ extern "C" { pub fn PyDict_New() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_GetItem")] pub fn PyDict_GetItem(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; + #[cfg_attr(PyPy, link_name = "PyPyDict_GetItemWithError")] pub fn PyDict_GetItemWithError(mp: *mut PyObject, key: *mut PyObject) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyDict_SetItem")] pub fn PyDict_SetItem(mp: *mut PyObject, key: *mut PyObject, item: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 15f71ba2e72..a6e682cf8df 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -5,13 +5,6 @@ use std::os::raw::{c_double, c_int}; // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) opaque_struct!(PyFloatObject); -#[cfg(not(Py_LIMITED_API))] -#[repr(C)] -pub struct PyFloatObject { - pub ob_base: PyObject, - pub ob_fval: c_double, -} - #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyFloat_Type")] @@ -43,12 +36,6 @@ extern "C" { pub fn PyFloat_AsDouble(arg1: *mut PyObject) -> c_double; } -#[cfg(not(Py_LIMITED_API))] -#[inline] -pub unsafe fn PyFloat_AS_DOUBLE(op: *mut PyObject) -> c_double { - (*(op as *mut PyFloatObject)).ob_fval -} - // skipped non-limited _PyFloat_Pack2 // skipped non-limited _PyFloat_Pack4 // skipped non-limited _PyFloat_Pack8 diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index 73ec459b90a..e4176778c7b 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -274,8 +274,6 @@ macro_rules! addr_of_mut_shim { pub use self::abstract_::*; pub use self::bltinmodule::*; pub use self::boolobject::*; -#[cfg(Py_3_11)] -pub use self::buffer::*; pub use self::bytearrayobject::*; pub use self::bytesobject::*; pub use self::ceval::*; @@ -308,6 +306,8 @@ pub use self::objimpl::*; pub use self::osmodule::*; #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] pub use self::pyarena::*; +#[cfg(Py_3_11)] +pub use self::pybuffer::*; pub use self::pycapsule::*; pub use self::pyerrors::*; pub use self::pyframe::*; @@ -335,8 +335,6 @@ mod abstract_; // skipped ast.h mod bltinmodule; mod boolobject; -#[cfg(Py_3_11)] -mod buffer; mod bytearrayobject; mod bytesobject; // skipped cellobject.h @@ -387,8 +385,9 @@ mod osmodule; // skipped py_curses.h #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] mod pyarena; +#[cfg(Py_3_11)] +mod pybuffer; mod pycapsule; -// skipped pydecimal.h // skipped pydtrace.h mod pyerrors; // skipped pyexpat.h @@ -402,6 +401,7 @@ mod pylifecycle; mod pymem; mod pyport; mod pystate; +// skipped pystats.h mod pythonrun; // skipped pystrhex.h // skipped pystrcmp.h diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 7fabcdf6f60..4c53dc7a441 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,5 +1,3 @@ -// FFI note: this file changed a lot between 3.6 and 3.10. -// Some missing definitions may not be marked "skipped". use crate::pyport::{Py_hash_t, Py_ssize_t}; use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; @@ -14,11 +12,24 @@ pub use crate::cpython::object::PyTypeObject; // _PyObject_HEAD_EXTRA: conditionally defined in PyObject_HEAD_INIT // _PyObject_EXTRA_INIT: conditionally defined in PyObject_HEAD_INIT +#[cfg(Py_3_12)] +pub const _Py_IMMORTAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + c_uint::MAX as Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (c_uint::MAX >> 2) as Py_ssize_t + } +}; + pub const PyObject_HEAD_INIT: PyObject = PyObject { #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_next: std::ptr::null_mut(), #[cfg(py_sys_config = "Py_TRACE_REFS")] _ob_prev: std::ptr::null_mut(), + #[cfg(Py_3_12)] + ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, + #[cfg(not(Py_3_12))] ob_refcnt: 1, #[cfg(PyPy)] ob_pypy_link: 0, @@ -28,6 +39,27 @@ pub const PyObject_HEAD_INIT: PyObject = PyObject { // skipped PyObject_VAR_HEAD // skipped Py_INVALID_SIZE +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(Py_3_12)] +/// This union is anonymous in CPython, so the name was given by PyO3 because +/// Rust unions need a name. +pub union PyObjectObRefcnt { + pub ob_refcnt: Py_ssize_t, + #[cfg(target_pointer_width = "64")] + pub ob_refcnt_split: [crate::PY_UINT32_T; 2], +} + +#[cfg(Py_3_12)] +impl std::fmt::Debug for PyObjectObRefcnt { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", unsafe { self.ob_refcnt }) + } +} + +#[cfg(not(Py_3_12))] +pub type PyObjectObRefcnt = Py_ssize_t; + #[repr(C)] #[derive(Copy, Clone, Debug)] pub struct PyObject { @@ -35,14 +67,13 @@ pub struct PyObject { pub _ob_next: *mut PyObject, #[cfg(py_sys_config = "Py_TRACE_REFS")] pub _ob_prev: *mut PyObject, - pub ob_refcnt: Py_ssize_t, + pub ob_refcnt: PyObjectObRefcnt, #[cfg(PyPy)] pub ob_pypy_link: Py_ssize_t, pub ob_type: *mut PyTypeObject, } // skipped _PyObject_CAST -// skipped _PyObject_CAST_CONST #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -52,18 +83,21 @@ pub struct PyVarObject { } // skipped _PyVarObject_CAST -// skipped _PyVarObject_CAST_CONST #[inline] pub unsafe fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int { (x == y).into() } -// skipped _Py_REFCNT: defined in Py_REFCNT +#[inline] +#[cfg(Py_3_12)] +pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + (*ob).ob_refcnt.ob_refcnt +} #[inline] +#[cfg(not(Py_3_12))] pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - assert!(!ob.is_null()); (*ob).ob_refcnt } @@ -72,9 +106,14 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { (*ob).ob_type } +// PyLong_Type defined in longobject.rs +// PyBool_Type defined in boolobject.rs + #[inline] pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { - (*(ob as *mut PyVarObject)).ob_size + debug_assert_ne!((*ob).ob_type, addr_of_mut_shim!(crate::PyLong_Type)); + debug_assert_ne!((*ob).ob_type, addr_of_mut_shim!(crate::PyBool_Type)); + (*ob.cast::()).ob_size } #[inline] @@ -82,6 +121,18 @@ pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } +#[inline(always)] +#[cfg(all(Py_3_12, target_pointer_width = "64"))] +pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int +} + +#[inline(always)] +#[cfg(all(Py_3_12, target_pointer_width = "32"))] +pub unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int +} + // skipped _Py_SET_REFCNT // skipped Py_SET_REFCNT // skipped _Py_SET_TYPE @@ -89,82 +140,51 @@ pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { // skipped _Py_SET_SIZE // skipped Py_SET_SIZE -pub type unaryfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; - -pub type binaryfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; - -pub type ternaryfunc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: *mut PyObject, - arg3: *mut PyObject, -) -> *mut PyObject; - -pub type inquiry = unsafe extern "C" fn(arg1: *mut PyObject) -> c_int; - -pub type lenfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> Py_ssize_t; - -pub type ssizeargfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t) -> *mut PyObject; - +pub type unaryfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type binaryfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> *mut PyObject; +pub type ternaryfunc = + unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type inquiry = unsafe extern "C" fn(*mut PyObject) -> c_int; +pub type lenfunc = unsafe extern "C" fn(*mut PyObject) -> Py_ssize_t; +pub type ssizeargfunc = unsafe extern "C" fn(*mut PyObject, Py_ssize_t) -> *mut PyObject; pub type ssizessizeargfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: Py_ssize_t) -> *mut PyObject; - -pub type ssizeobjargproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: Py_ssize_t, arg3: *mut PyObject) -> c_int; - -pub type ssizessizeobjargproc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: Py_ssize_t, - arg3: Py_ssize_t, - arg4: *mut PyObject, -) -> c_int; + unsafe extern "C" fn(*mut PyObject, Py_ssize_t, Py_ssize_t) -> *mut PyObject; +pub type ssizeobjargproc = unsafe extern "C" fn(*mut PyObject, Py_ssize_t, *mut PyObject) -> c_int; +pub type ssizessizeobjargproc = + unsafe extern "C" fn(*mut PyObject, Py_ssize_t, Py_ssize_t, arg4: *mut PyObject) -> c_int; +pub type objobjargproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; -pub type objobjargproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; - -pub type objobjproc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; +pub type objobjproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> c_int; pub type visitproc = unsafe extern "C" fn(object: *mut PyObject, arg: *mut c_void) -> c_int; pub type traverseproc = unsafe extern "C" fn(slf: *mut PyObject, visit: visitproc, arg: *mut c_void) -> c_int; -pub type freefunc = unsafe extern "C" fn(arg1: *mut c_void); -pub type destructor = unsafe extern "C" fn(arg1: *mut PyObject); -pub type getattrfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut c_char) -> *mut PyObject; -pub type getattrofunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject) -> *mut PyObject; -pub type setattrfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut c_char, arg3: *mut PyObject) -> c_int; -pub type setattrofunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type reprfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type hashfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> Py_hash_t; -pub type richcmpfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: c_int) -> *mut PyObject; -pub type getiterfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type iternextfunc = unsafe extern "C" fn(arg1: *mut PyObject) -> *mut PyObject; -pub type descrgetfunc = unsafe extern "C" fn( - arg1: *mut PyObject, - arg2: *mut PyObject, - arg3: *mut PyObject, -) -> *mut PyObject; -pub type descrsetfunc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type initproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject) -> c_int; -pub type newfunc = unsafe extern "C" fn( - arg1: *mut PyTypeObject, - arg2: *mut PyObject, - arg3: *mut PyObject, +pub type freefunc = unsafe extern "C" fn(*mut c_void); +pub type destructor = unsafe extern "C" fn(*mut PyObject); +pub type getattrfunc = unsafe extern "C" fn(*mut PyObject, *mut c_char) -> *mut PyObject; +pub type getattrofunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject) -> *mut PyObject; +pub type setattrfunc = unsafe extern "C" fn(*mut PyObject, *mut c_char, *mut PyObject) -> c_int; +pub type setattrofunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type reprfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type hashfunc = unsafe extern "C" fn(*mut PyObject) -> Py_hash_t; +pub type richcmpfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, c_int) -> *mut PyObject; +pub type getiterfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type iternextfunc = unsafe extern "C" fn(*mut PyObject) -> *mut PyObject; +pub type descrgetfunc = + unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type descrsetfunc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type initproc = unsafe extern "C" fn(*mut PyObject, *mut PyObject, *mut PyObject) -> c_int; +pub type newfunc = + unsafe extern "C" fn(*mut PyTypeObject, *mut PyObject, *mut PyObject) -> *mut PyObject; +pub type allocfunc = unsafe extern "C" fn(*mut PyTypeObject, Py_ssize_t) -> *mut PyObject; + +#[cfg(Py_3_8)] +pub type vectorcallfunc = unsafe extern "C" fn( + callable: *mut PyObject, + args: *const *mut PyObject, + nargsf: libc::size_t, + kwnames: *mut PyObject, ) -> *mut PyObject; -pub type allocfunc = - unsafe extern "C" fn(arg1: *mut PyTypeObject, arg2: Py_ssize_t) -> *mut PyObject; -#[cfg(Py_3_11)] -pub type getbufferproc = - unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer, arg3: c_int) -> c_int; -#[cfg(Py_3_11)] -pub type releasebufferproc = unsafe extern "C" fn(arg1: *mut PyObject, arg2: *mut crate::Py_buffer); #[repr(C)] #[derive(Copy, Clone)] @@ -220,9 +240,32 @@ extern "C" { #[cfg(any(Py_3_10, all(Py_3_9, not(Py_LIMITED_API))))] #[cfg_attr(PyPy, link_name = "PyPyType_GetModuleState")] pub fn PyType_GetModuleState(arg1: *mut PyTypeObject) -> *mut c_void; -} -extern "C" { + #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetName")] + pub fn PyType_GetName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_11)] + #[cfg_attr(PyPy, link_name = "PyPyType_GetQualName")] + pub fn PyType_GetQualName(arg1: *mut PyTypeObject) -> *mut PyObject; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyType_FromMetaclass")] + pub fn PyType_FromMetaclass( + metaclass: *mut PyTypeObject, + module: *mut PyObject, + spec: *mut PyType_Spec, + bases: *mut PyObject, + ) -> *mut PyObject; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetTypeData")] + pub fn PyObject_GetTypeData(obj: *mut PyObject, cls: *mut PyTypeObject) -> *mut c_void; + + #[cfg(Py_3_12)] + #[cfg_attr(PyPy, link_name = "PyPyObject_GetTypeDataSize")] + pub fn PyObject_GetTypeDataSize(cls: *mut PyTypeObject) -> Py_ssize_t; + #[cfg_attr(PyPy, link_name = "PyPyType_IsSubtype")] pub fn PyType_IsSubtype(a: *mut PyTypeObject, b: *mut PyTypeObject) -> c_int; } @@ -246,9 +289,7 @@ extern "C" { extern "C" { pub fn PyType_GetFlags(arg1: *mut PyTypeObject) -> c_ulong; -} -extern "C" { #[cfg_attr(PyPy, link_name = "PyPyType_Ready")] pub fn PyType_Ready(t: *mut PyTypeObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyType_GenericAlloc")] @@ -337,6 +378,15 @@ extern "C" { // Flag bits for printing: pub const Py_PRINT_RAW: c_int = 1; // No string quotes etc. +#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] +pub const _Py_TPFLAGS_STATIC_BUILTIN: c_ulong = 1 << 1; + +#[cfg(all(Py_3_12, not(Py_LIMITED_API)))] +pub const Py_TPFLAGS_MANAGED_WEAKREF: c_ulong = 1 << 3; + +#[cfg(all(Py_3_11, not(Py_LIMITED_API)))] +pub const Py_TPFLAGS_MANAGED_DICT: c_ulong = 1 << 4; + #[cfg(all(Py_3_10, not(Py_LIMITED_API)))] pub const Py_TPFLAGS_SEQUENCE: c_ulong = 1 << 5; @@ -356,7 +406,7 @@ pub const Py_TPFLAGS_HEAPTYPE: c_ulong = 1 << 9; pub const Py_TPFLAGS_BASETYPE: c_ulong = 1 << 10; /// Set if the type implements the vectorcall protocol (PEP 590) -#[cfg(all(Py_3_8, not(Py_LIMITED_API)))] +#[cfg(any(Py_3_12, all(Py_3_8, not(Py_LIMITED_API))))] pub const Py_TPFLAGS_HAVE_VECTORCALL: c_ulong = 1 << 11; // skipped non-limited _Py_TPFLAGS_HAVE_VECTORCALL @@ -374,15 +424,14 @@ const Py_TPFLAGS_HAVE_STACKLESS_EXTENSION: c_ulong = 0; #[cfg(Py_3_8)] pub const Py_TPFLAGS_METHOD_DESCRIPTOR: c_ulong = 1 << 17; -/// This flag does nothing in Python 3.10+ -pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; - pub const Py_TPFLAGS_VALID_VERSION_TAG: c_ulong = 1 << 19; /* Type is abstract and cannot be instantiated */ pub const Py_TPFLAGS_IS_ABSTRACT: c_ulong = 1 << 20; // skipped non-limited / 3.10 Py_TPFLAGS_HAVE_AM_SEND +#[cfg(Py_3_12)] +pub const Py_TPFLAGS_ITEMS_AT_END: c_ulong = 1 << 23; /* These flags are used to determine if a type is a subclass. */ pub const Py_TPFLAGS_LONG_SUBCLASS: c_ulong = 1 << 24; @@ -394,37 +443,161 @@ pub const Py_TPFLAGS_DICT_SUBCLASS: c_ulong = 1 << 29; pub const Py_TPFLAGS_BASE_EXC_SUBCLASS: c_ulong = 1 << 30; pub const Py_TPFLAGS_TYPE_SUBCLASS: c_ulong = 1 << 31; -pub const Py_TPFLAGS_DEFAULT: c_ulong = - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG; +pub const Py_TPFLAGS_DEFAULT: c_ulong = if cfg!(Py_3_10) { + Py_TPFLAGS_HAVE_STACKLESS_EXTENSION +} else { + Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | Py_TPFLAGS_HAVE_VERSION_TAG +}; pub const Py_TPFLAGS_HAVE_FINALIZE: c_ulong = 1; +pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -// skipped _Py_RefTotal -// skipped _Py_NegativeRefCount +#[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] +extern "C" { + pub fn _Py_NegativeRefCount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + #[cfg(Py_3_12)] + #[link_name = "_Py_IncRefTotal_DO_NOT_USE_THIS"] + fn _Py_INC_REFTOTAL(); + #[cfg(Py_3_12)] + #[link_name = "_Py_DecRefTotal_DO_NOT_USE_THIS"] + fn _Py_DEC_REFTOTAL(); +} extern "C" { #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] pub fn _Py_Dealloc(arg1: *mut PyObject); + + #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + pub fn Py_IncRef(o: *mut PyObject); + #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + pub fn Py_DecRef(o: *mut PyObject); + + #[cfg(Py_3_10)] + #[cfg_attr(PyPy, link_name = "_PyPy_IncRef")] + pub fn _Py_IncRef(o: *mut PyObject); + #[cfg(Py_3_10)] + #[cfg_attr(PyPy, link_name = "_PyPy_DecRef")] + pub fn _Py_DecRef(o: *mut PyObject); } -// Reference counting macros. -#[inline] +#[inline(always)] pub unsafe fn Py_INCREF(op: *mut PyObject) { - if cfg!(py_sys_config = "Py_REF_DEBUG") { - Py_IncRef(op) - } else { - (*op).ob_refcnt += 1 + #[cfg(any( + all(Py_LIMITED_API, Py_3_12), + all( + py_sys_config = "Py_REF_DEBUG", + Py_3_10, + not(all(Py_3_12, not(Py_LIMITED_API))) + ) + ))] + { + return _Py_IncRef(op); + } + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] + { + return Py_IncRef(op); + } + + #[cfg(any( + not(Py_LIMITED_API), + all(Py_LIMITED_API, not(Py_3_12)), + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)) + ))] + { + #[cfg(all(Py_3_12, target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; + let new_refcnt = cur_refcnt.wrapping_add(1); + if new_refcnt == 0 { + return; + } + (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; + } + + #[cfg(all(Py_3_12, target_pointer_width = "32"))] + { + if _Py_IsImmortal(op) != 0 { + return; + } + (*op).ob_refcnt.ob_refcnt += 1 + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt += 1 + } + + // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] + _Py_INC_REFTOTAL(); } } -#[inline] +#[inline(always)] +#[cfg_attr( + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), + track_caller +)] pub unsafe fn Py_DECREF(op: *mut PyObject) { - if cfg!(py_sys_config = "Py_REF_DEBUG") { - Py_DecRef(op) - } else { - (*op).ob_refcnt -= 1; - if (*op).ob_refcnt == 0 { - _Py_Dealloc(op) + #[cfg(any( + all(Py_LIMITED_API, Py_3_12), + all( + py_sys_config = "Py_REF_DEBUG", + Py_3_10, + not(all(Py_3_12, not(Py_LIMITED_API))) + ) + ))] + { + return _Py_DecRef(op); + } + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_3_10)))] + { + return Py_DecRef(op); + } + + #[cfg(any( + not(Py_LIMITED_API), + all(Py_LIMITED_API, not(Py_3_12)), + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)) + ))] + { + #[cfg(Py_3_12)] + if _Py_IsImmortal(op) != 0 { + return; + } + + // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(all(py_sys_config = "Py_REF_DEBUG", Py_3_12))] + _Py_DEC_REFTOTAL(); + + #[cfg(Py_3_12)] + { + (*op).ob_refcnt.ob_refcnt -= 1; + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + if (*op).ob_refcnt.ob_refcnt < 0 { + let location = std::panic::Location::caller(); + _Py_NegativeRefcount(location.file(), location.line(), op); + } + + if (*op).ob_refcnt.ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt -= 1; + + if (*op).ob_refcnt == 0 { + _Py_Dealloc(op); + } } } } @@ -453,14 +626,9 @@ pub unsafe fn Py_XDECREF(op: *mut PyObject) { } extern "C" { - #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] - pub fn Py_IncRef(o: *mut PyObject); - #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] - pub fn Py_DecRef(o: *mut PyObject); - - #[cfg(Py_3_10)] + #[cfg(all(Py_3_10, Py_LIMITED_API))] pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(Py_3_10)] + #[cfg(all(Py_3_10, Py_LIMITED_API))] pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; } @@ -480,6 +648,18 @@ pub unsafe fn _Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { obj } +#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { + _Py_NewRef(obj) +} + +#[cfg(all(Py_3_10, not(Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { + _Py_XNewRef(obj) +} + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "_PyPy_NoneStruct")] @@ -554,5 +734,5 @@ pub unsafe fn PyType_Check(op: *mut PyObject) -> c_int { #[inline] pub unsafe fn PyType_CheckExact(op: *mut PyObject) -> c_int { - (Py_TYPE(op) == addr_of_mut_shim!(PyType_Type)) as c_int + Py_IS_TYPE(op, addr_of_mut_shim!(PyType_Type)) } diff --git a/pyo3-ffi/src/buffer.rs b/pyo3-ffi/src/pybuffer.rs similarity index 95% rename from pyo3-ffi/src/buffer.rs rename to pyo3-ffi/src/pybuffer.rs index bfa48c086cb..20f92fb6d2b 100644 --- a/pyo3-ffi/src/buffer.rs +++ b/pyo3-ffi/src/pybuffer.rs @@ -53,6 +53,9 @@ impl Py_buffer { } } +pub type getbufferproc = unsafe extern "C" fn(*mut PyObject, *mut crate::Py_buffer, c_int) -> c_int; +pub type releasebufferproc = unsafe extern "C" fn(*mut PyObject, *mut crate::Py_buffer); + /* Return 1 if the getbuffer function is available, otherwise return 0. */ extern "C" { #[cfg(not(PyPy))] diff --git a/pyo3-ffi/src/pyerrors.rs b/pyo3-ffi/src/pyerrors.rs index 91d7e378799..9da00ea390e 100644 --- a/pyo3-ffi/src/pyerrors.rs +++ b/pyo3-ffi/src/pyerrors.rs @@ -13,12 +13,14 @@ extern "C" { pub fn PyErr_Occurred() -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyErr_Clear")] pub fn PyErr_Clear(); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_GetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Fetch")] pub fn PyErr_Fetch( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg_attr(Py_3_12, deprecated(note = "Use PyErr_SetRaisedException() instead."))] #[cfg_attr(PyPy, link_name = "PyPyErr_Restore")] pub fn PyErr_Restore(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyErr_GetExcInfo")] @@ -35,12 +37,22 @@ extern "C" { pub fn PyErr_GivenExceptionMatches(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyErr_ExceptionMatches")] pub fn PyErr_ExceptionMatches(arg1: *mut PyObject) -> c_int; + #[cfg_attr( + Py_3_12, + deprecated( + note = "Use PyErr_GetRaisedException() instead, to avoid any possible de-normalization." + ) + )] #[cfg_attr(PyPy, link_name = "PyPyErr_NormalizeException")] pub fn PyErr_NormalizeException( arg1: *mut *mut PyObject, arg2: *mut *mut PyObject, arg3: *mut *mut PyObject, ); + #[cfg(Py_3_12)] + pub fn PyErr_GetRaisedException() -> *mut PyObject; + #[cfg(Py_3_12)] + pub fn PyErr_SetRaisedException(exc: *mut PyObject); #[cfg_attr(PyPy, link_name = "PyPyException_SetTraceback")] pub fn PyException_SetTraceback(arg1: *mut PyObject, arg2: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyException_GetTraceback")] @@ -87,7 +99,7 @@ pub unsafe fn PyUnicodeDecodeError_Create( end: Py_ssize_t, reason: *const c_char, ) -> *mut PyObject { - crate::PyObject_CallFunction( + crate::_PyObject_CallFunction_SizeT( PyExc_UnicodeDecodeError, b"sy#nns\0".as_ptr().cast::(), encoding, diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index 4bf668dcbf5..741b0db7bf8 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,9 @@ +pub type PY_UINT32_T = u32; +pub type PY_UINT64_T = u64; + +pub type PY_INT32_T = i32; +pub type PY_INT64_T = i64; + pub type Py_uintptr_t = ::libc::uintptr_t; pub type Py_intptr_t = ::libc::intptr_t; pub type Py_ssize_t = ::libc::ssize_t; @@ -7,3 +13,13 @@ pub type Py_uhash_t = ::libc::size_t; pub const PY_SSIZE_T_MIN: Py_ssize_t = std::isize::MIN as Py_ssize_t; pub const PY_SSIZE_T_MAX: Py_ssize_t = std::isize::MAX as Py_ssize_t; + +#[cfg(target_endian = "big")] +pub const PY_BIG_ENDIAN: usize = 1; +#[cfg(target_endian = "big")] +pub const PY_LITTLE_ENDIAN: usize = 0; + +#[cfg(target_endian = "little")] +pub const PY_BIG_ENDIAN: usize = 0; +#[cfg(target_endian = "little")] +pub const PY_LITTLE_ENDIAN: usize = 1; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index 8bae6652d07..d2fd39e497d 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -1,4 +1,4 @@ -#[cfg(not(PyPy))] +#[cfg(any(not(PyPy), Py_3_9))] use crate::moduleobject::PyModuleDef; use crate::object::PyObject; use std::os::raw::c_int; @@ -28,13 +28,17 @@ extern "C" { #[cfg(not(PyPy))] pub fn PyInterpreterState_GetID(arg1: *mut PyInterpreterState) -> i64; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + #[cfg_attr(PyPy, link_name = "PyPyState_AddModule")] pub fn PyState_AddModule(arg1: *mut PyObject, arg2: *mut PyModuleDef) -> c_int; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + #[cfg_attr(PyPy, link_name = "PyPyState_RemoveModule")] pub fn PyState_RemoveModule(arg1: *mut PyModuleDef) -> c_int; - #[cfg(not(PyPy))] + #[cfg(any(not(PyPy), Py_3_9))] // only on PyPy since 3.9 + // only has PyPy prefix since 3.10 + #[cfg_attr(all(PyPy, Py_3_10), link_name = "PyPyState_FindModule")] pub fn PyState_FindModule(arg1: *mut PyModuleDef) -> *mut PyObject; #[cfg_attr(PyPy, link_name = "PyPyThreadState_New")] diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index 196cf5354e5..e5f20de0058 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -15,6 +15,9 @@ extern "C" { pub fn PyErr_PrintEx(arg1: c_int); #[cfg_attr(PyPy, link_name = "PyPyErr_Display")] pub fn PyErr_Display(arg1: *mut PyObject, arg2: *mut PyObject, arg3: *mut PyObject); + + #[cfg(Py_3_12)] + pub fn PyErr_DisplayException(exc: *mut PyObject); } // skipped PyOS_InputHook diff --git a/pyo3-ffi/src/unicodeobject.rs b/pyo3-ffi/src/unicodeobject.rs index 86075475475..5ce6496834c 100644 --- a/pyo3-ffi/src/unicodeobject.rs +++ b/pyo3-ffi/src/unicodeobject.rs @@ -59,6 +59,8 @@ extern "C" { pub fn PyUnicode_AsUCS4Copy(unicode: *mut PyObject) -> *mut Py_UCS4; #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetLength")] pub fn PyUnicode_GetLength(unicode: *mut PyObject) -> Py_ssize_t; + #[cfg(not(Py_3_12))] + #[deprecated(note = "Removed in Python 3.12")] #[cfg_attr(PyPy, link_name = "PyPyUnicode_GetSize")] pub fn PyUnicode_GetSize(unicode: *mut PyObject) -> Py_ssize_t; pub fn PyUnicode_ReadChar(unicode: *mut PyObject, index: Py_ssize_t) -> Py_UCS4; diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index e08919cfe8d..96d8b475118 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.19.0" +version = "0.19.2" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index d26f13b211f..1182a78ad03 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -187,7 +187,7 @@ pub fn take_pyo3_options(attrs: &mut Vec) -> Result { pub name: &'a syn::Ident, - pub by_ref: &'a Option, - pub mutability: &'a Option, pub ty: &'a syn::Type, pub optional: Option<&'a syn::Type>, pub default: Option, @@ -40,20 +37,13 @@ impl<'a> FnArg<'a> { } let arg_attrs = PyFunctionArgPyO3Attributes::from_attrs(&mut cap.attrs)?; - let (ident, by_ref, mutability) = match &*cap.pat { - syn::Pat::Ident(syn::PatIdent { - ident, - by_ref, - mutability, - .. - }) => (ident, by_ref, mutability), + let ident = match &*cap.pat { + syn::Pat::Ident(syn::PatIdent { ident, .. }) => ident, other => return Err(handle_argument_error(other)), }; Ok(FnArg { name: ident, - by_ref, - mutability, ty: &cap.ty, optional: utils::option_type_argument(&cap.ty), default: None, @@ -112,43 +102,36 @@ pub enum FnType { } impl FnType { - pub fn self_conversion( - &self, - cls: Option<&syn::Type>, - error_mode: ExtractErrorMode, - ) -> TokenStream { + pub fn self_arg(&self, cls: Option<&syn::Type>, error_mode: ExtractErrorMode) -> TokenStream { match self { - FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => st.receiver( - cls.expect("no class given for Fn with a \"self\" receiver"), - error_mode, - ), + FnType::Getter(st) | FnType::Setter(st) | FnType::Fn(st) => { + let mut receiver = st.receiver( + cls.expect("no class given for Fn with a \"self\" receiver"), + error_mode, + ); + syn::Token![,](Span::call_site()).to_tokens(&mut receiver); + receiver + } FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => { quote!() } FnType::FnClass | FnType::FnNewClass => { quote! { - let _slf = _pyo3::types::PyType::from_type_ptr(_py, _slf as *mut _pyo3::ffi::PyTypeObject); + _pyo3::types::PyType::from_type_ptr(py, _slf as *mut _pyo3::ffi::PyTypeObject), } } FnType::FnModule => { quote! { - let _slf = _py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf); + py.from_borrowed_ptr::<_pyo3::types::PyModule>(_slf), } } } } - - pub fn self_arg(&self) -> TokenStream { - match self { - FnType::FnNew | FnType::FnStatic | FnType::ClassAttribute => quote!(), - _ => quote!(_slf,), - } - } } #[derive(Clone, Debug)] pub enum SelfType { - Receiver { mutable: bool }, + Receiver { mutable: bool, span: Span }, TryFromPyCell(Span), } @@ -158,43 +141,54 @@ pub enum ExtractErrorMode { Raise, } +impl ExtractErrorMode { + pub fn handle_error(self, py: &syn::Ident, extract: TokenStream) -> TokenStream { + match self { + ExtractErrorMode::Raise => quote! { #extract? }, + ExtractErrorMode::NotImplemented => quote! { + match #extract { + ::std::result::Result::Ok(value) => value, + ::std::result::Result::Err(_) => { return _pyo3::callback::convert(#py, #py.NotImplemented()); }, + } + }, + } + } +} + impl SelfType { pub fn receiver(&self, cls: &syn::Type, error_mode: ExtractErrorMode) -> TokenStream { - let cell = match error_mode { - ExtractErrorMode::Raise => { - quote! { _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>()? } - } - ExtractErrorMode::NotImplemented => { - quote! { - match _py.from_borrowed_ptr::<_pyo3::PyAny>(_slf).downcast::<_pyo3::PyCell<#cls>>() { - ::std::result::Result::Ok(cell) => cell, - ::std::result::Result::Err(_) => return _pyo3::callback::convert(_py, _py.NotImplemented()), - } - } - } - }; + let py = syn::Ident::new("py", Span::call_site()); + let slf = syn::Ident::new("_slf", Span::call_site()); match self { - SelfType::Receiver { mutable: false } => { - quote! { - let _cell = #cell; - let _ref = _cell.try_borrow()?; - let _slf: &#cls = &*_ref; - } - } - SelfType::Receiver { mutable: true } => { - quote! { - let _cell = #cell; - let mut _ref = _cell.try_borrow_mut()?; - let _slf: &mut #cls = &mut *_ref; - } + SelfType::Receiver { span, mutable } => { + let method = if *mutable { + syn::Ident::new("extract_pyclass_ref_mut", *span) + } else { + syn::Ident::new("extract_pyclass_ref", *span) + }; + error_mode.handle_error( + &py, + quote_spanned! { *span => + _pyo3::impl_::extract_argument::#method::<#cls>( + #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf), + &mut { _pyo3::impl_::extract_argument::FunctionArgumentHolder::INIT }, + ) + }, + ) } SelfType::TryFromPyCell(span) => { - let _slf = quote! { _slf }; - quote_spanned! { *span => - let _cell = #cell; - #[allow(clippy::useless_conversion)] // In case _slf is PyCell - let #_slf = ::std::convert::TryFrom::try_from(_cell)?; - } + error_mode.handle_error( + &py, + quote_spanned! { *span => + #py.from_borrowed_ptr::<_pyo3::PyAny>(#slf).downcast::<_pyo3::PyCell<#cls>>() + .map_err(::std::convert::Into::<_pyo3::PyErr>::into) + .and_then( + #[allow(clippy::useless_conversion)] // In case slf is PyCell + |cell| ::std::convert::TryFrom::try_from(cell).map_err(::std::convert::Into::into) + ) + + } + ) } } } @@ -260,8 +254,9 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result { ) => { bail_spanned!(recv.span() => RECEIVER_BY_VALUE_ERR); } - syn::FnArg::Receiver(syn::Receiver { mutability, .. }) => Ok(SelfType::Receiver { - mutable: mutability.is_some(), + syn::FnArg::Receiver(recv) => Ok(SelfType::Receiver { + mutable: recv.mutability.is_some(), + span: recv.span(), }), syn::FnArg::Typed(syn::PatType { ty, .. }) => { if let syn::Type::ImplTrait(_) = &**ty { @@ -426,17 +421,12 @@ impl<'a> FnSpec<'a> { cls: Option<&syn::Type>, ) -> Result { let deprecations = &self.deprecations; - let self_conversion = self.tp.self_conversion(cls, ExtractErrorMode::Raise); - let self_arg = self.tp.self_arg(); - let py = syn::Ident::new("_py", Span::call_site()); + let self_arg = self.tp.self_arg(cls, ExtractErrorMode::Raise); + let py = syn::Ident::new("py", Span::call_site()); let func_name = &self.name; let rust_call = |args: Vec| { - quote! { - _pyo3::impl_::pymethods::OkWrap::wrap(function(#self_arg #(#args),*), #py) - .map(|obj| _pyo3::conversion::IntoPyPointer::into_ptr(obj)) - .map_err(::core::convert::Into::into) - } + quotes::map_result_into_ptr(quotes::ok_wrap(quote! { function(#self_arg #(#args),*) })) }; let rust_name = if let Some(cls) = cls { @@ -453,6 +443,7 @@ impl<'a> FnSpec<'a> { } else { rust_call(vec![]) }; + quote! { unsafe fn #ident<'py>( #py: _pyo3::Python<'py>, @@ -460,7 +451,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #call } } @@ -478,7 +468,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #arg_convert #call } @@ -496,7 +485,6 @@ impl<'a> FnSpec<'a> { ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { let function = #rust_name; // Shadow the function name to avoid #3017 #deprecations - #self_conversion #arg_convert #call } diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index cb2fda00c0f..b1dd100324b 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Code generation for the function that initializes a python module and adds classes and function. use crate::{ @@ -122,7 +121,7 @@ pub fn process_functions_in_module( let name = &func.sig.ident; let statements: Vec = syn::parse_quote! { #wrapped_function - #module_name.add_function(#krate::impl_::pyfunction::wrap_pyfunction_impl(&#name::DEF, #module_name)?)?; + #module_name.add_function(#krate::impl_::pyfunction::_wrap_pyfunction(&#name::DEF, #module_name)?)?; }; stmts.extend(statements); } diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 32a9be5d73a..b1e8538122c 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ method::{FnArg, FnSpec}, pyfunction::FunctionSignature, diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 7e7ac6b615f..b4c412e78ae 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1,7 +1,6 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; +use crate::attributes::kw::frozen; use crate::attributes::{ self, kw, take_pyo3_options, CrateAttribute, ExtendsAttribute, FreelistAttribute, ModuleAttribute, NameAttribute, NameLitStr, TextSignatureAttribute, @@ -357,7 +356,7 @@ fn impl_class( cls, args, methods_type, - descriptors_to_items(cls, field_options)?, + descriptors_to_items(cls, args.options.frozen, field_options)?, vec![], ) .doc(doc) @@ -676,6 +675,7 @@ fn extract_variant_data(variant: &mut syn::Variant) -> syn::Result, field_options: Vec<(&syn::Field, FieldPyO3Options)>, ) -> syn::Result> { let ty = syn::parse_quote!(#cls); @@ -702,7 +702,8 @@ fn descriptors_to_items( items.push(getter); } - if options.set.is_some() { + if let Some(set) = options.set { + ensure_spanned!(frozen.is_none(), set.span() => "cannot use `#[pyo3(set)]` on a `frozen` class"); let setter = impl_py_setter_def( &ty, PropertyType::Descriptor { @@ -1069,7 +1070,7 @@ impl<'a> PyClassImplsBuilder<'a> { quote! { impl _pyo3::impl_::pyclass::PyClassWithFreeList for #cls { #[inline] - fn get_free_list(_py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { + fn get_free_list(py: _pyo3::Python<'_>) -> &mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> { static mut FREELIST: *mut _pyo3::impl_::freelist::FreeList<*mut _pyo3::ffi::PyObject> = 0 as *mut _; unsafe { if FREELIST.is_null() { diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 8f56b3e916a..53188c7c072 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::{ attributes::{ self, get_pyo3_options, take_attributes, take_pyo3_options, CrateAttribute, diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index d0a1b6157cf..fd961aea190 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::collections::HashSet; use crate::{ @@ -99,7 +97,7 @@ pub fn impl_methods( let mut implemented_proto_fragments = HashSet::new(); - for iimpl in impls.iter_mut() { + for iimpl in impls { match iimpl { syn::ImplItem::Method(meth) => { let mut fun_options = PyFunctionOptions::from_attrs(&mut meth.attrs)?; diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 9689d863a44..0658e28af7a 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1,11 +1,9 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::borrow::Cow; use crate::attributes::NameAttribute; use crate::method::{CallingConvention, ExtractErrorMode}; use crate::utils::{ensure_not_async_fn, PythonDoc}; -use crate::{deprecations::Deprecations, utils}; +use crate::{deprecations::Deprecations, quotes, utils}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, pyfunction::PyFunctionOptions, @@ -206,7 +204,7 @@ pub fn gen_py_method( GeneratedPyMethod::Proto(impl_call_slot(cls, method.spec)?) } PyMethodProtoKind::Traverse => { - GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec.name)) + GeneratedPyMethod::Proto(impl_traverse_slot(cls, spec)?) } PyMethodProtoKind::SlotFragment(slot_fragment_def) => { let proto = slot_fragment_def.generate_pyproto_fragment(cls, spec)?; @@ -400,14 +398,23 @@ fn impl_call_slot(cls: &syn::Type, mut spec: FnSpec<'_>) -> Result MethodAndSlotDef { +fn impl_traverse_slot(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { + if let (Some(py_arg), _) = split_off_python_arg(&spec.signature.arguments) { + return Err(syn::Error::new_spanned(py_arg.ty, "__traverse__ may not take `Python`. \ + Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. \ + Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, \ + i.e. `Python::with_gil` will panic.")); + } + + let rust_fn_ident = spec.name; + let associated_method = quote! { pub unsafe extern "C" fn __pymethod_traverse__( slf: *mut _pyo3::ffi::PyObject, visit: _pyo3::ffi::visitproc, arg: *mut ::std::os::raw::c_void, ) -> ::std::os::raw::c_int { - _pyo3::impl_::pymethods::call_traverse_impl::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) + _pyo3::impl_::pymethods::_call_traverse::<#cls>(slf, #cls::#rust_fn_ident, visit, arg) } }; let slot_def = quote! { @@ -416,10 +423,10 @@ fn impl_traverse_slot(cls: &syn::Type, rust_fn_ident: &syn::Ident) -> MethodAndS pfunc: #cls::__pymethod_traverse__ as _pyo3::ffi::traverseproc as _ } }; - MethodAndSlotDef { + Ok(MethodAndSlotDef { associated_method, slot_def, - } + }) } fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { @@ -439,13 +446,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> _pyo3::PyResult<_pyo3::PyObject> { let function = #cls::#name; // Shadow the method name to avoid #3017 #deprecations - _pyo3::impl_::pymethods::OkWrap::wrap(#fncall, py) - .map_err(::core::convert::Into::into) + #body } }; @@ -464,8 +471,13 @@ fn impl_py_class_attribute(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result) -> syn::Result { +fn impl_call_setter( + cls: &syn::Type, + spec: &FnSpec<'_>, + self_type: &SelfType, +) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise); if args.is_empty() { bail_spanned!(spec.name.span() => "setter function expected to have one argument"); @@ -478,9 +490,9 @@ fn impl_call_setter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { - // named struct field - quote!({ _slf.#ident = _val; }) - } - PropertyType::Descriptor { field_index, .. } => { - // tuple struct field - let index = syn::Index::from(field_index); - quote!({ _slf.#index = _val; }) - } - PropertyType::Function { spec, .. } => impl_call_setter(cls, spec)?, - }; - - let slf = match property_type { - PropertyType::Descriptor { .. } => { - SelfType::Receiver { mutable: true }.receiver(cls, ExtractErrorMode::Raise) - } - PropertyType::Function { self_type, .. } => { - self_type.receiver(cls, ExtractErrorMode::Raise) + let slf = SelfType::Receiver { + mutable: true, + span: Span::call_site(), + } + .receiver(cls, ExtractErrorMode::Raise); + if let Some(ident) = &field.ident { + // named struct field + quote!({ #slf.#ident = _val; }) + } else { + // tuple struct field + let index = syn::Index::from(field_index); + quote!({ #slf.#index = _val; }) + } } + PropertyType::Function { + spec, self_type, .. + } => impl_call_setter(cls, spec, self_type)?, }; let wrapper_ident = match property_type { @@ -548,19 +556,18 @@ pub fn impl_py_setter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - _py: _pyo3::Python<'_>, + py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject, _value: *mut _pyo3::ffi::PyObject, ) -> _pyo3::PyResult<::std::os::raw::c_int> { - #slf - let _value = _py + let _value = py .from_borrowed_ptr_or_opt(_value) .ok_or_else(|| { _pyo3::exceptions::PyAttributeError::new_err("can't delete attribute") })?; let _val = _pyo3::FromPyObject::extract(_value)?; - _pyo3::callback::convert(_py, #setter_impl) + _pyo3::callback::convert(py, #setter_impl) } }; @@ -582,8 +589,13 @@ pub fn impl_py_setter_def( }) } -fn impl_call_getter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { +fn impl_call_getter( + cls: &syn::Type, + spec: &FnSpec<'_>, + self_type: &SelfType, +) -> syn::Result { let (py_arg, args) = split_off_python_arg(&spec.signature.arguments); + let slf = self_type.receiver(cls, ExtractErrorMode::Raise); ensure_spanned!( args.is_empty(), args[0].ty.span() => "getter function can only have one argument (of type pyo3::Python)" @@ -591,9 +603,9 @@ fn impl_call_getter(cls: &syn::Type, spec: &FnSpec<'_>) -> syn::Result { - // named struct field - quote!(::std::clone::Clone::clone(&(_slf.#ident))) - } - PropertyType::Descriptor { field_index, .. } => { - // tuple struct field - let index = syn::Index::from(field_index); - quote!(::std::clone::Clone::clone(&(_slf.#index))) - } - PropertyType::Function { spec, .. } => impl_call_getter(cls, spec)?, - }; - - let slf = match property_type { - PropertyType::Descriptor { .. } => { - SelfType::Receiver { mutable: false }.receiver(cls, ExtractErrorMode::Raise) - } - PropertyType::Function { self_type, .. } => { - self_type.receiver(cls, ExtractErrorMode::Raise) - } - }; - - let conversion = match property_type { - PropertyType::Descriptor { .. } => { - quote! { - let item: _pyo3::Py<_pyo3::PyAny> = _pyo3::IntoPy::into_py(item, _py); - ::std::result::Result::Ok(_pyo3::conversion::IntoPyPointer::into_ptr(item)) + let slf = SelfType::Receiver { + mutable: false, + span: Span::call_site(), } + .receiver(cls, ExtractErrorMode::Raise); + let field_token = if let Some(ident) = &field.ident { + // named struct field + ident.to_token_stream() + } else { + // tuple struct field + syn::Index::from(field_index).to_token_stream() + }; + quotes::map_result_into_ptr(quotes::ok_wrap(quote! { + ::std::clone::Clone::clone(&(#slf.#field_token)) + })) } // Forward to `IntoPyCallbackOutput`, to handle `#[getter]`s returning results. - PropertyType::Function { .. } => { + PropertyType::Function { + spec, self_type, .. + } => { + let call = impl_call_getter(cls, spec, self_type)?; quote! { - _pyo3::callback::convert(_py, item) + _pyo3::callback::convert(py, #call) } } }; @@ -676,12 +678,10 @@ pub fn impl_py_getter_def( let associated_method = quote! { #cfg_attrs unsafe fn #wrapper_ident( - _py: _pyo3::Python<'_>, + py: _pyo3::Python<'_>, _slf: *mut _pyo3::ffi::PyObject ) -> _pyo3::PyResult<*mut _pyo3::ffi::PyObject> { - #slf - let item = #getter_impl; - #conversion + #body } }; @@ -941,8 +941,7 @@ impl Ty { #ident.to_borrowed_any(#py) }, ), - Ty::CompareOp => handle_error( - extract_error_mode, + Ty::CompareOp => extract_error_mode.handle_error( py, quote! { _pyo3::class::basic::CompareOp::from_raw(#ident) @@ -951,8 +950,7 @@ impl Ty { ), Ty::PySsizeT => { let ty = arg.ty; - handle_error( - extract_error_mode, + extract_error_mode.handle_error( py, quote! { ::std::convert::TryInto::<#ty>::try_into(#ident).map_err(|e| _pyo3::exceptions::PyValueError::new_err(e.to_string())) @@ -965,30 +963,13 @@ impl Ty { } } -fn handle_error( - extract_error_mode: ExtractErrorMode, - py: &syn::Ident, - extract: TokenStream, -) -> TokenStream { - match extract_error_mode { - ExtractErrorMode::Raise => quote! { #extract? }, - ExtractErrorMode::NotImplemented => quote! { - match #extract { - ::std::result::Result::Ok(value) => value, - ::std::result::Result::Err(_) => { return _pyo3::callback::convert(#py, #py.NotImplemented()); }, - } - }, - } -} - fn extract_object( extract_error_mode: ExtractErrorMode, py: &syn::Ident, name: &str, source: TokenStream, ) -> TokenStream { - handle_error( - extract_error_mode, + extract_error_mode.handle_error( py, quote! { _pyo3::impl_::extract_argument::extract_argument( @@ -1098,7 +1079,7 @@ impl SlotDef { spec.name.span() => format!("`{}` must be `unsafe fn`", method_name) ); } - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) @@ -1158,19 +1139,14 @@ fn generate_method_body( extract_error_mode: ExtractErrorMode, return_mode: Option<&ReturnMode>, ) -> Result { - let self_conversion = spec.tp.self_conversion(Some(cls), extract_error_mode); - let self_arg = spec.tp.self_arg(); + let self_arg = spec.tp.self_arg(Some(cls), extract_error_mode); let rust_name = spec.name; let args = extract_proto_arguments(py, spec, arguments, extract_error_mode)?; let call = quote! { _pyo3::callback::convert(#py, #cls::#rust_name(#self_arg #(#args),*)) }; - let body = if let Some(return_mode) = return_mode { + Ok(if let Some(return_mode) = return_mode { return_mode.return_call_output(py, call) } else { call - }; - Ok(quote! { - #self_conversion - #body }) } @@ -1211,7 +1187,7 @@ impl SlotFragmentDef { let fragment_trait = format_ident!("PyClass{}SlotFragment", fragment); let method = syn::Ident::new(fragment, Span::call_site()); let wrapper_ident = format_ident!("__pymethod_{}__", fragment); - let py = syn::Ident::new("_py", Span::call_site()); + let py = syn::Ident::new("py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) .map(|i| format_ident!("arg{}", i)) diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs new file mode 100644 index 00000000000..b3b51404cf6 --- /dev/null +++ b/pyo3-macros-backend/src/quotes.rs @@ -0,0 +1,15 @@ +use proc_macro2::TokenStream; +use quote::quote; + +pub(crate) fn ok_wrap(obj: TokenStream) -> TokenStream { + quote! { + _pyo3::impl_::pymethods::OkWrap::wrap(#obj, py) + .map_err(::core::convert::Into::into) + } +} + +pub(crate) fn map_result_into_ptr(result: TokenStream) -> TokenStream { + quote! { + #result.map(_pyo3::IntoPyPointer::into_ptr) + } +} diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index c9e02d85fb7..c37dba579e7 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use proc_macro2::{Span, TokenStream}; use quote::ToTokens; use syn::{punctuated::Punctuated, spanned::Spanned, Token}; @@ -80,7 +79,7 @@ pub fn get_doc(attrs: &[syn::Attribute], mut text_signature: Option) -> let mut first = true; let mut current_part = text_signature.unwrap_or_default(); - for attr in attrs.iter() { + for attr in attrs { if attr.path.is_ident("doc") { if let Ok(DocArgs { _eq_token, diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 6c531c0fecf..b4e1aceff0e 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.19.0" +version = "0.19.2" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,4 +22,4 @@ abi3 = ["pyo3-macros-backend/abi3"] proc-macro2 = { version = "1", default-features = false } quote = "1" syn = { version = "1.0.85", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.19.2" } diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 387934310b9..37c7e6e9b99 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! This crate declares only the proc macro attributes, as a crate defining proc macro attributes //! must not contain any other public items. diff --git a/pyproject.toml b/pyproject.toml index f53001e2f93..1892c2aecf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ exclude = ''' [tool.towncrier] filename = "CHANGELOG.md" -version = "0.19.0" +version = "0.19.2" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/pytests/noxfile.py b/pytests/noxfile.py index 2eff279391d..bab55868011 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -1,21 +1,24 @@ import nox -import platform +from nox.command import CommandFailed nox.options.sessions = ["test"] @nox.session -def test(session): +def test(session: nox.Session): session.install("-rrequirements-dev.txt") - if platform.system() == "Linux" and platform.python_implementation() == "CPython": - session.install("numpy>=1.16") + try: + session.install("--only-binary=numpy", "numpy>=1.16") + except CommandFailed: + # No binary wheel for numpy available on this platform + pass session.install("maturin") session.run_always("maturin", "develop") session.run("pytest", *session.posargs) @nox.session -def bench(session): +def bench(session: nox.Session): session.install("-rrequirements-dev.txt") session.install(".") session.run("pytest", "--benchmark-enable", "--benchmark-only", *session.posargs) diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 9d2fb926c66..dfebfe31173 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["maturin>=0.13,<0.14"] +requires = ["maturin>=1,<2"] build-backend = "maturin" [tool.pytest.ini_options] diff --git a/pytests/src/dict_iter.rs b/pytests/src/dict_iter.rs index 35f3ad8d6b0..5f5992b6efc 100644 --- a/pytests/src/dict_iter.rs +++ b/pytests/src/dict_iter.rs @@ -22,7 +22,7 @@ impl DictSize { fn iter_dict(&mut self, _py: Python<'_>, dict: &PyDict) -> PyResult { let mut seen = 0u32; - for (sym, values) in dict.iter() { + for (sym, values) in dict { seen += 1; println!( "{:4}/{:4} iterations:{}=>{}", diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 3ee61b343b0..b9c7a5beb48 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -16,6 +16,7 @@ impl EmptyClass { /// This is for demonstrating how to return a value from __next__ #[pyclass] +#[derive(Default)] struct PyClassIter { count: usize, } diff --git a/pytests/tests/test_sequence.py b/pytests/tests/test_sequence.py index 91aac50ac72..b943ba48fad 100644 --- a/pytests/tests/test_sequence.py +++ b/pytests/tests/test_sequence.py @@ -1,5 +1,4 @@ import pytest -import platform from pyo3_pytests import sequence @@ -21,21 +20,15 @@ def test_vec_from_str(): sequence.vec_to_vec_pystring("123") -@pytest.mark.skipif( - platform.system() != "Linux" or platform.python_implementation() != "CPython", - reason="Binary NumPy wheels are not available for all platforms and Python implementations", -) def test_vec_from_array(): - import numpy + # binary numpy wheel not available on all platforms + numpy = pytest.importorskip("numpy") assert sequence.vec_to_vec_i32(numpy.array([1, 2, 3])) == [1, 2, 3] -@pytest.mark.skipif( - platform.system() != "Linux" or platform.python_implementation() != "CPython", - reason="Binary NumPy wheels are not available for all platforms and Python implementations", -) def test_rust_array_from_array(): - import numpy + # binary numpy wheel not available on all platforms + numpy = pytest.importorskip("numpy") assert sequence.array_to_array_i32(numpy.array([1, 2, 3])) == [1, 2, 3] diff --git a/src/buffer.rs b/src/buffer.rs index d3be9ee6187..eb48a7564c6 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -193,14 +193,13 @@ impl PyBuffer { pub fn get(obj: &PyAny) -> PyResult> { // TODO: use nightly API Box::new_uninit() once stable let mut buf = Box::new(mem::MaybeUninit::uninit()); - let buf: Box = unsafe { - err::error_on_minusone( - obj.py(), - ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO), - )?; + let buf: Box = { + err::error_on_minusone(obj.py(), unsafe { + ffi::PyObject_GetBuffer(obj.as_ptr(), buf.as_mut_ptr(), ffi::PyBUF_FULL_RO) + })?; // Safety: buf is initialized by PyObject_GetBuffer. // TODO: use nightly API Box::assume_init() once stable - mem::transmute(buf) + unsafe { mem::transmute(buf) } }; // Create PyBuffer immediately so that if validation checks fail, the PyBuffer::drop code // will call PyBuffer_Release (thus avoiding any leaks). @@ -469,7 +468,7 @@ impl PyBuffer { /// you can use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_to_slice(&self, py: Python<'_>, target: &mut [T]) -> PyResult<()> { - self.copy_to_slice_impl(py, target, b'C') + self._copy_to_slice(py, target, b'C') } /// Copies the buffer elements to the specified slice. @@ -482,10 +481,10 @@ impl PyBuffer { /// you can use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_to_fortran_slice(&self, py: Python<'_>, target: &mut [T]) -> PyResult<()> { - self.copy_to_slice_impl(py, target, b'F') + self._copy_to_slice(py, target, b'F') } - fn copy_to_slice_impl(&self, py: Python<'_>, target: &mut [T], fort: u8) -> PyResult<()> { + fn _copy_to_slice(&self, py: Python<'_>, target: &mut [T], fort: u8) -> PyResult<()> { if mem::size_of_val(target) != self.len_bytes() { return Err(PyBufferError::new_err(format!( "slice to copy to (of length {}) does not match buffer length of {}", @@ -493,22 +492,20 @@ impl PyBuffer { self.item_count() ))); } - unsafe { - err::error_on_minusone( - py, - ffi::PyBuffer_ToContiguous( - target.as_ptr() as *mut raw::c_void, - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - self.0.len, - fort as std::os::raw::c_char, - ), + + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_ToContiguous( + target.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + self.0.len, + fort as std::os::raw::c_char, ) - } + }) } /// Copies the buffer elements to a newly allocated vector. @@ -516,7 +513,7 @@ impl PyBuffer { /// /// Fails if the buffer format is not compatible with type `T`. pub fn to_vec(&self, py: Python<'_>) -> PyResult> { - self.to_vec_impl(py, b'C') + self._to_vec(py, b'C') } /// Copies the buffer elements to a newly allocated vector. @@ -524,32 +521,30 @@ impl PyBuffer { /// /// Fails if the buffer format is not compatible with type `T`. pub fn to_fortran_vec(&self, py: Python<'_>) -> PyResult> { - self.to_vec_impl(py, b'F') + self._to_vec(py, b'F') } - fn to_vec_impl(&self, py: Python<'_>, fort: u8) -> PyResult> { + fn _to_vec(&self, py: Python<'_>, fort: u8) -> PyResult> { let item_count = self.item_count(); let mut vec: Vec = Vec::with_capacity(item_count); - unsafe { - // Copy the buffer into the uninitialized space in the vector. - // Due to T:Copy, we don't need to be concerned with Drop impls. - err::error_on_minusone( - py, - ffi::PyBuffer_ToContiguous( - vec.as_ptr() as *mut raw::c_void, - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - self.0.len, - fort as std::os::raw::c_char, - ), - )?; - // set vector length to mark the now-initialized space as usable - vec.set_len(item_count); - } + + // Copy the buffer into the uninitialized space in the vector. + // Due to T:Copy, we don't need to be concerned with Drop impls. + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_ToContiguous( + vec.as_ptr() as *mut raw::c_void, + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + self.0.len, + fort as std::os::raw::c_char, + ) + })?; + // set vector length to mark the now-initialized space as usable + unsafe { vec.set_len(item_count) }; Ok(vec) } @@ -564,7 +559,7 @@ impl PyBuffer { /// use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_from_slice(&self, py: Python<'_>, source: &[T]) -> PyResult<()> { - self.copy_from_slice_impl(py, source, b'C') + self._copy_from_slice(py, source, b'C') } /// Copies the specified slice into the buffer. @@ -578,10 +573,10 @@ impl PyBuffer { /// use `::is_compatible_format(buf.format())`. /// Alternatively, `match buffer::ElementType::from_format(buf.format())`. pub fn copy_from_fortran_slice(&self, py: Python<'_>, source: &[T]) -> PyResult<()> { - self.copy_from_slice_impl(py, source, b'F') + self._copy_from_slice(py, source, b'F') } - fn copy_from_slice_impl(&self, py: Python<'_>, source: &[T], fort: u8) -> PyResult<()> { + fn _copy_from_slice(&self, py: Python<'_>, source: &[T], fort: u8) -> PyResult<()> { if self.readonly() { return Err(PyBufferError::new_err("cannot write to read-only buffer")); } else if mem::size_of_val(source) != self.len_bytes() { @@ -591,29 +586,27 @@ impl PyBuffer { self.item_count() ))); } - unsafe { - err::error_on_minusone( - py, - ffi::PyBuffer_FromContiguous( - #[cfg(Py_3_11)] - &*self.0, - #[cfg(not(Py_3_11))] - { - &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer - }, - #[cfg(Py_3_11)] - { - source.as_ptr() as *const raw::c_void - }, - #[cfg(not(Py_3_11))] - { - source.as_ptr() as *mut raw::c_void - }, - self.0.len, - fort as std::os::raw::c_char, - ), + + err::error_on_minusone(py, unsafe { + ffi::PyBuffer_FromContiguous( + #[cfg(Py_3_11)] + &*self.0, + #[cfg(not(Py_3_11))] + { + &*self.0 as *const ffi::Py_buffer as *mut ffi::Py_buffer + }, + #[cfg(Py_3_11)] + { + source.as_ptr() as *const raw::c_void + }, + #[cfg(not(Py_3_11))] + { + source.as_ptr() as *mut raw::c_void + }, + self.0.len, + fort as std::os::raw::c_char, ) - } + }) } /// Releases the buffer object, freeing the reference to the Python object diff --git a/src/callback.rs b/src/callback.rs index 6d59253730e..611b1787478 100644 --- a/src/callback.rs +++ b/src/callback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Utilities for a Python callable object that invokes a Rust function. use crate::err::{PyErr, PyResult}; diff --git a/src/conversion.rs b/src/conversion.rs index c5bda16fd7c..85e9f04eb63 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Defines conversions between Rust and Python types. use crate::err::{self, PyDowncastError, PyResult}; #[cfg(feature = "experimental-inspect")] diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 374fc161763..ae43ed610ef 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -221,8 +221,8 @@ impl FromPyObject<'_> for NaiveDateTime { impl ToPyObject for DateTime { fn to_object(&self, py: Python<'_>) -> PyObject { - let date = self.naive_utc().date(); - let time = self.naive_utc().time(); + let date = self.naive_local().date(); + let time = self.naive_local().time(); let yy = date.year(); let mm = date.month() as u8; let dd = date.day() as u8; @@ -251,7 +251,7 @@ impl IntoPy for DateTime { impl FromPyObject<'_> for DateTime { fn extract(ob: &PyAny) -> PyResult> { let dt: &PyDateTime = ob.downcast()?; - let ms = dt.get_fold() as u32 * 1_000_000 + dt.get_microsecond(); + let ms = dt.get_microsecond(); let h = dt.get_hour().into(); let m = dt.get_minute().into(); let s = dt.get_second().into(); @@ -266,7 +266,8 @@ impl FromPyObject<'_> for DateTime { NaiveTime::from_hms_micro_opt(h, m, s, ms) .ok_or_else(|| PyValueError::new_err("invalid or out-of-range time"))?, ); - Ok(DateTime::from_utc(dt, tz)) + // `FixedOffset` cannot have ambiguities so we don't have to worry about DST folds and such + Ok(DateTime::from_local(dt, tz)) } } @@ -612,7 +613,7 @@ mod tests { .and_hms_micro_opt(hour, minute, ssecond, ms) .unwrap(); let datetime = - DateTime::::from_utc(datetime, offset).to_object(py); + DateTime::::from_local(datetime, offset).to_object(py); let datetime: &PyDateTime = datetime.extract(py).unwrap(); let py_tz = offset.to_object(py); let py_tz = py_tz.downcast(py).unwrap(); @@ -681,41 +682,36 @@ mod tests { check_utc("fold", 2014, 5, 6, 7, 8, 9, 1_999_999, 999_999, true); check_utc("non fold", 2014, 5, 6, 7, 8, 9, 999_999, 999_999, false); - let check_fixed_offset = - |name: &'static str, year, month, day, hour, minute, second, ms, py_ms, fold| { - Python::with_gil(|py| { - let offset = FixedOffset::east_opt(3600).unwrap(); - let py_tz = offset.to_object(py); - let py_tz = py_tz.downcast(py).unwrap(); - let py_datetime = PyDateTime::new_with_fold( - py, - year, - month as u8, - day as u8, - hour as u8, - minute as u8, - second as u8, - py_ms, - Some(py_tz), - fold, - ) + let check_fixed_offset = |year, month, day, hour, minute, second, ms| { + Python::with_gil(|py| { + let offset = FixedOffset::east_opt(3600).unwrap(); + let py_tz = offset.to_object(py); + let py_tz = py_tz.downcast(py).unwrap(); + let py_datetime = PyDateTime::new_with_fold( + py, + year, + month as u8, + day as u8, + hour as u8, + minute as u8, + second as u8, + ms, + Some(py_tz), + false, // No such thing as fold for fixed offset timezones + ) + .unwrap(); + let py_datetime: DateTime = py_datetime.extract().unwrap(); + let datetime = NaiveDate::from_ymd_opt(year, month, day) + .unwrap() + .and_hms_micro_opt(hour, minute, second, ms) .unwrap(); - let py_datetime: DateTime = py_datetime.extract().unwrap(); - let datetime = NaiveDate::from_ymd_opt(year, month, day) - .unwrap() - .and_hms_micro_opt(hour, minute, second, ms) - .unwrap(); - let datetime = DateTime::::from_utc(datetime, offset); - assert_eq!( - py_datetime, datetime, - "{}: {} != {}", - name, datetime, py_datetime - ); - }) - }; + let datetime = DateTime::::from_local(datetime, offset); - check_fixed_offset("fold", 2014, 5, 6, 7, 8, 9, 1_999_999, 999_999, true); - check_fixed_offset("non fold", 2014, 5, 6, 7, 8, 9, 999_999, 999_999, false); + assert_eq!(py_datetime, datetime, "{} != {}", datetime, py_datetime); + }) + }; + + check_fixed_offset(2014, 5, 6, 7, 8, 9, 999_999); Python::with_gil(|py| { let py_tz = Utc.to_object(py); @@ -850,10 +846,38 @@ mod tests { #[cfg(all(test, not(target_arch = "wasm32")))] mod proptests { use super::*; + use crate::types::IntoPyDict; use proptest::prelude::*; proptest! { + + // Range is limited to 1970 to 2038 due to windows limitations + #[test] + fn test_pyo3_offset_fixed_frompyobject_created_in_python(timestamp in 0..(i32::MAX as i64), timedelta in -86399i32..=86399i32) { + Python::with_gil(|py| { + + let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py); + let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); + let t = py.eval(&code, Some(globals), None).unwrap(); + + // Get ISO 8601 string from python + let py_iso_str = t.call_method0("isoformat").unwrap(); + + // Get ISO 8601 string from rust + let t = t.extract::>().unwrap(); + // Python doesn't print the seconds of the offset if they are 0 + let rust_iso_str = if timedelta % 60 == 0 { + t.format("%Y-%m-%dT%H:%M:%S%:z").to_string() + } else { + t.format("%Y-%m-%dT%H:%M:%S%::z").to_string() + }; + + // They should be equal + assert_eq!(py_iso_str.to_string(), rust_iso_str); + }) + } + #[test] fn test_duration_roundtrip(days in -999999999i64..=999999999i64) { // Test roundtrip convertion rust->python->rust for all allowed @@ -947,7 +971,7 @@ mod tests { hour in 0u32..=24u32, min in 0u32..=60u32, sec in 0u32..=60u32, - micro in 0u32..=2_000_000u32, + micro in 0u32..=1_000_000u32, offset_secs in -86399i32..=86399i32 ) { Python::with_gil(|py| { diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index c7a99ce0dcc..1c59fd19e7e 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -62,7 +62,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = hashbrown::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index b6ed7a2ea20..706f5c4835f 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -130,7 +130,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = indexmap::IndexMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/conversions/num_bigint.rs b/src/conversions/num_bigint.rs index a8aec454040..52ff2149813 100644 --- a/src/conversions/num_bigint.rs +++ b/src/conversions/num_bigint.rs @@ -1,8 +1,4 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - -#![cfg(all(feature = "num-bigint", not(any(Py_LIMITED_API))))] +#![cfg(feature = "num-bigint")] //! Conversions to and from [num-bigint](https://docs.rs/num-bigint)’s [`BigInt`] and [`BigUint`] types. //! //! This is useful for converting Python integers when they may not fit in Rust's built-in integer types. @@ -57,15 +53,16 @@ //! ``` use crate::{ - err, ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyErr, PyObject, PyResult, - Python, ToPyObject, + ffi, types::*, AsPyPointer, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, + ToPyObject, }; use num_bigint::{BigInt, BigUint}; use std::os::raw::{c_int, c_uchar}; +#[cfg(not(Py_LIMITED_API))] unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { - err::error_on_minusone( + crate::err::error_on_minusone( ob.py(), ffi::_PyLong_AsByteArray( ob.as_ptr() as *mut ffi::PyLongObject, @@ -77,13 +74,33 @@ unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyRe ) } +#[cfg(Py_LIMITED_API)] +unsafe fn extract(ob: &PyLong, buffer: &mut [c_uchar], is_signed: c_int) -> PyResult<()> { + use crate::intern; + let py = ob.py(); + let kwargs = if is_signed != 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(intern!(py, "signed"), true)?; + Some(kwargs) + } else { + None + }; + let bytes_obj = ob + .getattr(intern!(py, "to_bytes"))? + .call((buffer.len(), "little"), kwargs)?; + let bytes: &PyBytes = bytes_obj.downcast_unchecked(); + buffer.copy_from_slice(bytes.as_bytes()); + Ok(()) +} + macro_rules! bigint_conversion { ($rust_ty: ty, $is_signed: expr, $to_bytes: path, $from_bytes: path) => { #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] impl ToPyObject for $rust_ty { + #[cfg(not(Py_LIMITED_API))] fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); unsafe { - let bytes = $to_bytes(self); let obj = ffi::_PyLong_FromByteArray( bytes.as_ptr() as *const c_uchar, bytes.len(), @@ -93,6 +110,23 @@ macro_rules! bigint_conversion { PyObject::from_owned_ptr(py, obj) } } + + #[cfg(Py_LIMITED_API)] + fn to_object(&self, py: Python<'_>) -> PyObject { + let bytes = $to_bytes(self); + let bytes_obj = PyBytes::new(py, &bytes); + let kwargs = if $is_signed > 0 { + let kwargs = PyDict::new(py); + kwargs.set_item(crate::intern!(py, "signed"), true).unwrap(); + Some(kwargs) + } else { + None + }; + py.get_type::() + .call_method("from_bytes", (bytes_obj, "little"), kwargs) + .expect("int.from_bytes() failed during to_object()") // FIXME: #1813 or similar + .into() + } } #[cfg_attr(docsrs, doc(cfg(feature = "num-bigint")))] @@ -109,14 +143,33 @@ macro_rules! bigint_conversion { unsafe { let num: Py = Py::from_owned_ptr_or_err(py, ffi::PyNumber_Index(ob.as_ptr()))?; - let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); - let n_bytes = if n_bits == (-1isize as usize) { - return Err(PyErr::fetch(py)); - } else if n_bits == 0 { - 0 - } else { - (n_bits - 1 + $is_signed) / 8 + 1 + + let n_bytes = { + cfg_if::cfg_if! { + if #[cfg(not(Py_LIMITED_API))] { + // fast path + let n_bits = ffi::_PyLong_NumBits(num.as_ptr()); + if n_bits == (-1isize as usize) { + return Err(crate::PyErr::fetch(py)); + } else if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } else { + // slow path + let n_bits_obj = num.getattr(py, crate::intern!(py, "bit_length"))?.call0(py)?; + let n_bits_int: &PyLong = n_bits_obj.downcast_unchecked(py); + let n_bits = n_bits_int.extract::()?; + if n_bits == 0 { + 0 + } else { + (n_bits - 1 + $is_signed) / 8 + 1 + } + } + } }; + if n_bytes <= 128 { let mut buffer = [0; 128]; extract(num.as_ref(py), &mut buffer[..n_bytes], $is_signed)?; diff --git a/src/conversions/num_complex.rs b/src/conversions/num_complex.rs index 217d862a542..df6b54b45bc 100644 --- a/src/conversions/num_complex.rs +++ b/src/conversions/num_complex.rs @@ -152,6 +152,18 @@ macro_rules! complex_conversion { #[cfg(any(Py_LIMITED_API, PyPy))] unsafe { + let obj = if obj.is_instance_of::() { + obj + } else if let Some(method) = + obj.lookup_special(crate::intern!(obj.py(), "__complex__"))? + { + method.call0()? + } else { + // `obj` might still implement `__float__` or `__index__`, which will be + // handled by `PyComplex_{Real,Imag}AsDouble`, including propagating any + // errors if those methods don't exist / raise exceptions. + obj + }; let ptr = obj.as_ptr(); let real = ffi::PyComplex_RealAsDouble(ptr); if real == -1.0 { @@ -172,6 +184,7 @@ complex_conversion!(f64); #[cfg(test)] mod tests { use super::*; + use crate::types::PyModule; #[test] fn from_complex() { @@ -197,4 +210,131 @@ mod tests { assert!(obj.extract::>(py).is_err()); }); } + #[test] + fn from_python_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + def __complex__(self): return 3.0+1.2j +class B: + def __float__(self): return 3.0 +class C: + def __index__(self): return 3 + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + // Before Python 3.8, `__index__` wasn't tried by `float`/`complex`. + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_inherited_magic() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class First: pass +class ComplexMixin: + def __complex__(self): return 3.0+1.2j +class FloatMixin: + def __float__(self): return 3.0 +class IndexMixin: + def __index__(self): return 3 +class A(First, ComplexMixin): pass +class B(First, FloatMixin): pass +class C(First, IndexMixin): pass + "#, + "test.py", + "test", + ) + .unwrap(); + let from_complex = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + from_complex.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + let from_float = module.getattr("B").unwrap().call0().unwrap(); + assert_eq!( + from_float.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + #[cfg(Py_3_8)] + { + let from_index = module.getattr("C").unwrap().call0().unwrap(); + assert_eq!( + from_index.extract::>().unwrap(), + Complex::new(3.0, 0.0) + ); + } + }) + } + #[test] + fn from_python_noncallable_descriptor_magic() { + // Functions and lambdas implement the descriptor protocol in a way that makes + // `type(inst).attr(inst)` equivalent to `inst.attr()` for methods, but this isn't the only + // way the descriptor protocol might be implemented. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class A: + @property + def __complex__(self): + return lambda: 3.0+1.2j + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } + #[test] + fn from_python_nondescriptor_magic() { + // Magic methods don't need to implement the descriptor protocol, if they're callable. + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class MyComplex: + def __call__(self): return 3.0+1.2j +class A: + __complex__ = MyComplex() + "#, + "test.py", + "test", + ) + .unwrap(); + let obj = module.getattr("A").unwrap().call0().unwrap(); + assert_eq!( + obj.extract::>().unwrap(), + Complex::new(3.0, 1.2) + ); + }) + } } diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs new file mode 100755 index 00000000000..ca3c8728f9b --- /dev/null +++ b/src/conversions/std/ipaddr.rs @@ -0,0 +1,110 @@ +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use crate::exceptions::PyValueError; +use crate::sync::GILOnceCell; +use crate::types::PyType; +use crate::{intern, FromPyObject, IntoPy, Py, PyAny, PyObject, PyResult, Python, ToPyObject}; + +impl FromPyObject<'_> for IpAddr { + fn extract(obj: &PyAny) -> PyResult { + match obj.getattr(intern!(obj.py(), "packed")) { + Ok(packed) => { + if let Ok(packed) = packed.extract::<[u8; 4]>() { + Ok(IpAddr::V4(Ipv4Addr::from(packed))) + } else if let Ok(packed) = packed.extract::<[u8; 16]>() { + Ok(IpAddr::V6(Ipv6Addr::from(packed))) + } else { + Err(PyValueError::new_err("invalid packed length")) + } + } + Err(_) => { + // We don't have a .packed attribute, so we try to construct an IP from str(). + obj.str()?.to_str()?.parse().map_err(PyValueError::new_err) + } + } + } +} + +impl ToPyObject for Ipv4Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV4_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV4_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv4Address") + .expect("failed to load ipaddress.IPv4Address") + .call1((u32::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv4Address") + .to_object(py) + } +} + +impl ToPyObject for Ipv6Addr { + fn to_object(&self, py: Python<'_>) -> PyObject { + static IPV6_ADDRESS: GILOnceCell> = GILOnceCell::new(); + IPV6_ADDRESS + .get_or_try_init_type_ref(py, "ipaddress", "IPv6Address") + .expect("failed to load ipaddress.IPv6Address") + .call1((u128::from_be_bytes(self.octets()),)) + .expect("failed to construct ipaddress.IPv6Address") + .to_object(py) + } +} + +impl ToPyObject for IpAddr { + fn to_object(&self, py: Python<'_>) -> PyObject { + match self { + IpAddr::V4(ip) => ip.to_object(py), + IpAddr::V6(ip) => ip.to_object(py), + } + } +} + +impl IntoPy for IpAddr { + fn into_py(self, py: Python<'_>) -> PyObject { + self.to_object(py) + } +} + +#[cfg(test)] +mod test_ipaddr { + use std::str::FromStr; + + use crate::types::PyString; + + use super::*; + + #[test] + fn test_roundtrip() { + Python::with_gil(|py| { + fn roundtrip(py: Python<'_>, ip: &str) { + let ip = IpAddr::from_str(ip).unwrap(); + let py_cls = if ip.is_ipv4() { + "IPv4Address" + } else { + "IPv6Address" + }; + + let pyobj = ip.into_py(py); + let repr = pyobj.as_ref(py).repr().unwrap().to_string_lossy(); + assert_eq!(repr, format!("{}('{}')", py_cls, ip)); + + let ip2: IpAddr = pyobj.extract(py).unwrap(); + assert_eq!(ip, ip2); + } + roundtrip(py, "127.0.0.1"); + roundtrip(py, "::1"); + roundtrip(py, "0.0.0.0"); + }); + } + + #[test] + fn test_from_pystring() { + Python::with_gil(|py| { + let py_str = PyString::new(py, "0:0:0:0:0:0:0:1"); + let ip: IpAddr = py_str.to_object(py).extract(py).unwrap(); + assert_eq!(ip, IpAddr::from_str("::1").unwrap()); + + let py_str = PyString::new(py, "invalid"); + assert!(py_str.to_object(py).extract::(py).is_err()); + }); + } +} diff --git a/src/conversions/std/map.rs b/src/conversions/std/map.rs index f7e9b58ce91..f79b415b9fa 100644 --- a/src/conversions/std/map.rs +++ b/src/conversions/std/map.rs @@ -74,7 +74,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = collections::HashMap::with_capacity_and_hasher(dict.len(), S::default()); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) @@ -94,7 +94,7 @@ where fn extract(ob: &'source PyAny) -> Result { let dict: &PyDict = ob.downcast()?; let mut ret = collections::BTreeMap::new(); - for (k, v) in dict.iter() { + for (k, v) in dict { ret.insert(K::extract(k)?, V::extract(v)?); } Ok(ret) diff --git a/src/conversions/std/mod.rs b/src/conversions/std/mod.rs index 6021c395288..f5e917d08ea 100644 --- a/src/conversions/std/mod.rs +++ b/src/conversions/std/mod.rs @@ -1,4 +1,5 @@ mod array; +mod ipaddr; mod map; mod num; mod osstr; diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 89e86b8a080..3427942ee11 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -178,16 +178,18 @@ mod fast_128bit_int_conversion { } impl IntoPy for $rust_type { fn into_py(self, py: Python<'_>) -> PyObject { + // Always use little endian + let bytes = self.to_le_bytes(); unsafe { - // Always use little endian - let bytes = self.to_le_bytes(); - let obj = ffi::_PyLong_FromByteArray( - bytes.as_ptr() as *const std::os::raw::c_uchar, - bytes.len(), - 1, - $is_signed, - ); - PyObject::from_owned_ptr(py, obj) + PyObject::from_owned_ptr( + py, + ffi::_PyLong_FromByteArray( + bytes.as_ptr() as *const std::os::raw::c_uchar, + bytes.len(), + 1, + $is_signed, + ), + ) } } @@ -199,23 +201,20 @@ mod fast_128bit_int_conversion { impl<'source> FromPyObject<'source> for $rust_type { fn extract(ob: &'source PyAny) -> PyResult<$rust_type> { - unsafe { - let num = ffi::PyNumber_Index(ob.as_ptr()); - if num.is_null() { - return Err(PyErr::fetch(ob.py())); - } - let mut buffer = [0; std::mem::size_of::<$rust_type>()]; - let ok = ffi::_PyLong_AsByteArray( - num as *mut ffi::PyLongObject, + let num = unsafe { + PyObject::from_owned_ptr_or_err(ob.py(), ffi::PyNumber_Index(ob.as_ptr()))? + }; + let mut buffer = [0; std::mem::size_of::<$rust_type>()]; + crate::err::error_on_minusone(ob.py(), unsafe { + ffi::_PyLong_AsByteArray( + num.as_ptr() as *mut ffi::PyLongObject, buffer.as_mut_ptr(), buffer.len(), 1, $is_signed, - ); - ffi::Py_DECREF(num); - crate::err::error_on_minusone(ob.py(), ok)?; - Ok(<$rust_type>::from_le_bytes(buffer)) - } + ) + })?; + Ok(<$rust_type>::from_le_bytes(buffer)) } #[cfg(feature = "experimental-inspect")] @@ -248,19 +247,17 @@ mod slow_128bit_int_conversion { impl IntoPy for $rust_type { fn into_py(self, py: Python<'_>) -> PyObject { - let lower = self as u64; - let upper = (self >> SHIFT) as $half_type; + let lower = (self as u64).into_py(py); + let upper = ((self >> SHIFT) as $half_type).into_py(py); + let shift = SHIFT.into_py(py); unsafe { let shifted = PyObject::from_owned_ptr( py, - ffi::PyNumber_Lshift( - upper.into_py(py).as_ptr(), - SHIFT.into_py(py).as_ptr(), - ), + ffi::PyNumber_Lshift(upper.as_ptr(), shift.as_ptr()), ); PyObject::from_owned_ptr( py, - ffi::PyNumber_Or(shifted.as_ptr(), lower.into_py(py).as_ptr()), + ffi::PyNumber_Or(shifted.as_ptr(), lower.as_ptr()), ) } } @@ -280,9 +277,10 @@ mod slow_128bit_int_conversion { -1 as _, ffi::PyLong_AsUnsignedLongLongMask(ob.as_ptr()), )? as $rust_type; + let shift = SHIFT.into_py(py); let shifted = PyObject::from_owned_ptr_or_err( py, - ffi::PyNumber_Rshift(ob.as_ptr(), SHIFT.into_py(py).as_ptr()), + ffi::PyNumber_Rshift(ob.as_ptr(), shift.as_ptr()), )?; let upper: $half_type = shifted.extract(py)?; Ok((<$rust_type>::from(upper) << SHIFT) | lower) diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index 12d00ffc431..f91822a874f 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -1,6 +1,4 @@ use crate::types::PyString; -#[cfg(windows)] -use crate::PyErr; use crate::{ ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyObject, PyResult, Python, ToPyObject, }; @@ -92,9 +90,7 @@ impl FromPyObject<'_> for OsString { // ourselves let size = unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), std::ptr::null_mut(), 0) }; - if size == -1 { - return Err(PyErr::fetch(ob.py())); - } + crate::err::error_on_minusone(ob.py(), size)?; let mut buffer = vec![0; size as usize]; let bytes_read = diff --git a/src/derive_utils.rs b/src/derive_utils.rs index 25ea7f89fa7..4ccb38f901b 100644 --- a/src/derive_utils.rs +++ b/src/derive_utils.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Functionality for the code generated by the derive backend use crate::{types::PyModule, Python}; diff --git a/src/err/err_state.rs b/src/err/err_state.rs index bf4fb3fdfb2..a7408efdba2 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -2,25 +2,53 @@ use crate::{ exceptions::{PyBaseException, PyTypeError}, ffi, types::{PyTraceback, PyType}, - AsPyPointer, IntoPy, IntoPyPointer, Py, PyObject, Python, + AsPyPointer, IntoPy, IntoPyPointer, Py, PyAny, PyObject, PyTypeInfo, Python, }; #[derive(Clone)] pub(crate) struct PyErrStateNormalized { + #[cfg(not(Py_3_12))] pub ptype: Py, pub pvalue: Py, + #[cfg(not(Py_3_12))] pub ptraceback: Option>, } +impl PyErrStateNormalized { + #[cfg(not(Py_3_12))] + pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { + self.ptype.as_ref(py) + } + + #[cfg(Py_3_12)] + pub(crate) fn ptype<'py>(&'py self, py: Python<'py>) -> &'py PyType { + self.pvalue.as_ref(py).get_type() + } + + #[cfg(not(Py_3_12))] + pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + self.ptraceback + .as_ref() + .map(|traceback| traceback.as_ref(py)) + } + + #[cfg(Py_3_12)] + pub(crate) fn ptraceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { + unsafe { py.from_owned_ptr_or_opt(ffi::PyException_GetTraceback(self.pvalue.as_ptr())) } + } +} + +pub(crate) struct PyErrStateLazyFnOutput { + pub(crate) ptype: PyObject, + pub(crate) pvalue: PyObject, +} + +pub(crate) type PyErrStateLazyFn = + dyn for<'py> FnOnce(Python<'py>) -> PyErrStateLazyFnOutput + Send + Sync; + pub(crate) enum PyErrState { - LazyTypeAndValue { - ptype: for<'py> fn(Python<'py>) -> &PyType, - pvalue: Box FnOnce(Python<'py>) -> PyObject + Send + Sync>, - }, - LazyValue { - ptype: Py, - pvalue: Box FnOnce(Python<'py>) -> PyObject + Send + Sync>, - }, + Lazy(Box), + #[cfg(not(Py_3_12))] FfiTuple { ptype: PyObject, pvalue: Option, @@ -44,35 +72,48 @@ where } } -pub(crate) fn boxed_args( - args: impl PyErrArguments + 'static, -) -> Box FnOnce(Python<'py>) -> PyObject + Send + Sync> { - Box::new(|py| args.arguments(py)) -} - impl PyErrState { + pub(crate) fn lazy(ptype: &PyAny, args: impl PyErrArguments + 'static) -> Self { + let ptype = ptype.into(); + PyErrState::Lazy(Box::new(move |py| PyErrStateLazyFnOutput { + ptype, + pvalue: args.arguments(py), + })) + } + + pub(crate) fn normalized(pvalue: &PyBaseException) -> Self { + Self::Normalized(PyErrStateNormalized { + #[cfg(not(Py_3_12))] + ptype: pvalue.get_type().into(), + pvalue: pvalue.into(), + #[cfg(not(Py_3_12))] + ptraceback: unsafe { + Py::from_owned_ptr_or_opt( + pvalue.py(), + ffi::PyException_GetTraceback(pvalue.as_ptr()), + ) + }, + }) + } + + #[cfg(not(Py_3_12))] pub(crate) fn into_ffi_tuple( self, py: Python<'_>, ) -> (*mut ffi::PyObject, *mut ffi::PyObject, *mut ffi::PyObject) { match self { - PyErrState::LazyTypeAndValue { ptype, pvalue } => { - let ty = ptype(py); - if unsafe { ffi::PyExceptionClass_Check(ty.as_ptr()) } == 0 { - Self::exceptions_must_derive_from_base_exception(py).into_ffi_tuple(py) - } else { - ( - ptype(py).into_ptr(), - pvalue(py).into_ptr(), - std::ptr::null_mut(), + PyErrState::Lazy(lazy) => { + let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py); + if unsafe { ffi::PyExceptionClass_Check(ptype.as_ptr()) } == 0 { + PyErrState::lazy( + PyTypeError::type_object(py), + "exceptions must derive from BaseException", ) + .into_ffi_tuple(py) + } else { + (ptype.into_ptr(), pvalue.into_ptr(), std::ptr::null_mut()) } } - PyErrState::LazyValue { ptype, pvalue } => ( - ptype.into_ptr(), - pvalue(py).into_ptr(), - std::ptr::null_mut(), - ), PyErrState::FfiTuple { ptype, pvalue, @@ -86,11 +127,57 @@ impl PyErrState { } } - #[inline] - pub(crate) fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> Self { - PyErrState::LazyValue { - ptype: py.get_type::().into(), - pvalue: boxed_args("exceptions must derive from BaseException"), + #[cfg(not(Py_3_12))] + pub(crate) fn normalize(self, py: Python<'_>) -> PyErrStateNormalized { + let (mut ptype, mut pvalue, mut ptraceback) = self.into_ffi_tuple(py); + + unsafe { + ffi::PyErr_NormalizeException(&mut ptype, &mut pvalue, &mut ptraceback); + PyErrStateNormalized { + ptype: Py::from_owned_ptr_or_opt(py, ptype).expect("Exception type missing"), + pvalue: Py::from_owned_ptr_or_opt(py, pvalue).expect("Exception value missing"), + ptraceback: Py::from_owned_ptr_or_opt(py, ptraceback), + } + } + } + + #[cfg(Py_3_12)] + pub(crate) fn normalize(self, py: Python<'_>) -> PyErrStateNormalized { + // To keep the implementation simple, just write the exception into the interpreter, + // which will cause it to be normalized + self.restore(py); + // Safety: self.restore(py) will set the raised exception + let pvalue = unsafe { Py::from_owned_ptr(py, ffi::PyErr_GetRaisedException()) }; + PyErrStateNormalized { pvalue } + } + + #[cfg(not(Py_3_12))] + pub(crate) fn restore(self, py: Python<'_>) { + let (ptype, pvalue, ptraceback) = self.into_ffi_tuple(py); + unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) } + } + + #[cfg(Py_3_12)] + pub(crate) fn restore(self, py: Python<'_>) { + match self { + PyErrState::Lazy(lazy) => { + let PyErrStateLazyFnOutput { ptype, pvalue } = lazy(py); + unsafe { + if ffi::PyExceptionClass_Check(ptype.as_ptr()) == 0 { + ffi::PyErr_SetString( + PyTypeError::type_object_raw(py).cast(), + "exceptions must derive from BaseException\0" + .as_ptr() + .cast(), + ) + } else { + ffi::PyErr_SetObject(ptype.as_ptr(), pvalue.as_ptr()) + } + } + } + PyErrState::Normalized(PyErrStateNormalized { pvalue }) => unsafe { + ffi::PyErr_SetRaisedException(pvalue.into_ptr()) + }, } } } diff --git a/src/err/mod.rs b/src/err/mod.rs index 88b03986159..a824a383c18 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::panic::PanicException; use crate::type_object::PyTypeInfo; use crate::types::{PyTraceback, PyType}; @@ -11,22 +9,22 @@ use crate::{AsPyPointer, IntoPy, IntoPyPointer, Py, PyAny, PyObject, Python, ToP use std::borrow::Cow; use std::cell::UnsafeCell; use std::ffi::CString; -use std::os::raw::c_int; mod err_state; mod impls; pub use err_state::PyErrArguments; -use err_state::{boxed_args, PyErrState, PyErrStateNormalized}; +use err_state::{PyErrState, PyErrStateLazyFnOutput, PyErrStateNormalized}; /// Represents a Python exception. /// -/// Python exceptions can be raised in a "lazy" fashion, where the full Python object for the -/// exception is not created until needed. The process of creating the full object is known -/// as "normalization". An exception which has not yet been created is known as "unnormalized". +/// To avoid needing access to [`Python`] in `Into` conversions to create `PyErr` (thus improving +/// compatibility with `?` and other Rust errors) this type supports creating exceptions instances +/// in a lazy fashion, where the full Python object for the exception is created only when needed. /// -/// This struct builds upon that design, supporting all lazily-created Python exceptions and also -/// supporting exceptions lazily-created from Rust. +/// Accessing the contained exception in any way, such as with [`value`](PyErr::value), +/// [`get_type`](PyErr::get_type), or [`is_instance`](PyErr::is_instance) will create the full +/// exception object if it was not already created. pub struct PyErr { // Safety: can only hand out references when in the "normalized" state. Will never change // after normalization. @@ -72,12 +70,13 @@ impl PyErr { /// * any other value: the exception instance will be created using the equivalent to the Python /// expression `T(value)` /// - /// This error will be stored in an unnormalized state. This avoids the need for the Python GIL + /// This exception instance will be initialized lazily. This avoids the need for the Python GIL /// to be held, but requires `args` to be `Send` and `Sync`. If `args` is not `Send` or `Sync`, /// consider using [`PyErr::from_value`] instead. /// - /// If an error occurs during normalization (for example if `T` is not a Python type which - /// extends from `BaseException`), then a different error may be produced during normalization. + /// If `T` does not inherit from `BaseException`, then a `TypeError` will be returned. + /// + /// If calling T's constructor with `args` raises an exception, that exception will be returned. /// /// # Examples /// @@ -120,10 +119,12 @@ impl PyErr { T: PyTypeInfo, A: PyErrArguments + Send + Sync + 'static, { - PyErr::from_state(PyErrState::LazyTypeAndValue { - ptype: T::type_object, - pvalue: boxed_args(args), - }) + PyErr::from_state(PyErrState::Lazy(Box::new(move |py| { + PyErrStateLazyFnOutput { + ptype: T::type_object(py).into(), + pvalue: args.arguments(py), + } + }))) } /// Constructs a new PyErr from the given Python type and arguments. @@ -133,28 +134,19 @@ impl PyErr { /// /// `args` is either a tuple or a single value, with the same meaning as in [`PyErr::new`]. /// - /// If an error occurs during normalization (for example if `T` is not a Python type which - /// extends from `BaseException`), then a different error may be produced during normalization. + /// If `ty` does not inherit from `BaseException`, then a `TypeError` will be returned. /// - /// This error will be stored in an unnormalized state. + /// If calling `ty` with `args` raises an exception, that exception will be returned. pub fn from_type(ty: &PyType, args: A) -> PyErr where A: PyErrArguments + Send + Sync + 'static, { - if unsafe { ffi::PyExceptionClass_Check(ty.as_ptr()) } == 0 { - return exceptions_must_derive_from_base_exception(ty.py()); - } - - PyErr::from_state(PyErrState::LazyValue { - ptype: ty.into(), - pvalue: boxed_args(args), - }) + PyErr::from_state(PyErrState::lazy(ty, args)) } /// Creates a new PyErr. /// - /// If `obj` is a Python exception object, the PyErr will contain that object. The error will be - /// in a normalized state. + /// If `obj` is a Python exception object, the PyErr will contain that object. /// /// If `obj` is a Python exception type object, this is equivalent to `PyErr::from_type(obj, ())`. /// @@ -184,22 +176,12 @@ impl PyErr { /// }); /// ``` pub fn from_value(obj: &PyAny) -> PyErr { - let ptr = obj.as_ptr(); - - let state = if unsafe { ffi::PyExceptionInstance_Check(ptr) } != 0 { - PyErrState::Normalized(PyErrStateNormalized { - ptype: obj.get_type().into(), - pvalue: unsafe { Py::from_borrowed_ptr(obj.py(), obj.as_ptr()) }, - ptraceback: None, - }) - } else if unsafe { ffi::PyExceptionClass_Check(obj.as_ptr()) } != 0 { - PyErrState::FfiTuple { - ptype: obj.into(), - pvalue: None, - ptraceback: None, - } + let state = if let Ok(obj) = obj.downcast::() { + PyErrState::normalized(obj) } else { - return exceptions_must_derive_from_base_exception(obj.py()); + // Assume obj is Type[Exception]; let later normalization handle if this + // is not the case + PyErrState::lazy(obj, obj.py().None()) }; PyErr::from_state(state) @@ -207,8 +189,6 @@ impl PyErr { /// Returns the type of this exception. /// - /// The object will be normalized first if needed. - /// /// # Examples /// ```rust /// use pyo3::{exceptions::PyTypeError, types::PyType, PyErr, Python}; @@ -219,13 +199,11 @@ impl PyErr { /// }); /// ``` pub fn get_type<'py>(&'py self, py: Python<'py>) -> &'py PyType { - self.normalized(py).ptype.as_ref(py) + self.normalized(py).ptype(py) } /// Returns the value of this exception. /// - /// The object will be normalized first if needed. - /// /// # Examples /// /// ```rust @@ -246,13 +224,18 @@ impl PyErr { // NB technically this causes one reference count increase and decrease in quick succession // on pvalue, but it's probably not worth optimizing this right now for the additional code // complexity. - self.normalized(py).pvalue.clone_ref(py) + let normalized = self.normalized(py); + let exc = normalized.pvalue.clone_ref(py); + if let Some(tb) = normalized.ptraceback(py) { + unsafe { + ffi::PyException_SetTraceback(exc.as_ptr(), tb.as_ptr()); + } + } + exc } /// Returns the traceback of this exception object. /// - /// The object will be normalized first if needed. - /// /// # Examples /// ```rust /// use pyo3::{exceptions::PyTypeError, Python}; @@ -263,10 +246,7 @@ impl PyErr { /// }); /// ``` pub fn traceback<'py>(&'py self, py: Python<'py>) -> Option<&'py PyTraceback> { - self.normalized(py) - .ptraceback - .as_ref() - .map(|obj| obj.as_ref(py)) + self.normalized(py).ptraceback(py) } /// Gets whether an error is present in the Python interpreter's global state. @@ -285,6 +265,11 @@ impl PyErr { /// expected to have been set, for example from [`PyErr::occurred`] or by an error return value /// from a C FFI function, use [`PyErr::fetch`]. pub fn take(py: Python<'_>) -> Option { + Self::_take(py) + } + + #[cfg(not(Py_3_12))] + fn _take(py: Python<'_>) -> Option { let (ptype, pvalue, ptraceback) = unsafe { let mut ptype: *mut ffi::PyObject = std::ptr::null_mut(); let mut pvalue: *mut ffi::PyObject = std::ptr::null_mut(); @@ -292,9 +277,9 @@ impl PyErr { ffi::PyErr_Fetch(&mut ptype, &mut pvalue, &mut ptraceback); // Convert to Py immediately so that any references are freed by early return. - let ptype = Py::from_owned_ptr_or_opt(py, ptype); - let pvalue = Py::from_owned_ptr_or_opt(py, pvalue); - let ptraceback = Py::from_owned_ptr_or_opt(py, ptraceback); + let ptype = PyObject::from_owned_ptr_or_opt(py, ptype); + let pvalue = PyObject::from_owned_ptr_or_opt(py, pvalue); + let ptraceback = PyObject::from_owned_ptr_or_opt(py, ptraceback); // A valid exception state should always have a non-null ptype, but the other two may be // null. @@ -316,23 +301,19 @@ impl PyErr { (ptype, pvalue, ptraceback) }; - if ptype.as_ptr() == PanicException::type_object(py).as_ptr() { - let msg: String = pvalue + if ptype.as_ptr() == PanicException::type_object_raw(py).cast() { + let msg = pvalue .as_ref() - .and_then(|obj| obj.extract(py).ok()) + .and_then(|obj| obj.as_ref(py).str().ok()) + .map(|py_str| py_str.to_string_lossy().into()) .unwrap_or_else(|| String::from("Unwrapped panic from Python code")); - eprintln!( - "--- PyO3 is resuming a panic after fetching a PanicException from Python. ---" - ); - eprintln!("Python stack trace below:"); - - unsafe { - ffi::PyErr_Restore(ptype.into_ptr(), pvalue.into_ptr(), ptraceback.into_ptr()); - ffi::PyErr_PrintEx(0); - } - - std::panic::resume_unwind(Box::new(msg)) + let state = PyErrState::FfiTuple { + ptype, + pvalue, + ptraceback, + }; + Self::print_panic_and_unwind(py, state, msg) } Some(PyErr::from_state(PyErrState::FfiTuple { @@ -342,6 +323,35 @@ impl PyErr { })) } + #[cfg(Py_3_12)] + fn _take(py: Python<'_>) -> Option { + let pvalue = unsafe { + py.from_owned_ptr_or_opt::(ffi::PyErr_GetRaisedException()) + }?; + if pvalue.get_type().as_ptr() == PanicException::type_object_raw(py).cast() { + let msg: String = pvalue + .str() + .map(|py_str| py_str.to_string_lossy().into()) + .unwrap_or_else(|_| String::from("Unwrapped panic from Python code")); + Self::print_panic_and_unwind(py, PyErrState::normalized(pvalue), msg) + } + + Some(PyErr::from_state(PyErrState::normalized(pvalue))) + } + + fn print_panic_and_unwind(py: Python<'_>, state: PyErrState, msg: String) -> ! { + eprintln!("--- PyO3 is resuming a panic after fetching a PanicException from Python. ---"); + eprintln!("Python stack trace below:"); + + state.restore(py); + + unsafe { + ffi::PyErr_PrintEx(0); + } + + std::panic::resume_unwind(Box::new(msg)) + } + /// Equivalent to [PyErr::take], but when no error is set: /// - Panics in debug mode. /// - Returns a `SystemError` in release mode. @@ -421,13 +431,32 @@ impl PyErr { } /// Prints a standard traceback to `sys.stderr`. + pub fn display(&self, py: Python<'_>) { + #[cfg(Py_3_12)] + unsafe { + ffi::PyErr_DisplayException(self.value(py).as_ptr()) + } + + #[cfg(not(Py_3_12))] + unsafe { + ffi::PyErr_Display( + self.get_type(py).as_ptr(), + self.value(py).as_ptr(), + self.traceback(py) + .map_or(std::ptr::null_mut(), PyTraceback::as_ptr), + ) + } + } + + /// Calls `sys.excepthook` and then prints a standard traceback to `sys.stderr`. pub fn print(&self, py: Python<'_>) { self.clone_ref(py).restore(py); unsafe { ffi::PyErr_PrintEx(0) } } - /// Prints a standard traceback to `sys.stderr`, and sets - /// `sys.last_{type,value,traceback}` attributes to this exception's data. + /// Calls `sys.excepthook` and then prints a standard traceback to `sys.stderr`. + /// + /// Additionally sets `sys.last_{type,value,traceback,exc}` attributes to this exception. pub fn print_and_set_sys_last_vars(&self, py: Python<'_>) { self.clone_ref(py).restore(py); unsafe { ffi::PyErr_PrintEx(1) } @@ -441,15 +470,13 @@ impl PyErr { where T: ToPyObject, { - unsafe { - ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), exc.to_object(py).as_ptr()) != 0 - } + self.is_instance(py, exc.to_object(py).as_ref(py)) } /// Returns true if the current exception is instance of `T`. #[inline] pub fn is_instance(&self, py: Python<'_>, ty: &PyAny) -> bool { - unsafe { ffi::PyErr_GivenExceptionMatches(self.type_ptr(py), ty.as_ptr()) != 0 } + (unsafe { ffi::PyErr_GivenExceptionMatches(self.get_type(py).as_ptr(), ty.as_ptr()) }) != 0 } /// Returns true if the current exception is instance of `T`. @@ -465,15 +492,10 @@ impl PyErr { /// This is the opposite of `PyErr::fetch()`. #[inline] pub fn restore(self, py: Python<'_>) { - let state = match self.state.into_inner() { - Some(state) => state, - // Safety: restore takes `self` by value so nothing else is accessing this err - // and the invariant is that state is always defined except during make_normalized - None => unsafe { std::hint::unreachable_unchecked() }, - }; - - let (ptype, pvalue, ptraceback) = state.into_ffi_tuple(py); - unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) } + self.state + .into_inner() + .expect("PyErr state should never be invalid outside of normalization") + .restore(py) } /// Reports the error as unraisable. @@ -533,16 +555,13 @@ impl PyErr { /// ``` pub fn warn(py: Python<'_>, category: &PyAny, message: &str, stacklevel: i32) -> PyResult<()> { let message = CString::new(message)?; - unsafe { - error_on_minusone( - py, - ffi::PyErr_WarnEx( - category.as_ptr(), - message.as_ptr(), - stacklevel as ffi::Py_ssize_t, - ), + error_on_minusone(py, unsafe { + ffi::PyErr_WarnEx( + category.as_ptr(), + message.as_ptr(), + stacklevel as ffi::Py_ssize_t, ) - } + }) } /// Issues a warning message, with more control over the warning attributes. @@ -573,19 +592,16 @@ impl PyErr { None => std::ptr::null_mut(), Some(obj) => obj.as_ptr(), }; - unsafe { - error_on_minusone( - py, - ffi::PyErr_WarnExplicit( - category.as_ptr(), - message.as_ptr(), - filename.as_ptr(), - lineno, - module_ptr, - registry, - ), + error_on_minusone(py, unsafe { + ffi::PyErr_WarnExplicit( + category.as_ptr(), + message.as_ptr(), + filename.as_ptr(), + lineno, + module_ptr, + registry, ) - } + }) } /// Clone the PyErr. This requires the GIL, which is why PyErr does not implement Clone. @@ -612,18 +628,21 @@ impl PyErr { /// Return the cause (either an exception instance, or None, set by `raise ... from ...`) /// associated with the exception, as accessible from Python through `__cause__`. pub fn cause(&self, py: Python<'_>) -> Option { - let ptr = unsafe { ffi::PyException_GetCause(self.value(py).as_ptr()) }; - let obj = unsafe { py.from_owned_ptr_or_opt::(ptr) }; + let value = self.value(py); + let obj = + unsafe { py.from_owned_ptr_or_opt::(ffi::PyException_GetCause(value.as_ptr())) }; obj.map(Self::from_value) } /// Set the cause associated with the exception, pass `None` to clear it. pub fn set_cause(&self, py: Python<'_>, cause: Option) { + let value = self.value(py); + let cause = cause.map(|err| err.into_value(py)); unsafe { // PyException_SetCause _steals_ a reference to cause, so must use .into_ptr() ffi::PyException_SetCause( - self.value(py).as_ptr(), - cause.map_or(std::ptr::null_mut(), |err| err.into_value(py).into_ptr()), + value.as_ptr(), + cause.map_or(std::ptr::null_mut(), IntoPyPointer::into_ptr), ); } } @@ -635,19 +654,6 @@ impl PyErr { } } - /// Returns borrowed reference to this Err's type - fn type_ptr(&self, py: Python<'_>) -> *mut ffi::PyObject { - match unsafe { &*self.state.get() } { - // In lazy type case, normalize before returning ptype in case the type is not a valid - // exception type. - Some(PyErrState::LazyTypeAndValue { .. }) => self.normalized(py).ptype.as_ptr(), - Some(PyErrState::LazyValue { ptype, .. }) => ptype.as_ptr(), - Some(PyErrState::FfiTuple { ptype, .. }) => ptype.as_ptr(), - Some(PyErrState::Normalized(n)) => n.ptype.as_ptr(), - None => panic!("Cannot access exception type while normalizing"), - } - } - #[inline] fn normalized(&self, py: Python<'_>) -> &PyErrStateNormalized { if let Some(PyErrState::Normalized(n)) = unsafe { @@ -673,17 +679,10 @@ impl PyErr { .take() .expect("Cannot normalize a PyErr while already normalizing it.") }; - let (mut ptype, mut pvalue, mut ptraceback) = state.into_ffi_tuple(py); unsafe { - ffi::PyErr_NormalizeException(&mut ptype, &mut pvalue, &mut ptraceback); let self_state = &mut *self.state.get(); - *self_state = Some(PyErrState::Normalized(PyErrStateNormalized { - ptype: Py::from_owned_ptr_or_opt(py, ptype).expect("Exception type missing"), - pvalue: Py::from_owned_ptr_or_opt(py, pvalue).expect("Exception value missing"), - ptraceback: Py::from_owned_ptr_or_opt(py, ptraceback), - })); - + *self_state = Some(PyErrState::Normalized(state.normalize(py))); match self_state { Some(PyErrState::Normalized(n)) => n, _ => unreachable!(), @@ -792,24 +791,38 @@ pub fn panic_after_error(_py: Python<'_>) -> ! { /// Returns Ok if the error code is not -1. #[inline] -pub fn error_on_minusone(py: Python<'_>, result: c_int) -> PyResult<()> { - if result != -1 { +pub(crate) fn error_on_minusone(py: Python<'_>, result: T) -> PyResult<()> { + if result != T::MINUS_ONE { Ok(()) } else { Err(PyErr::fetch(py)) } } -#[inline] -fn exceptions_must_derive_from_base_exception(py: Python<'_>) -> PyErr { - PyErr::from_state(PyErrState::exceptions_must_derive_from_base_exception(py)) +pub(crate) trait SignedInteger: Eq { + const MINUS_ONE: Self; } +macro_rules! impl_signed_integer { + ($t:ty) => { + impl SignedInteger for $t { + const MINUS_ONE: Self = -1; + } + }; +} + +impl_signed_integer!(i8); +impl_signed_integer!(i16); +impl_signed_integer!(i32); +impl_signed_integer!(i64); +impl_signed_integer!(i128); +impl_signed_integer!(isize); + #[cfg(test)] mod tests { use super::PyErrState; - use crate::exceptions; - use crate::{PyErr, Python}; + use crate::exceptions::{self, PyTypeError, PyValueError}; + use crate::{PyErr, PyTypeInfo, Python}; #[test] fn no_error() { @@ -836,6 +849,7 @@ mod tests { assert!(err.is_instance_of::(py)); err.restore(py); let err = PyErr::fetch(py); + assert!(err.is_instance_of::(py)); assert_eq!( err.to_string(), @@ -869,6 +883,25 @@ mod tests { }); } + #[test] + #[should_panic(expected = "new panic")] + #[cfg(not(Py_3_12))] + fn fetching_normalized_panic_exception_resumes_unwind() { + use crate::panic::PanicException; + + Python::with_gil(|py| { + let err: PyErr = PanicException::new_err("new panic"); + // Restoring an error doesn't normalize it before Python 3.12, + // so we have to explicitly test this case. + let _ = err.normalized(py); + err.restore(py); + assert!(PyErr::occurred(py)); + + // should resume unwind + let _ = PyErr::fetch(py); + }); + } + #[test] fn err_debug() { // Debug representation should be like the following (without the newlines): @@ -923,6 +956,25 @@ mod tests { is_sync::(); } + #[test] + fn test_pyerr_matches() { + Python::with_gil(|py| { + let err = PyErr::new::("foo"); + assert!(err.matches(py, PyValueError::type_object(py))); + + assert!(err.matches( + py, + (PyValueError::type_object(py), PyTypeError::type_object(py)) + )); + + assert!(!err.matches(py, PyTypeError::type_object(py))); + + // String is not a valid exception class, so we should get a TypeError + let err: PyErr = PyErr::from_type(crate::types::PyString::type_object(py), "foo"); + assert!(err.matches(py, PyTypeError::type_object(py))); + }) + } + #[test] fn test_pyerr_cause() { Python::with_gil(|py| { diff --git a/src/exceptions.rs b/src/exceptions.rs index d92e8b35dc3..48a09865666 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Exception and warning types defined by Python. //! //! The structs in this module represent Python's built-in exceptions and @@ -262,13 +260,13 @@ macro_rules! create_exception_type_object { } macro_rules! impl_native_exception ( - ($name:ident, $exc_name:ident, $doc:expr, $layout:path) => ( + ($name:ident, $exc_name:ident, $doc:expr, $layout:path $(, #checkfunction=$checkfunction:path)?) => ( #[doc = $doc] #[allow(clippy::upper_case_acronyms)] pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, *($crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject)); + $crate::pyobject_native_type!($name, $layout, *($crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject) $(, #checkfunction=$checkfunction)?); ); ($name:ident, $exc_name:ident, $doc:expr) => ( impl_native_exception!($name, $exc_name, $doc, $crate::ffi::PyBaseExceptionObject); @@ -361,7 +359,9 @@ Python::with_gil(|py| { impl_native_exception!( PyBaseException, PyExc_BaseException, - native_doc!("BaseException") + native_doc!("BaseException"), + ffi::PyBaseExceptionObject, + #checkfunction=ffi::PyExceptionInstance_Check ); impl_native_exception!(PyException, PyExc_Exception, native_doc!("Exception")); impl_native_exception!( @@ -726,7 +726,7 @@ impl_native_exception!( #[cfg(test)] macro_rules! test_exception { - ($exc_ty:ident $(, $constructor:expr)?) => { + ($exc_ty:ident $(, |$py:tt| $constructor:expr )?) => { #[allow(non_snake_case)] #[test] fn $exc_ty () { @@ -737,7 +737,7 @@ macro_rules! test_exception { let err: $crate::PyErr = { None $( - .or(Some($constructor(py))) + .or(Some({ let $py = py; $constructor })) )? .unwrap_or($exc_ty::new_err("a test exception")) }; @@ -772,12 +772,12 @@ pub mod asyncio { test_exception!(CancelledError); test_exception!(InvalidStateError); test_exception!(TimeoutError); - test_exception!(IncompleteReadError, |_| { - IncompleteReadError::new_err(("partial", "expected")) - }); - test_exception!(LimitOverrunError, |_| { - LimitOverrunError::new_err(("message", "consumed")) - }); + test_exception!(IncompleteReadError, |_| IncompleteReadError::new_err(( + "partial", "expected" + ))); + test_exception!(LimitOverrunError, |_| LimitOverrunError::new_err(( + "message", "consumed" + ))); test_exception!(QueueEmpty); test_exception!(QueueFull); } @@ -813,20 +813,20 @@ mod tests { let err: PyErr = gaierror::new_err(()); let socket = py .import("socket") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not import socket"); let d = PyDict::new(py); d.set_item("socket", socket) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); d.set_item("exc", err) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); py.run("assert isinstance(exc, socket.gaierror)", None, Some(d)) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("assertion failed"); }); } @@ -837,15 +837,15 @@ mod tests { let err: PyErr = MessageError::new_err(()); let email = py .import("email") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not import email"); let d = PyDict::new(py); d.set_item("email", email) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); d.set_item("exc", err) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("could not setitem"); py.run( @@ -853,7 +853,7 @@ mod tests { None, Some(d), ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .expect("assertion failed"); }); } @@ -1035,9 +1035,10 @@ mod tests { }); } #[cfg(Py_3_11)] - test_exception!(PyBaseExceptionGroup, |_| { - PyBaseExceptionGroup::new_err(("msg", vec![PyValueError::new_err("err")])) - }); + test_exception!(PyBaseExceptionGroup, |_| PyBaseExceptionGroup::new_err(( + "msg", + vec![PyValueError::new_err("err")] + ))); test_exception!(PyBaseException); test_exception!(PyException); test_exception!(PyStopAsyncIteration); @@ -1074,10 +1075,9 @@ mod tests { let err = std::str::from_utf8(invalid_utf8).expect_err("should be invalid utf8"); PyErr::from_value(PyUnicodeDecodeError::new_utf8(py, invalid_utf8, err).unwrap()) }); - test_exception!(PyUnicodeEncodeError, |py: Python<'_>| { - py.eval("chr(40960).encode('ascii')", None, None) - .unwrap_err() - }); + test_exception!(PyUnicodeEncodeError, |py| py + .eval("chr(40960).encode('ascii')", None, None) + .unwrap_err()); test_exception!(PyUnicodeTranslateError, |_| { PyUnicodeTranslateError::new_err(("\u{3042}", 0, 1, "ouch")) }); diff --git a/src/ffi/mod.rs b/src/ffi/mod.rs index 81d6f38e4b7..ce108223fbe 100644 --- a/src/ffi/mod.rs +++ b/src/ffi/mod.rs @@ -21,7 +21,7 @@ //! //! [capi]: https://docs.python.org/3/c-api/index.html -#[cfg(all(not(Py_LIMITED_API), test))] +#[cfg(test)] mod tests; // reexport raw bindings exposed in pyo3_ffi diff --git a/src/ffi/tests.rs b/src/ffi/tests.rs index 97e838cf527..14f76cb4fee 100644 --- a/src/ffi/tests.rs +++ b/src/ffi/tests.rs @@ -1,9 +1,15 @@ use crate::ffi::*; -use crate::{types::PyDict, AsPyPointer, IntoPy, Py, PyAny, Python}; - -use crate::types::PyString; +use crate::{AsPyPointer, Python}; + +#[cfg(not(Py_LIMITED_API))] +use crate::{ + types::{PyDict, PyString}, + IntoPy, Py, PyAny, +}; +#[cfg(not(any(Py_3_12, Py_LIMITED_API)))] use libc::wchar_t; +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_datetime_fromtimestamp() { @@ -24,6 +30,7 @@ fn test_datetime_fromtimestamp() { }) } +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_date_fromtimestamp() { @@ -44,6 +51,7 @@ fn test_date_fromtimestamp() { }) } +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[test] fn test_utc_timezone() { @@ -64,18 +72,15 @@ fn test_utc_timezone() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset() { use crate::types::PyDelta; Python::with_gil(|py| { - let tz: &PyAny = unsafe { - PyDateTime_IMPORT(); - py.from_borrowed_ptr(PyTimeZone_FromOffset( - PyDelta::new(py, 0, 100, 0, false).unwrap().as_ptr(), - )) - }; + let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); + let tz: &PyAny = unsafe { py.from_borrowed_ptr(PyTimeZone_FromOffset(delta.as_ptr())) }; crate::py_run!( py, tz, @@ -85,17 +90,19 @@ fn test_timezone_from_offset() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg(feature = "macros")] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons fn test_timezone_from_offset_and_name() { use crate::types::PyDelta; Python::with_gil(|py| { + let delta = PyDelta::new(py, 0, 100, 0, false).unwrap(); + let tzname = PyString::new(py, "testtz"); let tz: &PyAny = unsafe { - PyDateTime_IMPORT(); py.from_borrowed_ptr(PyTimeZone_FromOffsetAndName( - PyDelta::new(py, 0, 100, 0, false).unwrap().as_ptr(), - PyString::new(py, "testtz").as_ptr(), + delta.as_ptr(), + tzname.as_ptr(), )) }; crate::py_run!( @@ -107,6 +114,7 @@ fn test_timezone_from_offset_and_name() { } #[test] +#[cfg(not(Py_LIMITED_API))] fn ascii_object_bitfield() { let ob_base: PyObject = unsafe { std::mem::zeroed() }; @@ -116,6 +124,7 @@ fn ascii_object_bitfield() { #[cfg(not(PyPy))] hash: 0, state: 0u32, + #[cfg(not(Py_3_12))] wstr: std::ptr::null_mut() as *mut wchar_t, }; @@ -124,9 +133,12 @@ fn ascii_object_bitfield() { assert_eq!(o.kind(), 0); assert_eq!(o.compact(), 0); assert_eq!(o.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 0); - for i in 0..4 { + let interned_count = if cfg!(Py_3_12) { 2 } else { 4 }; + + for i in 0..interned_count { o.set_interned(i); assert_eq!(o.interned(), i); } @@ -142,12 +154,15 @@ fn ascii_object_bitfield() { o.set_ascii(1); assert_eq!(o.ascii(), 1); + #[cfg(not(Py_3_12))] o.set_ready(1); + #[cfg(not(Py_3_12))] assert_eq!(o.ready(), 1); } } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ascii() { Python::with_gil(|py| { @@ -163,6 +178,7 @@ fn ascii() { assert_eq!(ascii.kind(), PyUnicode_1BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 1); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 1); @@ -188,6 +204,7 @@ fn ascii() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(Py_3_10, allow(deprecated))] fn ucs4() { Python::with_gil(|py| { @@ -203,6 +220,7 @@ fn ucs4() { assert_eq!(ascii.kind(), PyUnicode_4BYTE_KIND); assert_eq!(ascii.compact(), 1); assert_eq!(ascii.ascii(), 0); + #[cfg(not(Py_3_12))] assert_eq!(ascii.ready(), 1); assert_eq!(PyUnicode_IS_ASCII(ptr), 0); @@ -230,6 +248,7 @@ fn ucs4() { } #[test] +#[cfg(not(Py_LIMITED_API))] #[cfg_attr(target_arch = "wasm32", ignore)] // DateTime import fails on wasm for mysterious reasons #[cfg(not(PyPy))] fn test_get_tzinfo() { @@ -270,3 +289,40 @@ fn test_get_tzinfo() { ); }) } + +#[test] +fn test_inc_dec_ref() { + Python::with_gil(|py| { + let obj = py.eval("object()", None, None).unwrap(); + + let ref_count = obj.get_refcnt(); + let ptr = obj.as_ptr(); + + unsafe { Py_INCREF(ptr) }; + + assert_eq!(obj.get_refcnt(), ref_count + 1); + + unsafe { Py_DECREF(ptr) }; + + assert_eq!(obj.get_refcnt(), ref_count); + }) +} + +#[test] +#[cfg(Py_3_12)] +fn test_inc_dec_ref_immortal() { + Python::with_gil(|py| { + let obj = py.None(); + + let ref_count = obj.get_refcnt(py); + let ptr = obj.as_ptr(); + + unsafe { Py_INCREF(ptr) }; + + assert_eq!(obj.get_refcnt(py), ref_count); + + unsafe { Py_DECREF(ptr) }; + + assert_eq!(obj.get_refcnt(py), ref_count); + }) +} diff --git a/src/gil.rs b/src/gil.rs index fa96a910797..69b511c9ff3 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -1,12 +1,14 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Interaction with Python's global interpreter lock use crate::impl_::not_send::{NotSend, NOT_SEND}; use crate::{ffi, Python}; use parking_lot::{const_mutex, Mutex, Once}; -use std::cell::{Cell, RefCell}; -use std::{mem, ptr::NonNull, sync::atomic}; +use std::cell::Cell; +#[cfg(debug_assertions)] +use std::cell::RefCell; +#[cfg(not(debug_assertions))] +use std::cell::UnsafeCell; +use std::{mem, ptr::NonNull}; static START: Once = Once::new(); @@ -35,7 +37,10 @@ thread_local_const_init! { static GIL_COUNT: Cell = const { Cell::new(0) }; /// Temporarily hold objects that will be released when the GILPool drops. - static OWNED_OBJECTS: RefCell>> = const { RefCell::new(Vec::new()) }; + #[cfg(debug_assertions)] + static OWNED_OBJECTS: RefCell = const { RefCell::new(Vec::new()) }; + #[cfg(not(debug_assertions))] + static OWNED_OBJECTS: UnsafeCell = const { UnsafeCell::new(Vec::new()) }; } const GIL_LOCKED_DURING_TRAVERSE: isize = -1; @@ -240,7 +245,6 @@ type PyObjVec = Vec>; /// Thread-safe storage for objects which were inc_ref / dec_ref while the GIL was not held. struct ReferencePool { - dirty: atomic::AtomicBool, // .0 is INCREFs, .1 is DECREFs pointer_ops: Mutex<(PyObjVec, PyObjVec)>, } @@ -248,30 +252,27 @@ struct ReferencePool { impl ReferencePool { const fn new() -> Self { Self { - dirty: atomic::AtomicBool::new(false), pointer_ops: const_mutex((Vec::new(), Vec::new())), } } fn register_incref(&self, obj: NonNull) { self.pointer_ops.lock().0.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn register_decref(&self, obj: NonNull) { self.pointer_ops.lock().1.push(obj); - self.dirty.store(true, atomic::Ordering::Release); } fn update_counts(&self, _py: Python<'_>) { - let prev = self.dirty.swap(false, atomic::Ordering::Acquire); - if !prev { + let mut ops = self.pointer_ops.lock(); + if ops.0.is_empty() && ops.1.is_empty() { return; } - let mut ops = self.pointer_ops.lock(); let (increfs, decrefs) = mem::take(&mut *ops); drop(ops); + // Always increase reference counts first - as otherwise objects which have a // nonzero total reference count might be incorrectly dropped by Python during // this update. @@ -379,7 +380,16 @@ impl GILPool { // Update counts of PyObjects / Py that have been cloned or dropped since last acquisition POOL.update_counts(Python::assume_gil_acquired()); GILPool { - start: OWNED_OBJECTS.try_with(|o| o.borrow().len()).ok(), + start: OWNED_OBJECTS + .try_with(|owned_objects| { + #[cfg(debug_assertions)] + let len = owned_objects.borrow().len(); + #[cfg(not(debug_assertions))] + // SAFETY: This is not re-entrant. + let len = unsafe { (*owned_objects.get()).len() }; + len + }) + .ok(), _not_send: NOT_SEND, } } @@ -393,18 +403,21 @@ impl GILPool { impl Drop for GILPool { fn drop(&mut self) { - if let Some(obj_len_start) = self.start { - let dropping_obj = OWNED_OBJECTS.with(|holder| { - // `holder` must be dropped before calling Py_DECREF, or Py_DECREF may call - // `GILPool::drop` recursively, resulting in invalid borrowing. - let mut holder = holder.borrow_mut(); - if obj_len_start < holder.len() { - holder.split_off(obj_len_start) + if let Some(start) = self.start { + let owned_objects = OWNED_OBJECTS.with(|owned_objects| { + #[cfg(debug_assertions)] + let mut owned_objects = owned_objects.borrow_mut(); + #[cfg(not(debug_assertions))] + // SAFETY: `OWNED_OBJECTS` is released before calling Py_DECREF, + // or Py_DECREF may call `GILPool::drop` recursively, resulting in invalid borrowing. + let owned_objects = unsafe { &mut *owned_objects.get() }; + if start < owned_objects.len() { + owned_objects.split_off(start) } else { Vec::new() } }); - for obj in dropping_obj { + for obj in owned_objects { unsafe { ffi::Py_DECREF(obj.as_ptr()); } @@ -453,7 +466,15 @@ pub unsafe fn register_decref(obj: NonNull) { pub unsafe fn register_owned(_py: Python<'_>, obj: NonNull) { debug_assert!(gil_is_acquired()); // Ignores the error in case this function called from `atexit`. - let _ = OWNED_OBJECTS.try_with(|holder| holder.borrow_mut().push(obj)); + let _ = OWNED_OBJECTS.try_with(|owned_objects| { + #[cfg(debug_assertions)] + owned_objects.borrow_mut().push(obj); + #[cfg(not(debug_assertions))] + // SAFETY: This is not re-entrant. + unsafe { + (*owned_objects.get()).push(obj); + } + }); } /// Increments pyo3's internal GIL count - to be called whenever GILPool or GILGuard is created. @@ -489,7 +510,7 @@ mod tests { use crate::{ffi, gil, AsPyPointer, IntoPyPointer, PyObject, Python, ToPyObject}; #[cfg(not(target_arch = "wasm32"))] use parking_lot::{const_mutex, Condvar, Mutex}; - use std::{ptr::NonNull, sync::atomic::Ordering}; + use std::ptr::NonNull; fn get_object(py: Python<'_>) -> PyObject { // Convenience function for getting a single unique object, using `new_pool` so as to leave @@ -502,11 +523,27 @@ mod tests { } fn owned_object_count() -> usize { - OWNED_OBJECTS.with(|holder| holder.borrow().len()) + #[cfg(debug_assertions)] + let len = OWNED_OBJECTS.with(|owned_objects| owned_objects.borrow().len()); + #[cfg(not(debug_assertions))] + let len = OWNED_OBJECTS.with(|owned_objects| unsafe { (*owned_objects.get()).len() }); + len + } + + fn pool_inc_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .0 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } - fn pool_not_dirty() -> bool { - !POOL.dirty.load(Ordering::SeqCst) + fn pool_dec_refs_does_not_contain(obj: &PyObject) -> bool { + !POOL + .pointer_ops + .lock() + .1 + .contains(&unsafe { NonNull::new_unchecked(obj.as_ptr()) }) } #[cfg(not(target_arch = "wasm32"))] @@ -584,13 +621,13 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // With the GIL held, reference cound will be decreased immediately. drop(reference); assert_eq!(obj.get_refcnt(py), 1); - assert!(pool_not_dirty()); + assert!(pool_dec_refs_does_not_contain(&obj)); }); } @@ -603,7 +640,7 @@ mod tests { let reference = obj.clone_ref(py); assert_eq!(obj.get_refcnt(py), 2); - assert!(pool_not_dirty()); + assert!(pool_inc_refs_does_not_contain(&obj)); // Drop reference in a separate thread which doesn't have the GIL. std::thread::spawn(move || drop(reference)).join().unwrap(); diff --git a/src/impl_/extract_argument.rs b/src/impl_/extract_argument.rs index dd361e4b538..56af8921f11 100644 --- a/src/impl_/extract_argument.rs +++ b/src/impl_/extract_argument.rs @@ -728,14 +728,14 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); + let kwargs = [("foo".to_object(py).into_ref(py), 0u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), - [("foo".to_object(py).into_ref(py), 0u8)] - .into_py_dict(py) - .as_ptr(), + args.as_ptr(), + kwargs.as_ptr(), &mut [], ) .unwrap_err() @@ -759,14 +759,14 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); + let kwargs = [(1u8.to_object(py).into_ref(py), 1u8)].into_py_dict(py); let err = unsafe { function_description .extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), - [(1u8.to_object(py).into_ref(py), 1u8)] - .into_py_dict(py) - .as_ptr(), + args.as_ptr(), + kwargs.as_ptr(), &mut [], ) .unwrap_err() @@ -790,11 +790,12 @@ mod tests { }; Python::with_gil(|py| { + let args = PyTuple::new(py, Vec::<&PyAny>::new()); let mut output = [None, None]; let err = unsafe { function_description.extract_arguments_tuple_dict::( py, - PyTuple::new(py, Vec::<&PyAny>::new()).as_ptr(), + args.as_ptr(), std::ptr::null_mut(), &mut output, ) diff --git a/src/impl_/freelist.rs b/src/impl_/freelist.rs index d5e3d1f8143..955a50a3549 100644 --- a/src/impl_/freelist.rs +++ b/src/impl_/freelist.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Support for [free allocation lists][1]. //! //! This can improve performance for types that are often created and deleted in quick succession. diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index 312144a69c0..b56077b4dab 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -122,7 +122,7 @@ impl Default for PyClassImplCollector { impl Clone for PyClassImplCollector { fn clone(&self) -> Self { - Self::new() + *self } } diff --git a/src/impl_/pyclass/lazy_type_object.rs b/src/impl_/pyclass/lazy_type_object.rs index 551b052ae71..0e606b588d0 100644 --- a/src/impl_/pyclass/lazy_type_object.rs +++ b/src/impl_/pyclass/lazy_type_object.rs @@ -202,8 +202,9 @@ fn initialize_tp_dict( // We hold the GIL: the dictionary update can be considered atomic from // the POV of other threads. for (key, val) in items { - let ret = unsafe { ffi::PyObject_SetAttrString(type_object, key.as_ptr(), val.into_ptr()) }; - crate::err::error_on_minusone(py, ret)?; + crate::err::error_on_minusone(py, unsafe { + ffi::PyObject_SetAttrString(type_object, key.as_ptr(), val.into_ptr()) + })?; } Ok(()) } diff --git a/src/impl_/pyfunction.rs b/src/impl_/pyfunction.rs index 95d8350d270..14cdbd48f85 100644 --- a/src/impl_/pyfunction.rs +++ b/src/impl_/pyfunction.rs @@ -2,7 +2,7 @@ use crate::{derive_utils::PyFunctionArguments, types::PyCFunction, PyResult}; pub use crate::impl_::pymethods::PyMethodDef; -pub fn wrap_pyfunction_impl<'a>( +pub fn _wrap_pyfunction<'a>( method_def: &PyMethodDef, py_or_module: impl Into>, ) -> PyResult<&'a PyCFunction> { diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 60db3fbb1bf..98089d209d9 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -247,7 +247,7 @@ impl PySetterDef { /// Calls an implementation of __traverse__ for tp_traverse #[doc(hidden)] -pub unsafe fn call_traverse_impl( +pub unsafe fn _call_traverse( slf: *mut ffi::PyObject, impl_: fn(&T, PyVisit<'_>) -> Result<(), PyTraverseError>, visit: ffi::visitproc, diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index efe50e20727..1fcdebc25e7 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -18,7 +18,7 @@ use crate::{ pub unsafe fn module_init( f: for<'py> unsafe fn(Python<'py>) -> PyResult>, ) -> *mut ffi::PyObject { - trampoline_inner(|py| f(py).map(|module| module.into_ptr())) + trampoline(|py| f(py).map(|module| module.into_ptr())) } #[inline] @@ -28,7 +28,7 @@ pub unsafe fn noargs( f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject) -> PyResult<*mut ffi::PyObject>, ) -> *mut ffi::PyObject { debug_assert!(args.is_null()); - trampoline_inner(|py| f(py, slf)) + trampoline(|py| f(py, slf)) } macro_rules! trampoline { @@ -38,7 +38,7 @@ macro_rules! trampoline { $($arg_names: $arg_types,)* f: for<'py> unsafe fn (Python<'py>, $($arg_types),*) -> PyResult<$ret>, ) -> $ret { - trampoline_inner(|py| f(py, $($arg_names,)*)) + trampoline(|py| f(py, $($arg_names,)*)) } } } @@ -131,7 +131,7 @@ pub unsafe fn releasebufferproc( buf: *mut ffi::Py_buffer, f: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject, *mut ffi::Py_buffer) -> PyResult<()>, ) { - trampoline_inner_unraisable(|py| f(py, slf, buf), slf) + trampoline_unraisable(|py| f(py, slf, buf), slf) } #[inline] @@ -143,7 +143,7 @@ pub(crate) unsafe fn dealloc( // so pass null_mut() to the context. // // (Note that we don't allow the implementation `f` to fail.) - trampoline_inner_unraisable( + trampoline_unraisable( |py| { f(py, slf); Ok(()) @@ -168,7 +168,7 @@ trampoline!( /// Panics during execution are trapped so that they don't propagate through any /// outer FFI boundary. #[inline] -pub(crate) fn trampoline_inner(body: F) -> R +pub(crate) fn trampoline(body: F) -> R where F: for<'py> FnOnce(Python<'py>) -> PyResult + UnwindSafe, R: PyCallbackOutput, @@ -214,7 +214,7 @@ where /// /// ctx must be either a valid ffi::PyObject or NULL #[inline] -unsafe fn trampoline_inner_unraisable(body: F, ctx: *mut ffi::PyObject) +unsafe fn trampoline_unraisable(body: F, ctx: *mut ffi::PyObject) where F: for<'py> FnOnce(Python<'py>) -> PyResult<()> + UnwindSafe, { diff --git a/src/instance.rs b/src/instance.rs index d3927768ec9..aa9cbf34f01 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::conversion::PyTryFrom; use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::gil; @@ -651,12 +650,9 @@ impl Py { let attr_name = attr_name.into_py(py); let value = value.into_py(py); - unsafe { - err::error_on_minusone( - py, - ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()), - ) - } + err::error_on_minusone(py, unsafe { + ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()) + }) } /// Calls the object. @@ -1241,7 +1237,7 @@ a = A() Python::with_gil(|py| { let v = py .eval("...", None, None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap() .to_object(py); diff --git a/src/lib.rs b/src/lib.rs index ded43370148..8114e791339 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -504,6 +504,7 @@ pub mod doc_test { "guide/src/migration.md" => guide_migration_md, "guide/src/module.md" => guide_module_md, "guide/src/parallelism.md" => guide_parallelism_md, + "guide/src/performance.md" => guide_performance_md, "guide/src/python_from_rust.md" => guide_python_from_rust_md, "guide/src/python_typing_hints.md" => guide_python_typing_hints_md, "guide/src/rust_cpython.md" => guide_rust_cpython_md, diff --git a/src/macros.rs b/src/macros.rs index d9237e2c959..560d43da1ca 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -125,12 +125,12 @@ macro_rules! wrap_pyfunction { ($function:path) => { &|py_or_module| { use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, py_or_module) + $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, py_or_module) } }; ($function:path, $py_or_module:expr) => {{ use $function as wrapped_pyfunction; - $crate::impl_::pyfunction::wrap_pyfunction_impl(&wrapped_pyfunction::DEF, $py_or_module) + $crate::impl_::pyfunction::_wrap_pyfunction(&wrapped_pyfunction::DEF, $py_or_module) }}; } diff --git a/src/marker.rs b/src/marker.rs index c6d6fc40f96..8912f664b69 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - //! Fundamental properties of objects tied to the Python interpreter. //! //! The Python interpreter is not threadsafe. To protect the Python interpreter in multithreaded @@ -880,8 +876,7 @@ impl<'py> Python<'py> { /// [1]: https://docs.python.org/3/c-api/exceptions.html?highlight=pyerr_checksignals#c.PyErr_CheckSignals /// [2]: https://docs.python.org/3/library/signal.html pub fn check_signals(self) -> PyResult<()> { - let v = unsafe { ffi::PyErr_CheckSignals() }; - err::error_on_minusone(self, v) + err::error_on_minusone(self, unsafe { ffi::PyErr_CheckSignals() }) } /// Create a new pool for managing PyO3's owned references. @@ -944,6 +939,72 @@ impl<'py> Python<'py> { } } +impl Python<'_> { + /// Creates a scope using a new pool for managing PyO3's owned references. + /// + /// This is a safe alterantive to [`new_pool`][Self::new_pool] as + /// it limits the closure to using the new GIL token at the cost of + /// being unable to capture existing GIL-bound references. + /// + /// Note that on stable Rust, this API suffers from the same the `SendWrapper` loophole + /// as [`allow_threads`][Self::allow_threads], c.f. the documentation of the [`Ungil`] trait, + /// + /// # Examples + /// + /// ```rust + /// # use pyo3::prelude::*; + /// Python::with_gil(|py| { + /// // Some long-running process like a webserver, which never releases the GIL. + /// loop { + /// // Create a new scope, so that PyO3 can clear memory at the end of the loop. + /// py.with_pool(|py| { + /// // do stuff... + /// }); + /// # break; // Exit the loop so that doctest terminates! + /// } + /// }); + /// ``` + /// + /// The `Ungil` bound on the closure does prevent hanging on to existing GIL-bound references + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// # use pyo3::types::PyString; + /// + /// Python::with_gil(|py| { + /// let old_str = PyString::new(py, "a message from the past"); + /// + /// py.with_pool(|_py| { + /// print!("{:?}", old_str); + /// }); + /// }); + /// ``` + /// + /// or continuing to use the old GIL token + /// + /// ```compile_fail + /// # use pyo3::prelude::*; + /// + /// Python::with_gil(|old_py| { + /// old_py.with_pool(|_new_py| { + /// let _none = old_py.None(); + /// }); + /// }); + /// ``` + #[inline] + pub fn with_pool(&self, f: F) -> R + where + F: for<'py> FnOnce(Python<'py>) -> R + Ungil, + { + // SAFETY: The closure is `Ungil`, + // i.e. it does not capture any GIL-bound references + // and accesses only the newly created GIL token. + let pool = unsafe { GILPool::new() }; + + f(pool.python()) + } +} + impl<'unbound> Python<'unbound> { /// Unsafely creates a Python token with an unbounded lifetime. /// @@ -980,7 +1041,7 @@ mod tests { // Make sure builtin names are accessible let v: i32 = py .eval("min(1, 2)", None, None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap() .extract() .unwrap(); @@ -1097,7 +1158,10 @@ mod tests { Python::with_gil(|py| { assert_eq!(py.Ellipsis().to_string(), "Ellipsis"); - let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap(); + let v = py + .eval("...", None, None) + .map_err(|e| e.display(py)) + .unwrap(); assert!(v.eq(py.Ellipsis()).unwrap()); }); diff --git a/src/prelude.rs b/src/prelude.rs index a110ea5c1f5..ca0b0cf38db 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! PyO3's prelude. //! //! The purpose of this module is to alleviate imports of many commonly used items of the PyO3 crate diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index bc2058f50f0..09aec7a4418 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -7,7 +7,7 @@ use crate::{ }, impl_::{ pymethods::{get_doc, get_name, Getter, Setter}, - trampoline::trampoline_inner, + trampoline::trampoline, }, types::PyType, Py, PyClass, PyGetterDef, PyMethodDefType, PyResult, PySetterDef, PyTypeInfo, Python, @@ -413,7 +413,7 @@ unsafe extern "C" fn no_constructor_defined( _args: *mut ffi::PyObject, _kwds: *mut ffi::PyObject, ) -> *mut ffi::PyObject { - trampoline_inner(|_| { + trampoline(|_| { Err(crate::exceptions::PyTypeError::new_err( "No constructor defined", )) @@ -513,7 +513,7 @@ impl GetSetDefType { ) -> *mut ffi::PyObject { // Safety: PyO3 sets the closure when constructing the ffi getter so this cast should always be valid let getter: Getter = std::mem::transmute(closure); - trampoline_inner(|py| getter(py, slf)) + trampoline(|py| getter(py, slf)) } (Some(getter), None, closure as Getter as _) } @@ -525,7 +525,7 @@ impl GetSetDefType { ) -> c_int { // Safety: PyO3 sets the closure when constructing the ffi setter so this cast should always be valid let setter: Setter = std::mem::transmute(closure); - trampoline_inner(|py| setter(py, slf, value)) + trampoline(|py| setter(py, slf, value)) } (None, Some(setter), closure as Setter as _) } @@ -535,7 +535,7 @@ impl GetSetDefType { closure: *mut c_void, ) -> *mut ffi::PyObject { let getset: &GetterAndSetter = &*(closure as *const GetterAndSetter); - trampoline_inner(|py| (getset.getter)(py, slf)) + trampoline(|py| (getset.getter)(py, slf)) } unsafe extern "C" fn getset_setter( @@ -544,7 +544,7 @@ impl GetSetDefType { closure: *mut c_void, ) -> c_int { let getset: &GetterAndSetter = &*(closure as *const GetterAndSetter); - trampoline_inner(|py| (getset.setter)(py, slf, value)) + trampoline(|py| (getset.setter)(py, slf, value)) } ( Some(getset_getter), diff --git a/src/pyclass/gc.rs b/src/pyclass/gc.rs index 900027f7bfb..7878ccf5ca8 100644 --- a/src/pyclass/gc.rs +++ b/src/pyclass/gc.rs @@ -38,7 +38,11 @@ impl<'p> PyVisit<'p> { /// Creates the PyVisit from the arguments to tp_traverse #[doc(hidden)] - pub unsafe fn from_raw(visit: ffi::visitproc, arg: *mut c_void, _py: Python<'p>) -> Self { - Self { visit, arg, _py } + pub unsafe fn from_raw(visit: ffi::visitproc, arg: *mut c_void, py: Python<'p>) -> Self { + Self { + visit, + arg, + _py: py, + } } } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 015f5791579..0f606853f74 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -1,7 +1,7 @@ //! Contains initialization utilities for `#[pyclass]`. use crate::callback::IntoPyCallbackOutput; use crate::impl_::pyclass::{PyClassBaseType, PyClassDict, PyClassThreadChecker, PyClassWeakRef}; -use crate::{ffi, PyCell, PyClass, PyErr, PyResult, Python}; +use crate::{ffi, IntoPyPointer, Py, PyCell, PyClass, PyErr, PyResult, Python}; use crate::{ ffi::PyTypeObject, pycell::{ @@ -137,9 +137,14 @@ impl PyObjectInit for PyNativeTypeInitializer { /// ); /// }); /// ``` -pub struct PyClassInitializer { - init: T, - super_init: ::Initializer, +pub struct PyClassInitializer(PyClassInitializerImpl); + +enum PyClassInitializerImpl { + Existing(Py), + New { + init: T, + super_init: ::Initializer, + }, } impl PyClassInitializer { @@ -147,7 +152,7 @@ impl PyClassInitializer { /// /// It is recommended to use `add_subclass` instead of this method for most usage. pub fn new(init: T, super_init: ::Initializer) -> Self { - Self { init, super_init } + Self(PyClassInitializerImpl::New { init, super_init }) } /// Constructs a new initializer from an initializer for the base class. @@ -245,13 +250,18 @@ impl PyObjectInit for PyClassInitializer { contents: MaybeUninit>, } - let obj = self.super_init.into_new_object(py, subtype)?; + let (init, super_init) = match self.0 { + PyClassInitializerImpl::Existing(value) => return Ok(value.into_ptr()), + PyClassInitializerImpl::New { init, super_init } => (init, super_init), + }; + + let obj = super_init.into_new_object(py, subtype)?; let cell: *mut PartiallyInitializedPyCell = obj as _; std::ptr::write( (*cell).contents.as_mut_ptr(), PyCellContents { - value: ManuallyDrop::new(UnsafeCell::new(self.init)), + value: ManuallyDrop::new(UnsafeCell::new(init)), borrow_checker: ::Storage::new(), thread_checker: T::ThreadChecker::new(), dict: T::Dict::INIT, @@ -287,6 +297,13 @@ where } } +impl From> for PyClassInitializer { + #[inline] + fn from(value: Py) -> PyClassInitializer { + PyClassInitializer(PyClassInitializerImpl::Existing(value)) + } +} + // Implementation used by proc macros to allow anything convertible to PyClassInitializer to be // the return value of pyclass #[new] method (optionally wrapped in `Result`). impl IntoPyCallbackOutput> for U diff --git a/src/sync.rs b/src/sync.rs index 0f5a51631d3..3cb4206d239 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,5 +1,5 @@ //! Synchronization mechanisms based on the Python GIL. -use crate::{types::PyString, Py, Python}; +use crate::{types::PyString, types::PyType, Py, PyErr, Python}; use std::cell::UnsafeCell; /// Value with concurrent access protected by the GIL. @@ -169,6 +169,21 @@ impl GILOnceCell { } } +impl GILOnceCell> { + /// Get a reference to the contained Python type, initializing it if needed. + /// + /// This is a shorthand method for `get_or_init` which imports the type from Python on init. + pub(crate) fn get_or_try_init_type_ref<'py>( + &'py self, + py: Python<'py>, + module_name: &str, + attr_name: &str, + ) -> Result<&'py PyType, PyErr> { + self.get_or_try_init(py, || py.import(module_name)?.getattr(attr_name)?.extract()) + .map(|ty| ty.as_ref(py)) + } +} + /// Interns `text` as a Python string and stores a reference to it in static storage. /// /// A reference to the same Python string is returned on each invocation. diff --git a/src/test_hygiene/pyclass.rs b/src/test_hygiene/pyclass.rs index 0b535abe860..4d07009cad6 100644 --- a/src/test_hygiene/pyclass.rs +++ b/src/test_hygiene/pyclass.rs @@ -56,6 +56,6 @@ pub struct Foo4 { field: i32, #[pyo3(get, set)] - #[cfg(any(not(FALSE)))] + #[cfg(not(FALSE))] field: u32, } diff --git a/src/type_object.rs b/src/type_object.rs index f36abd94788..3098b07217f 100644 --- a/src/type_object.rs +++ b/src/type_object.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors //! Python type object information use crate::types::{PyAny, PyType}; diff --git a/src/types/any.rs b/src/types/any.rs index afdeb6ab573..5065cfd934c 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,7 +1,7 @@ use crate::class::basic::CompareOp; use crate::conversion::{AsPyPointer, FromPyObject, IntoPy, IntoPyPointer, PyTryFrom, ToPyObject}; use crate::err::{PyDowncastError, PyErr, PyResult}; -use crate::exceptions::PyTypeError; +use crate::exceptions::{PyAttributeError, PyTypeError}; use crate::type_object::PyTypeInfo; #[cfg(not(PyPy))] use crate::types::PySuper; @@ -79,14 +79,37 @@ impl PyAny { /// /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used /// to intern `attr_name`. + /// + /// # Example: `intern!`ing the attribute name + /// + /// ``` + /// # use pyo3::{intern, pyfunction, types::PyModule, Python, PyResult}; + /// # + /// #[pyfunction] + /// fn has_version(sys: &PyModule) -> PyResult { + /// sys.hasattr(intern!(sys.py(), "version")) + /// } + /// # + /// # Python::with_gil(|py| { + /// # let sys = py.import("sys").unwrap(); + /// # has_version(sys).unwrap(); + /// # }); + /// ``` pub fn hasattr(&self, attr_name: N) -> PyResult where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult { + // PyObject_HasAttr suppresses all exceptions, which was the behaviour of `hasattr` in Python 2. + // Use an implementation which suppresses only AttributeError, which is consistent with `hasattr` in Python 3. + match any._getattr(attr_name) { + Ok(_) => Ok(true), + Err(err) if err.is_instance_of::(any.py()) => Ok(false), + Err(e) => Err(e), + } + } - unsafe { Ok(ffi::PyObject_HasAttr(self.as_ptr(), attr_name.as_ptr()) != 0) } + inner(self, attr_name.into_py(self.py())) } /// Retrieves an attribute value. @@ -115,12 +138,65 @@ impl PyAny { where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); + fn inner(any: &PyAny, attr_name: Py) -> PyResult<&PyAny> { + any._getattr(attr_name) + .map(|object| object.into_ref(any.py())) + } + + inner(self, attr_name.into_py(self.py())) + } + fn _getattr(&self, attr_name: Py) -> PyResult { unsafe { - let ret = ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()); - py.from_owned_ptr_or_err(ret) + Py::from_owned_ptr_or_err( + self.py(), + ffi::PyObject_GetAttr(self.as_ptr(), attr_name.as_ptr()), + ) + } + } + + /// Retrieve an attribute value, skipping the instance dictionary during the lookup but still + /// binding the object to the instance. + /// + /// This is useful when trying to resolve Python's "magic" methods like `__getitem__`, which + /// are looked up starting from the type object. This returns an `Option` as it is not + /// typically a direct error for the special lookup to fail, as magic methods are optional in + /// many situations in which they might be called. + /// + /// To avoid repeated temporary allocations of Python strings, the [`intern!`] macro can be used + /// to intern `attr_name`. + #[allow(dead_code)] // Currently only used with num-complex+abi3, so dead without that. + pub(crate) fn lookup_special(&self, attr_name: N) -> PyResult> + where + N: IntoPy>, + { + let py = self.py(); + let self_type = self.get_type(); + let attr = if let Ok(attr) = self_type.getattr(attr_name) { + attr + } else { + return Ok(None); + }; + + // Manually resolve descriptor protocol. + if cfg!(Py_3_10) + || unsafe { ffi::PyType_HasFeature(attr.get_type_ptr(), ffi::Py_TPFLAGS_HEAPTYPE) } != 0 + { + // This is the preferred faster path, but does not work on static types (generally, + // types defined in extension modules) before Python 3.10. + unsafe { + let descr_get_ptr = ffi::PyType_GetSlot(attr.get_type_ptr(), ffi::Py_tp_descr_get); + if descr_get_ptr.is_null() { + return Ok(Some(attr)); + } + let descr_get: ffi::descrgetfunc = std::mem::transmute(descr_get_ptr); + let ret = descr_get(attr.as_ptr(), self.as_ptr(), self_type.as_ptr()); + py.from_owned_ptr_or_err(ret).map(Some) + } + } else if let Ok(descr_get) = attr.get_type().getattr(crate::intern!(py, "__get__")) { + descr_get.call1((attr, self, self_type)).map(Some) + } else { + Ok(Some(attr)) } } @@ -151,14 +227,14 @@ impl PyAny { N: IntoPy>, V: ToPyObject, { - let py = self.py(); - let attr_name = attr_name.into_py(py); - let value = value.to_object(py); - - unsafe { - let ret = ffi::PyObject_SetAttr(self.as_ptr(), attr_name.as_ptr(), value.as_ptr()); - err::error_on_minusone(py, ret) + fn inner(any: &PyAny, attr_name: Py, value: PyObject) -> PyResult<()> { + err::error_on_minusone(any.py(), unsafe { + ffi::PyObject_SetAttr(any.as_ptr(), attr_name.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, attr_name.into_py(py), value.to_object(py)) } /// Deletes an attribute. @@ -171,13 +247,13 @@ impl PyAny { where N: IntoPy>, { - let py = self.py(); - let attr_name = attr_name.into_py(py); - - unsafe { - let ret = ffi::PyObject_DelAttr(self.as_ptr(), attr_name.as_ptr()); - err::error_on_minusone(py, ret) + fn inner(any: &PyAny, attr_name: Py) -> PyResult<()> { + err::error_on_minusone(any.py(), unsafe { + ffi::PyObject_DelAttr(any.as_ptr(), attr_name.as_ptr()) + }) } + + inner(self, attr_name.into_py(self.py())) } /// Returns an [`Ordering`] between `self` and `other`. @@ -293,13 +369,17 @@ impl PyAny { where O: ToPyObject, { - unsafe { - self.py().from_owned_ptr_or_err(ffi::PyObject_RichCompare( - self.as_ptr(), - other.to_object(self.py()).as_ptr(), - compare_op as c_int, - )) + fn inner(slf: &PyAny, other: PyObject, compare_op: CompareOp) -> PyResult<&PyAny> { + unsafe { + slf.py().from_owned_ptr_or_err(ffi::PyObject_RichCompare( + slf.as_ptr(), + other.as_ptr(), + compare_op as c_int, + )) + } } + + inner(self, other.to_object(self.py()), compare_op) } /// Tests whether this object is less than another. @@ -691,12 +771,14 @@ impl PyAny { where K: ToPyObject, { - unsafe { - self.py().from_owned_ptr_or_err(ffi::PyObject_GetItem( - self.as_ptr(), - key.to_object(self.py()).as_ptr(), - )) + fn inner(slf: &PyAny, key: PyObject) -> PyResult<&PyAny> { + unsafe { + slf.py() + .from_owned_ptr_or_err(ffi::PyObject_GetItem(slf.as_ptr(), key.as_ptr())) + } } + + inner(self, key.to_object(self.py())) } /// Sets a collection item value. @@ -707,17 +789,14 @@ impl PyAny { K: ToPyObject, V: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyObject_SetItem( - self.as_ptr(), - key.to_object(py).as_ptr(), - value.to_object(py).as_ptr(), - ), - ) + fn inner(slf: &PyAny, key: PyObject, value: PyObject) -> PyResult<()> { + err::error_on_minusone(slf.py(), unsafe { + ffi::PyObject_SetItem(slf.as_ptr(), key.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, key.to_object(py), value.to_object(py)) } /// Deletes an item from the collection. @@ -727,12 +806,13 @@ impl PyAny { where K: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyObject_DelItem(self.as_ptr(), key.to_object(self.py()).as_ptr()), - ) + fn inner(slf: &PyAny, key: PyObject) -> PyResult<()> { + err::error_on_minusone(slf.py(), unsafe { + ffi::PyObject_DelItem(slf.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Takes an object and returns an iterator for it. @@ -823,6 +903,44 @@ impl PyAny { ::try_from(self) } + /// Downcast this `PyAny` to a concrete Python type or pyclass (but not a subclass of it). + /// + /// It is almost always better to use [`PyAny::downcast`] because it accounts for Python + /// subtyping. Use this method only when you do not want to allow subtypes. + /// + /// The advantage of this method over [`PyAny::downcast`] is that it is faster. The implementation + /// of `downcast_exact` uses the equivalent of the Python expression `type(self) is T`, whereas + /// `downcast` uses `isinstance(self, T)`. + /// + /// For extracting a Rust-only type, see [`PyAny::extract`](struct.PyAny.html#method.extract). + /// + /// # Example: Downcasting to a specific Python object but not a subtype + /// + /// ```rust + /// use pyo3::prelude::*; + /// use pyo3::types::{PyBool, PyLong}; + /// + /// Python::with_gil(|py| { + /// let b = PyBool::new(py, true); + /// assert!(b.is_instance_of::()); + /// let any: &PyAny = b.as_ref(); + /// + /// // `bool` is a subtype of `int`, so `downcast` will accept a `bool` as an `int` + /// // but `downcast_exact` will not. + /// assert!(any.downcast::().is_ok()); + /// assert!(any.downcast_exact::().is_err()); + /// + /// assert!(any.downcast_exact::().is_ok()); + /// }); + /// ``` + #[inline] + pub fn downcast_exact<'p, T>(&'p self) -> Result<&'p T, PyDowncastError<'_>> + where + T: PyTryFrom<'p>, + { + ::try_from_exact(self) + } + /// Converts this `PyAny` to a concrete Python type without checking validity. /// /// # Safety @@ -876,11 +994,8 @@ impl PyAny { /// This is equivalent to the Python expression `hash(self)`. pub fn hash(&self) -> PyResult { let v = unsafe { ffi::PyObject_Hash(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v) } /// Returns the length of the sequence or mapping. @@ -888,11 +1003,8 @@ impl PyAny { /// This is equivalent to the Python expression `len(self)`. pub fn len(&self) -> PyResult { let v = unsafe { ffi::PyObject_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns the list of attributes of this object. @@ -974,9 +1086,82 @@ impl PyAny { #[cfg(test)] mod tests { use crate::{ - types::{IntoPyDict, PyBool, PyList, PyLong, PyModule}, + types::{IntoPyDict, PyAny, PyBool, PyList, PyLong, PyModule}, Python, ToPyObject, }; + + #[test] + fn test_lookup_special() { + Python::with_gil(|py| { + let module = PyModule::from_code( + py, + r#" +class CustomCallable: + def __call__(self): + return 1 + +class SimpleInt: + def __int__(self): + return 1 + +class InheritedInt(SimpleInt): pass + +class NoInt: pass + +class NoDescriptorInt: + __int__ = CustomCallable() + +class InstanceOverrideInt: + def __int__(self): + return 1 +instance_override = InstanceOverrideInt() +instance_override.__int__ = lambda self: 2 + +class ErrorInDescriptorInt: + @property + def __int__(self): + raise ValueError("uh-oh!") + +class NonHeapNonDescriptorInt: + # A static-typed callable that doesn't implement `__get__`. These are pretty hard to come by. + __int__ = int + "#, + "test.py", + "test", + ) + .unwrap(); + + let int = crate::intern!(py, "__int__"); + let eval_int = + |obj: &PyAny| obj.lookup_special(int)?.unwrap().call0()?.extract::(); + + let simple = module.getattr("SimpleInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(simple).unwrap(), 1); + let inherited = module.getattr("InheritedInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(inherited).unwrap(), 1); + let no_descriptor = module.getattr("NoDescriptorInt").unwrap().call0().unwrap(); + assert_eq!(eval_int(no_descriptor).unwrap(), 1); + let missing = module.getattr("NoInt").unwrap().call0().unwrap(); + assert!(missing.lookup_special(int).unwrap().is_none()); + // Note the instance override should _not_ call the instance method that returns 2, + // because that's not how special lookups are meant to work. + let instance_override = module.getattr("instance_override").unwrap(); + assert_eq!(eval_int(instance_override).unwrap(), 1); + let descriptor_error = module + .getattr("ErrorInDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert!(descriptor_error.lookup_special(int).is_err()); + let nonheap_nondescriptor = module + .getattr("NonHeapNonDescriptorInt") + .unwrap() + .call0() + .unwrap(); + assert_eq!(eval_int(nonheap_nondescriptor).unwrap(), 0); + }) + } + #[test] fn test_call_for_non_existing_method() { Python::with_gil(|py| { @@ -1051,6 +1236,44 @@ class SimpleClass: }); } + #[test] + fn test_hasattr() { + Python::with_gil(|py| { + let x = 5.to_object(py).into_ref(py); + assert!(x.is_instance_of::()); + + assert!(x.hasattr("to_bytes").unwrap()); + assert!(!x.hasattr("bbbbbbytes").unwrap()); + }) + } + + #[cfg(feature = "macros")] + #[test] + fn test_hasattr_error() { + use crate::exceptions::PyValueError; + use crate::prelude::*; + + #[pyclass(crate = "crate")] + struct GetattrFail; + + #[pymethods(crate = "crate")] + impl GetattrFail { + fn __getattr__(&self, attr: PyObject) -> PyResult { + Err(PyValueError::new_err(attr)) + } + } + + Python::with_gil(|py| { + let obj = Py::new(py, GetattrFail).unwrap(); + let obj = obj.as_ref(py).as_ref(); + + assert!(obj + .hasattr("foo") + .unwrap_err() + .is_instance_of::(py)); + }) + } + #[test] fn test_nan_eq() { Python::with_gil(|py| { @@ -1226,7 +1449,10 @@ class SimpleClass: #[test] fn test_is_ellipsis() { Python::with_gil(|py| { - let v = py.eval("...", None, None).map_err(|e| e.print(py)).unwrap(); + let v = py + .eval("...", None, None) + .map_err(|e| e.display(py)) + .unwrap(); assert!(v.is_ellipsis()); diff --git a/src/types/boolobject.rs b/src/types/boolobject.rs index 80c453a6522..919810d2ddc 100644 --- a/src/types/boolobject.rs +++ b/src/types/boolobject.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ diff --git a/src/types/bytearray.rs b/src/types/bytearray.rs index 927c450085c..c11af6d71f8 100644 --- a/src/types/bytearray.rs +++ b/src/types/bytearray.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{PyErr, PyResult}; use crate::{ffi, AsPyPointer, Py, PyAny, Python}; use std::os::raw::c_char; diff --git a/src/types/capsule.rs b/src/types/capsule.rs index 7da59e735e3..152ac0ffc61 100644 --- a/src/types/capsule.rs +++ b/src/types/capsule.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::Python; use crate::{ffi, AsPyPointer, PyAny}; use crate::{pyobject_native_type_core, PyErr, PyResult}; diff --git a/src/types/code.rs b/src/types/code.rs index fc7e3e9f83e..c0d0ce83926 100644 --- a/src/types/code.rs +++ b/src/types/code.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/datetime.rs b/src/types/datetime.rs index 2c995356e2c..8d5aae1f977 100644 --- a/src/types/datetime.rs +++ b/src/types/datetime.rs @@ -257,7 +257,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.DateTimeType, ); py.from_owned_ptr_or_err(ptr) @@ -294,7 +294,7 @@ impl PyDateTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), c_int::from(fold), api.DateTimeType, ); @@ -399,7 +399,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), api.TimeType, ); py.from_owned_ptr_or_err(ptr) @@ -423,7 +423,7 @@ impl PyTime { c_int::from(minute), c_int::from(second), microsecond as c_int, - opt_to_pyobj(py, tzinfo), + opt_to_pyobj(tzinfo), fold as c_int, api.TimeType, ); @@ -535,12 +535,12 @@ impl PyDeltaAccess for PyDelta { } } -// Utility function -fn opt_to_pyobj(py: Python<'_>, opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { - // Convenience function for unpacking Options to either an Object or None +// Utility function which returns a borrowed reference to either +// the underlying tzinfo or None. +fn opt_to_pyobj(opt: Option<&PyTzInfo>) -> *mut ffi::PyObject { match opt { Some(tzi) => tzi.as_ptr(), - None => py.None().as_ptr(), + None => unsafe { ffi::Py_None() }, } } diff --git a/src/types/dict.rs b/src/types/dict.rs index 0cf7e6ca7b6..80d187c175c 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -1,13 +1,10 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use super::PyMapping; use crate::err::{self, PyErr, PyResult}; use crate::ffi::Py_ssize_t; use crate::types::{PyAny, PyList}; -use crate::{ffi, AsPyPointer, Python, ToPyObject}; #[cfg(not(PyPy))] -use crate::{IntoPyPointer, PyObject}; -use std::ptr::NonNull; +use crate::IntoPyPointer; +use crate::{ffi, AsPyPointer, PyObject, Python, ToPyObject}; /// Represents a Python `dict`. #[repr(transparent)] @@ -71,14 +68,11 @@ impl PyDict { /// this keeps the last entry seen. #[cfg(not(PyPy))] pub fn from_sequence(py: Python<'_>, seq: PyObject) -> PyResult<&PyDict> { - unsafe { - let dict = py.from_owned_ptr::(ffi::PyDict_New()); - err::error_on_minusone( - py, - ffi::PyDict_MergeFromSeq2(dict.into_ptr(), seq.into_ptr(), 1), - )?; - Ok(dict) - } + let dict = Self::new(py); + err::error_on_minusone(py, unsafe { + ffi::PyDict_MergeFromSeq2(dict.into_ptr(), seq.into_ptr(), 1) + })?; + Ok(dict) } /// Returns a new dictionary that contains the same key-value pairs as self. @@ -127,13 +121,15 @@ impl PyDict { where K: ToPyObject, { - unsafe { - match ffi::PyDict_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(dict: &PyDict, key: PyObject) -> PyResult { + match unsafe { ffi::PyDict_Contains(dict.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(dict.py())), } } + + inner(self, key.to_object(self.py())) } /// Gets an item from the dictionary. @@ -145,13 +141,20 @@ impl PyDict { where K: ToPyObject, { - unsafe { - let ptr = ffi::PyDict_GetItem(self.as_ptr(), key.to_object(self.py()).as_ptr()); - NonNull::new(ptr).map(|p| { - // PyDict_GetItem return s borrowed ptr, must make it owned for safety (see #890). - self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())) - }) + fn inner(dict: &PyDict, key: PyObject) -> Option<&PyAny> { + let py = dict.py(); + // PyDict_GetItem returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + unsafe { + PyObject::from_borrowed_ptr_or_opt( + py, + ffi::PyDict_GetItem(dict.as_ptr(), key.as_ptr()), + ) + } + .map(|pyobject| pyobject.into_ref(py)) } + + inner(self, key.to_object(self.py())) } /// Gets an item from the dictionary, @@ -159,20 +162,26 @@ impl PyDict { /// returns `Ok(None)` if item is not present, or `Err(PyErr)` if an error occurs. /// /// To get a `KeyError` for non-existing keys, use `PyAny::get_item_with_error`. - #[cfg(not(PyPy))] pub fn get_item_with_error(&self, key: K) -> PyResult> where K: ToPyObject, { - unsafe { - let ptr = - ffi::PyDict_GetItemWithError(self.as_ptr(), key.to_object(self.py()).as_ptr()); - if !ffi::PyErr_Occurred().is_null() { - return Err(PyErr::fetch(self.py())); + fn inner(dict: &PyDict, key: PyObject) -> PyResult> { + let py = dict.py(); + // PyDict_GetItemWithError returns a borrowed ptr, must make it owned for safety (see #890). + // PyObject::from_borrowed_ptr_or_opt will take ownership in this way. + unsafe { + PyObject::from_borrowed_ptr_or_opt( + py, + ffi::PyDict_GetItemWithError(dict.as_ptr(), key.as_ptr()), + ) } - - Ok(NonNull::new(ptr).map(|p| self.py().from_owned_ptr(ffi::_Py_NewRef(p.as_ptr())))) + .map(|pyobject| Ok(pyobject.into_ref(py))) + .or_else(|| PyErr::take(py).map(Err)) + .transpose() } + + inner(self, key.to_object(self.py())) } /// Sets an item value. @@ -183,17 +192,14 @@ impl PyDict { K: ToPyObject, V: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyDict_SetItem( - self.as_ptr(), - key.to_object(py).as_ptr(), - value.to_object(py).as_ptr(), - ), - ) + fn inner(dict: &PyDict, key: PyObject, value: PyObject) -> PyResult<()> { + err::error_on_minusone(dict.py(), unsafe { + ffi::PyDict_SetItem(dict.as_ptr(), key.as_ptr(), value.as_ptr()) + }) } + + let py = self.py(); + inner(self, key.to_object(py), value.to_object(py)) } /// Deletes an item. @@ -203,13 +209,13 @@ impl PyDict { where K: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyDict_DelItem(self.as_ptr(), key.to_object(py).as_ptr()), - ) + fn inner(dict: &PyDict, key: PyObject) -> PyResult<()> { + err::error_on_minusone(dict.py(), unsafe { + ffi::PyDict_DelItem(dict.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Returns a list of dict keys. @@ -264,7 +270,9 @@ impl PyDict { /// to use `self.update(other.as_mapping())`, note: `PyDict::as_mapping` is a zero-cost conversion. pub fn update(&self, other: &PyMapping) -> PyResult<()> { let py = self.py(); - unsafe { err::error_on_minusone(py, ffi::PyDict_Update(self.as_ptr(), other.as_ptr())) } + err::error_on_minusone(py, unsafe { + ffi::PyDict_Update(self.as_ptr(), other.as_ptr()) + }) } /// Add key/value pairs from another dictionary to this one only when they do not exist in this. @@ -277,7 +285,9 @@ impl PyDict { /// so should have the same performance as `update`. pub fn update_if_missing(&self, other: &PyMapping) -> PyResult<()> { let py = self.py(); - unsafe { err::error_on_minusone(py, ffi::PyDict_Merge(self.as_ptr(), other.as_ptr(), 0)) } + err::error_on_minusone(py, unsafe { + ffi::PyDict_Merge(self.as_ptr(), other.as_ptr(), 0) + }) } } @@ -581,14 +591,14 @@ mod tests { fn test_set_item_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; - let none = py.None(); - cnt = none.get_refcnt(py); - let _dict = [(10, none)].into_py_dict(py); + cnt = obj.get_refcnt(); + let _dict = [(10, obj)].into_py_dict(py); } { - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); } }); } @@ -644,7 +654,7 @@ mod tests { // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; let mut value_sum = 0; - for el in dict.items().iter() { + for el in dict.items() { let tuple = el.downcast::().unwrap(); key_sum += tuple.get_item(0).unwrap().extract::().unwrap(); value_sum += tuple.get_item(1).unwrap().extract::().unwrap(); @@ -665,7 +675,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut key_sum = 0; - for el in dict.keys().iter() { + for el in dict.keys() { key_sum += el.extract::().unwrap(); } assert_eq!(7 + 8 + 9, key_sum); @@ -683,7 +693,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); // Can't just compare against a vector of tuples since we don't have a guaranteed ordering. let mut values_sum = 0; - for el in dict.values().iter() { + for el in dict.values() { values_sum += el.extract::().unwrap(); } assert_eq!(32 + 42 + 123, values_sum); @@ -701,7 +711,7 @@ mod tests { let dict: &PyDict = ob.downcast(py).unwrap(); let mut key_sum = 0; let mut value_sum = 0; - for (key, value) in dict.iter() { + for (key, value) in dict { key_sum += key.extract::().unwrap(); value_sum += value.extract::().unwrap(); } @@ -721,7 +731,7 @@ mod tests { let ob = v.to_object(py); let dict: &PyDict = ob.downcast(py).unwrap(); - for (key, value) in dict.iter() { + for (key, value) in dict { dict.set_item(key, value.extract::().unwrap() + 7) .unwrap(); } diff --git a/src/types/floatob.rs b/src/types/floatob.rs index ccf73dd8cde..f56bcb1f9fd 100644 --- a/src/types/floatob.rs +++ b/src/types/floatob.rs @@ -1,10 +1,8 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython #[cfg(feature = "experimental-inspect")] use crate::inspect::types::TypeInfo; use crate::{ - ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyObject, PyResult, Python, ToPyObject, + ffi, AsPyPointer, FromPyObject, IntoPy, PyAny, PyErr, PyNativeType, PyObject, PyResult, Python, + ToPyObject, }; use std::os::raw::c_double; @@ -32,7 +30,16 @@ impl PyFloat { /// Gets the value of this float. pub fn value(&self) -> c_double { - unsafe { ffi::PyFloat_AsDouble(self.as_ptr()) } + #[cfg(not(Py_LIMITED_API))] + unsafe { + // Safety: self is PyFloat object + ffi::PyFloat_AS_DOUBLE(self.as_ptr()) + } + + #[cfg(Py_LIMITED_API)] + unsafe { + ffi::PyFloat_AsDouble(self.as_ptr()) + } } } @@ -57,6 +64,15 @@ impl<'source> FromPyObject<'source> for f64 { // PyFloat_AsDouble returns -1.0 upon failure #![allow(clippy::float_cmp)] fn extract(obj: &'source PyAny) -> PyResult { + // On non-limited API, .value() uses PyFloat_AS_DOUBLE which + // allows us to have an optimized fast path for the case when + // we have exactly a `float` object (it's not worth going through + // `isinstance` machinery for subclasses). + #[cfg(not(Py_LIMITED_API))] + if let Ok(float) = obj.downcast_exact::() { + return Ok(float.value()); + } + let v = unsafe { ffi::PyFloat_AsDouble(obj.as_ptr()) }; if v == -1.0 { @@ -104,11 +120,7 @@ impl<'source> FromPyObject<'source> for f32 { #[cfg(test)] mod tests { - #[cfg(not(Py_LIMITED_API))] - use crate::ffi::PyFloat_AS_DOUBLE; - #[cfg(not(Py_LIMITED_API))] - use crate::AsPyPointer; - use crate::{Python, ToPyObject}; + use crate::{types::PyFloat, Python, ToPyObject}; macro_rules! num_to_py_object_and_back ( ($func_name:ident, $t1:ty, $t2:ty) => ( @@ -130,15 +142,14 @@ mod tests { num_to_py_object_and_back!(to_from_f32, f32, f32); num_to_py_object_and_back!(int_to_float, i32, f64); - #[cfg(not(Py_LIMITED_API))] #[test] - fn test_as_double_macro() { + fn test_float_value() { use assert_approx_eq::assert_approx_eq; Python::with_gil(|py| { let v = 1.23f64; - let obj = v.to_object(py); - assert_approx_eq!(v, unsafe { PyFloat_AS_DOUBLE(obj.as_ptr()) }); + let obj = PyFloat::new(py, 1.23); + assert_approx_eq!(v, obj.value()); }); } } diff --git a/src/types/frame.rs b/src/types/frame.rs index c16e143987d..160f3b3b6ed 100644 --- a/src/types/frame.rs +++ b/src/types/frame.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2022-present PyO3 Project and Contributors - use crate::ffi; use crate::PyAny; diff --git a/src/types/frozenset.rs b/src/types/frozenset.rs index 5b728784a30..ff590495ba4 100644 --- a/src/types/frozenset.rs +++ b/src/types/frozenset.rs @@ -1,16 +1,48 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, PyObject, + Py, PyObject, }; use crate::{ffi, AsPyPointer, PyAny, Python, ToPyObject}; use std::ptr; +/// Allows building a Python `frozenset` one item at a time +pub struct PyFrozenSetBuilder<'py> { + py_frozen_set: &'py PyFrozenSet, +} + +impl<'py> PyFrozenSetBuilder<'py> { + /// Create a new `FrozenSetBuilder`. + /// Since this allocates a `PyFrozenSet` internally it may + /// panic when running out of memory. + pub fn new(py: Python<'py>) -> PyResult> { + Ok(PyFrozenSetBuilder { + py_frozen_set: PyFrozenSet::empty(py)?, + }) + } + + /// Adds an element to the set. + pub fn add(&mut self, key: K) -> PyResult<()> + where + K: ToPyObject, + { + fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult<()> { + err::error_on_minusone(frozenset.py(), unsafe { + ffi::PySet_Add(frozenset.as_ptr(), key.as_ptr()) + }) + } + + inner(self.py_frozen_set, key.to_object(self.py_frozen_set.py())) + } + + /// Finish building the set and take ownership of its current value + pub fn finalize(self) -> &'py PyFrozenSet { + self.py_frozen_set + } +} + /// Represents a Python `frozenset` #[repr(transparent)] pub struct PyFrozenSet(PyAny); @@ -65,13 +97,15 @@ impl PyFrozenSet { where K: ToPyObject, { - unsafe { - match ffi::PySet_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(frozenset: &PyFrozenSet, key: PyObject) -> PyResult { + match unsafe { ffi::PySet_Contains(frozenset.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(frozenset.py())), } } + + inner(self, key.to_object(self.py())) } /// Returns an iterator of values in this frozen set. @@ -168,7 +202,7 @@ pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, ) -> PyResult> { - fn new_from_iter_inner( + fn inner( py: Python<'_>, elements: &mut dyn Iterator, ) -> PyResult> { @@ -179,16 +213,14 @@ pub(crate) fn new_from_iter( let ptr = set.as_ptr(); for obj in elements { - unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; - } + err::error_on_minusone(py, unsafe { ffi::PySet_Add(ptr, obj.as_ptr()) })?; } Ok(set) } let mut iter = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter_inner(py, &mut iter) + inner(py, &mut iter) } #[cfg(test)] @@ -228,7 +260,7 @@ mod tests { let set = PyFrozenSet::new(py, &[1]).unwrap(); // iter method - for el in set.iter() { + for el in set { assert_eq!(1i32, el.extract::().unwrap()); } @@ -238,4 +270,25 @@ mod tests { } }); } + + #[test] + fn test_frozenset_builder() { + use super::PyFrozenSetBuilder; + + Python::with_gil(|py| { + let mut builder = PyFrozenSetBuilder::new(py).unwrap(); + + // add an item + builder.add(1).unwrap(); + builder.add(2).unwrap(); + builder.add(2).unwrap(); + + // finalize it + let set = builder.finalize(); + + assert!(set.contains(1).unwrap()); + assert!(set.contains(2).unwrap()); + assert!(!set.contains(3).unwrap()); + }); + } } diff --git a/src/types/iterator.rs b/src/types/iterator.rs index cfbbd31b6dc..7b411bde765 100644 --- a/src/types/iterator.rs +++ b/src/types/iterator.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, AsPyPointer, IntoPyPointer, Py, PyAny, PyErr, PyNativeType, PyResult, Python}; use crate::{PyDowncastError, PyTryFrom}; @@ -59,6 +55,12 @@ impl<'p> Iterator for &'p PyIterator { None => PyErr::take(py).map(Err), } } + + #[cfg(not(Py_LIMITED_API))] + fn size_hint(&self) -> (usize, Option) { + let hint = unsafe { ffi::PyObject_LengthHint(self.0.as_ptr(), 0) }; + (hint.max(0) as usize, None) + } } // PyIter_Check does not exist in the limited API until 3.8 @@ -153,31 +155,30 @@ mod tests { #[test] fn iter_item_refcnt() { Python::with_gil(|py| { - let obj; - let none; let count; - { + let obj = py.eval("object()", None, None).unwrap(); + let list = { let _pool = unsafe { GILPool::new() }; - let l = PyList::empty(py); - none = py.None(); - l.append(10).unwrap(); - l.append(&none).unwrap(); - count = none.get_refcnt(py); - obj = l.to_object(py); - } + let list = PyList::empty(py); + list.append(10).unwrap(); + list.append(obj).unwrap(); + count = obj.get_refcnt(); + list.to_object(py) + }; { let _pool = unsafe { GILPool::new() }; - let inst = obj.as_ref(py); + let inst = list.as_ref(py); let mut it = inst.iter().unwrap(); assert_eq!( 10_i32, it.next().unwrap().unwrap().extract::<'_, i32>().unwrap() ); - assert!(it.next().unwrap().unwrap().is_none()); + assert!(it.next().unwrap().unwrap().is(obj)); + assert!(it.next().is_none()); } - assert_eq!(count, none.get_refcnt(py)); + assert_eq!(count, obj.get_refcnt()); }); } @@ -317,4 +318,15 @@ def fibonacci(target): ); }); } + + #[test] + #[cfg(not(Py_LIMITED_API))] + fn length_hint_becomes_size_hint_lower_bound() { + Python::with_gil(|py| { + let list = py.eval("[1, 2, 3]", None, None).unwrap(); + let iter = list.iter().unwrap(); + let hint = iter.size_hint(); + assert_eq!(hint, (3, None)); + }); + } } diff --git a/src/types/list.rs b/src/types/list.rs index 3e801276b95..91de4eb418b 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use std::convert::TryInto; use crate::err::{self, PyResult}; @@ -171,16 +167,13 @@ impl PyList { where I: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyList_SetItem( - self.as_ptr(), - get_ssize_index(index), - item.to_object(self.py()).into_ptr(), - ), - ) + fn inner(list: &PyList, index: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_SetItem(list.as_ptr(), get_ssize_index(index), item.into_ptr()) + }) } + + inner(self, index, item.to_object(self.py())) } /// Deletes the `index`th element of self. @@ -196,17 +189,14 @@ impl PyList { /// This is equivalent to the Python statement `self[low:high] = v`. #[inline] pub fn set_slice(&self, low: usize, high: usize, seq: &PyAny) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PyList_SetSlice( - self.as_ptr(), - get_ssize_index(low), - get_ssize_index(high), - seq.as_ptr(), - ), + err::error_on_minusone(self.py(), unsafe { + ffi::PyList_SetSlice( + self.as_ptr(), + get_ssize_index(low), + get_ssize_index(high), + seq.as_ptr(), ) - } + }) } /// Deletes the slice from `low` to `high` from `self`. @@ -222,13 +212,13 @@ impl PyList { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyList_Append(self.as_ptr(), item.to_object(py).as_ptr()), - ) + fn inner(list: &PyList, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_Append(list.as_ptr(), item.as_ptr()) + }) } + + inner(self, item.to_object(self.py())) } /// Inserts an item at the specified index. @@ -238,17 +228,13 @@ impl PyList { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PyList_Insert( - self.as_ptr(), - get_ssize_index(index), - item.to_object(py).as_ptr(), - ), - ) + fn inner(list: &PyList, index: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(list.py(), unsafe { + ffi::PyList_Insert(list.as_ptr(), get_ssize_index(index), item.as_ptr()) + }) } + + inner(self, index, item.to_object(self.py())) } /// Determines if self contains `value`. @@ -283,12 +269,12 @@ impl PyList { /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. pub fn sort(&self) -> PyResult<()> { - unsafe { err::error_on_minusone(self.py(), ffi::PyList_Sort(self.as_ptr())) } + err::error_on_minusone(self.py(), unsafe { ffi::PyList_Sort(self.as_ptr()) }) } /// Reverses the list in-place. Equivalent to the Python expression `l.reverse()`. pub fn reverse(&self) -> PyResult<()> { - unsafe { err::error_on_minusone(self.py(), ffi::PyList_Reverse(self.as_ptr())) } + err::error_on_minusone(self.py(), unsafe { ffi::PyList_Reverse(self.as_ptr()) }) } /// Return a new tuple containing the contents of the list; equivalent to the Python expression `tuple(list)`. @@ -409,18 +395,18 @@ mod tests { #[test] fn test_set_item_refcnt() { Python::with_gil(|py| { + let obj = py.eval("object()", None, None).unwrap(); let cnt; { let _pool = unsafe { crate::GILPool::new() }; let v = vec![2]; let ob = v.to_object(py); let list: &PyList = ob.downcast(py).unwrap(); - let none = py.None(); - cnt = none.get_refcnt(py); - list.set_item(0, none).unwrap(); + cnt = obj.get_refcnt(); + list.set_item(0, obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } @@ -445,15 +431,15 @@ mod tests { fn test_insert_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; let list = PyList::empty(py); - let none = py.None(); - cnt = none.get_refcnt(py); - list.insert(0, none).unwrap(); + cnt = obj.get_refcnt(); + list.insert(0, obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } @@ -471,14 +457,14 @@ mod tests { fn test_append_refcnt() { Python::with_gil(|py| { let cnt; + let obj = py.eval("object()", None, None).unwrap(); { let _pool = unsafe { crate::GILPool::new() }; let list = PyList::empty(py); - let none = py.None(); - cnt = none.get_refcnt(py); - list.append(none).unwrap(); + cnt = obj.get_refcnt(); + list.append(obj).unwrap(); } - assert_eq!(cnt, py.None().get_refcnt(py)); + assert_eq!(cnt, obj.get_refcnt()); }); } @@ -488,7 +474,7 @@ mod tests { let v = vec![2, 3, 5, 7]; let list = PyList::new(py, &v); let mut idx = 0; - for el in list.iter() { + for el in list { assert_eq!(v[idx], el.extract::().unwrap()); idx += 1; } diff --git a/src/types/mapping.rs b/src/types/mapping.rs index 602302a953b..b947f619a22 100644 --- a/src/types/mapping.rs +++ b/src/types/mapping.rs @@ -1,6 +1,4 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - -use crate::err::{PyDowncastError, PyErr, PyResult}; +use crate::err::{PyDowncastError, PyResult}; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::{PyAny, PyDict, PySequence, PyType}; @@ -19,11 +17,8 @@ impl PyMapping { #[inline] pub fn len(&self) -> PyResult { let v = unsafe { ffi::PyMapping_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns whether the mapping is empty. diff --git a/src/types/mod.rs b/src/types/mod.rs index 385d0af68dc..06a677ab62d 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - //! Various types defined by the Python interpreter such as `int`, `str` and `tuple`. pub use self::any::PyAny; @@ -21,7 +19,7 @@ pub use self::dict::{PyDictItems, PyDictKeys, PyDictValues}; pub use self::floatob::PyFloat; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::frame::PyFrame; -pub use self::frozenset::PyFrozenSet; +pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder}; pub use self::function::PyCFunction; #[cfg(all(not(Py_LIMITED_API), not(PyPy)))] pub use self::function::PyFunction; diff --git a/src/types/module.rs b/src/types/module.rs index 970a22ff0ce..b6b25eacb95 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::callback::IntoPyCallbackOutput; use crate::err::{PyErr, PyResult}; use crate::exceptions; @@ -102,7 +98,7 @@ impl PyModule { /// let code = include_str!("../../assets/script.py"); /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, code, "example", "example")?; + /// PyModule::from_code(py, code, "example.py", "example")?; /// Ok(()) /// })?; /// # Ok(()) @@ -121,7 +117,7 @@ impl PyModule { /// let code = std::fs::read_to_string("assets/script.py")?; /// /// Python::with_gil(|py| -> PyResult<()> { - /// PyModule::from_code(py, &code, "example", "example")?; + /// PyModule::from_code(py, &code, "example.py", "example")?; /// Ok(()) /// })?; /// Ok(()) diff --git a/src/types/num.rs b/src/types/num.rs index 21f6d72d3cd..522517155f8 100644 --- a/src/types/num.rs +++ b/src/types/num.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::{ffi, PyAny}; /// Represents a Python `int` object. diff --git a/src/types/sequence.rs b/src/types/sequence.rs index ccc2d895b72..04570459bbc 100644 --- a/src/types/sequence.rs +++ b/src/types/sequence.rs @@ -1,4 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors use crate::err::{self, PyDowncastError, PyErr, PyResult}; use crate::exceptions::PyTypeError; #[cfg(feature = "experimental-inspect")] @@ -7,7 +6,7 @@ use crate::internal_tricks::get_ssize_index; use crate::sync::GILOnceCell; use crate::type_object::PyTypeInfo; use crate::types::{PyAny, PyList, PyString, PyTuple, PyType}; -use crate::{ffi, PyNativeType, ToPyObject}; +use crate::{ffi, PyNativeType, PyObject, ToPyObject}; use crate::{AsPyPointer, IntoPyPointer, Py, Python}; use crate::{FromPyObject, PyTryFrom}; @@ -24,11 +23,8 @@ impl PySequence { #[inline] pub fn len(&self) -> PyResult { let v = unsafe { ffi::PySequence_Size(self.as_ptr()) }; - if v == -1 { - Err(PyErr::fetch(self.py())) - } else { - Ok(v as usize) - } + crate::err::error_on_minusone(self.py(), v)?; + Ok(v as usize) } /// Returns whether the sequence is empty. @@ -129,17 +125,13 @@ impl PySequence { where I: ToPyObject, { - let py = self.py(); - unsafe { - err::error_on_minusone( - py, - ffi::PySequence_SetItem( - self.as_ptr(), - get_ssize_index(i), - item.to_object(py).as_ptr(), - ), - ) + fn inner(seq: &PySequence, i: usize, item: PyObject) -> PyResult<()> { + err::error_on_minusone(seq.py(), unsafe { + ffi::PySequence_SetItem(seq.as_ptr(), get_ssize_index(i), item.as_ptr()) + }) } + + inner(self, i, item.to_object(self.py())) } /// Deletes the `i`th element of self. @@ -147,12 +139,9 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i]`. #[inline] pub fn del_item(&self, i: usize) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_DelItem(self.as_ptr(), get_ssize_index(i)), - ) - } + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_DelItem(self.as_ptr(), get_ssize_index(i)) + }) } /// Assigns the sequence `v` to the slice of `self` from `i1` to `i2`. @@ -160,17 +149,14 @@ impl PySequence { /// This is equivalent to the Python statement `self[i1:i2] = v`. #[inline] pub fn set_slice(&self, i1: usize, i2: usize, v: &PyAny) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_SetSlice( - self.as_ptr(), - get_ssize_index(i1), - get_ssize_index(i2), - v.as_ptr(), - ), + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_SetSlice( + self.as_ptr(), + get_ssize_index(i1), + get_ssize_index(i2), + v.as_ptr(), ) - } + }) } /// Deletes the slice from `i1` to `i2` from `self`. @@ -178,12 +164,9 @@ impl PySequence { /// This is equivalent to the Python statement `del self[i1:i2]`. #[inline] pub fn del_slice(&self, i1: usize, i2: usize) -> PyResult<()> { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySequence_DelSlice(self.as_ptr(), get_ssize_index(i1), get_ssize_index(i2)), - ) - } + err::error_on_minusone(self.py(), unsafe { + ffi::PySequence_DelSlice(self.as_ptr(), get_ssize_index(i1), get_ssize_index(i2)) + }) } /// Returns the number of occurrences of `value` in self, that is, return the @@ -194,13 +177,13 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Count(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(self.py())) - } else { + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Count(seq.as_ptr(), value.as_ptr()) }; + crate::err::error_on_minusone(seq.py(), r)?; Ok(r as usize) } + + inner(self, value.to_object(self.py())) } /// Determines if self contains `value`. @@ -211,13 +194,16 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Contains(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - match r { - 0 => Ok(false), - 1 => Ok(true), - _ => Err(PyErr::fetch(self.py())), + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Contains(seq.as_ptr(), value.as_ptr()) }; + match r { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(PyErr::fetch(seq.py())), + } } + + inner(self, value.to_object(self.py())) } /// Returns the first index `i` for which `self[i] == value`. @@ -228,13 +214,13 @@ impl PySequence { where V: ToPyObject, { - let r = - unsafe { ffi::PySequence_Index(self.as_ptr(), value.to_object(self.py()).as_ptr()) }; - if r == -1 { - Err(PyErr::fetch(self.py())) - } else { + fn inner(seq: &PySequence, value: PyObject) -> PyResult { + let r = unsafe { ffi::PySequence_Index(seq.as_ptr(), value.as_ptr()) }; + crate::err::error_on_minusone(seq.py(), r)?; Ok(r as usize) } + + inner(self, value.to_object(self.py())) } /// Returns a fresh list based on the Sequence. @@ -787,7 +773,7 @@ mod tests { let seq = ob.downcast::(py).unwrap(); let repeat_seq = seq.repeat(3).unwrap(); assert_eq!(6, repeat_seq.len().unwrap()); - let repeated = vec!["foo", "bar", "foo", "bar", "foo", "bar"]; + let repeated = ["foo", "bar", "foo", "bar", "foo", "bar"]; for (el, rpt) in repeat_seq.iter().unwrap().zip(repeated.iter()) { assert_eq!(*rpt, el.unwrap().extract::().unwrap()); } diff --git a/src/types/set.rs b/src/types/set.rs index 36774b9a014..d4a64d7b581 100644 --- a/src/types/set.rs +++ b/src/types/set.rs @@ -1,11 +1,8 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// - #[cfg(Py_LIMITED_API)] use crate::types::PyIterator; use crate::{ err::{self, PyErr, PyResult}, - IntoPyPointer, Py, + Py, }; use crate::{ffi, AsPyPointer, PyAny, PyObject, Python, ToPyObject}; use std::ptr; @@ -74,13 +71,15 @@ impl PySet { where K: ToPyObject, { - unsafe { - match ffi::PySet_Contains(self.as_ptr(), key.to_object(self.py()).as_ptr()) { + fn inner(set: &PySet, key: PyObject) -> PyResult { + match unsafe { ffi::PySet_Contains(set.as_ptr(), key.as_ptr()) } { 1 => Ok(true), 0 => Ok(false), - _ => Err(PyErr::fetch(self.py())), + _ => Err(PyErr::fetch(set.py())), } } + + inner(self, key.to_object(self.py())) } /// Removes the element from the set if it is present. @@ -98,12 +97,13 @@ impl PySet { where K: ToPyObject, { - unsafe { - err::error_on_minusone( - self.py(), - ffi::PySet_Add(self.as_ptr(), key.to_object(self.py()).as_ptr()), - ) + fn inner(set: &PySet, key: PyObject) -> PyResult<()> { + err::error_on_minusone(set.py(), unsafe { + ffi::PySet_Add(set.as_ptr(), key.as_ptr()) + }) } + + inner(self, key.to_object(self.py())) } /// Removes and returns an arbitrary element from the set. @@ -241,10 +241,7 @@ pub(crate) fn new_from_iter( py: Python<'_>, elements: impl IntoIterator, ) -> PyResult> { - fn new_from_iter_inner( - py: Python<'_>, - elements: &mut dyn Iterator, - ) -> PyResult> { + fn inner(py: Python<'_>, elements: &mut dyn Iterator) -> PyResult> { let set: Py = unsafe { // We create the `Py` pointer because its Drop cleans up the set if user code panics. Py::from_owned_ptr_or_err(py, ffi::PySet_New(std::ptr::null_mut()))? @@ -252,16 +249,14 @@ pub(crate) fn new_from_iter( let ptr = set.as_ptr(); for obj in elements { - unsafe { - err::error_on_minusone(py, ffi::PySet_Add(ptr, obj.into_ptr()))?; - } + err::error_on_minusone(py, unsafe { ffi::PySet_Add(ptr, obj.as_ptr()) })?; } Ok(set) } let mut iter = elements.into_iter().map(|e| e.to_object(py)); - new_from_iter_inner(py, &mut iter) + inner(py, &mut iter) } #[cfg(test)] @@ -361,7 +356,7 @@ mod tests { let set = PySet::new(py, &[1]).unwrap(); // iter method - for el in set.iter() { + for el in set { assert_eq!(1i32, el.extract::<'_, i32>().unwrap()); } diff --git a/src/types/slice.rs b/src/types/slice.rs index 61574f6490e..fdf9f7a856b 100644 --- a/src/types/slice.rs +++ b/src/types/slice.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{PyErr, PyResult}; use crate::ffi::{self, Py_ssize_t}; use crate::{AsPyPointer, PyAny, PyObject, Python, ToPyObject}; @@ -56,6 +54,14 @@ impl PySlice { } } + /// Constructs a new full slice that is equivalent to `::`. + pub fn full(py: Python<'_>) -> &PySlice { + unsafe { + let ptr = ffi::PySlice_New(ffi::Py_None(), ffi::Py_None(), ffi::Py_None()); + py.from_owned_ptr(ptr) + } + } + /// Retrieves the start, stop, and step indices from the slice object, /// assuming a sequence of length `length`, and stores the length of the /// slice in its `slicelength` member. @@ -118,6 +124,34 @@ mod tests { }); } + #[test] + fn test_py_slice_full() { + Python::with_gil(|py| { + let slice = PySlice::full(py); + assert!(slice.getattr("start").unwrap().is_none(),); + assert!(slice.getattr("stop").unwrap().is_none(),); + assert!(slice.getattr("step").unwrap().is_none(),); + assert_eq!( + slice.indices(0).unwrap(), + PySliceIndices { + start: 0, + stop: 0, + step: 1, + slicelength: 0, + }, + ); + assert_eq!( + slice.indices(42).unwrap(), + PySliceIndices { + start: 0, + stop: 42, + step: 1, + slicelength: 42, + }, + ); + }); + } + #[test] fn test_py_slice_indices_new() { let start = 0; diff --git a/src/types/string.rs b/src/types/string.rs index 3998a6adf04..80b0665b402 100644 --- a/src/types/string.rs +++ b/src/types/string.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - #[cfg(not(Py_LIMITED_API))] use crate::exceptions::PyUnicodeDecodeError; use crate::types::PyBytes; @@ -298,7 +296,7 @@ mod tests { #[test] fn test_to_str_surrogate() { Python::with_gil(|py| { - let obj: PyObject = py.eval(r#"'\ud800'"#, None, None).unwrap().into(); + let obj: PyObject = py.eval(r"'\ud800'", None, None).unwrap().into(); let py_string: &PyString = obj.downcast(py).unwrap(); assert!(py_string.to_str().is_err()); }) @@ -318,7 +316,7 @@ mod tests { fn test_to_string_lossy() { Python::with_gil(|py| { let obj: PyObject = py - .eval(r#"'🐈 Hello \ud800World'"#, None, None) + .eval(r"'🐈 Hello \ud800World'", None, None) .unwrap() .into(); let py_string: &PyString = obj.downcast(py).unwrap(); diff --git a/src/types/traceback.rs b/src/types/traceback.rs index 7e4393eb98d..dfd318a7d30 100644 --- a/src/types/traceback.rs +++ b/src/types/traceback.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use crate::err::{error_on_minusone, PyResult}; use crate::ffi; use crate::types::PyString; @@ -67,7 +65,7 @@ impl PyTraceback { #[cfg(test)] mod tests { - use crate::Python; + use crate::{prelude::*, types::PyDict}; #[test] fn format_traceback() { @@ -82,4 +80,49 @@ mod tests { ); }) } + + #[test] + fn test_err_from_value() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + // Produce an error from python so that it has a traceback + py.run( + r" +try: + raise ValueError('raised exception') +except Exception as e: + err = e +", + None, + Some(locals), + ) + .unwrap(); + let err = PyErr::from_value(locals.get_item("err").unwrap()); + let traceback = err.value(py).getattr("__traceback__").unwrap(); + assert!(err.traceback(py).unwrap().is(traceback)); + }) + } + + #[test] + fn test_err_into_py() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + // Produce an error from python so that it has a traceback + py.run( + r" +def f(): + raise ValueError('raised exception') +", + None, + Some(locals), + ) + .unwrap(); + let f = locals.get_item("f").unwrap(); + let err = f.call0().unwrap_err(); + let traceback = err.traceback(py).unwrap(); + let err_object = err.clone_ref(py).into_py(py).into_ref(py); + + assert!(err_object.getattr("__traceback__").unwrap().is(traceback)); + }) + } } diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 3a124d3690e..429f2d0f848 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -1,5 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors - use std::convert::TryInto; use crate::ffi::{self, Py_ssize_t}; diff --git a/src/types/typeobject.rs b/src/types/typeobject.rs index 00ba62c3be1..ca4a5cdbef6 100644 --- a/src/types/typeobject.rs +++ b/src/types/typeobject.rs @@ -1,7 +1,3 @@ -// Copyright (c) 2017-present PyO3 Project and Contributors -// -// based on Daniel Grunwald's https://github.com/dgrunwald/rust-cpython - use crate::err::{self, PyResult}; use crate::{ffi, AsPyPointer, PyAny, PyTypeInfo, Python}; diff --git a/tests/test_append_to_inittab.rs b/tests/test_append_to_inittab.rs index 220a73912f1..e0a57da1b5c 100644 --- a/tests/test_append_to_inittab.rs +++ b/tests/test_append_to_inittab.rs @@ -26,7 +26,7 @@ assert module_with_functions.foo() == 123 None, None, ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }) } diff --git a/tests/test_class_attributes.rs b/tests/test_class_attributes.rs index 043971455b6..e07aa457de7 100644 --- a/tests/test_class_attributes.rs +++ b/tests/test_class_attributes.rs @@ -96,7 +96,7 @@ fn recursive_class_attributes() { } #[test] -#[cfg(panic = "unwind")] +#[cfg_attr(cfg_panic, cfg(panic = "unwind"))] fn test_fallible_class_attribute() { use pyo3::{exceptions::PyValueError, types::PyString}; diff --git a/tests/test_class_new.rs b/tests/test_class_new.rs index b9b0d152086..8cb426861db 100644 --- a/tests/test_class_new.rs +++ b/tests/test_class_new.rs @@ -2,6 +2,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::sync::GILOnceCell; use pyo3::types::IntoPyDict; #[pyclass] @@ -81,14 +82,14 @@ fn tuple_class_with_new() { #[pyclass] #[derive(Debug)] struct NewWithOneArg { - _data: i32, + data: i32, } #[pymethods] impl NewWithOneArg { #[new] fn new(arg: i32) -> NewWithOneArg { - NewWithOneArg { _data: arg } + NewWithOneArg { data: arg } } } @@ -99,14 +100,14 @@ fn new_with_one_arg() { let wrp = typeobj.call((42,), None).unwrap(); let obj = wrp.downcast::>().unwrap(); let obj_ref = obj.borrow(); - assert_eq!(obj_ref._data, 42); + assert_eq!(obj_ref.data, 42); }); } #[pyclass] struct NewWithTwoArgs { - _data1: i32, - _data2: i32, + data1: i32, + data2: i32, } #[pymethods] @@ -114,8 +115,8 @@ impl NewWithTwoArgs { #[new] fn new(arg1: i32, arg2: i32) -> Self { NewWithTwoArgs { - _data1: arg1, - _data2: arg2, + data1: arg1, + data2: arg2, } } } @@ -126,12 +127,12 @@ fn new_with_two_args() { let typeobj = py.get_type::(); let wrp = typeobj .call((10, 20), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); let obj = wrp.downcast::>().unwrap(); let obj_ref = obj.borrow(); - assert_eq!(obj_ref._data1, 10); - assert_eq!(obj_ref._data2, 20); + assert_eq!(obj_ref.data1, 10); + assert_eq!(obj_ref.data2, 20); }); } @@ -171,7 +172,7 @@ assert c.from_rust is False let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("SuperClass", super_cls).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } @@ -204,3 +205,62 @@ fn new_with_custom_error() { assert_eq!(err.to_string(), "ValueError: custom error"); }); } + +#[pyclass] +struct NewExisting { + #[pyo3(get)] + num: usize, +} + +#[pymethods] +impl NewExisting { + #[new] + fn new(py: pyo3::Python<'_>, val: usize) -> pyo3::Py { + static PRE_BUILT: GILOnceCell<[pyo3::Py; 2]> = GILOnceCell::new(); + let existing = PRE_BUILT.get_or_init(py, || { + [ + pyo3::PyCell::new(py, NewExisting { num: 0 }) + .unwrap() + .into(), + pyo3::PyCell::new(py, NewExisting { num: 1 }) + .unwrap() + .into(), + ] + }); + + if val < existing.len() { + return existing[val].clone_ref(py); + } + + pyo3::PyCell::new(py, NewExisting { num: val }) + .unwrap() + .into() + } +} + +#[test] +fn test_new_existing() { + Python::with_gil(|py| { + let typeobj = py.get_type::(); + + let obj1 = typeobj.call1((0,)).unwrap(); + let obj2 = typeobj.call1((0,)).unwrap(); + let obj3 = typeobj.call1((1,)).unwrap(); + let obj4 = typeobj.call1((1,)).unwrap(); + let obj5 = typeobj.call1((2,)).unwrap(); + let obj6 = typeobj.call1((2,)).unwrap(); + + assert!(obj1.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj2.getattr("num").unwrap().extract::().unwrap() == 0); + assert!(obj3.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj4.getattr("num").unwrap().extract::().unwrap() == 1); + assert!(obj5.getattr("num").unwrap().extract::().unwrap() == 2); + assert!(obj6.getattr("num").unwrap().extract::().unwrap() == 2); + + assert!(obj1.is(obj2)); + assert!(obj3.is(obj4)); + assert!(!obj1.is(obj3)); + assert!(!obj1.is(obj5)); + assert!(!obj5.is(obj6)); + }); +} diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 4625ea16431..5d933e13bc6 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -37,5 +37,5 @@ fn test_compile_errors() { t.compile_fail("tests/ui/not_send.rs"); t.compile_fail("tests/ui/not_send2.rs"); t.compile_fail("tests/ui/get_set_all.rs"); - t.compile_fail("tests/ui/traverse_bare_self.rs"); + t.compile_fail("tests/ui/traverse.rs"); } diff --git a/tests/test_datetime.rs b/tests/test_datetime.rs index f863afdce74..37b2d060303 100644 --- a/tests/test_datetime.rs +++ b/tests/test_datetime.rs @@ -86,7 +86,7 @@ fn test_time_check() { fn test_datetime_check() { Python::with_gil(|py| { let (obj, sub_obj, sub_sub_obj) = _get_subclasses(py, "datetime", "2018, 1, 1, 13, 30, 15") - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); unsafe { PyDateTime_IMPORT() } diff --git a/tests/test_dict_iter.rs b/tests/test_dict_iter.rs index 5d30d4c766a..dc32eb61fd7 100644 --- a/tests/test_dict_iter.rs +++ b/tests/test_dict_iter.rs @@ -8,7 +8,7 @@ fn iter_dict_nosegv() { const LEN: usize = 10_000_000; let dict = (0..LEN as u64).map(|i| (i, i * 2)).into_py_dict(py); let mut sum = 0; - for (k, _v) in dict.iter() { + for (k, _v) in dict { let i: u64 = k.extract().unwrap(); sum += i; } diff --git a/tests/test_field_cfg.rs b/tests/test_field_cfg.rs index dc84701c2b3..bd671641e5b 100644 --- a/tests/test_field_cfg.rs +++ b/tests/test_field_cfg.rs @@ -8,7 +8,12 @@ struct CfgClass { #[cfg(any())] pub a: u32, #[pyo3(get, set)] - #[cfg(all())] + // This is always true + #[cfg(any( + target_family = "unix", + target_family = "windows", + target_family = "wasm" + ))] pub b: u32, } diff --git a/tests/test_getter_setter.rs b/tests/test_getter_setter.rs index 222421ce43f..a94c739d36a 100644 --- a/tests/test_getter_setter.rs +++ b/tests/test_getter_setter.rs @@ -216,3 +216,23 @@ fn cell_getter_setter() { ); }); } + +#[test] +fn borrowed_value_with_lifetime_of_self() { + #[pyclass] + struct BorrowedValue {} + + #[pymethods] + impl BorrowedValue { + #[getter] + fn value(&self) -> &str { + "value" + } + } + + Python::with_gil(|py| { + let inst = Py::new(py, BorrowedValue {}).unwrap().to_object(py); + + py_run!(py, inst, "assert inst.value == 'value'"); + }); +} diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 8c46a4de4ae..aa4166a754d 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -26,7 +26,7 @@ fn subclass() { None, Some(d), ) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } diff --git a/tests/test_pep_587.rs b/tests/test_pep_587.rs index ca5c3f65ea2..24e1f07d2d8 100644 --- a/tests/test_pep_587.rs +++ b/tests/test_pep_587.rs @@ -29,7 +29,10 @@ fn test_default_interpreter() { unsafe { ffi::PyConfig_InitPythonConfig(&mut config) }; // Require manually calling _Py_InitializeMain to exercise more ffi code - config._init_main = 0; + #[allow(clippy::used_underscore_binding)] + { + config._init_main = 0; + } #[cfg(Py_3_10)] unsafe { diff --git a/tests/test_proto_methods.rs b/tests/test_proto_methods.rs index e04ce45a6ad..50584fbf9b6 100644 --- a/tests/test_proto_methods.rs +++ b/tests/test_proto_methods.rs @@ -14,14 +14,14 @@ struct EmptyClass; struct ExampleClass { #[pyo3(get, set)] value: i32, - _custom_attr: Option, + custom_attr: Option, } #[pymethods] impl ExampleClass { fn __getattr__(&self, py: Python<'_>, attr: &str) -> PyResult { if attr == "special_custom_attr" { - Ok(self._custom_attr.into_py(py)) + Ok(self.custom_attr.into_py(py)) } else { Err(PyAttributeError::new_err(attr.to_string())) } @@ -29,7 +29,7 @@ impl ExampleClass { fn __setattr__(&mut self, attr: &str, value: &PyAny) -> PyResult<()> { if attr == "special_custom_attr" { - self._custom_attr = Some(value.extract()?); + self.custom_attr = Some(value.extract()?); Ok(()) } else { Err(PyAttributeError::new_err(attr.to_string())) @@ -38,7 +38,7 @@ impl ExampleClass { fn __delattr__(&mut self, attr: &str) -> PyResult<()> { if attr == "special_custom_attr" { - self._custom_attr = None; + self.custom_attr = None; Ok(()) } else { Err(PyAttributeError::new_err(attr.to_string())) @@ -68,7 +68,7 @@ fn make_example(py: Python<'_>) -> &PyCell { py, ExampleClass { value: 5, - _custom_attr: Some(20), + custom_attr: Some(20), }, ) .unwrap() @@ -689,7 +689,7 @@ asyncio.run(main()) let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("Once", once).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } @@ -746,7 +746,7 @@ asyncio.run(main()) .set_item("AsyncIterator", py.get_type::()) .unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } @@ -815,7 +815,7 @@ assert c.counter.count == 1 let globals = PyModule::import(py, "__main__").unwrap().dict(); globals.set_item("Counter", counter).unwrap(); py.run(source, Some(globals), None) - .map_err(|e| e.print(py)) + .map_err(|e| e.display(py)) .unwrap(); }); } diff --git a/tests/ui/invalid_closure.stderr b/tests/ui/invalid_closure.stderr index 4844f766887..9a2a957592c 100644 --- a/tests/ui/invalid_closure.stderr +++ b/tests/ui/invalid_closure.stderr @@ -1,6 +1,8 @@ error[E0597]: `local_data` does not live long enough --> tests/ui/invalid_closure.rs:7:27 | +6 | let local_data = vec![0, 1, 2, 3, 4]; + | ---------- binding `local_data` declared here 7 | let ref_: &[u8] = &local_data; | ^^^^^^^^^^^ borrowed value does not live long enough ... diff --git a/tests/ui/invalid_frozen_pyclass_borrow.rs b/tests/ui/invalid_frozen_pyclass_borrow.rs index c7b2f27b5b1..1f18eab6170 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.rs +++ b/tests/ui/invalid_frozen_pyclass_borrow.rs @@ -6,6 +6,11 @@ pub struct Foo { field: u32, } +#[pymethods] +impl Foo { + fn mut_method(&mut self) {} +} + fn borrow_mut_fails(foo: Py, py: Python) { let borrow = foo.as_ref(py).borrow_mut(); } @@ -28,4 +33,10 @@ fn pyclass_get_of_mutable_class_fails(class: &PyCell) { class.get(); } +#[pyclass(frozen)] +pub struct SetOnFrozenClass { + #[pyo3(set)] + field: u32, +} + fn main() {} diff --git a/tests/ui/invalid_frozen_pyclass_borrow.stderr b/tests/ui/invalid_frozen_pyclass_borrow.stderr index b91d5c0cefb..5e09d512ae7 100644 --- a/tests/ui/invalid_frozen_pyclass_borrow.stderr +++ b/tests/ui/invalid_frozen_pyclass_borrow.stderr @@ -1,47 +1,77 @@ +error: cannot use `#[pyo3(set)]` on a `frozen` class + --> tests/ui/invalid_frozen_pyclass_borrow.rs:38:12 + | +38 | #[pyo3(set)] + | ^^^ + +error[E0271]: type mismatch resolving `::Frozen == False` + --> tests/ui/invalid_frozen_pyclass_borrow.rs:11:19 + | +11 | fn mut_method(&mut self) {} + | ^ expected `False`, found `True` + | +note: required by a bound in `extract_pyclass_ref_mut` + --> src/impl_/extract_argument.rs + | + | pub fn extract_pyclass_ref_mut<'a, 'py: 'a, T: PyClass>( + | ^^^^^^^^^^^^^^ required by this bound in `extract_pyclass_ref_mut` + error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:10:33 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:15:33 | -10 | let borrow = foo.as_ref(py).borrow_mut(); +15 | let borrow = foo.as_ref(py).borrow_mut(); | ^^^^^^^^^^ expected `False`, found `True` | note: required by a bound in `pyo3::PyCell::::borrow_mut` --> src/pycell.rs | + | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | ---------- required by a bound in this associated function + | where | T: PyClass, | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == False` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:20:35 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:25:35 | -20 | let borrow = child.as_ref(py).borrow_mut(); +25 | let borrow = child.as_ref(py).borrow_mut(); | ^^^^^^^^^^ expected `False`, found `True` | note: required by a bound in `pyo3::PyCell::::borrow_mut` --> src/pycell.rs | + | pub fn borrow_mut(&self) -> PyRefMut<'_, T> + | ---------- required by a bound in this associated function + | where | T: PyClass, | ^^^^^^^^^^^^^^ required by this bound in `PyCell::::borrow_mut` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:24:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:29:11 | -24 | class.get(); +29 | class.get(); | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::Py::::get` --> src/instance.rs | + | pub fn get(&self) -> &T + | --- required by a bound in this associated function + | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `Py::::get` error[E0271]: type mismatch resolving `::Frozen == True` - --> tests/ui/invalid_frozen_pyclass_borrow.rs:28:11 + --> tests/ui/invalid_frozen_pyclass_borrow.rs:33:11 | -28 | class.get(); +33 | class.get(); | ^^^ expected `True`, found `False` | note: required by a bound in `pyo3::PyCell::::get` --> src/pycell.rs | + | pub fn get(&self) -> &T + | --- required by a bound in this associated function + | where | T: PyClass + Sync, | ^^^^^^^^^^^^^ required by this bound in `PyCell::::get` diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 18547a10f80..395723cba1f 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -7,11 +7,27 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | required by a bound introduced by this call | = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` - = note: required because it appears within the type `PhantomData<*mut Python<'static>>` - = note: required because it appears within the type `NotSend` +note: required because it appears within the type `PhantomData<*mut Python<'static>>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `NotSend` + --> src/impl_/not_send.rs + | + | pub(crate) struct NotSend(PhantomData<*mut Python<'static>>); + | ^^^^^^^ = note: required because it appears within the type `(&GILGuard, NotSend)` - = note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` - = note: required because it appears within the type `Python<'_>` +note: required because it appears within the type `PhantomData<(&GILGuard, NotSend)>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `Python<'_>` + --> src/marker.rs + | + | pub struct Python<'py>(PhantomData<(&'py GILGuard, NotSend)>); + | ^^^^^^ = note: required for `&pyo3::Python<'_>` to implement `Send` note: required because it's used within this closure --> tests/ui/not_send.rs:4:22 @@ -22,5 +38,8 @@ note: required because it's used within this closure note: required by a bound in `pyo3::Python::<'py>::allow_threads` --> src/marker.rs | + | pub fn allow_threads(self, f: F) -> T + | ------------- required by a bound in this associated function + | where | F: Ungil + FnOnce() -> T, | ^^^^^ required by this bound in `Python::<'py>::allow_threads` diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index 8bfa6016d86..2e6db009bad 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -10,8 +10,16 @@ error[E0277]: `UnsafeCell` cannot be shared between threads safely | |_________^ `UnsafeCell` cannot be shared between threads safely | = help: within `&PyString`, the trait `Sync` is not implemented for `UnsafeCell` - = note: required because it appears within the type `PyAny` - = note: required because it appears within the type `PyString` +note: required because it appears within the type `PyAny` + --> src/types/any.rs + | + | pub struct PyAny(UnsafeCell); + | ^^^^^ +note: required because it appears within the type `PyString` + --> src/types/string.rs + | + | pub struct PyString(PyAny); + | ^^^^^^^^ = note: required because it appears within the type `&PyString` = note: required for `&&PyString` to implement `Send` note: required because it's used within this closure @@ -23,5 +31,8 @@ note: required because it's used within this closure note: required by a bound in `pyo3::Python::<'py>::allow_threads` --> src/marker.rs | + | pub fn allow_threads(self, f: F) -> T + | ------------- required by a bound in this associated function + | where | F: Ungil + FnOnce() -> T, | ^^^^^ required by this bound in `Python::<'py>::allow_threads` diff --git a/tests/ui/traverse.rs b/tests/ui/traverse.rs new file mode 100644 index 00000000000..034224951c9 --- /dev/null +++ b/tests/ui/traverse.rs @@ -0,0 +1,27 @@ +use pyo3::prelude::*; +use pyo3::PyVisit; +use pyo3::PyTraverseError; + +#[pyclass] +struct TraverseTriesToTakePyRef {} + +#[pymethods] +impl TraverseTriesToTakePyRef { + fn __traverse__(slf: PyRef, visit: PyVisit) {} +} + +#[pyclass] +struct Class; + +#[pymethods] +impl Class { + fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + Ok(()) + } + + fn __clear__(&mut self) { + } +} + + +fn main() {} diff --git a/tests/ui/traverse.stderr b/tests/ui/traverse.stderr new file mode 100644 index 00000000000..e2718c76e56 --- /dev/null +++ b/tests/ui/traverse.stderr @@ -0,0 +1,23 @@ +error: __traverse__ may not take `Python`. Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`. Most importantly, safe access to the GIL is prohibited inside implementations of `__traverse__`, i.e. `Python::with_gil` will panic. + --> tests/ui/traverse.rs:18:32 + | +18 | fn __traverse__(&self, py: Python<'_>, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + | ^^^^^^^^^^ + +error[E0308]: mismatched types + --> tests/ui/traverse.rs:9:6 + | +8 | #[pymethods] + | ------------ arguments to this function are incorrect +9 | impl TraverseTriesToTakePyRef { + | ______^ +10 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} + | |___________________^ expected fn pointer, found fn item + | + = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` + found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` +note: function defined here + --> src/impl_/pymethods.rs + | + | pub unsafe fn _call_traverse( + | ^^^^^^^^^^^^^^ diff --git a/tests/ui/traverse_bare_self.rs b/tests/ui/traverse_bare_self.rs deleted file mode 100644 index 5adc316e43f..00000000000 --- a/tests/ui/traverse_bare_self.rs +++ /dev/null @@ -1,12 +0,0 @@ -use pyo3::prelude::*; -use pyo3::PyVisit; - -#[pyclass] -struct TraverseTriesToTakePyRef {} - -#[pymethods] -impl TraverseTriesToTakePyRef { - fn __traverse__(slf: PyRef, visit: PyVisit) {} -} - -fn main() {} diff --git a/tests/ui/traverse_bare_self.stderr b/tests/ui/traverse_bare_self.stderr deleted file mode 100644 index aba76145dc3..00000000000 --- a/tests/ui/traverse_bare_self.stderr +++ /dev/null @@ -1,17 +0,0 @@ -error[E0308]: mismatched types - --> tests/ui/traverse_bare_self.rs:8:6 - | -7 | #[pymethods] - | ------------ arguments to this function are incorrect -8 | impl TraverseTriesToTakePyRef { - | ______^ -9 | | fn __traverse__(slf: PyRef, visit: PyVisit) {} - | |___________________^ expected fn pointer, found fn item - | - = note: expected fn pointer `for<'a, 'b> fn(&'a TraverseTriesToTakePyRef, PyVisit<'b>) -> Result<(), PyTraverseError>` - found fn item `for<'a, 'b> fn(pyo3::PyRef<'a, TraverseTriesToTakePyRef>, PyVisit<'b>) {TraverseTriesToTakePyRef::__traverse__}` -note: function defined here - --> src/impl_/pymethods.rs - | - | pub unsafe fn call_traverse_impl( - | ^^^^^^^^^^^^^^^^^^ diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml deleted file mode 100644 index 48a5a63fd3f..00000000000 --- a/xtask/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "xtask" -version = "0.1.0" -edition = "2018" -publish = false - -[[bin]] -name = "xtask" - -[dependencies] -anyhow = "1.0.51" - -# Clap 3 requires MSRV 1.54 -rustversion = "1.0" -structopt = { version = "0.3", default-features = false } -clap = { version = "2" } diff --git a/xtask/README.md b/xtask/README.md deleted file mode 100644 index 68d078e5595..00000000000 --- a/xtask/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Commands to test PyO3. - -To run these commands, you should be in PyO3's root directory, and run (for example) `cargo xtask ci`. - -``` -USAGE: - xtask.exe - -FLAGS: - -h, --help Prints help information - -V, --version Prints version information - -SUBCOMMANDS: - ci Runs everything - clippy Runs `clippy`, denying all warnings - coverage Runs `cargo llvm-cov` for the PyO3 codebase - default Only runs the fast things (this is used if no command is specified) - doc Attempts to render the documentation - fmt Checks Rust and Python code formatting with `rustfmt` and `black` - help Prints this message or the help of the given subcommand(s) - test Runs various variations on `cargo test` - test-py Runs the tests in examples/ and pytests/ -``` \ No newline at end of file diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs deleted file mode 100644 index f873816a017..00000000000 --- a/xtask/src/cli.rs +++ /dev/null @@ -1,216 +0,0 @@ -use crate::utils::*; -use anyhow::{ensure, Result}; -use std::io; -use std::process::{Command, Output}; -use std::time::Instant; -use structopt::StructOpt; - -pub const MSRV: &str = "1.48"; - -#[derive(StructOpt)] -pub enum Subcommand { - /// Only runs the fast things (this is used if no command is specified) - Default, - /// Runs everything - Ci, - /// Checks Rust and Python code formatting with `rustfmt` and `black` - Fmt, - /// Runs `clippy`, denying all warnings. - Clippy, - /// Attempts to render the documentation. - Doc(DocOpts), - /// Runs various variations on `cargo test` - Test, - /// Runs the tests in examples/ and pytests/ - TestPy, -} - -impl Default for Subcommand { - fn default() -> Self { - Self::Default - } -} - -#[derive(StructOpt)] -pub struct DocOpts { - /// Whether to run the docs using nightly rustdoc - #[structopt(long)] - pub stable: bool, - /// Whether to open the docs after rendering. - #[structopt(long)] - pub open: bool, - /// Whether to show the private and hidden API. - #[structopt(long)] - pub internal: bool, -} - -impl Default for DocOpts { - fn default() -> Self { - Self { - stable: true, - open: false, - internal: false, - } - } -} - -impl Subcommand { - pub fn execute(self) -> Result<()> { - print_metadata()?; - - let start = Instant::now(); - - match self { - Subcommand::Default => { - crate::fmt::rust::run()?; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - } - Subcommand::Ci => { - let installed = Installed::new()?; - crate::fmt::rust::run()?; - if installed.black { - crate::fmt::python::run()?; - } else { - Installed::warn_black() - }; - crate::clippy::run()?; - crate::test::run()?; - crate::doc::run(DocOpts::default())?; - if installed.nox { - crate::pytests::run(None)?; - } else { - Installed::warn_nox() - }; - installed.assert()? - } - - Subcommand::Doc(opts) => crate::doc::run(opts)?, - Subcommand::Fmt => { - crate::fmt::rust::run()?; - crate::fmt::python::run()?; - } - Subcommand::Clippy => crate::clippy::run()?, - Subcommand::TestPy => crate::pytests::run(None)?, - Subcommand::Test => crate::test::run()?, - }; - - let dt = start.elapsed().as_secs(); - let minutes = dt / 60; - let seconds = dt % 60; - println!("\nxtask finished in {}m {}s.", minutes, seconds); - - Ok(()) - } -} - -/// Run a command as a child process, inheriting stdin, stdout and stderr. -pub fn run(command: &mut Command) -> Result<()> { - let command_str = format_command(command); - let github_actions = std::env::var_os("GITHUB_ACTIONS").is_some(); - if github_actions { - println!("::group::Running: {}", command_str); - } else { - println!("Running: {}", command_str); - } - - let status = command.spawn()?.wait()?; - - ensure! { - status.success(), - "process did not run successfully ({exit}): {command}", - exit = match status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - }; - - if github_actions { - println!("::endgroup::") - } - Ok(()) -} - -/// Like `run`, but does not inherit stdin, stdout and stderr. -pub fn run_with_output(command: &mut Command) -> Result { - let command_str = format_command(command); - - println!("Running: {}", command_str); - - let output = command.output()?; - - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}:\n{stderr}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = command_str, - stderr = String::from_utf8_lossy(&output.stderr) - }; - - Ok(output) -} - -#[derive(Copy, Clone, Debug)] -pub struct Installed { - pub nox: bool, - pub black: bool, -} - -impl Installed { - pub fn new() -> anyhow::Result { - Ok(Self { - nox: Self::nox()?, - black: Self::black()?, - }) - } - - pub fn nox() -> anyhow::Result { - let output = std::process::Command::new("nox").arg("--version").output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_nox() { - eprintln!("Skipping: formatting Python code, because `nox` was not found"); - } - - pub fn black() -> anyhow::Result { - let output = std::process::Command::new("black") - .arg("--version") - .output(); - match output { - Ok(_) => Ok(true), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), - Err(other) => Err(other.into()), - } - } - - pub fn warn_black() { - eprintln!("Skipping: Python code formatting, because `black` was not found."); - } - - pub fn assert(&self) -> anyhow::Result<()> { - if self.nox && self.black { - Ok(()) - } else { - let mut err = - String::from("\n\nxtask was unable to run all tests due to some missing programs:"); - if !self.black { - err.push_str("\n`black` was not installed. (`pip install black`)"); - } - if !self.nox { - err.push_str("\n`nox` was not installed. (`pip install nox`)"); - } - - Err(anyhow::anyhow!(err)) - } - } -} diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs deleted file mode 100644 index eec5f5fdd0b..00000000000 --- a/xtask/src/clippy.rs +++ /dev/null @@ -1,25 +0,0 @@ -use crate::cli; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--features=full") - .arg("--all-targets") - .arg("--workspace") - .arg("--") - .arg("-Dwarnings"), - )?; - cli::run( - Command::new("cargo") - .arg("clippy") - .arg("--all-targets") - .arg("--workspace") - .arg("--features=abi3,full") - .arg("--") - .arg("-Dwarnings"), - )?; - - Ok(()) -} diff --git a/xtask/src/doc.rs b/xtask/src/doc.rs deleted file mode 100644 index cde2c882914..00000000000 --- a/xtask/src/doc.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::cli; -use crate::cli::DocOpts; -use std::process::Command; -//--cfg docsrs --Z unstable-options --document-hidden-items - -pub fn run(opts: DocOpts) -> anyhow::Result<()> { - let mut flags = Vec::new(); - - if !opts.stable { - flags.push("--cfg docsrs"); - } - if opts.internal { - flags.push("--Z unstable-options"); - flags.push("--document-hidden-items"); - } - flags.push("-Dwarnings"); - - std::env::set_var("RUSTDOCFLAGS", flags.join(" ")); - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .args(if opts.stable { None } else { Some("+nightly") }) - .arg("doc") - .arg("--lib") - .arg("--no-default-features") - .arg("--features=full") - .arg("--no-deps") - .arg("--workspace") - .args(if opts.internal { - &["--document-private-items"][..] - } else { - &["--exclude=pyo3-macros", "--exclude=pyo3-macros-backend"][..] - }) - .args(if opts.stable { - &[][..] - } else { - &["-Z", "unstable-options", "-Z", "rustdoc-scrape-examples"][..] - }) - .args(if opts.open { Some("--open") } else { None }), - )?; - - Ok(()) -} diff --git a/xtask/src/fmt.rs b/xtask/src/fmt.rs deleted file mode 100644 index 8bc745246bc..00000000000 --- a/xtask/src/fmt.rs +++ /dev/null @@ -1,23 +0,0 @@ -pub mod rust { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("fmt") - .arg("--all") - .arg("--") - .arg("--check"), - )?; - Ok(()) - } -} - -pub mod python { - use crate::cli; - use std::process::Command; - pub fn run() -> anyhow::Result<()> { - cli::run(Command::new("black").arg(".").arg("--check"))?; - Ok(()) - } -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs deleted file mode 100644 index 421742685ca..00000000000 --- a/xtask/src/main.rs +++ /dev/null @@ -1,23 +0,0 @@ -use clap::ErrorKind::MissingArgumentOrSubcommand; -use structopt::StructOpt; - -pub mod cli; -pub mod clippy; -pub mod doc; -pub mod fmt; -pub mod pytests; -pub mod test; -pub mod utils; - -fn main() -> anyhow::Result<()> { - // Avoid spewing backtraces all over the command line - // For some reason this is automatically enabled on nightly compilers... - std::env::set_var("RUST_LIB_BACKTRACE", "0"); - - match cli::Subcommand::from_args_safe() { - Ok(c) => c.execute()?, - Err(e) if e.kind == MissingArgumentOrSubcommand => cli::Subcommand::default().execute()?, - Err(e) => return Err(e.into()), - } - Ok(()) -} diff --git a/xtask/src/pytests.rs b/xtask/src/pytests.rs deleted file mode 100644 index 78744c69bd1..00000000000 --- a/xtask/src/pytests.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::cli; -use anyhow::Result; -use std::{path::Path, process::Command}; - -pub fn run<'a>(env: impl IntoIterator + Copy) -> Result<()> { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(Path::new("pytests").join("noxfile.py")) - .envs(env), - )?; - - for entry in std::fs::read_dir("examples")? { - let path = entry?.path(); - if path.is_dir() && path.join("noxfile.py").exists() { - cli::run( - Command::new("nox") - .arg("--non-interactive") - .arg("-f") - .arg(path.join("noxfile.py")) - .envs(env), - )?; - } - } - Ok(()) -} diff --git a/xtask/src/test.rs b/xtask/src/test.rs deleted file mode 100644 index b85f3e4a504..00000000000 --- a/xtask/src/test.rs +++ /dev/null @@ -1,81 +0,0 @@ -use crate::cli::{self, MSRV}; -use std::process::Command; - -pub fn run() -> anyhow::Result<()> { - cli::run( - Command::new("cargo") - .arg("test") - .arg("--lib") - .arg("--no-default-features") - .arg("--tests") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=full") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--features=abi3,full") - .arg("--quiet"), - )?; - - // If the MSRV toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg(MSRV), - )?; - - // Test MSRV - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg(format!("+{}", MSRV)) - .arg("test") - .arg("--no-default-features") - .arg("--features=full,auto-initialize") - .arg("--quiet"), - )?; - - // If the nightly toolchain is not installed, this will install it - cli::run( - Command::new("rustup") - .arg("toolchain") - .arg("install") - .arg("nightly"), - )?; - - cli::run( - Command::new(concat!(env!("CARGO_HOME"), "/bin/cargo")) - .arg("+nightly") - .arg("test") - .arg("--no-default-features") - .arg("--features=full,nightly") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--manifest-path=pyo3-ffi/Cargo.toml") - .arg("--quiet"), - )?; - - cli::run( - Command::new("cargo") - .arg("test") - .arg("--no-default-features") - .arg("--manifest-path=pyo3-build-config/Cargo.toml") - .arg("--quiet"), - )?; - - Ok(()) -} diff --git a/xtask/src/utils.rs b/xtask/src/utils.rs deleted file mode 100644 index 045697e7357..00000000000 --- a/xtask/src/utils.rs +++ /dev/null @@ -1,65 +0,0 @@ -use anyhow::ensure; -use std::process::Command; - -// Replacement for str.split_once() on Rust older than 1.52 -#[rustversion::before(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - let mut iter = s.splitn(2, pat); - Some((iter.next()?, iter.next()?)) -} - -#[rustversion::since(1.52)] -pub fn split_once(s: &str, pat: char) -> Option<(&str, &str)> { - s.split_once(pat) -} - -#[rustversion::since(1.57)] -pub fn format_command(command: &Command) -> String { - let mut buf = String::new(); - buf.push('`'); - buf.push_str(&command.get_program().to_string_lossy()); - for arg in command.get_args() { - buf.push(' '); - buf.push_str(&arg.to_string_lossy()); - } - buf.push('`'); - buf -} - -#[rustversion::before(1.57)] -pub fn format_command(command: &Command) -> String { - // Debug impl isn't as nice as the above, but will do on < 1.57 - format!("{:?}", command) -} - -pub fn get_output(command: &mut Command) -> anyhow::Result { - let output = command.output()?; - ensure! { - output.status.success(), - "process did not run successfully ({exit}): {command}", - exit = match output.status.code() { - Some(code) => format!("exit code {}", code), - None => "terminated by signal".into(), - }, - command = format_command(command), - }; - Ok(output) -} - -pub fn print_metadata() -> anyhow::Result<()> { - let rustc_output = std::process::Command::new("rustc") - .arg("--version") - .arg("--verbose") - .output()?; - let rustc_version = core::str::from_utf8(&rustc_output.stdout).unwrap(); - println!("Metadata: \n\n{}", rustc_version); - - let py_output = std::process::Command::new("python") - .arg("--version") - .arg("-V") - .output()?; - let py_version = core::str::from_utf8(&py_output.stdout).unwrap(); - println!("{}", py_version); - - Ok(()) -}