Merge branch 'main' into optimize-kwarg-extraction

Author: davidhewitt

.github/workflows/ci.yml

@@ -697,7 +697,8 @@ jobs:
         with:
           save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }}
       - run: python -m pip install --upgrade pip && pip install nox[uv]
-      - run: nox -s test-introspection
+        # TODO test will be fixed in https://github.com/PyO3/pyo3/pull/5450
+#      - run: nox -s test-introspection
     env:
       CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}
 

newsfragments/5444.fixed.md

@@ -0,0 +1 @@
+fix `OsStr` conversion for non-utf8 strings on windows

newsfragments/5445.added.md

@@ -0,0 +1 @@
+Implement `AsRef<[u8]>` for `PyBytes`

src/conversions/std/osstr.rs

@@ -110,6 +110,12 @@ impl FromPyObject<'_, '_> for OsString {
                 unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), std::ptr::null_mut(), 0) };
             crate::err::error_on_minusone(ob.py(), size)?;
 
+            debug_assert!(
+                size > 0,
+                "PyUnicode_AsWideChar should return at least 1 for null terminator"
+            );
+            let size = size - 1; // exclude null terminator
+
             let mut buffer = vec![0; size as usize];
             let bytes_read =
                 unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), buffer.as_mut_ptr(), size) };
@@ -169,7 +175,7 @@ impl<'py> IntoPyObject<'py> for &OsString {
 
 #[cfg(test)]
 mod tests {
-    use crate::types::{PyString, PyStringMethods};
+    use crate::types::{PyAnyMethods, PyString, PyStringMethods};
     use crate::{BoundObject, IntoPyObject, Python};
     use std::fmt::Debug;
     use std::{
@@ -181,7 +187,6 @@ mod tests {
     #[cfg(not(windows))]
     fn test_non_utf8_conversion() {
         Python::attach(|py| {
-            use crate::types::PyAnyMethods;
             #[cfg(not(target_os = "wasi"))]
             use std::os::unix::ffi::OsStrExt;
             #[cfg(target_os = "wasi")]
@@ -219,4 +224,32 @@ mod tests {
             test_roundtrip::<OsString>(py, os_str.to_os_string());
         });
     }
+
+    #[test]
+    #[cfg(windows)]
+    fn test_windows_non_utf8_osstring_roundtrip() {
+        use std::os::windows::ffi::{OsStrExt, OsStringExt};
+
+        Python::attach(|py| {
+            // Example: Unpaired surrogate (0xD800) is not valid UTF-8, but valid in Windows OsString
+            let wide: &[u16] = &['A' as u16, 0xD800, 'B' as u16]; // 'A', unpaired surrogate, 'B'
+            let os_str = OsString::from_wide(wide);
+
+            assert_eq!(os_str.to_string_lossy(), "A�B");
+
+            // This cannot be represented as UTF-8, so .to_str() would return None
+            assert!(os_str.to_str().is_none());
+
+            // Convert to Python and back
+            let py_str = os_str.as_os_str().into_pyobject(py).unwrap();
+            let os_str_2 = py_str.extract::<OsString>().unwrap();
+
+            // The roundtrip should preserve the original wide data
+            assert_eq!(os_str, os_str_2);
+
+            // Show that encode_wide is necessary: direct UTF-8 conversion would lose information
+            let encoded: Vec<u16> = os_str.encode_wide().collect();
+            assert_eq!(encoded, wide);
+        });
+    }
 }

src/types/bytes.rs

@@ -275,6 +275,20 @@ impl PartialEq<Borrowed<'_, '_, PyBytes>> for &'_ [u8] {
     }
 }
 
+impl<'a> AsRef<[u8]> for Borrowed<'a, '_, PyBytes> {
+    #[inline]
+    fn as_ref(&self) -> &'a [u8] {
+        self.as_bytes()
+    }
+}
+
+impl AsRef<[u8]> for Bound<'_, PyBytes> {
+    #[inline]
+    fn as_ref(&self) -> &[u8] {
+        self.as_bytes()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -381,4 +395,17 @@ mod tests {
             }
         })
     }
+
+    #[test]
+    fn test_as_ref_slice() {
+        Python::attach(|py| {
+            let b = b"hello, world";
+            let py_bytes = PyBytes::new(py, b);
+            let ref_bound: &[u8] = py_bytes.as_ref();
+            assert_eq!(ref_bound, b);
+            let py_bytes_borrowed = py_bytes.as_borrowed();
+            let ref_borrowed: &[u8] = py_bytes_borrowed.as_ref();
+            assert_eq!(ref_borrowed, b);
+        })
+    }
 }

tests/ui/invalid_closure.stderr

@@ -1,9 +1,9 @@
 error[E0597]: `local_data` does not live long enough
   --> tests/ui/invalid_closure.rs:7:27
    |
-6  |         let local_data = vec![0, 1, 2, 3, 4];
+ 6 |         let local_data = vec![0, 1, 2, 3, 4];
    |             ---------- binding `local_data` declared here
-7  |         let ref_: &[u8] = &local_data;
+ 7 |         let ref_: &[u8] = &local_data;
    |                           ^^^^^^^^^^^ borrowed value does not live long enough
 ...
 14 |         PyCFunction::new_closure(py, None, None, closure_fn)

tests/ui/invalid_frozen_pyclass_borrow.stderr

@@ -13,7 +13,7 @@ error[E0271]: type mismatch resolving `<Foo as PyClass>::Frozen == False`
 note: expected this to be `False`
   --> tests/ui/invalid_frozen_pyclass_borrow.rs:3:1
    |
-3  | #[pyclass(frozen)]
+ 3 | #[pyclass(frozen)]
    | ^^^^^^^^^^^^^^^^^^
 note: required by a bound in `extract_pyclass_ref_mut`
   --> src/impl_/extract_argument.rs
@@ -49,7 +49,7 @@ error[E0271]: type mismatch resolving `<Foo as PyClass>::Frozen == False`
 note: expected this to be `False`
   --> tests/ui/invalid_frozen_pyclass_borrow.rs:3:1
    |
-3  | #[pyclass(frozen)]
+ 3 | #[pyclass(frozen)]
    | ^^^^^^^^^^^^^^^^^^
 note: required by a bound in `pyo3::Bound::<'py, T>::borrow_mut`
   --> src/instance.rs

tests/ui/invalid_pyclass_generic.stderr

@@ -12,7 +12,7 @@ error[E0592]: duplicate definitions with name `__pymethod___class_getitem____`
 error[E0592]: duplicate definitions with name `__class_getitem__`
   --> tests/ui/invalid_pyclass_generic.rs:4:1
    |
-4  |   #[pyclass(generic)]
+ 4 |   #[pyclass(generic)]
    |   ^^^^^^^^^^^^^^^^^^^ duplicate definitions for `__class_getitem__`
 ...
 15 | /     pub fn __class_getitem__(
@@ -44,13 +44,13 @@ note: candidate #2 is defined in an impl for the type `ClassRedefinesClassGetIte
 error[E0034]: multiple applicable items in scope
   --> tests/ui/invalid_pyclass_generic.rs:4:1
    |
-4  | #[pyclass(generic)]
+ 4 | #[pyclass(generic)]
    | ^^^^^^^^^^^^^^^^^^^ multiple `__class_getitem__` found
    |
 note: candidate #1 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:4:1
    |
-4  | #[pyclass(generic)]
+ 4 | #[pyclass(generic)]
    | ^^^^^^^^^^^^^^^^^^^
 note: candidate #2 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:15:5
@@ -70,8 +70,6 @@ error[E0034]: multiple applicable items in scope
   |
   = note: candidate #1 is defined in an impl for the type `IntoPyObjectConverter<Result<T, E>>`
   = note: candidate #2 is defined in an impl for the type `IntoPyObjectConverter<T>`
-  = note: candidate #3 is defined in an impl for the type `UnknownReturnResultType<Result<T, E>>`
-  = note: candidate #4 is defined in an impl for the type `UnknownReturnType<T>`
   = note: this error originates in the attribute macro `pyclass` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 error[E0308]: mismatched types
@@ -94,12 +92,12 @@ error[E0034]: multiple applicable items in scope
 note: candidate #1 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:4:1
    |
-4  | #[pyclass(generic)]
+ 4 | #[pyclass(generic)]
    | ^^^^^^^^^^^^^^^^^^^
 note: candidate #2 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:7:1
    |
-7  | #[pymethods]
+ 7 | #[pymethods]
    | ^^^^^^^^^^^^
    = note: this error originates in the attribute macro `pyclass` which comes from the expansion of the attribute macro `pymethods` (in Nightly builds, run with -Z macro-backtrace for more info)
 
@@ -112,7 +110,7 @@ error[E0034]: multiple applicable items in scope
 note: candidate #1 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:4:1
    |
-4  | #[pyclass(generic)]
+ 4 | #[pyclass(generic)]
    | ^^^^^^^^^^^^^^^^^^^
 note: candidate #2 is defined in an impl for the type `ClassRedefinesClassGetItem`
   --> tests/ui/invalid_pyclass_generic.rs:15:5
@@ -132,5 +130,3 @@ error[E0034]: multiple applicable items in scope
    |
    = note: candidate #1 is defined in an impl for the type `IntoPyObjectConverter<Result<T, E>>`
    = note: candidate #2 is defined in an impl for the type `IntoPyObjectConverter<T>`
-   = note: candidate #3 is defined in an impl for the type `UnknownReturnResultType<Result<T, E>>`
-   = note: candidate #4 is defined in an impl for the type `UnknownReturnType<T>`

tests/ui/invalid_pymethod_enum.stderr

@@ -7,7 +7,7 @@ error[E0271]: type mismatch resolving `<ComplexEnum as PyClass>::Frozen == False
 note: expected this to be `False`
   --> tests/ui/invalid_pymethod_enum.rs:3:1
    |
-3  | #[pyclass]
+ 3 | #[pyclass]
    | ^^^^^^^^^^
 note: required by a bound in `extract_pyclass_ref_mut`
   --> src/impl_/extract_argument.rs

tests/ui/not_send2.stderr

@@ -1,11 +1,11 @@
 error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safely
   --> tests/ui/not_send2.rs:8:19
    |
-8  |           py.detach(|| {
+ 8 |           py.detach(|| {
    |  ____________------_^
    | |            |
    | |            required by a bound introduced by this call
-9  | |             println!("{:?}", string);
+ 9 | |             println!("{:?}", string);
 10 | |         });
    | |_________^ `*mut pyo3::Python<'static>` cannot be shared between threads safely
    |
@@ -39,7 +39,7 @@ note: required because it appears within the type `pyo3::Bound<'_, PyString>`
 note: required because it's used within this closure
   --> tests/ui/not_send2.rs:8:19
    |
-8  |         py.detach(|| {
+ 8 |         py.detach(|| {
    |                   ^^
    = note: required for `{closure@$DIR/tests/ui/not_send2.rs:8:19: 8:21}` to implement `Ungil`
 note: required by a bound in `pyo3::Python::<'py>::detach`