Support Result<T, E> across FFI when niche optimization can be used

compiler/rustc_feature/src/unstable.rs

@@ -585,6 +585,9 @@ declare_features! (
     (incomplete, repr128, "1.16.0", Some(56071)),
     /// Allows `repr(simd)` and importing the various simd intrinsics.
     (unstable, repr_simd, "1.4.0", Some(27731)),
+    /// Allows enums like Result<T, E> to be used across FFI, if T's niche value can
+    /// be used to describe E or vise-versa.
+    (unstable, result_ffi_guarantees, "CURRENT_RUSTC_VERSION", Some(110503)),
     /// Allows bounding the return type of AFIT/RPITIT.
     (incomplete, return_type_notation, "1.70.0", Some(109417)),
     /// Allows `extern "rust-cold"`.

compiler/rustc_lint/src/types.rs

@@ -1099,6 +1099,32 @@ fn get_nullable_type<'tcx>(
     })
 }
 
+/// A type is niche-optimization candidate iff:
+/// - Is a zero-sized type with alignment 1 (a “1-ZST”).
+/// - Has no fields.
+/// - Does not have the `#[non_exhaustive]` attribute.
+fn is_niche_optimization_candidate<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    ty: Ty<'tcx>,
+) -> bool {
+    if tcx.layout_of(param_env.and(ty)).is_ok_and(|layout| !layout.is_1zst()) {
+        return false;
+    }
+
+    match ty.kind() {
+        ty::Adt(ty_def, _) => {
+            let non_exhaustive = ty_def.is_variant_list_non_exhaustive();
+            let empty = (ty_def.is_struct() && ty_def.all_fields().next().is_none())
+                || (ty_def.is_enum() && ty_def.variants().is_empty());
+
+            !non_exhaustive && empty
+        }
+        ty::Tuple(tys) => tys.is_empty(),
+        _ => false,
+    }
+}
+
 /// Check if this enum can be safely exported based on the "nullable pointer optimization". If it
 /// can, return the type that `ty` can be safely converted to, otherwise return `None`.
 /// Currently restricted to function pointers, boxes, references, `core::num::NonZero`,
@@ -1115,6 +1141,22 @@ pub(crate) fn repr_nullable_ptr<'tcx>(
         let field_ty = match &ty_def.variants().raw[..] {
             [var_one, var_two] => match (&var_one.fields.raw[..], &var_two.fields.raw[..]) {
                 ([], [field]) | ([field], []) => field.ty(tcx, args),
+                ([field1], [field2]) => {
+                    if !tcx.features().result_ffi_guarantees {
+                        return None;
+                    }
+
+                    let ty1 = field1.ty(tcx, args);
+                    let ty2 = field2.ty(tcx, args);
+
+                    if is_niche_optimization_candidate(tcx, param_env, ty1) {
+                        ty2
+                    } else if is_niche_optimization_candidate(tcx, param_env, ty2) {
+                        ty1
+                    } else {
+                        return None;
+                    }
+                }
                 _ => return None,
             },
             _ => return None,
@@ -1200,7 +1242,6 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         args: GenericArgsRef<'tcx>,
     ) -> FfiResult<'tcx> {
         use FfiResult::*;
-
         let transparent_with_all_zst_fields = if def.repr().transparent() {
             if let Some(field) = transparent_newtype_field(self.cx.tcx, variant) {
                 // Transparent newtypes have at most one non-ZST field which needs to be checked..
@@ -1327,27 +1368,29 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                             return FfiSafe;
                         }
 
+                        if def.is_variant_list_non_exhaustive() && !def.did().is_local() {
+                            return FfiUnsafe {
+                                ty,
+                                reason: fluent::lint_improper_ctypes_non_exhaustive,
+                                help: None,
+                            };
+                        }
+
                         // Check for a repr() attribute to specify the size of the
                         // discriminant.
                         if !def.repr().c() && !def.repr().transparent() && def.repr().int.is_none()
                         {
-                            // Special-case types like `Option<extern fn()>`.
-                            if repr_nullable_ptr(self.cx.tcx, self.cx.param_env, ty, self.mode)
-                                .is_none()
+                            // Special-case types like `Option<extern fn()>` and `Result<extern fn(), ()>`
+                            if let Some(ty) =
+                                repr_nullable_ptr(self.cx.tcx, self.cx.param_env, ty, self.mode)
                             {
-                                return FfiUnsafe {
-                                    ty,
-                                    reason: fluent::lint_improper_ctypes_enum_repr_reason,
-                                    help: Some(fluent::lint_improper_ctypes_enum_repr_help),
-                                };
+                                return self.check_type_for_ffi(cache, ty);
                             }
-                        }
 
-                        if def.is_variant_list_non_exhaustive() && !def.did().is_local() {
                             return FfiUnsafe {
                                 ty,
-                                reason: fluent::lint_improper_ctypes_non_exhaustive,
-                                help: None,
+                                reason: fluent::lint_improper_ctypes_enum_repr_reason,
+                                help: Some(fluent::lint_improper_ctypes_enum_repr_help),
                             };
                         }
 

compiler/rustc_span/src/symbol.rs

@@ -1509,6 +1509,7 @@ symbols! {
         require,
         residual,
         result,
+        result_ffi_guarantees,
         resume,
         return_position_impl_trait_in_trait,
         return_type_notation,

src/doc/unstable-book/src/language-features/result-ffi-guarantees.md

@@ -0,0 +1,14 @@
+# `result_ffi_guarantees`
+
+The tracking issue for this feature is: [#110503]
+
+[#110503]: https://github.com/rust-lang/rust/issues/110503
+
+------------------------
+
+This feature adds the possibility of using `Result<T, E>` in FFI if T's niche
+value can be used to describe E or vise-versa.
+
+See [RFC 3391] for more information.
+
+[RFC 3391]: https://github.com/rust-lang/rfcs/blob/master/text/3391-result_ffi_guarantees.md

tests/ui/feature-gates/feature-gate-result_ffi_guarantees.rs

@@ -0,0 +1,99 @@
+#![allow(dead_code)]
+#![deny(improper_ctypes)]
+#![feature(ptr_internals)]
+
+use std::num;
+
+enum Z {}
+
+#[repr(transparent)]
+struct TransparentStruct<T>(T, std::marker::PhantomData<Z>);
+
+#[repr(transparent)]
+enum TransparentEnum<T> {
+    Variant(T, std::marker::PhantomData<Z>),
+}
+
+struct NoField;
+
+extern "C" {
+    fn result_ref_t(x: Result<&'static u8, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_fn_t(x: Result<extern "C" fn(), ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonnull_t(x: Result<std::ptr::NonNull<u8>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_unique_t(x: Result<std::ptr::Unique<u8>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u8_t(x: Result<num::NonZero<u8>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u16_t(x: Result<num::NonZero<u16>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u32_t(x: Result<num::NonZero<u32>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u64_t(x: Result<num::NonZero<u64>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_usize_t(x: Result<num::NonZero<usize>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i8_t(x: Result<num::NonZero<i8>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i16_t(x: Result<num::NonZero<i16>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i32_t(x: Result<num::NonZero<i32>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i64_t(x: Result<num::NonZero<i64>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_isize_t(x: Result<num::NonZero<isize>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_transparent_struct_t(x: Result<TransparentStruct<num::NonZero<u8>>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_transparent_enum_t(x: Result<TransparentEnum<num::NonZero<u8>>, ()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_phantom_t(x: Result<num::NonZero<u8>, std::marker::PhantomData<()>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_1zst_exhaustive_no_variant_t(x: Result<num::NonZero<u8>, Z>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_1zst_exhaustive_no_field_t(x: Result<num::NonZero<u8>, NoField>);
+    //~^ ERROR `extern` block uses type `Result
+
+    fn result_ref_e(x: Result<(), &'static u8>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_fn_e(x: Result<(), extern "C" fn()>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonnull_e(x: Result<(), std::ptr::NonNull<u8>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_unique_e(x: Result<(), std::ptr::Unique<u8>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u8_e(x: Result<(), num::NonZero<u8>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u16_e(x: Result<(), num::NonZero<u16>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u32_e(x: Result<(), num::NonZero<u32>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_u64_e(x: Result<(), num::NonZero<u64>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_usize_e(x: Result<(), num::NonZero<usize>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i8_e(x: Result<(), num::NonZero<i8>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i16_e(x: Result<(), num::NonZero<i16>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i32_e(x: Result<(), num::NonZero<i32>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_i64_e(x: Result<(), num::NonZero<i64>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_nonzero_isize_e(x: Result<(), num::NonZero<isize>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_transparent_struct_e(x: Result<(), TransparentStruct<num::NonZero<u8>>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_transparent_enum_e(x: Result<(), TransparentEnum<num::NonZero<u8>>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_phantom_e(x: Result<num::NonZero<u8>, std::marker::PhantomData<()>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_1zst_exhaustive_no_variant_e(x: Result<Z, num::NonZero<u8>>);
+    //~^ ERROR `extern` block uses type `Result
+    fn result_1zst_exhaustive_no_field_e(x: Result<NoField, num::NonZero<u8>>);
+    //~^ ERROR `extern` block uses type `Result
+}
+
+pub fn main() {}