Rollup merge of #124251 - scottmcm:unop-ptr-metadata, r=oli-obk

Add an intrinsic for `ptr::metadata`

The follow-up to #123840, so we can remove `PtrComponents` and `PtrRepr` from libcore entirely (well, after a bootstrap update).

As discussed in <https://rust-lang.zulipchat.com/#narrow/stream/189540-t-compiler.2Fwg-mir-opt/topic/.60ptr_metadata.60.20in.20MIR/near/435637808>, this introduces `UnOp::PtrMetadata` taking a raw pointer and returning the associated metadata value.

By no longer going through a `union`, this should also help future PRs better optimize pointer operations.

r? ``@oli-obk``

compiler/rustc_codegen_cranelift/src/base.rs

@@ -616,22 +616,34 @@ fn codegen_stmt<'tcx>(
                 Rvalue::UnaryOp(un_op, ref operand) => {
                     let operand = codegen_operand(fx, operand);
                     let layout = operand.layout();
-                    let val = operand.load_scalar(fx);
                     let res = match un_op {
-                        UnOp::Not => match layout.ty.kind() {
-                            ty::Bool => {
-                                let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
-                                CValue::by_val(res, layout)
+                        UnOp::Not => {
+                            let val = operand.load_scalar(fx);
+                            match layout.ty.kind() {
+                                ty::Bool => {
+                                    let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0);
+                                    CValue::by_val(res, layout)
+                                }
+                                ty::Uint(_) | ty::Int(_) => {
+                                    CValue::by_val(fx.bcx.ins().bnot(val), layout)
+                                }
+                                _ => unreachable!("un op Not for {:?}", layout.ty),
                             }
-                            ty::Uint(_) | ty::Int(_) => {
-                                CValue::by_val(fx.bcx.ins().bnot(val), layout)
+                        }
+                        UnOp::Neg => {
+                            let val = operand.load_scalar(fx);
+                            match layout.ty.kind() {
+                                ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
+                                ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
+                                _ => unreachable!("un op Neg for {:?}", layout.ty),
                             }
-                            _ => unreachable!("un op Not for {:?}", layout.ty),
-                        },
-                        UnOp::Neg => match layout.ty.kind() {
-                            ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
-                            ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
-                            _ => unreachable!("un op Neg for {:?}", layout.ty),
+                        }
+                        UnOp::PtrMetadata => match layout.abi {
+                            Abi::Scalar(_) => CValue::zst(dest_layout),
+                            Abi::ScalarPair(_, _) => {
+                                CValue::by_val(operand.load_scalar_pair(fx).1, dest_layout)
+                            }
+                            _ => bug!("Unexpected `PtrToMetadata` operand: {operand:?}"),
                         },
                     };
                     lval.write_cvalue(fx, res);

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -100,7 +100,7 @@ pub(crate) fn codegen_const_value<'tcx>(
     assert!(layout.is_sized(), "unsized const value");
 
     if layout.is_zst() {
-        return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout);
+        return CValue::zst(layout);
     }
 
     match const_val {

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -95,6 +95,14 @@ impl<'tcx> CValue<'tcx> {
         CValue(CValueInner::ByValPair(value, extra), layout)
     }
 
+    /// Create an instance of a ZST
+    ///
+    /// The is represented by a dangling pointer of suitable alignment.
+    pub(crate) fn zst(layout: TyAndLayout<'tcx>) -> CValue<'tcx> {
+        assert!(layout.is_zst());
+        CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout)
+    }
+
     pub(crate) fn layout(&self) -> TyAndLayout<'tcx> {
         self.1
     }

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -565,6 +565,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 for elem in place_ref.projection.iter() {
                     match elem {
                         mir::ProjectionElem::Field(ref f, _) => {
+                            debug_assert!(
+                                !o.layout.ty.is_any_ptr(),
+                                "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
+                                 but tried to access field {f:?} of pointer {o:?}",
+                            );
                             o = o.extract_field(bx, f.index());
                         }
                         mir::ProjectionElem::Index(_)

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -480,6 +480,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             cg_base = match *elem {
                 mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()),
                 mir::ProjectionElem::Field(ref field, _) => {
+                    debug_assert!(
+                        !cg_base.layout.ty.is_any_ptr(),
+                        "Bad PlaceRef: destructing pointers should use cast/PtrMetadata, \
+                         but tried to access field {field:?} of pointer {cg_base:?}",
+                    );
                     cg_base.project_field(bx, field.index())
                 }
                 mir::ProjectionElem::OpaqueCast(ty) => {

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -623,19 +623,36 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             mir::Rvalue::UnaryOp(op, ref operand) => {
                 let operand = self.codegen_operand(bx, operand);
-                let lloperand = operand.immediate();
                 let is_float = operand.layout.ty.is_floating_point();
-                let llval = match op {
-                    mir::UnOp::Not => bx.not(lloperand),
+                let (val, layout) = match op {
+                    mir::UnOp::Not => {
+                        let llval = bx.not(operand.immediate());
+                        (OperandValue::Immediate(llval), operand.layout)
+                    }
                     mir::UnOp::Neg => {
-                        if is_float {
-                            bx.fneg(lloperand)
+                        let llval = if is_float {
+                            bx.fneg(operand.immediate())
+                        } else {
+                            bx.neg(operand.immediate())
+                        };
+                        (OperandValue::Immediate(llval), operand.layout)
+                    }
+                    mir::UnOp::PtrMetadata => {
+                        debug_assert!(operand.layout.ty.is_unsafe_ptr());
+                        let (_, meta) = operand.val.pointer_parts();
+                        assert_eq!(operand.layout.fields.count() > 1, meta.is_some());
+                        if let Some(meta) = meta {
+                            (OperandValue::Immediate(meta), operand.layout.field(self.cx, 1))
                         } else {
-                            bx.neg(lloperand)
+                            (OperandValue::ZeroSized, bx.cx().layout_of(bx.tcx().types.unit))
                         }
                     }
                 };
-                OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout }
+                debug_assert!(
+                    val.is_expected_variant_for_type(self.cx, layout),
+                    "Made wrong variant {val:?} for type {layout:?}",
+                );
+                OperandRef { val, layout }
             }
 
             mir::Rvalue::Discriminant(ref place) => {
@@ -718,8 +735,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let values = op.val.immediates_or_place().left_or_else(|p| {
                         bug!("Field {field_idx:?} is {p:?} making {layout:?}");
                     });
-                    inputs.extend(values);
                     let scalars = self.value_kind(op.layout).scalars().unwrap();
+                    debug_assert_eq!(values.len(), scalars.len());
+                    inputs.extend(values);
                     input_scalars.extend(scalars);
                 }
 

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -207,7 +207,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         assert!(cast_to.ty.is_unsafe_ptr());
         // Handle casting any ptr to raw ptr (might be a fat ptr).
         if cast_to.size == src.layout.size {
-            // Thin or fat pointer that just hast the ptr kind of target type changed.
+            // Thin or fat pointer that just has the ptr kind of target type changed.
             return Ok(ImmTy::from_immediate(**src, cast_to));
         } else {
             // Casting the metadata away from a fat ptr.

compiler/rustc_const_eval/src/interpret/operator.rs

@@ -9,7 +9,7 @@ use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::sym;
 use tracing::trace;
 
-use super::{err_ub, throw_ub, ImmTy, InterpCx, Machine};
+use super::{err_ub, throw_ub, ImmTy, InterpCx, Machine, MemPlaceMeta};
 
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> {
@@ -415,11 +415,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         use rustc_middle::mir::UnOp::*;
 
         let layout = val.layout;
-        let val = val.to_scalar();
         trace!("Running unary op {:?}: {:?} ({})", un_op, val, layout.ty);
 
         match layout.ty.kind() {
             ty::Bool => {
+                let val = val.to_scalar();
                 let val = val.to_bool()?;
                 let res = match un_op {
                     Not => !val,
@@ -428,6 +428,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 Ok(ImmTy::from_bool(res, *self.tcx))
             }
             ty::Float(fty) => {
+                let val = val.to_scalar();
                 // No NaN adjustment here, `-` is a bitwise operation!
                 let res = match (un_op, fty) {
                     (Neg, FloatTy::F32) => Scalar::from_f32(-val.to_f32()?),
@@ -436,8 +437,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 };
                 Ok(ImmTy::from_scalar(res, layout))
             }
-            _ => {
-                assert!(layout.ty.is_integral());
+            _ if layout.ty.is_integral() => {
+                let val = val.to_scalar();
                 let val = val.to_bits(layout.size)?;
                 let res = match un_op {
                     Not => self.truncate(!val, layout), // bitwise negation, then truncate
@@ -450,9 +451,28 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                         // Truncate to target type.
                         self.truncate(res, layout)
                     }
+                    _ => span_bug!(self.cur_span(), "Invalid integer op {:?}", un_op),
                 };
                 Ok(ImmTy::from_uint(res, layout))
             }
+            ty::RawPtr(..) => {
+                assert_eq!(un_op, PtrMetadata);
+                let (_, meta) = val.to_scalar_and_meta();
+                Ok(match meta {
+                    MemPlaceMeta::Meta(scalar) => {
+                        let ty = un_op.ty(*self.tcx, val.layout.ty);
+                        let layout = self.layout_of(ty)?;
+                        ImmTy::from_scalar(scalar, layout)
+                    }
+                    MemPlaceMeta::None => {
+                        let unit_layout = self.layout_of(self.tcx.types.unit)?;
+                        ImmTy::uninit(unit_layout)
+                    }
+                })
+            }
+            _ => {
+                bug!("Unexpected unary op argument {val:?}")
+            }
         }
     }
 }

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -130,6 +130,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
         | sym::is_val_statically_known
         | sym::ptr_mask
         | sym::aggregate_raw_ptr
+        | sym::ptr_metadata
         | sym::ub_checks
         | sym::fadd_algebraic
         | sym::fsub_algebraic
@@ -576,6 +577,7 @@ pub fn check_intrinsic_type(
             // This type check is not particularly useful, but the `where` bounds
             // on the definition in `core` do the heavy lifting for checking it.
             sym::aggregate_raw_ptr => (3, 1, vec![param(1), param(2)], param(0)),
+            sym::ptr_metadata => (2, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)),
 
             sym::ub_checks => (0, 1, Vec::new(), tcx.types.bool),
 

compiler/rustc_middle/src/mir/syntax.rs

@@ -1434,6 +1434,13 @@ pub enum UnOp {
     Not,
     /// The `-` operator for negation
     Neg,
+    /// Get the metadata `M` from a `*const/mut impl Pointee<Metadata = M>`.
+    ///
+    /// For example, this will give a `()` from `*const i32`, a `usize` from
+    /// `*mut [u8]`, or a pointer to a vtable from a `*const dyn Foo`.
+    ///
+    /// Allowed only in [`MirPhase::Runtime`]; earlier it's an intrinsic.
+    PtrMetadata,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]

compiler/rustc_middle/src/mir/tcx.rs

@@ -180,7 +180,10 @@ impl<'tcx> Rvalue<'tcx> {
                 let rhs_ty = rhs.ty(local_decls, tcx);
                 op.ty(tcx, lhs_ty, rhs_ty)
             }
-            Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, ref operand) => operand.ty(local_decls, tcx),
+            Rvalue::UnaryOp(op, ref operand) => {
+                let arg_ty = operand.ty(local_decls, tcx);
+                op.ty(tcx, arg_ty)
+            }
             Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx),
             Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
                 tcx.types.usize
@@ -282,6 +285,27 @@ impl<'tcx> BinOp {
     }
 }
 
+impl<'tcx> UnOp {
+    pub fn ty(&self, tcx: TyCtxt<'tcx>, arg_ty: Ty<'tcx>) -> Ty<'tcx> {
+        match self {
+            UnOp::Not | UnOp::Neg => arg_ty,
+            UnOp::PtrMetadata => {
+                let pointee_ty = arg_ty
+                    .builtin_deref(true)
+                    .unwrap_or_else(|| bug!("PtrMetadata of non-dereferenceable ty {arg_ty:?}"));
+                if pointee_ty.is_trivially_sized(tcx) {
+                    tcx.types.unit
+                } else {
+                    let Some(metadata_def_id) = tcx.lang_items().metadata_type() else {
+                        bug!("No metadata_type lang item while looking at {arg_ty:?}")
+                    };
+                    Ty::new_projection(tcx, metadata_def_id, [pointee_ty])
+                }
+            }
+        }
+    }
+}
+
 impl BorrowKind {
     pub fn to_mutbl_lossy(self) -> hir::Mutability {
         match self {

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -1579,8 +1579,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 let formatted_op = match op {
                     UnOp::Not => "!",
                     UnOp::Neg => "-",
+                    UnOp::PtrMetadata => "PtrMetadata",
                 };
                 let parenthesized = match ct.kind() {
+                    _ if op == UnOp::PtrMetadata => true,
                     ty::ConstKind::Expr(Expr::UnOp(c_op, ..)) => c_op != op,
                     ty::ConstKind::Expr(_) => true,
                     _ => false,

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -212,6 +212,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                 Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
             },
             @call(mir_len, args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
+            @call(mir_ptr_metadata, args) => Ok(Rvalue::UnaryOp(UnOp::PtrMetadata, self.parse_operand(args[0])?)),
             @call(mir_copy_for_deref, args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
             ExprKind::Borrow { borrow_kind, arg } => Ok(
                 Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -316,6 +316,23 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
 
                         terminator.kind = TerminatorKind::Goto { target };
                     }
+                    sym::ptr_metadata => {
+                        let Ok([ptr]) = <[_; 1]>::try_from(std::mem::take(args)) else {
+                            span_bug!(
+                                terminator.source_info.span,
+                                "Wrong number of arguments for ptr_metadata intrinsic",
+                            );
+                        };
+                        let target = target.unwrap();
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
+                    }
                     _ => {}
                 }
             }

compiler/rustc_mir_transform/src/promote_consts.rs

@@ -464,7 +464,7 @@ impl<'tcx> Validator<'_, 'tcx> {
             Rvalue::UnaryOp(op, operand) => {
                 match op {
                     // These operations can never fail.
-                    UnOp::Neg | UnOp::Not => {}
+                    UnOp::Neg | UnOp::Not | UnOp::PtrMetadata => {}
                 }
 
                 self.validate_operand(operand)?;

compiler/rustc_mir_transform/src/validate.rs

@@ -1109,6 +1109,16 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             ty::Int(..) | ty::Uint(..) | ty::Bool
                         );
                     }
+                    UnOp::PtrMetadata => {
+                        if !matches!(self.mir_phase, MirPhase::Runtime(_)) {
+                            // It would probably be fine to support this in earlier phases,
+                            // but at the time of writing it's only ever introduced from intrinsic lowering,
+                            // so earlier things can just `bug!` on it.
+                            self.fail(location, "PtrMetadata should be in runtime MIR only");
+                        }
+
+                        check_kinds!(a, "Cannot PtrMetadata non-pointer type {:?}", ty::RawPtr(..));
+                    }
                 }
             }
             Rvalue::ShallowInitBox(operand, _) => {

compiler/rustc_smir/src/rustc_internal/internal.rs

@@ -10,7 +10,7 @@ use rustc_span::Symbol;
 use stable_mir::abi::Layout;
 use stable_mir::mir::alloc::AllocId;
 use stable_mir::mir::mono::{Instance, MonoItem, StaticDef};
-use stable_mir::mir::{BinOp, Mutability, Place, ProjectionElem, Safety};
+use stable_mir::mir::{BinOp, Mutability, Place, ProjectionElem, Safety, UnOp};
 use stable_mir::ty::{
     Abi, AdtDef, Binder, BoundRegionKind, BoundTyKind, BoundVariableKind, ClosureKind, Const,
     DynKind, ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FloatTy, FnSig,
@@ -582,6 +582,18 @@ impl RustcInternal for BinOp {
     }
 }
 
+impl RustcInternal for UnOp {
+    type T<'tcx> = rustc_middle::mir::UnOp;
+
+    fn internal<'tcx>(&self, _tables: &mut Tables<'_>, _tcx: TyCtxt<'tcx>) -> Self::T<'tcx> {
+        match self {
+            UnOp::Not => rustc_middle::mir::UnOp::Not,
+            UnOp::Neg => rustc_middle::mir::UnOp::Neg,
+            UnOp::PtrMetadata => rustc_middle::mir::UnOp::PtrMetadata,
+        }
+    }
+}
+
 impl<T> RustcInternal for &T
 where
     T: RustcInternal,