ABI compat check: detect unadjusted ABI mismatches

Author: RalfJung

compiler/rustc_abi/src/lib.rs

@@ -1700,7 +1700,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
 
     /// Checks if these two `Layout` are equal enough to be considered "the same for all function
     /// call ABIs". Note however that real ABIs depend on more details that are not reflected in the
-    /// `Layout`; the `PassMode` need to be compared as well.
+    /// `Layout`; the `PassMode` need to be compared as well. Also note that we assume
+    /// aggregates are passed via `PassMode::Indirect` or `PassMode::Cast`; more strict
+    /// checks would otherwise be required.
     pub fn eq_abi(&self, other: &Self) -> bool {
         // The one thing that we are not capturing here is that for unsized types, the metadata must
         // also have the same ABI, and moreover that the same metadata leads to the same size. The

compiler/rustc_target/src/abi/call/mod.rs

@@ -745,10 +745,25 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
 
     /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all
     /// function call ABIs".
-    pub fn eq_abi(&self, other: &Self) -> bool {
+    pub fn eq_abi(&self, other: &Self) -> bool
+    where
+        Ty: PartialEq,
+    {
         // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look
         // at the type.
-        self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode)
+        self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode) && {
+            // `fn_arg_sanity_check` accepts `PassMode::Direct` for some aggregates.
+            // That elevates any type difference to an ABI difference since we just use the
+            // full Rust type as the LLVM argument/return type.
+            if matches!(self.mode, PassMode::Direct(..))
+                && matches!(self.layout.abi, Abi::Aggregate { .. })
+            {
+                // For aggregates in `Direct` mode to be compatible, the types need to be equal.
+                self.layout.ty == other.layout.ty
+            } else {
+                true
+            }
+        }
     }
 }
 

compiler/rustc_target/src/spec/abi/mod.rs

@@ -53,6 +53,8 @@ pub enum Abi {
     },
     RustIntrinsic,
     RustCall,
+    /// *Not* a stable ABI, just directly use the Rust types to describe the ABI for LLVM. Even
+    /// normally ABI-compatible Rust types can become ABI-incompatible with this ABI!
     Unadjusted,
     /// For things unlikely to be called, where reducing register pressure in
     /// `extern "Rust"` callers is worth paying extra cost in the callee.

tests/ui/abi/compatibility.rs

@@ -40,9 +40,10 @@
 //@[loongarch64] compile-flags: --target loongarch64-unknown-linux-gnu
 //@[loongarch64] needs-llvm-components: loongarch
 //@[loongarch64] min-llvm-version: 18
-//@ revisions: wasm
-//@[wasm] compile-flags: --target wasm32-unknown-unknown
-//@[wasm] needs-llvm-components: webassembly
+//FIXME: wasm is disabled due to <https://github.com/rust-lang/rust/issues/115666>.
+//FIXME @ revisions: wasm
+//FIXME @[wasm] compile-flags: --target wasm32-unknown-unknown
+//FIXME @[wasm] needs-llvm-components: webassembly
 //@ revisions: wasip1
 //@[wasip1] compile-flags: --target wasm32-wasip1
 //@[wasip1] needs-llvm-components: webassembly