ICE: `rust abi shouldn't use on_stack`

[matthiaskrgr]

auto-reduced (treereduce-rust):

//@compile-flags: -Clink-dead-code=true
#![feature(rust_cold_cc)]
#[repr(C)]
struct F1(*const ());
#[repr(C)]
struct F2(*const ());
#[repr(C)]
struct F3(*const ());
#[repr(C)]
struct F4(*const ());
#[repr(C)]
struct F5(*const ());
#[repr(C)]
struct F6(*const ());

#[repr(C)]
struct B {
    f1: F1,
    f2: F2,
    f3: F3,
    f4: F4,
    f5: F5,
    f6: F6,
}

extern "rust-cold" fn foo(_: B) {}

fn main() {}

original:

//@ check-pass

#![recursion_limit = "5"]
#![allow(unused)]
#![deny(improper_ctypes)]

#[repr(C)]
struct F1(*const ());
#[repr(C)]
struct F2(*const ());
#[repr(C)]
struct F3(*const ());
#[repr(C)]
struct F4(*const ());
#[repr(C)]
struct F5(*const ());
#[repr(C)]
struct F6(*const ());

#[repr(C)]
struct B {
    f1: F1,
    f2: F2,
    f3: F3,
    f4: F4,
    f5: F5,
    f6: F6,
}

extern "rust-cold" fn foo(_: B) {}

fn main() {}

Version information

rustc 1.84.0-nightly (f7273e004 2024-11-12)
binary: rustc
commit-hash: f7273e0044ad8f35ad27282e4ab776af50b61a54
commit-date: 2024-11-12
host: x86_64-unknown-linux-gnu
release: 1.84.0-nightly
LLVM version: 19.1.3

Possibly related line of code:

rust/compiler/rustc_ty_utils/src/abi.rs

Lines 471 to 483 in f7273e0

	if arg.layout.is_zst() {
	// Casting closures to function pointers depends on ZST closure types being
	// omitted entirely in the calling convention.
	assert!(arg.is_ignore());
	}
	if let PassMode::Indirect { on_stack, .. } = arg.mode {
	assert!(!on_stack, "rust abi shouldn't use on_stack");
	}
	}
	match &arg.mode {
	PassMode::Ignore => {
	assert!(arg.layout.is_zst() || arg.layout.is_uninhabited());

Command:
/home/matthias/.rustup/toolchains/master/bin/rustc -Zcrate-attr=feature(rust_cold_cc) -Clink-dead-code=true

Program output

warning: struct `F1` is never constructed
 --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:2:8
  |
2 | struct F1(*const ());
  |        ^^
  |
  = note: `#[warn(dead_code)]` on by default

warning: struct `F2` is never constructed
 --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:4:8
  |
4 | struct F2(*const ());
  |        ^^

warning: struct `F3` is never constructed
 --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:6:8
  |
6 | struct F3(*const ());
  |        ^^

warning: struct `F4` is never constructed
 --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:8:8
  |
8 | struct F4(*const ());
  |        ^^

warning: struct `F5` is never constructed
  --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:10:8
   |
10 | struct F5(*const ());
   |        ^^

warning: struct `F6` is never constructed
  --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:12:8
   |
12 | struct F6(*const ());
   |        ^^

warning: struct `B` is never constructed
  --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:15:8
   |
15 | struct B {
   |        ^

warning: function `foo` is never used
  --> /tmp/icemaker_global_tempdir.96zRzYL4bT05/rustc_testrunner_tmpdir_reporting.H7K5B09iYkG9/mvce.rs:24:23
   |
24 | extern "rust-cold" fn foo(_: B) {}
   |                       ^^^

thread 'rustc' panicked at compiler/rustc_ty_utils/src/abi.rs:477:17:
rust abi shouldn't use on_stack
stack backtrace:
   0:     0x7b42e5444d7a - <std::sys::backtrace::BacktraceLock::print::DisplayBacktrace as core::fmt::Display>::fmt::h89eef71006a12021
   1:     0x7b42e5c04126 - core::fmt::write::ha3e9ca569d22f3a9
   2:     0x7b42e70d5e91 - std::io::Write::write_fmt::h99880bc7e97ca82b
   3:     0x7b42e5444bd2 - std::sys::backtrace::BacktraceLock::print::hfc03349f2a3f19d7
   4:     0x7b42e54470d6 - std::panicking::default_hook::{{closure}}::h70f5ed1f326e175f
   5:     0x7b42e5446f20 - std::panicking::default_hook::hb18598c1d85282d8
   6:     0x7b42e44d5901 - std[1d66e2c2164e10e5]::panicking::update_hook::<alloc[15deac6fe0f616b3]::boxed::Box<rustc_driver_impl[d4843947ea36b84c]::install_ice_hook::{closure#0}>>::{closure#0}
   7:     0x7b42e54477e8 - std::panicking::rust_panic_with_hook::hd24a003b24cf5cbe
   8:     0x7b42e5447586 - std::panicking::begin_panic_handler::{{closure}}::hdc473c19107af62a
   9:     0x7b42e5445229 - std::sys::backtrace::__rust_end_short_backtrace::hac257b6aa77e4692
  10:     0x7b42e544727c - rust_begin_unwind
  11:     0x7b42e1ed0bc0 - core::panicking::panic_fmt::hb7c6bf9f04f7c675
  12:     0x7b42e5f03fb9 - rustc_ty_utils[35d0d47d2c9173c5]::abi::fn_abi_new_uncached
  13:     0x7b42e5eed80f - rustc_ty_utils[35d0d47d2c9173c5]::abi::fn_abi_of_instance
  14:     0x7b42e5eec043 - rustc_query_impl[79133480aa573a39]::plumbing::__rust_begin_short_backtrace::<rustc_query_impl[79133480aa573a39]::query_impl::fn_abi_of_instance::dynamic_query::{closure#2}::{closure#0}, rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 16usize]>>
  15:     0x7b42e5ee9fe3 - rustc_query_system[f466237e30d2c716]::query::plumbing::try_execute_query::<rustc_query_impl[79133480aa573a39]::DynamicConfig<rustc_query_system[f466237e30d2c716]::query::caches::DefaultCache<rustc_middle[6fd9b5ee8acc4b83]::ty::ParamEnvAnd<(rustc_middle[6fd9b5ee8acc4b83]::ty::instance::Instance, &rustc_middle[6fd9b5ee8acc4b83]::ty::list::RawList<(), rustc_middle[6fd9b5ee8acc4b83]::ty::Ty>)>, rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 16usize]>>, false, false, false>, rustc_query_impl[79133480aa573a39]::plumbing::QueryCtxt, false>
  16:     0x7b42e5ee9bfa - rustc_query_impl[79133480aa573a39]::query_impl::fn_abi_of_instance::get_query_non_incr::__rust_end_short_backtrace
  17:     0x7b42e2e295ae - rustc_monomorphize[e87bc633dae94d23]::mono_checks::check_mono_item
  18:     0x7b42e63adeae - rustc_query_impl[79133480aa573a39]::plumbing::__rust_begin_short_backtrace::<rustc_query_impl[79133480aa573a39]::query_impl::check_mono_item::dynamic_query::{closure#2}::{closure#0}, rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 0usize]>>
  19:     0x7b42e63ad7ea - rustc_query_system[f466237e30d2c716]::query::plumbing::try_execute_query::<rustc_query_impl[79133480aa573a39]::DynamicConfig<rustc_query_system[f466237e30d2c716]::query::caches::DefaultCache<rustc_middle[6fd9b5ee8acc4b83]::ty::instance::Instance, rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 0usize]>>, false, false, false>, rustc_query_impl[79133480aa573a39]::plumbing::QueryCtxt, false>
  20:     0x7b42e63ad493 - rustc_query_impl[79133480aa573a39]::query_impl::check_mono_item::get_query_non_incr::__rust_end_short_backtrace
  21:     0x7b42e63b47dd - rustc_monomorphize[e87bc633dae94d23]::collector::collect_items_rec::{closure#0}
  22:     0x7b42e6371c43 - rustc_monomorphize[e87bc633dae94d23]::collector::collect_items_rec
  23:     0x7b42e6369ab4 - rustc_monomorphize[e87bc633dae94d23]::partitioning::collect_and_partition_mono_items
  24:     0x7b42e6c18d24 - rustc_query_impl[79133480aa573a39]::plumbing::__rust_begin_short_backtrace::<rustc_query_impl[79133480aa573a39]::query_impl::collect_and_partition_mono_items::dynamic_query::{closure#2}::{closure#0}, rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 24usize]>>
  25:     0x7b42e6c18d09 - <rustc_query_impl[79133480aa573a39]::query_impl::collect_and_partition_mono_items::dynamic_query::{closure#2} as core[b88d2412ee64a335]::ops::function::FnOnce<(rustc_middle[6fd9b5ee8acc4b83]::ty::context::TyCtxt, ())>>::call_once
  26:     0x7b42e6c188c9 - rustc_query_system[f466237e30d2c716]::query::plumbing::try_execute_query::<rustc_query_impl[79133480aa573a39]::DynamicConfig<rustc_query_system[f466237e30d2c716]::query::caches::SingleCache<rustc_middle[6fd9b5ee8acc4b83]::query::erase::Erased<[u8; 24usize]>>, false, false, false>, rustc_query_impl[79133480aa573a39]::plumbing::QueryCtxt, false>
  27:     0x7b42e6c185e0 - rustc_query_impl[79133480aa573a39]::query_impl::collect_and_partition_mono_items::get_query_non_incr::__rust_end_short_backtrace
  28:     0x7b42e6b339c7 - <rustc_codegen_llvm[b1599222c2b690f]::LlvmCodegenBackend as rustc_codegen_ssa[b3f0f00cfe7398f]::traits::backend::CodegenBackend>::codegen_crate
  29:     0x7b42e6cbd367 - <rustc_interface[8115c8704cecad87]::queries::Linker>::codegen_and_build_linker
  30:     0x7b42e6ad606a - rustc_interface[8115c8704cecad87]::interface::run_compiler::<core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>, rustc_driver_impl[d4843947ea36b84c]::run_compiler::{closure#0}>::{closure#1}
  31:     0x7b42e6b3d950 - std[1d66e2c2164e10e5]::sys::backtrace::__rust_begin_short_backtrace::<rustc_interface[8115c8704cecad87]::util::run_in_thread_with_globals<rustc_interface[8115c8704cecad87]::util::run_in_thread_pool_with_globals<rustc_interface[8115c8704cecad87]::interface::run_compiler<core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>, rustc_driver_impl[d4843947ea36b84c]::run_compiler::{closure#0}>::{closure#1}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>::{closure#0}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>
  32:     0x7b42e6b3dd6b - <<std[1d66e2c2164e10e5]::thread::Builder>::spawn_unchecked_<rustc_interface[8115c8704cecad87]::util::run_in_thread_with_globals<rustc_interface[8115c8704cecad87]::util::run_in_thread_pool_with_globals<rustc_interface[8115c8704cecad87]::interface::run_compiler<core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>, rustc_driver_impl[d4843947ea36b84c]::run_compiler::{closure#0}>::{closure#1}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>::{closure#0}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>::{closure#0}::{closure#0}, core[b88d2412ee64a335]::result::Result<(), rustc_span[b00456e1c008159a]::ErrorGuaranteed>>::{closure#1} as core[b88d2412ee64a335]::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
  33:     0x7b42e6b3e839 - std::sys::pal::unix::thread::Thread::new::thread_start::hfef00d5abafeaf0a
  34:     0x7b42e841639d - <unknown>
  35:     0x7b42e849b49c - <unknown>
  36:                0x0 - <unknown>

error: the compiler unexpectedly panicked. this is a bug.

note: we would appreciate a bug report: https://github.com/rust-lang/rust/issues/new?labels=C-bug%2C+I-ICE%2C+T-compiler&template=ice.md

note: please make sure that you have updated to the latest nightly

note: rustc 1.84.0-nightly (f7273e004 2024-11-12) running on x86_64-unknown-linux-gnu

note: compiler flags: -Z crate-attr=feature(rust_cold_cc) -C link-dead-code=true -Z dump-mir-dir=dir

query stack during panic:
#0 [fn_abi_of_instance] computing call ABI of `foo`
#1 [check_mono_item] monomorphization-time checking
end of query stack
warning: 8 warnings emitted

@rustbot label +F-rust_cold_cc

[cyrgani]

Reduced:

#![feature(rust_cold_cc)]
pub extern "rust-cold" fn foo(_: [usize; 3]) {}

[bjorn3]

Looks like we are forgetting to do ABI adjustments for extern "rust-cold" at

rust/compiler/rustc_ty_utils/src/abi.rs

Line 701 in ec239b8

if abi == ExternAbi::Rust || abi == ExternAbi::RustCall || abi == ExternAbi::RustIntrinsic {

[matthiaskrgr]

bisection points to #132729

[bjorn3]

Yeah, that PR made the ABI calculation sanity check a bit stricter causing it to catch this pre-existing bug.