Stack overflow in rustc using unboxed closure bounds

[MicahChalmer]

Attempting to compile this causes rustc to stack overflow:

fn g<F>(_:F) where F:FnOnce(Option<F>) {}

fn main() {
    g(|_| {  });
}

There are other stack overflow issues with recursive traits that seem like they are probably related (#15477 seems like the closest match--maybe #12511, #11824, others?) But they are all about declaring new traits, and here I have it overflowing using only the built in unboxed closure traits. (FnOnce can be replaced with Fn or FnMut and it still reproduces the crash, but I can't produce it with anything not involving a closure.) The use of the closure's own type can be either in a parameter or in the return type--this produces a similar crash:

fn g<F>(_:F) where F:Fn() -> Option<F> {}

fn main() {
    g(|_| { None });
}

The crash only happens when you actually try to use the function g--just declaring it is accepted (with an appropriate warning about dead code.)

Running a backtrace I found the following frames repeated until overflow:

    frame #6862: 0x0000000100b8248b librustc-4e7c5e5c.dylib`vec::Vec$LT$T$GT$.FromIterator$LT$T$GT$::from_iter::h124704758860088298 + 251
    frame #6863: 0x0000000100d761da librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::push_sig_to_string::h275a3a004b459dc4aww + 122
    frame #6864: 0x0000000100d75ea3 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::closure_to_string::hdda4570fec3a76c1wsw + 1427
    frame #6865: 0x0000000100a6e546 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::hcd6011aeb96823a6npw + 2534
    frame #6866: 0x0000000100d74ac6 librustc-4e7c5e5c.dylib`util::ppaux::parameterized::he217971be53b1ff5hMw + 1894
    frame #6867: 0x0000000100a6dcf9 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::hcd6011aeb96823a6npw + 409
    frame #6868: 0x0000000100a6e670 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::hcd6011aeb96823a6npw + 2832

and started with this:

    frame #6869: 0x0000000100b8248b librustc-4e7c5e5c.dylib`vec::Vec$LT$T$GT$.FromIterator$LT$T$GT$::from_iter::h124704758860088298 + 251
    frame #6870: 0x0000000100d761da librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::push_sig_to_string::h275a3a004b459dc4aww + 122
    frame #6871: 0x0000000100d75ea3 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::closure_to_string::hdda4570fec3a76c1wsw + 1427
    frame #6872: 0x0000000100a6e546 librustc-4e7c5e5c.dylib`util::ppaux::ty_to_string::hcd6011aeb96823a6npw + 2534
    frame #6873: 0x0000000100adfafc librustc-4e7c5e5c.dylib`util::ppaux::ty..TyS$LT$$u{27}tcx$GT$.Repr$LT$$u{27}tcx$GT$::repr::h4bf84b3400060659c6w + 60
    frame #6874: 0x000000010023f7cd librustc_trans-4e7c5e5c.dylib`trans::type_of::llvm_type_name::h1ac07dfb7b8f66d1ikp + 333
    frame #6875: 0x0000000100193552 librustc_trans-4e7c5e5c.dylib`trans::type_of::type_of::h464e2cce4cb54ac5m3o + 1426
    frame #6876: 0x000000010021088a librustc_trans-4e7c5e5c.dylib`trans::type_of::arg_type_of::h3beeef01d22e7f8bi2o + 106
    frame #6877: 0x000000010023e6ab librustc_trans-4e7c5e5c.dylib`trans::type_of::type_of_rust_fn::h206db2d255225a44TRo + 1915
    frame #6878: 0x0000000100245cd3 librustc_trans-4e7c5e5c.dylib`trans::base::decl_rust_fn::hd5446512607c304fL3r + 1555
    frame #6879: 0x00000001001cc294 librustc_trans-4e7c5e5c.dylib`trans::closure::get_or_create_declaration_if_unboxed_closure::hb28256a89df11ddckdy + 916
    frame #6880: 0x00000001001d2ef9 librustc_trans-4e7c5e5c.dylib`trans::expr::trans_rvalue_dps_unadjusted::h02efc13db25728504Ti + 5241
    frame #6881: 0x00000001001d1390 librustc_trans-4e7c5e5c.dylib`trans::expr::trans_unadjusted::h521c3e1191209f01Xki + 1344
    frame #6882: 0x000000010018bf46 librustc_trans-4e7c5e5c.dylib`trans::expr::trans::hd24c3b6c4b22a757nDh + 374
    frame #6883: 0x000000010019ce51 librustc_trans-4e7c5e5c.dylib`trans::callee::trans_args::h5c9c4058893bf80eGdh + 2129
    frame #6884: 0x00000001001cdc48 librustc_trans-4e7c5e5c.dylib`trans::callee::trans_call_inner::h15833327081339630702 + 3928
    frame #6885: 0x00000001001d3ae3 librustc_trans-4e7c5e5c.dylib`trans::expr::trans_rvalue_dps_unadjusted::h02efc13db25728504Ti + 8291
    frame #6886: 0x000000010018af9b librustc_trans-4e7c5e5c.dylib`trans::expr::trans_into::h1821ff8f2b47a0e9Tzh + 779
    frame #6887: 0x000000010018a2f8 librustc_trans-4e7c5e5c.dylib`trans::controlflow::trans_stmt_semi::hddfc9e0ddad39be5j3d + 440
    frame #6888: 0x000000010018b7de librustc_trans-4e7c5e5c.dylib`trans::controlflow::trans_block::h09ac232cbc7d52d4a4d + 1406
    frame #6889: 0x000000010025408f librustc_trans-4e7c5e5c.dylib`trans::base::trans_closure::h28cbb624b7c481c3R0t + 9871
    frame #6890: 0x00000001001771b7 librustc_trans-4e7c5e5c.dylib`trans::base::trans_fn::h73ff98724898bdfaubu + 919
    frame #6891: 0x000000010017285b librustc_trans-4e7c5e5c.dylib`trans::base::trans_item::ha984c77b599e4a24Pyu + 1163
    frame #6892: 0x000000010025a20c librustc_trans-4e7c5e5c.dylib`trans::base::trans_crate::he1ccece86ed431c3wuv + 5804
    frame #6893: 0x0000000100025774 librustc_driver-4e7c5e5c.dylib`driver::phase_4_translate_to_llvm::hc30a4046e6864f5auOa + 1044
    frame #6894: 0x000000010000711a librustc_driver-4e7c5e5c.dylib`driver::compile_input::h34c1c11c6899857fAba + 5706
    frame #6895: 0x00000001000c4716 librustc_driver-4e7c5e5c.dylib`run_compiler::h823f7b3bd358b28alac + 5222
    frame #6896: 0x00000001000c1a73 librustc_driver-4e7c5e5c.dylib`thunk::F.Invoke$LT$A$C$$u{20}R$GT$::invoke::h2577805882533131152 + 547
    frame #6897: 0x00000001000c08d8 librustc_driver-4e7c5e5c.dylib`rt::unwind::try::try_fn::h7233092926527859983 + 136
    frame #6898: 0x000000010397e2a9 libstd-4e7c5e5c.dylib`rust_try_inner + 9
    frame #6899: 0x000000010397e296 libstd-4e7c5e5c.dylib`rust_try + 6
    frame #6900: 0x00000001000c0f16 librustc_driver-4e7c5e5c.dylib`thunk::F.Invoke$LT$A$C$$u{20}R$GT$::invoke::h8652838825404240908 + 1046
    frame #6901: 0x0000000103901e92 libstd-4e7c5e5c.dylib`sys::thread::thread_start::h239f7d5cb0adebaaGKw + 146
    frame #6902: 0x00007fff9bbc32fc libsystem_pthread.dylib`_pthread_body + 131
    frame #6903: 0x00007fff9bbc3279 libsystem_pthread.dylib`_pthread_start + 176
    frame #6904: 0x00007fff9bbc14b1 libsystem_pthread.dylib`thread_start + 13

The presence of closure_to_string in the repeating stack frames makes me think it's something specific to closures, rather than just a special case of one of the already-filed issues affecting traits in general.

[Aatch]

This is effectively the same issue as #19596, as we currently don't detect or handle this case at all. The reason why you got a different message is because it's trying to print an error message.

Closing in favor of #19596,

[Aatch]

Actually, nevermind me, I misunderstood the issue. Re-opening.

[tamird]

Triage: still overflows.