From 00c4b72e6d6e25ff56eb59277e616ebd0f3cb310 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 29 Jun 2024 00:48:23 -0700 Subject: [PATCH] Avoid MIR bloat in inlining In 126578 we ended up with more binary size increases than expected. This change attempts to avoid inlining large things into small things, to avoid that kind of increase, in cases when top-down inlining will still be able to do that inlining later. --- .../rustc_mir_transform/src/cost_checker.rs | 31 +++ compiler/rustc_mir_transform/src/inline.rs | 50 +++- compiler/rustc_session/src/options.rs | 2 + library/alloc/src/raw_vec.rs | 17 ++ library/alloc/src/vec/mod.rs | 1 + ...inline_direct.Inline.after.panic-abort.mir | 22 ++ ...nline_direct.Inline.after.panic-unwind.mir | 22 ++ ...line_indirect.Inline.after.panic-abort.mir | 27 ++ ...ine_indirect.Inline.after.panic-unwind.mir | 27 ++ ...ic_not_inline.Inline.after.panic-abort.mir | 34 +++ ...c_not_inline.Inline.after.panic-unwind.mir | 34 +++ .../inline/inline_more_in_non_inline.rs | 46 ++++ .../inline_shims.drop.Inline.panic-abort.diff | 58 +---- ...erse_loop.PreCodegen.after.panic-abort.mir | 231 +++-------------- ...rse_loop.PreCodegen.after.panic-unwind.mir | 237 +++--------------- ...next_back.PreCodegen.after.panic-abort.mir | 195 +------------- ...ext_back.PreCodegen.after.panic-unwind.mir | 195 +------------- 17 files changed, 384 insertions(+), 845 deletions(-) create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir create mode 100644 tests/mir-opt/inline/inline_more_in_non_inline.rs diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 7e401b5482f3d..3333bebff3a6e 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -31,6 +31,37 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { CostChecker { tcx, param_env, callee_body, instance, penalty: 0, bonus: 0 } } + /// Add function-level costs not well-represented by the block-level costs. + /// + /// Needed because the `CostChecker` is used sometimes for just blocks, + /// and even the full `Inline` doesn't call `visit_body`, so there's nowhere + /// to put this logic in the visitor. + pub fn add_function_level_costs(&mut self) { + fn is_call_like(bbd: &BasicBlockData<'_>) -> bool { + use TerminatorKind::*; + match bbd.terminator().kind { + Call { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true, + + Goto { .. } + | SwitchInt { .. } + | UnwindResume + | UnwindTerminate(_) + | Return + | Unreachable => false, + + Yield { .. } | CoroutineDrop | FalseEdge { .. } | FalseUnwind { .. } => { + unreachable!() + } + } + } + + // If the only has one Call (or similar), inlining isn't increasing the total + // number of calls, so give extra encouragement to inlining that. + if self.callee_body.basic_blocks.iter().filter(|bbd| is_call_like(bbd)).count() == 1 { + self.bonus += CALL_PENALTY; + } + } + pub fn cost(&self) -> usize { usize::saturating_sub(self.penalty, self.bonus) } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 07482d0571a2f..6fa31c1174d0b 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -85,13 +85,18 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { } let param_env = tcx.param_env_reveal_all_normalized(def_id); + let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); let mut this = Inliner { tcx, param_env, - codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), + codegen_fn_attrs, history: Vec::new(), changed: false, + caller_is_inline_forwarder: matches!( + codegen_fn_attrs.inline, + InlineAttr::Hint | InlineAttr::Always + ) && body_is_forwarder(body), }; let blocks = START_BLOCK..body.basic_blocks.next_index(); this.process_blocks(body, blocks); @@ -111,6 +116,9 @@ struct Inliner<'tcx> { history: Vec, /// Indicates that the caller body has been modified. changed: bool, + /// Indicates that the caller is #[inline] and just calls another function, + /// and thus we can inline less into it as it'll be inlined itself. + caller_is_inline_forwarder: bool, } impl<'tcx> Inliner<'tcx> { @@ -485,7 +493,9 @@ impl<'tcx> Inliner<'tcx> { ) -> Result<(), &'static str> { let tcx = self.tcx; - let mut threshold = if cross_crate_inlinable { + let mut threshold = if self.caller_is_inline_forwarder { + self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30) + } else if cross_crate_inlinable { self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100) } else { self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50) @@ -504,6 +514,8 @@ impl<'tcx> Inliner<'tcx> { let mut checker = CostChecker::new(self.tcx, self.param_env, Some(callsite.callee), callee_body); + checker.add_function_level_costs(); + // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); @@ -1091,3 +1103,37 @@ fn try_instance_mir<'tcx>( } Ok(tcx.instance_mir(instance)) } + +fn body_is_forwarder(body: &Body<'_>) -> bool { + let TerminatorKind::Call { target, .. } = body.basic_blocks[START_BLOCK].terminator().kind + else { + return false; + }; + if let Some(target) = target { + let TerminatorKind::Return = body.basic_blocks[target].terminator().kind else { + return false; + }; + } + + let max_blocks = if !body.is_polymorphic { + 2 + } else if target.is_none() { + 3 + } else { + 4 + }; + if body.basic_blocks.len() > max_blocks { + return false; + } + + body.basic_blocks.iter_enumerated().all(|(bb, bb_data)| { + bb == START_BLOCK + || matches!( + bb_data.terminator().kind, + TerminatorKind::Return + | TerminatorKind::Drop { .. } + | TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + ) + }) +} diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 80f7ca544f3c5..2e4421d50e313 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1766,6 +1766,8 @@ options! { "enable LLVM inlining (default: yes)"), inline_mir: Option = (None, parse_opt_bool, [TRACKED], "enable MIR inlining (default: no)"), + inline_mir_forwarder_threshold: Option = (None, parse_opt_number, [TRACKED], + "inlining threshold when the caller is a simple forwarding function (default: 30)"), inline_mir_hint_threshold: Option = (None, parse_opt_number, [TRACKED], "inlining threshold for functions with inline hint (default: 100)"), inline_mir_preserve_debug: Option = (None, parse_opt_bool, [TRACKED], diff --git a/library/alloc/src/raw_vec.rs b/library/alloc/src/raw_vec.rs index 1134c7f833e2b..7b7dae5a057f0 100644 --- a/library/alloc/src/raw_vec.rs +++ b/library/alloc/src/raw_vec.rs @@ -429,6 +429,7 @@ impl RawVec { /// /// Aborts on OOM. #[cfg(not(no_global_oom_handling))] + #[inline] pub fn shrink_to_fit(&mut self, cap: usize) { if let Err(err) = self.shrink(cap) { handle_error(err); @@ -511,9 +512,25 @@ impl RawVec { } #[cfg(not(no_global_oom_handling))] + #[inline] fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> { assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity"); + // SAFETY: Just checked this isn't trying to grow + unsafe { self.shrink_unchecked(cap) } + } + /// `shrink`, but without the capacity check. + /// + /// This is split out so that `shrink` can inline the check, since it + /// optimizes out in things like `shrink_to_fit`, without needing to + /// also inline all this code, as doing that ends up failing the + /// `vec-shrink-panic` codegen test when `shrink_to_fit` ends up being too + /// big for LLVM to be willing to inline. + /// + /// # Safety + /// `cap <= self.capacity()` + #[cfg(not(no_global_oom_handling))] + unsafe fn shrink_unchecked(&mut self, cap: usize) -> Result<(), TryReserveError> { let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) }; // See current_memory() why this assert is here const { assert!(mem::size_of::() % mem::align_of::() == 0) }; diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 743429d26dbfb..f1706e31bb80a 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -1101,6 +1101,7 @@ impl Vec { /// ``` #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] + #[inline] pub fn shrink_to_fit(&mut self) { // The capacity is never less than the length, and there's nothing to do when // they are equal, so we can avoid the panic case in `RawVec::shrink_to_fit` diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..522f772c6f48e --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-abort.mir @@ -0,0 +1,22 @@ +// MIR for `marked_inline_direct` after Inline + +fn marked_inline_direct(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + _2 = call_twice(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..722b02eeba806 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_direct.Inline.after.panic-unwind.mir @@ -0,0 +1,22 @@ +// MIR for `marked_inline_direct` after Inline + +fn marked_inline_direct(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + _2 = call_twice(move _3) -> [return: bb1, unwind continue]; + } + + bb1: { + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..63b91cbce2d60 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-abort.mir @@ -0,0 +1,27 @@ +// MIR for `marked_inline_indirect` after Inline + +fn marked_inline_indirect(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined marked_inline_direct) { + let _4: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + _4 = call_twice(move _3) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..7c84e98dd516c --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.marked_inline_indirect.Inline.after.panic-unwind.mir @@ -0,0 +1,27 @@ +// MIR for `marked_inline_indirect` after Inline + +fn marked_inline_indirect(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined marked_inline_direct) { + let _4: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + _4 = call_twice(move _3) -> [return: bb1, unwind continue]; + } + + bb1: { + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir new file mode 100644 index 0000000000000..989014b8b476d --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-abort.mir @@ -0,0 +1,34 @@ +// MIR for `monomorphic_not_inline` after Inline + +fn monomorphic_not_inline(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined call_twice) { + let _4: (); + let _5: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + StorageLive(_5); + _4 = other_thing(_3) -> [return: bb2, unwind unreachable]; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } + + bb2: { + _5 = other_thing(move _3) -> [return: bb1, unwind unreachable]; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir new file mode 100644 index 0000000000000..bd200719bbbe9 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.monomorphic_not_inline.Inline.after.panic-unwind.mir @@ -0,0 +1,34 @@ +// MIR for `monomorphic_not_inline` after Inline + +fn monomorphic_not_inline(_1: i32) -> () { + debug x => _1; + let mut _0: (); + let _2: (); + let mut _3: i32; + scope 1 (inlined call_twice) { + let _4: (); + let _5: (); + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + StorageLive(_4); + StorageLive(_5); + _4 = other_thing(_3) -> [return: bb2, unwind continue]; + } + + bb1: { + StorageDead(_5); + StorageDead(_4); + StorageDead(_3); + StorageDead(_2); + _0 = const (); + return; + } + + bb2: { + _5 = other_thing(move _3) -> [return: bb1, unwind continue]; + } +} diff --git a/tests/mir-opt/inline/inline_more_in_non_inline.rs b/tests/mir-opt/inline/inline_more_in_non_inline.rs new file mode 100644 index 0000000000000..5968b97047096 --- /dev/null +++ b/tests/mir-opt/inline/inline_more_in_non_inline.rs @@ -0,0 +1,46 @@ +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY +//@ compile-flags: -O --crate-type lib + +// To avoid MIR blow-up, don't inline large callees into simple shim callers, +// but *do* inline other trivial things. + +extern "Rust" { + fn other_thing(x: i32); +} + +#[inline] +unsafe fn call_twice(x: i32) { + unsafe { + other_thing(x); + other_thing(x); + } +} + +// EMIT_MIR inline_more_in_non_inline.monomorphic_not_inline.Inline.after.mir +#[no_mangle] +pub unsafe fn monomorphic_not_inline(x: i32) { + // CHECK-LABEL: monomorphic_not_inline + // CHECK: other_thing + // CHECK: other_thing + unsafe { call_twice(x) }; +} + +// EMIT_MIR inline_more_in_non_inline.marked_inline_direct.Inline.after.mir +#[inline] +pub unsafe fn marked_inline_direct(x: i32) { + // CHECK-LABEL: marked_inline_direct + // CHECK-NOT: other_thing + // CHECK: call_twice + // CHECK-NOT: other_thing + unsafe { call_twice(x) }; +} + +// EMIT_MIR inline_more_in_non_inline.marked_inline_indirect.Inline.after.mir +#[inline] +pub unsafe fn marked_inline_indirect(x: i32) { + // CHECK-LABEL: marked_inline_indirect + // CHECK-NOT: other_thing + // CHECK: call_twice + // CHECK-NOT: other_thing + unsafe { marked_inline_direct(x) }; +} diff --git a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff index 45ce933a55ad2..2a36ccaab110a 100644 --- a/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff +++ b/tests/mir-opt/inline/inline_shims.drop.Inline.panic-abort.diff @@ -11,30 +11,10 @@ + scope 1 (inlined std::ptr::drop_in_place::> - shim(Some(Vec))) { + let mut _6: &mut std::vec::Vec; + let mut _7: (); -+ scope 2 (inlined as Drop>::drop) { -+ let mut _8: *mut [A]; -+ let mut _9: *mut A; -+ let mut _10: usize; -+ scope 3 (inlined Vec::::as_mut_ptr) { -+ let mut _11: &alloc::raw_vec::RawVec; -+ scope 4 (inlined alloc::raw_vec::RawVec::::ptr) { -+ let mut _13: std::ptr::NonNull; -+ scope 5 (inlined Unique::::as_ptr) { -+ scope 6 (inlined NonNull::::as_ptr) { -+ let mut _12: *const A; -+ } -+ } -+ } -+ } -+ scope 7 (inlined slice_from_raw_parts_mut::) { -+ scope 8 (inlined std::ptr::from_raw_parts_mut::<[A], A>) { -+ } -+ } -+ } + } -+ scope 9 (inlined std::ptr::drop_in_place::> - shim(Some(Option))) { -+ let mut _14: isize; -+ let mut _15: isize; ++ scope 2 (inlined std::ptr::drop_in_place::> - shim(Some(Option))) { ++ let mut _8: isize; ++ let mut _9: isize; + } bb0: { @@ -45,24 +25,7 @@ + StorageLive(_6); + StorageLive(_7); + _6 = &mut (*_4); -+ StorageLive(_8); -+ StorageLive(_9); -+ StorageLive(_11); -+ _11 = &((*_6).0: alloc::raw_vec::RawVec); -+ StorageLive(_13); -+ _13 = ((((*_6).0: alloc::raw_vec::RawVec).0: std::ptr::Unique).0: std::ptr::NonNull); -+ StorageLive(_12); -+ _12 = (_13.0: *const A); -+ _9 = move _12 as *mut A (PtrToPtr); -+ StorageDead(_12); -+ StorageDead(_13); -+ StorageDead(_11); -+ StorageLive(_10); -+ _10 = ((*_6).1: usize); -+ _8 = *mut [A] from (_9, _10); -+ StorageDead(_10); -+ StorageDead(_9); -+ _7 = std::ptr::drop_in_place::<[A]>(move _8) -> [return: bb2, unwind unreachable]; ++ _7 = as Drop>::drop(move _6) -> [return: bb2, unwind unreachable]; } bb1: { @@ -73,20 +36,19 @@ StorageLive(_5); _5 = _2; - _0 = std::ptr::drop_in_place::>(move _5) -> [return: bb2, unwind unreachable]; -+ StorageLive(_14); -+ StorageLive(_15); -+ _14 = discriminant((*_5)); -+ switchInt(move _14) -> [0: bb3, otherwise: bb4]; ++ StorageLive(_8); ++ StorageLive(_9); ++ _8 = discriminant((*_5)); ++ switchInt(move _8) -> [0: bb3, otherwise: bb4]; } bb2: { -+ StorageDead(_8); + drop(((*_4).0: alloc::raw_vec::RawVec)) -> [return: bb1, unwind unreachable]; + } + + bb3: { -+ StorageDead(_15); -+ StorageDead(_14); ++ StorageDead(_9); ++ StorageDead(_8); StorageDead(_5); return; + } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir index 79c5bcfe9cd5c..fbb887fe76a59 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-abort.mir @@ -7,90 +7,19 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _11: std::slice::Iter<'_, T>; let mut _12: std::iter::Rev>; let mut _13: std::iter::Rev>; - let mut _37: std::option::Option<&T>; - let mut _39: &impl Fn(&T); - let mut _40: (&T,); - let _41: (); + let mut _15: std::option::Option<&T>; + let mut _16: isize; + let mut _18: &impl Fn(&T); + let mut _19: (&T,); + let _20: (); scope 1 { debug iter => _13; - let _38: &T; + let _17: &T; scope 2 { - debug x => _38; + debug x => _17; } scope 17 (inlined > as Iterator>::next) { - scope 18 (inlined as DoubleEndedIterator>::next_back) { - let mut _14: *const *const T; - let mut _15: *const std::ptr::NonNull; - let mut _20: bool; - let mut _21: *const T; - let _36: &T; - scope 19 { - let _16: std::ptr::NonNull; - let _22: usize; - scope 20 { - } - scope 21 { - scope 25 (inlined as PartialEq>::eq) { - let mut _17: std::ptr::NonNull; - scope 26 (inlined NonNull::::as_ptr) { - let mut _18: *const T; - } - scope 27 (inlined NonNull::::as_ptr) { - let mut _19: *const T; - } - } - } - scope 22 (inlined std::ptr::const_ptr::::addr) { - scope 23 (inlined std::ptr::const_ptr::::cast::<()>) { - } - } - scope 24 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) { - let _29: std::ptr::NonNull; - scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) { - let mut _23: *mut *const T; - let mut _24: *mut std::ptr::NonNull; - let mut _25: std::ptr::NonNull; - let mut _28: std::ptr::NonNull; - let mut _30: *mut *const T; - let mut _31: *mut usize; - let mut _32: usize; - let mut _33: usize; - scope 30 { - scope 31 { - } - scope 32 { - scope 35 (inlined NonNull::::sub) { - scope 36 (inlined core::num::::unchecked_neg) { - scope 37 (inlined core::ub_checks::check_language_ub) { - scope 38 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 39 (inlined NonNull::::offset) { - let mut _26: *const T; - let mut _27: *const T; - } - } - } - scope 33 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 34 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 40 (inlined NonNull::::as_ref::<'_>) { - let mut _34: std::ptr::NonNull; - scope 41 (inlined NonNull::::as_ptr) { - let mut _35: *const T; - } - scope 42 (inlined std::ptr::mut_ptr::::cast_const) { - } - } - } - } + let mut _14: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -178,147 +107,45 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_37); - StorageLive(_22); - StorageLive(_21); - StorageLive(_16); - StorageLive(_36); - StorageLive(_20); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6]; + StorageLive(_15); + StorageLive(_14); + _14 = &mut (_13.0: std::slice::Iter<'_, T>); + _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind unreachable]; } bb5: { - StorageLive(_15); - StorageLive(_14); - _14 = &raw const ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _15 = _14 as *const std::ptr::NonNull (PtrToPtr); StorageDead(_14); - _16 = (*_15); - StorageDead(_15); - StorageLive(_18); - StorageLive(_19); - StorageLive(_17); - _17 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - _18 = (_17.0: *const T); - StorageDead(_17); - _19 = (_16.0: *const T); - _20 = Eq(_18, _19); - StorageDead(_19); - StorageDead(_18); - goto -> bb7; + _16 = discriminant(_15); + switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - _21 = ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _22 = _21 as usize (Transmute); - _20 = Eq(_22, const 0_usize); - goto -> bb7; + StorageDead(_15); + StorageDead(_13); + drop(_2) -> [return: bb7, unwind unreachable]; } bb7: { - switchInt(move _20) -> [0: bb8, otherwise: bb16]; + return; } bb8: { - StorageLive(_35); - StorageLive(_29); - StorageLive(_31); - StorageLive(_24); - switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb13]; + _17 = ((_15 as Some).0: &T); + StorageLive(_18); + _18 = &_2; + StorageLive(_19); + _19 = (_17,); + _20 = >::call(move _18, move _19) -> [return: bb9, unwind unreachable]; } bb9: { - StorageLive(_23); - _23 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _24 = _23 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_23); - StorageLive(_28); - _25 = (*_24); - switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11]; - } - - bb10: { - StorageLive(_27); - StorageLive(_26); - _26 = (_25.0: *const T); - _27 = Offset(move _26, const -1_isize); - StorageDead(_26); - _28 = NonNull:: { pointer: move _27 }; - StorageDead(_27); - goto -> bb12; - } - - bb11: { - _28 = _25; - goto -> bb12; - } - - bb12: { - (*_24) = move _28; - StorageDead(_28); - _29 = (*_24); - goto -> bb14; - } - - bb13: { - StorageLive(_30); - _30 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _31 = _30 as *mut usize (PtrToPtr); - StorageDead(_30); - StorageLive(_33); - StorageLive(_32); - _32 = (*_31); - _33 = SubUnchecked(move _32, const 1_usize); - StorageDead(_32); - (*_31) = move _33; - StorageDead(_33); - _29 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - goto -> bb14; - } - - bb14: { - StorageDead(_24); - StorageDead(_31); - StorageLive(_34); - _34 = _29; - _35 = (_34.0: *const T); - StorageDead(_34); - _36 = &(*_35); - StorageDead(_29); - StorageDead(_35); - _37 = Option::<&T>::Some(_36); - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - _38 = ((_37 as Some).0: &T); - StorageLive(_39); - _39 = &_2; - StorageLive(_40); - _40 = (_38,); - _41 = >::call(move _39, move _40) -> [return: bb15, unwind unreachable]; - } - - bb15: { - StorageDead(_40); - StorageDead(_39); - StorageDead(_37); + StorageDead(_19); + StorageDead(_18); + StorageDead(_15); goto -> bb4; } - bb16: { - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - StorageDead(_37); - StorageDead(_13); - drop(_2) -> [return: bb17, unwind unreachable]; - } - - bb17: { - return; + bb10: { + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir index 7c107a23f9e0b..db9409f72ab1a 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir @@ -7,90 +7,19 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { let mut _11: std::slice::Iter<'_, T>; let mut _12: std::iter::Rev>; let mut _13: std::iter::Rev>; - let mut _37: std::option::Option<&T>; - let mut _39: &impl Fn(&T); - let mut _40: (&T,); - let _41: (); + let mut _15: std::option::Option<&T>; + let mut _16: isize; + let mut _18: &impl Fn(&T); + let mut _19: (&T,); + let _20: (); scope 1 { debug iter => _13; - let _38: &T; + let _17: &T; scope 2 { - debug x => _38; + debug x => _17; } scope 17 (inlined > as Iterator>::next) { - scope 18 (inlined as DoubleEndedIterator>::next_back) { - let mut _14: *const *const T; - let mut _15: *const std::ptr::NonNull; - let mut _20: bool; - let mut _21: *const T; - let _36: &T; - scope 19 { - let _16: std::ptr::NonNull; - let _22: usize; - scope 20 { - } - scope 21 { - scope 25 (inlined as PartialEq>::eq) { - let mut _17: std::ptr::NonNull; - scope 26 (inlined NonNull::::as_ptr) { - let mut _18: *const T; - } - scope 27 (inlined NonNull::::as_ptr) { - let mut _19: *const T; - } - } - } - scope 22 (inlined std::ptr::const_ptr::::addr) { - scope 23 (inlined std::ptr::const_ptr::::cast::<()>) { - } - } - scope 24 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 28 (inlined std::slice::Iter::<'_, T>::next_back_unchecked) { - let _29: std::ptr::NonNull; - scope 29 (inlined std::slice::Iter::<'_, T>::pre_dec_end) { - let mut _23: *mut *const T; - let mut _24: *mut std::ptr::NonNull; - let mut _25: std::ptr::NonNull; - let mut _28: std::ptr::NonNull; - let mut _30: *mut *const T; - let mut _31: *mut usize; - let mut _32: usize; - let mut _33: usize; - scope 30 { - scope 31 { - } - scope 32 { - scope 35 (inlined NonNull::::sub) { - scope 36 (inlined core::num::::unchecked_neg) { - scope 37 (inlined core::ub_checks::check_language_ub) { - scope 38 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 39 (inlined NonNull::::offset) { - let mut _26: *const T; - let mut _27: *const T; - } - } - } - scope 33 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 34 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 40 (inlined NonNull::::as_ref::<'_>) { - let mut _34: std::ptr::NonNull; - scope 41 (inlined NonNull::::as_ptr) { - let mut _35: *const T; - } - scope 42 (inlined std::ptr::mut_ptr::::cast_const) { - } - } - } - } + let mut _14: &mut std::slice::Iter<'_, T>; } } scope 3 (inlined core::slice::::iter) { @@ -178,155 +107,53 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { } bb4: { - StorageLive(_37); - StorageLive(_22); - StorageLive(_21); - StorageLive(_16); - StorageLive(_36); - StorageLive(_20); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb6]; + StorageLive(_15); + StorageLive(_14); + _14 = &mut (_13.0: std::slice::Iter<'_, T>); + _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind: bb11]; } bb5: { - StorageLive(_15); - StorageLive(_14); - _14 = &raw const ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _15 = _14 as *const std::ptr::NonNull (PtrToPtr); StorageDead(_14); - _16 = (*_15); - StorageDead(_15); - StorageLive(_18); - StorageLive(_19); - StorageLive(_17); - _17 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - _18 = (_17.0: *const T); - StorageDead(_17); - _19 = (_16.0: *const T); - _20 = Eq(_18, _19); - StorageDead(_19); - StorageDead(_18); - goto -> bb7; + _16 = discriminant(_15); + switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; } bb6: { - _21 = ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _22 = _21 as usize (Transmute); - _20 = Eq(_22, const 0_usize); - goto -> bb7; + StorageDead(_15); + StorageDead(_13); + drop(_2) -> [return: bb7, unwind continue]; } bb7: { - switchInt(move _20) -> [0: bb8, otherwise: bb18]; + return; } bb8: { - StorageLive(_35); - StorageLive(_29); - StorageLive(_31); - StorageLive(_24); - switchInt(const ::IS_ZST) -> [0: bb9, otherwise: bb13]; + _17 = ((_15 as Some).0: &T); + StorageLive(_18); + _18 = &_2; + StorageLive(_19); + _19 = (_17,); + _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11]; } bb9: { - StorageLive(_23); - _23 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _24 = _23 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_23); - StorageLive(_28); - _25 = (*_24); - switchInt(const ::IS_ZST) -> [0: bb10, otherwise: bb11]; + StorageDead(_19); + StorageDead(_18); + StorageDead(_15); + goto -> bb4; } bb10: { - StorageLive(_27); - StorageLive(_26); - _26 = (_25.0: *const T); - _27 = Offset(move _26, const -1_isize); - StorageDead(_26); - _28 = NonNull:: { pointer: move _27 }; - StorageDead(_27); - goto -> bb12; - } - - bb11: { - _28 = _25; - goto -> bb12; - } - - bb12: { - (*_24) = move _28; - StorageDead(_28); - _29 = (*_24); - goto -> bb14; - } - - bb13: { - StorageLive(_30); - _30 = &raw mut ((_13.0: std::slice::Iter<'_, T>).1: *const T); - _31 = _30 as *mut usize (PtrToPtr); - StorageDead(_30); - StorageLive(_33); - StorageLive(_32); - _32 = (*_31); - _33 = SubUnchecked(move _32, const 1_usize); - StorageDead(_32); - (*_31) = move _33; - StorageDead(_33); - _29 = ((_13.0: std::slice::Iter<'_, T>).0: std::ptr::NonNull); - goto -> bb14; + unreachable; } - bb14: { - StorageDead(_24); - StorageDead(_31); - StorageLive(_34); - _34 = _29; - _35 = (_34.0: *const T); - StorageDead(_34); - _36 = &(*_35); - StorageDead(_29); - StorageDead(_35); - _37 = Option::<&T>::Some(_36); - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - _38 = ((_37 as Some).0: &T); - StorageLive(_39); - _39 = &_2; - StorageLive(_40); - _40 = (_38,); - _41 = >::call(move _39, move _40) -> [return: bb15, unwind: bb16]; + bb11 (cleanup): { + drop(_2) -> [return: bb12, unwind terminate(cleanup)]; } - bb15: { - StorageDead(_40); - StorageDead(_39); - StorageDead(_37); - goto -> bb4; - } - - bb16 (cleanup): { - drop(_2) -> [return: bb17, unwind terminate(cleanup)]; - } - - bb17 (cleanup): { + bb12 (cleanup): { resume; } - - bb18: { - StorageDead(_20); - StorageDead(_36); - StorageDead(_16); - StorageDead(_21); - StorageDead(_22); - StorageDead(_37); - StorageDead(_13); - drop(_2) -> [return: bb19, unwind continue]; - } - - bb19: { - return; - } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir index 2df346c081c8e..78f96bf419559 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-abort.mir @@ -3,205 +3,12 @@ fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> { debug it => _1; let mut _0: std::option::Option<&mut T>; - scope 1 (inlined as DoubleEndedIterator>::next_back) { - let mut _2: *const *mut T; - let mut _3: *const std::ptr::NonNull; - let mut _8: bool; - let mut _9: *mut T; - let mut _25: &mut T; - scope 2 { - let _4: std::ptr::NonNull; - let _10: usize; - scope 3 { - } - scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; - scope 9 (inlined NonNull::::as_ptr) { - let mut _6: *const T; - } - scope 10 (inlined NonNull::::as_ptr) { - let mut _7: *const T; - } - } - } - scope 5 (inlined std::ptr::mut_ptr::::addr) { - scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) { - } - } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 11 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) { - let mut _17: std::ptr::NonNull; - scope 12 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) { - let mut _11: *mut *mut T; - let mut _12: *mut std::ptr::NonNull; - let mut _13: std::ptr::NonNull; - let mut _16: std::ptr::NonNull; - let mut _18: *mut *mut T; - let mut _19: *mut usize; - let mut _20: usize; - let mut _21: usize; - scope 13 { - scope 14 { - } - scope 15 { - scope 18 (inlined NonNull::::sub) { - scope 19 (inlined core::num::::unchecked_neg) { - scope 20 (inlined core::ub_checks::check_language_ub) { - scope 21 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 22 (inlined NonNull::::offset) { - let mut _14: *const T; - let mut _15: *const T; - } - } - } - scope 16 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 17 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 23 (inlined NonNull::::as_mut::<'_>) { - let mut _22: std::ptr::NonNull; - let mut _24: *mut T; - scope 24 (inlined NonNull::::as_ptr) { - let mut _23: *const T; - } - } - } - } bb0: { - StorageLive(_10); - StorageLive(_9); - StorageLive(_4); - StorageLive(_25); - StorageLive(_8); - switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; + _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind unreachable]; } bb1: { - StorageLive(_3); - StorageLive(_2); - _2 = &raw const ((*_1).1: *mut T); - _3 = _2 as *const std::ptr::NonNull (PtrToPtr); - StorageDead(_2); - _4 = (*_3); - StorageDead(_3); - StorageLive(_6); - StorageLive(_7); - StorageLive(_5); - _5 = ((*_1).0: std::ptr::NonNull); - _6 = (_5.0: *const T); - StorageDead(_5); - _7 = (_4.0: *const T); - _8 = Eq(_6, _7); - StorageDead(_7); - StorageDead(_6); - goto -> bb3; - } - - bb2: { - _9 = ((*_1).1: *mut T); - _10 = _9 as usize (Transmute); - _8 = Eq(_10, const 0_usize); - goto -> bb3; - } - - bb3: { - switchInt(move _8) -> [0: bb4, otherwise: bb11]; - } - - bb4: { - StorageLive(_24); - StorageLive(_17); - StorageLive(_19); - StorageLive(_12); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb9]; - } - - bb5: { - StorageLive(_11); - _11 = &raw mut ((*_1).1: *mut T); - _12 = _11 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_11); - StorageLive(_16); - _13 = (*_12); - switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7]; - } - - bb6: { - StorageLive(_15); - StorageLive(_14); - _14 = (_13.0: *const T); - _15 = Offset(move _14, const -1_isize); - StorageDead(_14); - _16 = NonNull:: { pointer: move _15 }; - StorageDead(_15); - goto -> bb8; - } - - bb7: { - _16 = _13; - goto -> bb8; - } - - bb8: { - (*_12) = move _16; - StorageDead(_16); - _17 = (*_12); - goto -> bb10; - } - - bb9: { - StorageLive(_18); - _18 = &raw mut ((*_1).1: *mut T); - _19 = _18 as *mut usize (PtrToPtr); - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - _20 = (*_19); - _21 = SubUnchecked(move _20, const 1_usize); - StorageDead(_20); - (*_19) = move _21; - StorageDead(_21); - _17 = ((*_1).0: std::ptr::NonNull); - goto -> bb10; - } - - bb10: { - StorageDead(_12); - StorageDead(_19); - StorageLive(_22); - _22 = _17; - StorageLive(_23); - _23 = (_22.0: *const T); - _24 = move _23 as *mut T (PtrToPtr); - StorageDead(_23); - StorageDead(_22); - _25 = &mut (*_24); - StorageDead(_17); - StorageDead(_24); - _0 = Option::<&mut T>::Some(_25); - goto -> bb12; - } - - bb11: { - _0 = const {transmute(0x0000000000000000): Option<&mut T>}; - goto -> bb12; - } - - bb12: { - StorageDead(_8); - StorageDead(_25); - StorageDead(_4); - StorageDead(_9); - StorageDead(_10); return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir index 2df346c081c8e..dfe5e206fadaf 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir @@ -3,205 +3,12 @@ fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut T> { debug it => _1; let mut _0: std::option::Option<&mut T>; - scope 1 (inlined as DoubleEndedIterator>::next_back) { - let mut _2: *const *mut T; - let mut _3: *const std::ptr::NonNull; - let mut _8: bool; - let mut _9: *mut T; - let mut _25: &mut T; - scope 2 { - let _4: std::ptr::NonNull; - let _10: usize; - scope 3 { - } - scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; - scope 9 (inlined NonNull::::as_ptr) { - let mut _6: *const T; - } - scope 10 (inlined NonNull::::as_ptr) { - let mut _7: *const T; - } - } - } - scope 5 (inlined std::ptr::mut_ptr::::addr) { - scope 6 (inlined std::ptr::mut_ptr::::cast::<()>) { - } - } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } - } - scope 11 (inlined std::slice::IterMut::<'_, T>::next_back_unchecked) { - let mut _17: std::ptr::NonNull; - scope 12 (inlined std::slice::IterMut::<'_, T>::pre_dec_end) { - let mut _11: *mut *mut T; - let mut _12: *mut std::ptr::NonNull; - let mut _13: std::ptr::NonNull; - let mut _16: std::ptr::NonNull; - let mut _18: *mut *mut T; - let mut _19: *mut usize; - let mut _20: usize; - let mut _21: usize; - scope 13 { - scope 14 { - } - scope 15 { - scope 18 (inlined NonNull::::sub) { - scope 19 (inlined core::num::::unchecked_neg) { - scope 20 (inlined core::ub_checks::check_language_ub) { - scope 21 (inlined core::ub_checks::check_language_ub::runtime) { - } - } - } - scope 22 (inlined NonNull::::offset) { - let mut _14: *const T; - let mut _15: *const T; - } - } - } - scope 16 (inlined std::ptr::mut_ptr::::cast::) { - } - scope 17 (inlined std::ptr::mut_ptr::::cast::>) { - } - } - } - scope 23 (inlined NonNull::::as_mut::<'_>) { - let mut _22: std::ptr::NonNull; - let mut _24: *mut T; - scope 24 (inlined NonNull::::as_ptr) { - let mut _23: *const T; - } - } - } - } bb0: { - StorageLive(_10); - StorageLive(_9); - StorageLive(_4); - StorageLive(_25); - StorageLive(_8); - switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; + _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind continue]; } bb1: { - StorageLive(_3); - StorageLive(_2); - _2 = &raw const ((*_1).1: *mut T); - _3 = _2 as *const std::ptr::NonNull (PtrToPtr); - StorageDead(_2); - _4 = (*_3); - StorageDead(_3); - StorageLive(_6); - StorageLive(_7); - StorageLive(_5); - _5 = ((*_1).0: std::ptr::NonNull); - _6 = (_5.0: *const T); - StorageDead(_5); - _7 = (_4.0: *const T); - _8 = Eq(_6, _7); - StorageDead(_7); - StorageDead(_6); - goto -> bb3; - } - - bb2: { - _9 = ((*_1).1: *mut T); - _10 = _9 as usize (Transmute); - _8 = Eq(_10, const 0_usize); - goto -> bb3; - } - - bb3: { - switchInt(move _8) -> [0: bb4, otherwise: bb11]; - } - - bb4: { - StorageLive(_24); - StorageLive(_17); - StorageLive(_19); - StorageLive(_12); - switchInt(const ::IS_ZST) -> [0: bb5, otherwise: bb9]; - } - - bb5: { - StorageLive(_11); - _11 = &raw mut ((*_1).1: *mut T); - _12 = _11 as *mut std::ptr::NonNull (PtrToPtr); - StorageDead(_11); - StorageLive(_16); - _13 = (*_12); - switchInt(const ::IS_ZST) -> [0: bb6, otherwise: bb7]; - } - - bb6: { - StorageLive(_15); - StorageLive(_14); - _14 = (_13.0: *const T); - _15 = Offset(move _14, const -1_isize); - StorageDead(_14); - _16 = NonNull:: { pointer: move _15 }; - StorageDead(_15); - goto -> bb8; - } - - bb7: { - _16 = _13; - goto -> bb8; - } - - bb8: { - (*_12) = move _16; - StorageDead(_16); - _17 = (*_12); - goto -> bb10; - } - - bb9: { - StorageLive(_18); - _18 = &raw mut ((*_1).1: *mut T); - _19 = _18 as *mut usize (PtrToPtr); - StorageDead(_18); - StorageLive(_21); - StorageLive(_20); - _20 = (*_19); - _21 = SubUnchecked(move _20, const 1_usize); - StorageDead(_20); - (*_19) = move _21; - StorageDead(_21); - _17 = ((*_1).0: std::ptr::NonNull); - goto -> bb10; - } - - bb10: { - StorageDead(_12); - StorageDead(_19); - StorageLive(_22); - _22 = _17; - StorageLive(_23); - _23 = (_22.0: *const T); - _24 = move _23 as *mut T (PtrToPtr); - StorageDead(_23); - StorageDead(_22); - _25 = &mut (*_24); - StorageDead(_17); - StorageDead(_24); - _0 = Option::<&mut T>::Some(_25); - goto -> bb12; - } - - bb11: { - _0 = const {transmute(0x0000000000000000): Option<&mut T>}; - goto -> bb12; - } - - bb12: { - StorageDead(_8); - StorageDead(_25); - StorageDead(_4); - StorageDead(_9); - StorageDead(_10); return; } }