diff --git a/Cargo.lock b/Cargo.lock index e5ea9fee8a6d8..79e17ff3aaf9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3826,6 +3826,7 @@ dependencies = [ "rustc_macros", "rustc_metadata", "rustc_middle", + "rustc_mir_transform", "rustc_monomorphize", "rustc_query_system", "rustc_serialize", diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 6d26ca0b899b2..58ed4869e4152 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -5,6 +5,7 @@ use cranelift_codegen::CodegenError; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; use cranelift_module::ModuleError; use rustc_ast::InlineAsmOptions; +use rustc_codegen_ssa::mir::make_codegen_mir; use rustc_index::IndexVec; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::ty::adjustment::PointerCoercion; @@ -40,7 +41,8 @@ pub(crate) fn codegen_fn<'tcx>( let symbol_name = tcx.symbol_name(instance).name.to_string(); let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); - let mir = tcx.instance_mir(instance.def); + let mir = make_codegen_mir(tcx, instance); + let mir = &mir; let _mir_guard = crate::PrintOnPanic(|| { let mut buf = Vec::new(); with_no_trimmed_paths!({ @@ -302,19 +304,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { .generic_activity("codegen prelude") .run(|| crate::abi::codegen_fn_prelude(fx, start_block)); - let reachable_blocks = traversal::mono_reachable_as_bitset(fx.mir, fx.tcx, fx.instance); - for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); - if !reachable_blocks.contains(bb) { - // We want to skip this block, because it's not reachable. But we still create - // the block so terminators in other blocks can reference it. - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - continue; - } - if bb_data.is_cleanup { // Unwinding after panicking is not supported continue; diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 21d0cd2d30f2a..a0518d82b267c 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -283,7 +283,7 @@ pub(crate) fn create_wrapper_function( unwind_context.add_function(wrapper_func_id, &ctx, module.isa()); } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { +pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { pub(crate) cx: &'clif mut crate::CodegenCx, pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, @@ -294,7 +294,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { pub(crate) instance: Instance<'tcx>, pub(crate) symbol_name: String, - pub(crate) mir: &'tcx Body<'tcx>, + pub(crate) mir: &'m Body<'tcx>, pub(crate) fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, pub(crate) bcx: FunctionBuilder<'clif>, diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 3771fc6b0a271..a1cae61b6b9bf 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -27,6 +27,7 @@ rustc_index = { path = "../rustc_index" } rustc_macros = { path = "../rustc_macros" } rustc_metadata = { path = "../rustc_metadata" } rustc_middle = { path = "../rustc_middle" } +rustc_mir_transform = { path = "../rustc_mir_transform" } rustc_monomorphize = { path = "../rustc_monomorphize" } rustc_query_system = { path = "../rustc_query_system" } rustc_serialize = { path = "../rustc_serialize" } diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs index 0577ba32ffdcc..1991ae07e94e9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs +++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs @@ -13,8 +13,8 @@ use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; use rustc_middle::{bug, span_bug}; use tracing::debug; -pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( - fx: &FunctionCx<'a, 'tcx, Bx>, +pub fn non_ssa_locals<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( + fx: &FunctionCx<'body, 'a, 'tcx, Bx>, ) -> BitSet { let mir = fx.mir; let dominators = mir.basic_blocks.dominators(); @@ -69,9 +69,9 @@ enum LocalKind { SSA(DefLocation), } -struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { - fx: &'mir FunctionCx<'a, 'tcx, Bx>, - dominators: &'mir Dominators, +struct LocalAnalyzer<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { + fx: &'body FunctionCx<'body, 'a, 'tcx, Bx>, + dominators: &'body Dominators, locals: IndexVec, } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index bd9704b37aea9..c14704fa8f020 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -39,17 +39,17 @@ enum MergingSucc { /// Used by `FunctionCx::codegen_terminator` for emitting common patterns /// e.g., creating a basic block, calling a function, etc. -struct TerminatorCodegenHelper<'tcx> { +struct TerminatorCodegenHelper<'body, 'tcx> { bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &'body mir::Terminator<'tcx>, } -impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { +impl<'body, 'a, 'tcx> TerminatorCodegenHelper<'body, 'tcx> { /// Returns the appropriate `Funclet` for the current funclet, if on MSVC, /// either already previously cached, or newly created, by `landing_pad_for`. fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>( &self, - fx: &'b mut FunctionCx<'a, 'tcx, Bx>, + fx: &'b mut FunctionCx<'body, 'a, 'tcx, Bx>, ) -> Option<&'b Bx::Funclet> { let cleanup_kinds = fx.cleanup_kinds.as_ref()?; let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?; @@ -75,7 +75,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// stuff in it or next to it. fn llbb_with_cleanup>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, target: mir::BasicBlock, ) -> Bx::BasicBlock { let (needs_landing_pad, is_cleanupret) = self.llbb_characteristics(fx, target); @@ -99,7 +99,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn llbb_characteristics>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, target: mir::BasicBlock, ) -> (bool, bool) { if let Some(ref cleanup_kinds) = fx.cleanup_kinds { @@ -124,7 +124,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { fn funclet_br>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, bx: &mut Bx, target: mir::BasicBlock, mergeable_succ: bool, @@ -153,7 +153,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// return destination `destination` and the unwind action `unwind`. fn do_call>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, @@ -272,7 +272,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Generates inline assembly with optional `destination` and `unwind`. fn do_inlineasm>( &self, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, bx: &mut Bx, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperandRef<'tcx, Bx>], @@ -339,9 +339,13 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } /// Codegen implementations for some terminator variants. -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { /// Generates code for a `Resume` terminator. - fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, bx: &mut Bx) { + fn codegen_resume_terminator( + &mut self, + helper: TerminatorCodegenHelper<'body, 'tcx>, + bx: &mut Bx, + ) { if let Some(funclet) = helper.funclet(self) { bx.cleanup_ret(funclet, None); } else { @@ -358,7 +362,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_switchint_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, discr: &mir::Operand<'tcx>, targets: &SwitchTargets, @@ -498,7 +502,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { #[tracing::instrument(level = "trace", skip(self, helper, bx))] fn codegen_drop_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, source_info: &mir::SourceInfo, location: mir::Place<'tcx>, @@ -640,7 +644,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_assert_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, cond: &mir::Operand<'tcx>, @@ -719,7 +723,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_terminate_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, reason: UnwindTerminateReason, @@ -749,7 +753,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { /// Returns `Some` if this is indeed a panic intrinsic and codegen is done. fn codegen_panic_intrinsic( &mut self, - helper: &TerminatorCodegenHelper<'tcx>, + helper: &TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, intrinsic: Option, instance: Option>, @@ -818,7 +822,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_call_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, func: &mir::Operand<'tcx>, @@ -1170,7 +1174,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn codegen_asm_terminator( &mut self, - helper: TerminatorCodegenHelper<'tcx>, + helper: TerminatorCodegenHelper<'body, 'tcx>, bx: &mut Bx, terminator: &mir::Terminator<'tcx>, template: &[ast::InlineAsmTemplatePiece], @@ -1254,7 +1258,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn codegen_block(&mut self, mut bb: mir::BasicBlock) { let llbb = match self.try_llbb(bb) { Some(llbb) => llbb, @@ -1309,7 +1313,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &mut self, bx: &mut Bx, bb: mir::BasicBlock, - terminator: &'tcx mir::Terminator<'tcx>, + terminator: &'body mir::Terminator<'tcx>, ) -> MergingSucc { debug!("codegen_terminator: {:?}", terminator); diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 822f5c2c44a2e..23ff0025da054 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -10,7 +10,7 @@ use rustc_target::abi::Abi; use super::FunctionCx; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn eval_mir_constant_to_operand( &self, bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs index 721872772287b..4f8b6bb945601 100644 --- a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs @@ -5,7 +5,7 @@ use rustc_middle::mir::SourceScope; use super::FunctionCx; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn codegen_coverage(&self, bx: &mut Bx, kind: &CoverageKind, scope: SourceScope) { // Determine the instance that coverage data was originally generated for. let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) { diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 6b89636b6540c..aed767892c5ee 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -203,7 +203,7 @@ fn calculate_debuginfo_offset< DebugInfoOffset { direct_offset, indirect_offsets, result: place } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) { bx.set_span(source_info.span); if let Some(dbg_loc) = self.dbg_loc(source_info) { diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index f88deaa7abca6..94bfc94d4064a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -54,7 +54,7 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx.memset(dst, val, size, align, flags); } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { /// In the `Err` case, returns the instance that should be called instead. pub fn codegen_intrinsic_call( bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs index c7f63eab8298d..06e85db9abe23 100644 --- a/compiler/rustc_codegen_ssa/src/mir/locals.rs +++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs @@ -34,7 +34,7 @@ impl<'tcx, V> Locals<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub(super) fn initialize_locals(&mut self, values: Vec>) { assert!(self.locals.values.is_empty()); // FIXME(#115215): After #115025 get's merged this might not be necessary diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs index e8da98428829e..db63165405c70 100644 --- a/compiler/rustc_codegen_ssa/src/mir/mod.rs +++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs @@ -43,10 +43,10 @@ enum CachedLlbb { } /// Master context for codegenning from MIR. -pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { +pub struct FunctionCx<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { instance: Instance<'tcx>, - mir: &'tcx mir::Body<'tcx>, + mir: &'body mir::Body<'tcx>, debug_context: Option>, @@ -115,14 +115,27 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> { caller_location: Option>, } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn monomorphize(&self, value: T) -> T where T: Copy + TypeFoldable>, { - debug!("monomorphize: self.instance={:?}", self.instance); + value + } +} + +struct MonoCx<'tcx> { + tcx: TyCtxt<'tcx>, + instance: Instance<'tcx>, +} + +impl<'tcx> MonoCx<'tcx> { + fn monomorphize(&self, value: T) -> T + where + T: TypeFoldable>, + { self.instance.instantiate_mir_and_normalize_erasing_regions( - self.cx.tcx(), + self.tcx, ty::ParamEnv::reveal_all(), ty::EarlyBinder::bind(value), ) @@ -155,7 +168,172 @@ impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> { } } -/////////////////////////////////////////////////////////////////////////// +use rustc_index::Idx; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::visit::Visitor; + +struct Reachable { + locals: BitSet, +} + +impl mir::visit::Visitor<'_> for Reachable { + fn visit_local( + &mut self, + local: mir::Local, + _context: mir::visit::PlaceContext, + _location: mir::Location, + ) { + self.locals.insert(local); + } +} + +struct LocalRenamer<'tcx> { + tcx: TyCtxt<'tcx>, + replacements: Vec, +} + +impl<'tcx> mir::visit::MutVisitor<'tcx> for LocalRenamer<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local( + &mut self, + local: &mut mir::Local, + _context: mir::visit::PlaceContext, + _location: mir::Location, + ) { + *local = self.replacements[local.index()]; + } +} + +#[instrument(level = "debug", skip(tcx))] +pub fn make_codegen_mir<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> mir::Body<'tcx> { + let mcx = MonoCx { tcx, instance }; + + let mir = tcx.instance_mir(instance.def); + + // Create our new list of basic blocks. Initialy all blocks are turned into unreachables. + let mut blocks: IndexVec<_, _> = mir + .basic_blocks + .iter() + .map(|block| mir::BasicBlockData { + statements: Vec::new(), + terminator: Some(mir::Terminator { + source_info: block.terminator().source_info, + kind: mir::TerminatorKind::Unreachable, + }), + is_cleanup: block.is_cleanup, + }) + .collect(); + + // Figure out which blocks are reachable, and which are mentioned. + // As we encounter reachable blocks, monomorphize them, and record which Locals they use. + let mut reachable = Reachable { locals: BitSet::new_empty(mir.local_decls.len()) }; + for idx in 0..(mir.arg_count + 1) { + let local = mir::Local::new(idx); + reachable.locals.insert(local); + } + let mut it = traversal::mono_reachable::(mir, tcx, instance); + while let Some((bb, block)) = it.next() { + let block = mcx.monomorphize(block.into_owned()); + reachable.visit_basic_block_data(bb, &block); + blocks[bb] = block; + } + let reachable_blocks = it.visited(); + + // Delete all blocks which are not mentioned, and deduplicate non-cleanup unreachable blocks + // that remain. + let mut block_replacements: Vec<_> = (0..blocks.len()).map(mir::BasicBlock::new).collect(); + let mut orig_index = 0; + let mut used_index = 0; + blocks.raw.retain(|_block| { + let orig_bb = mir::BasicBlock::new(orig_index); + // Block is not mentioned. Delete it. + if !reachable_blocks.contains(orig_bb) { + orig_index += 1; + return false; + } + + let used_bb = mir::BasicBlock::new(used_index); + block_replacements[orig_index] = used_bb; + used_index += 1; + orig_index += 1; + true + }); + + let mut local_replacements: Vec<_> = (0..mir.local_decls.len()).map(mir::Local::new).collect(); + let mut used_index = 0; + let local_decls = mir + .local_decls + .iter_enumerated() + .filter_map(|(local, decl)| { + let decl = decl.clone(); + if !reachable.locals.contains(local) { + None + } else { + local_replacements[local.index()] = mir::Local::new(used_index); + used_index += 1; + Some(mcx.monomorphize(decl)) + } + }) + .collect(); + + let mut renamer = LocalRenamer { tcx, replacements: local_replacements }; + + for (bb, block) in blocks.iter_enumerated_mut() { + for target in block.terminator_mut().successors_mut() { + *target = block_replacements[target.index()]; + } + renamer.visit_basic_block_data(bb, block); + } + + let basic_blocks = mir::BasicBlocks::new(blocks); + + let var_debug_info = mir + .var_debug_info + .iter() + .filter_map(|info| match info.value { + mir::VarDebugInfoContents::Place(place) => { + if reachable.locals.contains(place.local) { + let place = mir::Place { + local: renamer.replacements[place.local.index()], + projection: place.projection, + }; + let mut info = info.clone(); + info.value = mir::VarDebugInfoContents::Place(place); + Some(info) + } else { + None + } + } + _ => Some(info.clone()), + }) + .map(|info| mcx.monomorphize(info)) + .collect(); + + mir::Body { + basic_blocks, + arg_count: mir.arg_count, + coroutine: mir.coroutine.clone(), + coverage_branch_info: mir.coverage_branch_info.clone(), + function_coverage_info: mir.function_coverage_info.clone(), + injection_phase: mir.injection_phase, + is_polymorphic: mir.is_polymorphic, + local_decls, + mentioned_items: mir.mentioned_items.clone(), + pass_count: mir.pass_count, + phase: mir.phase, + required_consts: mir.required_consts.clone(), + source: mir.source, + source_scopes: mcx.monomorphize(mir.source_scopes.clone()), // Required for coverage + span: mir.span, + spread_arg: mir.spread_arg, + tainted_by_errors: mir.tainted_by_errors, + user_type_annotations: mir.user_type_annotations.clone(), + var_debug_info, + } +} #[instrument(level = "debug", skip(cx))] pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( @@ -165,8 +343,10 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( assert!(!instance.args.has_infer()); let llfn = cx.get_fn(instance); + let tcx = cx.tcx(); - let mir = cx.tcx().instance_mir(instance.def); + let mir = make_codegen_mir(tcx, instance); + let mir = &mir; let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); debug!("fn_abi: {:?}", fn_abi); @@ -268,26 +448,18 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( // So drop the builder of `start_llbb` to avoid having two at the same time. drop(start_bx); - let reachable_blocks = traversal::mono_reachable_as_bitset(mir, cx.tcx(), instance); - // Codegen the body of each block using reverse postorder for (bb, _) in traversal::reverse_postorder(mir) { - if reachable_blocks.contains(bb) { - fx.codegen_block(bb); - } else { - // We want to skip this block, because it's not reachable. But we still create - // the block so terminators in other blocks can reference it. - fx.codegen_block_as_unreachable(bb); - } + fx.codegen_block(bb); } } /// Produces, for each argument, a `Value` pointing at the /// argument's value. As arguments are places, these are always /// indirect. -fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( +fn arg_local_refs<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, - fx: &mut FunctionCx<'a, 'tcx, Bx>, + fx: &mut FunctionCx<'body, 'a, 'tcx, Bx>, memory_locals: &BitSet, ) -> Vec> { let mir = fx.mir; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index cc0e913965067..f917631cfb8a2 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -551,7 +551,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index 449fd9ae0db9c..fb429967c2324 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -446,7 +446,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub fn codegen_place( &mut self, diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index ad6b3f1159dec..6001b7e8b3ff0 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -19,7 +19,7 @@ use rustc_target::abi::{self, FieldIdx, FIRST_VARIANT}; use arrayvec::ArrayVec; use tracing::{debug, instrument}; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { #[instrument(level = "trace", skip(self, bx))] pub fn codegen_rvalue( &mut self, @@ -1027,7 +1027,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool { match *rvalue { mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => { diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs index 27494f48b099b..77edba7e8c209 100644 --- a/compiler/rustc_codegen_ssa/src/mir/statement.rs +++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs @@ -7,7 +7,7 @@ use super::FunctionCx; use super::LocalRef; use crate::traits::*; -impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { +impl<'body, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'body, 'a, 'tcx, Bx> { #[instrument(level = "debug", skip(self, bx))] pub fn codegen_statement(&mut self, bx: &mut Bx, statement: &mir::Statement<'tcx>) { self.set_debug_loc(bx, statement.source_info); diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index 245e9096bad47..94fa2d98e064a 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -289,27 +289,15 @@ pub fn reverse_postorder<'a, 'tcx>( /// of `if ::CONST`, as well as [`NullOp::UbChecks`]. /// /// [`NullOp::UbChecks`]: rustc_middle::mir::NullOp::UbChecks -pub fn mono_reachable<'a, 'tcx>( +pub fn mono_reachable<'a, 'tcx, const REWRITE: bool>( body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, -) -> MonoReachable<'a, 'tcx> { +) -> MonoReachable<'a, 'tcx, REWRITE> { MonoReachable::new(body, tcx, instance) } -/// [`MonoReachable`] internally accumulates a [`BitSet`] of visited blocks. This is just a -/// convenience function to run that traversal then extract its set of reached blocks. -pub fn mono_reachable_as_bitset<'a, 'tcx>( - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, - instance: Instance<'tcx>, -) -> BitSet { - let mut iter = mono_reachable(body, tcx, instance); - while let Some(_) = iter.next() {} - iter.visited -} - -pub struct MonoReachable<'a, 'tcx> { +pub struct MonoReachable<'a, 'tcx, const REWRITE: bool> { body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, @@ -320,12 +308,12 @@ pub struct MonoReachable<'a, 'tcx> { worklist: BitSet, } -impl<'a, 'tcx> MonoReachable<'a, 'tcx> { +impl<'a, 'tcx, const REWRITE: bool> MonoReachable<'a, 'tcx, REWRITE> { pub fn new( body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, - ) -> MonoReachable<'a, 'tcx> { + ) -> MonoReachable<'a, 'tcx, REWRITE> { let mut worklist = BitSet::new_empty(body.basic_blocks.len()); worklist.insert(START_BLOCK); MonoReachable { @@ -344,29 +332,58 @@ impl<'a, 'tcx> MonoReachable<'a, 'tcx> { } } } + + pub fn visited(self) -> BitSet { + self.visited + } } -impl<'a, 'tcx> Iterator for MonoReachable<'a, 'tcx> { - type Item = (BasicBlock, &'a BasicBlockData<'tcx>); +impl<'a, 'tcx, const REWRITE: bool> Iterator for MonoReachable<'a, 'tcx, REWRITE> { + type Item = (BasicBlock, Cow<'a, BasicBlockData<'tcx>>); - fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> { + fn next(&mut self) -> Option { while let Some(idx) = self.worklist.iter().next() { self.worklist.remove(idx); if !self.visited.insert(idx) { continue; } - let data = &self.body[idx]; + let mut data = Cow::Borrowed(&self.body[idx]); if let Some((bits, targets)) = - Body::try_const_mono_switchint(self.tcx, self.instance, data) + Body::try_const_mono_switchint(self.tcx, self.instance, &data) { let target = targets.target_for_value(bits); + if REWRITE { + let mut block = data.into_owned(); + block.terminator_mut().kind = TerminatorKind::Goto { target }; + data = Cow::Owned(block); + } self.add_work([target]); - } else { - self.add_work(data.terminator().successors()); + return Some((idx, data)); } + if REWRITE && self.tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { + if let TerminatorKind::SwitchInt { discr, targets } = &data.terminator().kind { + let mut it = targets.iter(); + if it.len() == 2 + && self.body.basic_blocks[targets.otherwise()].is_empty_unreachable() + { + let first = it.next().unwrap(); + let second = it.next().unwrap(); + let terminator = TerminatorKind::SwitchInt { + discr: discr.clone(), + targets: SwitchTargets::new([first].into_iter(), second.1), + }; + let mut block = data.into_owned(); + block.terminator_mut().kind = terminator; + data = Cow::Owned(block); + } + } + } + + self.add_work(data.terminator().successors()); + return Some((idx, data)); } diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index 61680dbfaf52d..20457c051521d 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1284,8 +1284,8 @@ fn collect_items_of_instance<'tcx>( }; if mode == CollectionMode::UsedItems { - for (bb, data) in traversal::mono_reachable(body, tcx, instance) { - collector.visit_basic_block_data(bb, data) + for (bb, data) in traversal::mono_reachable::(body, tcx, instance) { + collector.visit_basic_block_data(bb, &data) } } diff --git a/tests/codegen/constant-branch.rs b/tests/codegen/constant-branch.rs index a2710cc4b2586..b201553c06583 100644 --- a/tests/codegen/constant-branch.rs +++ b/tests/codegen/constant-branch.rs @@ -7,10 +7,11 @@ // CHECK-LABEL: @if_bool #[no_mangle] pub fn if_bool() { - // CHECK: br label %{{.+}} + // CHECK-NEXT: start: + // CHECK-NEXT: ret void + _ = if true { 0 } else { 1 }; - // CHECK: br label %{{.+}} _ = if false { 0 } else { 1 }; } @@ -28,23 +29,21 @@ pub fn if_constant_int_eq() { // CHECK-LABEL: @if_constant_match #[no_mangle] pub fn if_constant_match() { - // CHECK: br label %{{.+}} + // CHECK-NEXT: start: + // CHECK-NEXT: ret void + _ = match 1 { 1 => 2, 2 => 3, _ => 4, }; - // CHECK: br label %{{.+}} _ = match 1 { 2 => 3, _ => 4, }; - // CHECK: br label %[[MINUS1:.+]] _ = match -1 { - // CHECK: [[MINUS1]]: - // CHECK: store i32 1 -1 => 1, _ => 0, } diff --git a/tests/codegen/skip-mono-inside-if-false.rs b/tests/codegen/skip-mono-inside-if-false.rs index 8b95de99dd3bd..3a51eaa1cc7ae 100644 --- a/tests/codegen/skip-mono-inside-if-false.rs +++ b/tests/codegen/skip-mono-inside-if-false.rs @@ -13,13 +13,9 @@ pub fn demo_for_i32() { // CHECK-LABEL: ; skip_mono_inside_if_false::generic_impl // CHECK: start: -// CHECK-NEXT: br label %[[ELSE_BRANCH:bb[0-9]+]] -// CHECK: [[ELSE_BRANCH]]: // CHECK-NEXT: call skip_mono_inside_if_false::small_impl -// CHECK: bb{{[0-9]+}}: +// CHECK-NEXT: call void @_ZN25skip_mono_inside_if_false10small_impl // CHECK-NEXT: ret void -// CHECK: bb{{[0-9+]}}: -// CHECK-NEXT: unreachable fn generic_impl() { trait MagicTrait {