Auto merge of #148040 - saethlin:trivial-consts, r=oli-obk

Add a fast path for lowering trivial consts

The objective of this PR is to improve compilation performance for crates that define a lot of trivial consts. This is a flamegraph of a build of a library crate that is just 100,000 trivial consts, taken from a nightly compiler:
<img width="842" height="280" alt="2025-10-25-164005_842x280_scrot" src="https://github.com/user-attachments/assets/e5400aaf-03bd-4461-b905-054aa82ca60f" />
My objective is to target all of the cycles in `eval_to_const_value_raw` that are not part of `mir_built`, because if you look at the `mir_built` for a trivial const, we already have the value available.

In this PR, the definition of a trivial const is this:
```rust
const A: usize = 0;
```
Specifically, we look for if the `mir_built` body is a single basic block containing one assign statement and a return terminator, where the assign statement assigns an `Operand::Constant(Const::Val)`. The MIR dumps for these look like:
```
const A: usize = {
    let mut _0: usize;

    bb0: {
        _0 = const 0_usize;
        return;
    }
}
```

The implementation is built around a new query, `trivial_const(LocalDefId) -> Option<(ConstValue, Ty)>` which returns the contents of the `Const::Val` in the `mir_built` if the `LocalDefId` is a trivial const.

Then I added _debug_ assertions to the beginning of `mir_for_ctfe` and `mir_promoted` to prevent trying to get the body of a trivial const, because that would defeat the optimization here. But these are deliberately _debug_ assertions because the consequence of failing the assertion is that compilation is slow, not corrupt. If we made these hard assertions, I'm sure there are obscure scenarios people will run into where the compiler would ICE instead of continuing on compilation, just a bit slower. I'd like to know about those, but I do not think serving up an ICE is worth it.

With the assertions in place, I just added logic around all the places they were hit, to skip over trying to analyze the bodies of trivial consts.

In the future, I'd like to see this work extended by:
* Pushing detection of trivial consts before MIR building
* Including DefKind::Static and DefKind::InlineConst
* Including consts like `_1 = const 0_usize; _0 = &_1`, which would make a lot of promoteds into trivial consts
* Handling less-trivial consts like `const A: usize = B`, which have `Operand::Constant(Const::Unevaluated)`

Author: bors

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -7,7 +7,7 @@ use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
 use rustc_middle::mir::{self, ConstAlloc, ConstValue};
 use rustc_middle::query::TyCtxtAt;
-use rustc_middle::ty::layout::HasTypingEnv;
+use rustc_middle::ty::layout::{HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, throw_inval};
@@ -24,13 +24,11 @@ use crate::interpret::{
 };
 use crate::{CTRL_C_RECEIVED, errors};
 
-// Returns a pointer to where the result lives
-#[instrument(level = "trace", skip(ecx, body))]
-fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+fn setup_for_eval<'tcx>(
     ecx: &mut CompileTimeInterpCx<'tcx>,
     cid: GlobalId<'tcx>,
-    body: &'tcx mir::Body<'tcx>,
-) -> InterpResult<'tcx, R> {
+    layout: TyAndLayout<'tcx>,
+) -> InterpResult<'tcx, (InternKind, MPlaceTy<'tcx>)> {
     let tcx = *ecx.tcx;
     assert!(
         cid.promoted.is_some()
@@ -46,7 +44,6 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         "Unexpected DefKind: {:?}",
         ecx.tcx.def_kind(cid.instance.def_id())
     );
-    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
     assert!(layout.is_sized());
 
     let intern_kind = if cid.promoted.is_some() {
@@ -58,12 +55,25 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         }
     };
 
-    let ret = if let InternKind::Static(_) = intern_kind {
-        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
+    let return_place = if let InternKind::Static(_) = intern_kind {
+        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)
     } else {
-        ecx.allocate(layout, MemoryKind::Stack)?
+        ecx.allocate(layout, MemoryKind::Stack)
     };
 
+    return_place.map(|ret| (intern_kind, ret))
+}
+
+#[instrument(level = "trace", skip(ecx, body))]
+fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    body: &'tcx mir::Body<'tcx>,
+) -> InterpResult<'tcx, R> {
+    let tcx = *ecx.tcx;
+    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
+    let (intern_kind, ret) = setup_for_eval(ecx, cid, layout)?;
+
     trace!(
         "eval_body_using_ecx: pushing stack frame for global: {}{}",
         with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
@@ -87,6 +97,31 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
         }
     }
 
+    intern_and_validate(ecx, cid, intern_kind, ret)
+}
+
+#[instrument(level = "trace", skip(ecx))]
+fn eval_trivial_const_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    val: ConstValue,
+    ty: Ty<'tcx>,
+) -> InterpResult<'tcx, R> {
+    let layout = ecx.layout_of(ty)?;
+    let (intern_kind, return_place) = setup_for_eval(ecx, cid, layout)?;
+
+    let opty = ecx.const_val_to_op(val, ty, Some(layout))?;
+    ecx.copy_op(&opty, &return_place)?;
+
+    intern_and_validate(ecx, cid, intern_kind, return_place)
+}
+
+fn intern_and_validate<'tcx, R: InterpretationResult<'tcx>>(
+    ecx: &mut CompileTimeInterpCx<'tcx>,
+    cid: GlobalId<'tcx>,
+    intern_kind: InternKind,
+    ret: MPlaceTy<'tcx>,
+) -> InterpResult<'tcx, R> {
     // Intern the result
     let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
 
@@ -292,6 +327,9 @@ pub fn eval_to_const_value_raw_provider<'tcx>(
     tcx: TyCtxt<'tcx>,
     key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
 ) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
+    if let Some((value, _ty)) = tcx.trivial_const(key.value.instance.def_id()) {
+        return Ok(value);
+    }
     tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
 }
 
@@ -368,10 +406,14 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
         // so we have to reject reading mutable global memory.
         CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
     );
-    let res = ecx.load_mir(cid.instance.def, cid.promoted);
-    res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
-        .report_err()
-        .map_err(|error| report_eval_error(&ecx, cid, error))
+
+    let result = if let Some((value, ty)) = tcx.trivial_const(def) {
+        eval_trivial_const_using_ecx(&mut ecx, cid, value, ty)
+    } else {
+        ecx.load_mir(cid.instance.def, cid.promoted)
+            .and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
+    };
+    result.report_err().map_err(|error| report_eval_error(&ecx, cid, error))
 }
 
 #[inline(always)]

compiler/rustc_interface/src/passes.rs

@@ -1088,9 +1088,15 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
 
     sess.time("MIR_borrow_checking", || {
         tcx.par_hir_body_owners(|def_id| {
-            if !tcx.is_typeck_child(def_id.to_def_id()) {
+            let not_typeck_child = !tcx.is_typeck_child(def_id.to_def_id());
+            if not_typeck_child {
                 // Child unsafety and borrowck happens together with the parent
                 tcx.ensure_ok().check_unsafety(def_id);
+            }
+            if tcx.is_trivial_const(def_id) {
+                return;
+            }
+            if not_typeck_child {
                 tcx.ensure_ok().mir_borrowck(def_id);
                 tcx.ensure_ok().check_transmutes(def_id);
             }
@@ -1198,7 +1204,9 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) {
     if tcx.sess.opts.unstable_opts.validate_mir {
         sess.time("ensuring_final_MIR_is_computable", || {
             tcx.par_hir_body_owners(|def_id| {
-                tcx.instance_mir(ty::InstanceKind::Item(def_id.into()));
+                if !tcx.is_trivial_const(def_id) {
+                    tcx.instance_mir(ty::InstanceKind::Item(def_id.into()));
+                }
             });
         });
     }

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -240,6 +240,7 @@ provide! { tcx, def_id, other, cdata,
     thir_abstract_const => { table }
     optimized_mir => { table }
     mir_for_ctfe => { table }
+    trivial_const => { table }
     closure_saved_names_of_captured_variables => { table }
     mir_coroutine_witnesses => { table }
     promoted_mir => { table }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1791,8 +1791,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     record!(self.tables.mir_coroutine_witnesses[def_id.to_def_id()] <- witnesses);
                 }
             }
+            let mut is_trivial = false;
             if encode_const {
-                record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id));
+                if let Some((val, ty)) = tcx.trivial_const(def_id) {
+                    is_trivial = true;
+                    record!(self.tables.trivial_const[def_id.to_def_id()] <- (val, ty));
+                } else {
+                    is_trivial = false;
+                    record!(self.tables.mir_for_ctfe[def_id.to_def_id()] <- tcx.mir_for_ctfe(def_id));
+                }
 
                 // FIXME(generic_const_exprs): this feels wrong to have in `encode_mir`
                 let abstract_const = tcx.thir_abstract_const(def_id);
@@ -1810,7 +1817,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     }
                 }
             }
-            record!(self.tables.promoted_mir[def_id.to_def_id()] <- tcx.promoted_mir(def_id));
+            if !is_trivial {
+                record!(self.tables.promoted_mir[def_id.to_def_id()] <- tcx.promoted_mir(def_id));
+            }
 
             if self.tcx.is_coroutine(def_id.to_def_id())
                 && let Some(witnesses) = tcx.mir_coroutine_witnesses(def_id)
@@ -2234,6 +2243,9 @@ fn prefetch_mir(tcx: TyCtxt<'_>) {
 
     let reachable_set = tcx.reachable_set(());
     par_for_each_in(tcx.mir_keys(()), |&&def_id| {
+        if tcx.is_trivial_const(def_id) {
+            return;
+        }
         let (encode_const, encode_opt) = should_encode_mir(tcx, reachable_set, def_id);
 
         if encode_const {

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -29,6 +29,7 @@ use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
 use rustc_middle::middle::lib_features::FeatureStability;
 use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir;
+use rustc_middle::mir::ConstValue;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::{self, Ty, TyCtxt, UnusedGenericParams};
 use rustc_middle::util::Providers;
@@ -426,6 +427,7 @@ define_tables! {
     object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
     optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
     mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
+    trivial_const: Table<DefIndex, LazyValue<(ConstValue, Ty<'static>)>>,
     closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
     mir_coroutine_witnesses: Table<DefIndex, LazyValue<mir::CoroutineLayout<'static>>>,
     promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,

compiler/rustc_metadata/src/rmeta/parameterized.rs

@@ -102,6 +102,7 @@ trivially_parameterized_over_tcx! {
     rustc_middle::middle::lib_features::FeatureStability,
     rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
     rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::ConstValue,
     rustc_middle::ty::AnonConstKind,
     rustc_middle::ty::AssocContainer,
     rustc_middle::ty::AsyncDestructor,

compiler/rustc_middle/src/mir/graphviz.rs

@@ -16,6 +16,7 @@ where
 
     let mirs = def_ids
         .iter()
+        .filter(|def_id| !tcx.is_trivial_const(*def_id))
         .flat_map(|def_id| {
             if tcx.is_const_fn(*def_id) {
                 vec![tcx.optimized_mir(*def_id), tcx.mir_for_ctfe(*def_id)]

compiler/rustc_middle/src/mir/pretty.rs

@@ -353,8 +353,16 @@ pub fn write_mir_pretty<'tcx>(
             // are shared between mir_for_ctfe and optimized_mir
             writer.write_mir_fn(tcx.mir_for_ctfe(def_id), w)?;
         } else {
-            let instance_mir = tcx.instance_mir(ty::InstanceKind::Item(def_id));
-            render_body(w, instance_mir)?;
+            if let Some((val, ty)) = tcx.trivial_const(def_id) {
+                ty::print::with_forced_impl_filename_line! {
+                    // see notes on #41697 elsewhere
+                    write!(w, "const {}", tcx.def_path_str(def_id))?
+                }
+                writeln!(w, ": {} = const {};", ty, Const::Val(val, ty))?;
+            } else {
+                let instance_mir = tcx.instance_mir(ty::InstanceKind::Item(def_id));
+                render_body(w, instance_mir)?;
+            }
         }
     }
     Ok(())

compiler/rustc_middle/src/query/erase.rs

@@ -160,6 +160,10 @@ impl EraseType for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
     type Result = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
 }
 
+impl EraseType for Option<(mir::ConstValue, Ty<'_>)> {
+    type Result = [u8; size_of::<Option<(mir::ConstValue, Ty<'_>)>>()];
+}
+
 impl EraseType for EvalToValTreeResult<'_> {
     type Result = [u8; size_of::<EvalToValTreeResult<'static>>()];
 }

compiler/rustc_middle/src/query/mod.rs

@@ -2719,6 +2719,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query trivial_const(def_id: DefId) -> Option<(mir::ConstValue, Ty<'tcx>)> {
+        desc { |tcx| "checking if `{}` is a trivial const", tcx.def_path_str(def_id) }
+        cache_on_disk_if { def_id.is_local() }
+        separate_provide_extern
+    }
+
     /// Checks for the nearest `#[sanitize(xyz = "off")]` or
     /// `#[sanitize(xyz = "on")]` on this def and any enclosing defs, up to the
     /// crate root.