perf: Put the expression stuff in the expression store behind an Option<Box>

crates/hir-def/src/expr_store/body.rs

@@ -36,6 +36,7 @@ pub struct Body {
 impl ops::Deref for Body {
     type Target = ExpressionStore;
 
+    #[inline]
     fn deref(&self) -> &Self::Target {
         &self.store
     }
@@ -61,6 +62,7 @@ pub struct BodySourceMap {
 impl ops::Deref for BodySourceMap {
     type Target = ExpressionStoreSourceMap;
 
+    #[inline]
     fn deref(&self) -> &Self::Target {
         &self.store
     }
@@ -102,9 +104,7 @@ impl Body {
             }
         };
         let module = def.module(db);
-        let (body, mut source_map) =
-            lower_body(db, def, file_id, module, params, body, is_async_fn);
-        source_map.store.shrink_to_fit();
+        let (body, source_map) = lower_body(db, def, file_id, module, params, body, is_async_fn);
 
         (Arc::new(body), Arc::new(source_map))
     }

crates/hir-def/src/expr_store/lower/asm.rs

@@ -273,7 +273,7 @@ impl ExprCollector<'_> {
             Expr::InlineAsm(InlineAsm { operands: operands.into_boxed_slice(), options }),
             syntax_ptr,
         );
-        self.source_map
+        self.store
             .template_map
             .get_or_insert_with(Default::default)
             .asm_to_captures

crates/hir-def/src/expr_store/lower/path/tests.rs

@@ -23,7 +23,7 @@ fn lower_path(path: ast::Path) -> (TestDB, ExpressionStore, Option<Path>) {
     let mut ctx =
         ExprCollector::new(&db, crate_def_map(&db, krate).root_module_id(), file_id.into());
     let lowered_path = ctx.lower_path(path, &mut ExprCollector::impl_trait_allocator);
-    let store = ctx.store.finish();
+    let (store, _) = ctx.store.finish();
     (db, store, lowered_path)
 }
 

crates/hir-def/src/expr_store/pretty.rs

@@ -902,7 +902,7 @@ impl Printer<'_> {
                         let mut same_name = false;
                         if let Pat::Bind { id, subpat: None } = &self.store[arg.pat] {
                             if let Binding { name, mode: BindingAnnotation::Unannotated, .. } =
-                                &self.store.bindings[*id]
+                                &self.store.assert_expr_only().bindings[*id]
                             {
                                 if name.as_str() == field_name {
                                     same_name = true;
@@ -1063,7 +1063,7 @@ impl Printer<'_> {
     }
 
     fn print_binding(&mut self, id: BindingId) {
-        let Binding { name, mode, .. } = &self.store.bindings[id];
+        let Binding { name, mode, .. } = &self.store.assert_expr_only().bindings[id];
         let mode = match mode {
             BindingAnnotation::Unannotated => "",
             BindingAnnotation::Mutable => "mut ",

crates/hir-def/src/expr_store/scope.rs

@@ -106,7 +106,9 @@ impl ExprScopes {
         let mut scopes = ExprScopes {
             scopes: Arena::default(),
             scope_entries: Arena::default(),
-            scope_by_expr: ArenaMap::with_capacity(body.exprs.len()),
+            scope_by_expr: ArenaMap::with_capacity(
+                body.expr_only.as_ref().map_or(0, |it| it.exprs.len()),
+            ),
         };
         let mut root = scopes.root_scope();
         if let Some(self_param) = body.self_param {
@@ -179,7 +181,7 @@ impl ExprScopes {
         binding: BindingId,
         hygiene: HygieneId,
     ) {
-        let Binding { name, .. } = &store.bindings[binding];
+        let Binding { name, .. } = &store[binding];
         let entry = self.scope_entries.alloc(ScopeEntry { name: name.clone(), binding, hygiene });
         self.scopes[scope].entries =
             IdxRange::new_inclusive(self.scopes[scope].entries.start()..=entry);
@@ -251,7 +253,7 @@ fn compute_expr_scopes(
     scope: &mut ScopeId,
 ) {
     let make_label =
-        |label: &Option<LabelId>| label.map(|label| (label, store.labels[label].name.clone()));
+        |label: &Option<LabelId>| label.map(|label| (label, store[label].name.clone()));
 
     let compute_expr_scopes = |scopes: &mut ExprScopes, expr: ExprId, scope: &mut ScopeId| {
         compute_expr_scopes(expr, store, scopes, scope)
@@ -534,9 +536,8 @@ fn foo() {
         };
 
         let resolved = scopes.resolve_name_in_scope(expr_scope, &name_ref.as_name()).unwrap();
-        let pat_src = source_map
-            .pat_syntax(*source_map.binding_definitions[resolved.binding()].first().unwrap())
-            .unwrap();
+        let pat_src =
+            source_map.pat_syntax(source_map.patterns_for_binding(resolved.binding())[0]).unwrap();
 
         let local_name = pat_src.value.syntax_node_ptr().to_node(file.syntax());
         assert_eq!(local_name.text_range(), expected_name.syntax().text_range());

crates/hir-def/src/expr_store/tests/body.rs

@@ -508,9 +508,9 @@ fn f() {
 }
     "#,
     );
-    assert_eq!(body.bindings.len(), 1, "should have a binding for `B`");
+    assert_eq!(body.assert_expr_only().bindings.len(), 1, "should have a binding for `B`");
     assert_eq!(
-        body.bindings[BindingId::from_raw(RawIdx::from_u32(0))].name.as_str(),
+        body[BindingId::from_raw(RawIdx::from_u32(0))].name.as_str(),
         "B",
         "should have a binding for `B`",
     );
@@ -566,6 +566,7 @@ const fn f(x: i32) -> i32 {
     );
 
     let mtch_arms = body
+        .assert_expr_only()
         .exprs
         .iter()
         .find_map(|(_, expr)| {
@@ -578,10 +579,10 @@ const fn f(x: i32) -> i32 {
         .unwrap();
 
     let MatchArm { pat, .. } = mtch_arms[1];
-    match body.pats[pat] {
+    match body[pat] {
         Pat::Range { start, end } => {
-            let hir_start = &body.exprs[start.unwrap()];
-            let hir_end = &body.exprs[end.unwrap()];
+            let hir_start = &body[start.unwrap()];
+            let hir_end = &body[end.unwrap()];
 
             assert!(matches!(hir_start, Expr::Path { .. }));
             assert!(matches!(hir_end, Expr::Path { .. }));

crates/hir-def/src/signatures.rs

@@ -779,14 +779,10 @@ impl VariantFields {
                 Arc::new(VariantFields { fields, store: Arc::new(store), shape }),
                 Arc::new(source_map),
             ),
-            None => (
-                Arc::new(VariantFields {
-                    fields: Arena::default(),
-                    store: ExpressionStore::empty_singleton(),
-                    shape,
-                }),
-                ExpressionStoreSourceMap::empty_singleton(),
-            ),
+            None => {
+                let (store, source_map) = ExpressionStore::empty_singleton();
+                (Arc::new(VariantFields { fields: Arena::default(), store, shape }), source_map)
+            }
         }
     }
 
@@ -878,7 +874,7 @@ fn lower_fields<Field: ast::HasAttrs + ast::HasVisibility>(
                 idx += 1;
             }
             Err(cfg) => {
-                col.source_map.diagnostics.push(
+                col.store.diagnostics.push(
                     crate::expr_store::ExpressionStoreDiagnostics::InactiveCode {
                         node: InFile::new(fields.file_id, SyntaxNodePtr::new(field.syntax())),
                         cfg,
@@ -891,9 +887,9 @@ fn lower_fields<Field: ast::HasAttrs + ast::HasVisibility>(
     if !has_fields {
         return None;
     }
-    let store = col.store.finish();
+    let (store, source_map) = col.store.finish();
     arena.shrink_to_fit();
-    Some((arena, store, col.source_map))
+    Some((arena, store, source_map))
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -980,7 +976,7 @@ impl EnumVariants {
             if !matches!(variant.shape, FieldsShape::Unit) {
                 let body = db.body(v.into());
                 // A variant with explicit discriminant
-                if body.exprs[body.body_expr] != crate::hir::Expr::Missing {
+                if !matches!(body[body.body_expr], crate::hir::Expr::Missing) {
                     return false;
                 }
             }

crates/hir-ty/src/consteval.rs

@@ -281,7 +281,7 @@ pub(crate) fn const_eval_discriminant_variant(
     let def = variant_id.into();
     let body = db.body(def);
     let loc = variant_id.lookup(db);
-    if body.exprs[body.body_expr] == Expr::Missing {
+    if matches!(body[body.body_expr], Expr::Missing) {
         let prev_idx = loc.index.checked_sub(1);
         let value = match prev_idx {
             Some(prev_idx) => {
@@ -334,7 +334,7 @@ pub(crate) fn eval_to_const(
         // Type checking clousres need an isolated body (See the above FIXME). Bail out early to prevent panic.
         return unknown_const(infer[expr].clone());
     }
-    if let Expr::Path(p) = &ctx.body.exprs[expr] {
+    if let Expr::Path(p) = &ctx.body[expr] {
         let resolver = &ctx.resolver;
         if let Some(c) =
             path_to_const(db, resolver, p, mode, || ctx.generics(), debruijn, infer[expr].clone())

crates/hir-ty/src/diagnostics/decl_check.rs

@@ -226,11 +226,10 @@ impl<'a> DeclValidator<'a> {
         let body = self.db.body(func.into());
         let edition = self.edition(func);
         let mut pats_replacements = body
-            .pats
-            .iter()
+            .pats()
             .filter_map(|(pat_id, pat)| match pat {
                 Pat::Bind { id, .. } => {
-                    let bind_name = &body.bindings[*id].name;
+                    let bind_name = &body[*id].name;
                     let mut suggested_text = to_lower_snake_case(bind_name.as_str())?;
                     if is_raw_identifier(&suggested_text, edition) {
                         suggested_text.insert_str(0, "r#");

crates/hir-ty/src/diagnostics/expr.rs

@@ -101,7 +101,7 @@ impl ExprValidator {
             self.check_for_trailing_return(body.body_expr, &body);
         }
 
-        for (id, expr) in body.exprs.iter() {
+        for (id, expr) in body.exprs() {
             if let Some((variant, missed_fields, true)) =
                 record_literal_missing_fields(db, &self.infer, id, expr)
             {
@@ -132,7 +132,7 @@ impl ExprValidator {
             }
         }
 
-        for (id, pat) in body.pats.iter() {
+        for (id, pat) in body.pats() {
             if let Some((variant, missed_fields, true)) =
                 record_pattern_missing_fields(db, &self.infer, id, pat)
             {
@@ -389,7 +389,7 @@ impl ExprValidator {
         if !self.validate_lints {
             return;
         }
-        match &body.exprs[body_expr] {
+        match &body[body_expr] {
             Expr::Block { statements, tail, .. } => {
                 let last_stmt = tail.or_else(|| match statements.last()? {
                     Statement::Expr { expr, .. } => Some(*expr),
@@ -428,7 +428,7 @@ impl ExprValidator {
             if else_branch.is_none() {
                 return;
             }
-            if let Expr::Block { statements, tail, .. } = &self.body.exprs[*then_branch] {
+            if let Expr::Block { statements, tail, .. } = &self.body[*then_branch] {
                 let last_then_expr = tail.or_else(|| match statements.last()? {
                     Statement::Expr { expr, .. } => Some(*expr),
                     _ => None,

crates/hir-ty/src/diagnostics/match_check.rs

@@ -151,7 +151,7 @@ impl<'a> PatCtxt<'a> {
             hir_def::hir::Pat::Bind { id, subpat, .. } => {
                 let bm = self.infer.binding_modes[pat];
                 ty = &self.infer[id];
-                let name = &self.body.bindings[id].name;
+                let name = &self.body[id].name;
                 match (bm, ty.kind(Interner)) {
                     (BindingMode::Ref(_), TyKind::Ref(.., rty)) => ty = rty,
                     (BindingMode::Ref(_), _) => {

crates/hir-ty/src/diagnostics/unsafe_check.rs

@@ -217,7 +217,7 @@ impl<'db> UnsafeVisitor<'db> {
     }
 
     fn walk_pat(&mut self, current: PatId) {
-        let pat = &self.body.pats[current];
+        let pat = &self.body[current];
 
         if self.inside_union_destructure {
             match pat {
@@ -264,7 +264,7 @@ impl<'db> UnsafeVisitor<'db> {
     }
 
     fn walk_expr(&mut self, current: ExprId) {
-        let expr = &self.body.exprs[current];
+        let expr = &self.body[current];
         let inside_assignment = mem::replace(&mut self.inside_assignment, false);
         match expr {
             &Expr::Call { callee, .. } => {
@@ -284,7 +284,7 @@ impl<'db> UnsafeVisitor<'db> {
                 self.resolver.reset_to_guard(guard);
             }
             Expr::Ref { expr, rawness: Rawness::RawPtr, mutability: _ } => {
-                match self.body.exprs[*expr] {
+                match self.body[*expr] {
                     // Do not report unsafe for `addr_of[_mut]!(EXTERN_OR_MUT_STATIC)`,
                     // see https://github.com/rust-lang/rust/pull/125834.
                     Expr::Path(_) => return,

crates/hir-ty/src/infer/mutability.rs

@@ -273,7 +273,7 @@ impl InferenceContext<'_> {
     fn pat_bound_mutability(&self, pat: PatId) -> Mutability {
         let mut r = Mutability::Not;
         self.body.walk_bindings_in_pat(pat, |b| {
-            if self.body.bindings[b].mode == BindingAnnotation::RefMut {
+            if self.body[b].mode == BindingAnnotation::RefMut {
                 r = Mutability::Mut;
             }
         });

crates/hir-ty/src/infer/pat.rs

@@ -459,7 +459,7 @@ impl InferenceContext<'_> {
         expected: &Ty,
         decl: Option<DeclContext>,
     ) -> Ty {
-        let Binding { mode, .. } = self.body.bindings[binding];
+        let Binding { mode, .. } = self.body[binding];
         let mode = if mode == BindingAnnotation::Unannotated {
             default_bm
         } else {
@@ -639,7 +639,7 @@ impl InferenceContext<'_> {
 pub(super) fn contains_explicit_ref_binding(body: &Body, pat_id: PatId) -> bool {
     let mut res = false;
     body.walk_pats(pat_id, &mut |pat| {
-        res |= matches!(body[pat], Pat::Bind { id, .. } if body.bindings[id].mode == BindingAnnotation::Ref);
+        res |= matches!(body[pat], Pat::Bind { id, .. } if body[id].mode == BindingAnnotation::Ref);
     });
     res
 }

crates/hir-ty/src/layout/tests.rs

@@ -119,8 +119,7 @@ fn eval_expr(
         .unwrap();
     let hir_body = db.body(function_id.into());
     let b = hir_body
-        .bindings
-        .iter()
+        .bindings()
         .find(|x| x.1.name.display_no_db(file_id.edition(&db)).to_smolstr() == "goal")
         .unwrap()
         .0;

crates/hir-ty/src/mir.rs

@@ -1212,10 +1212,9 @@ impl MirSpan {
         match *self {
             MirSpan::ExprId(expr) => matches!(body[expr], Expr::Ref { .. }),
             // FIXME: Figure out if this is correct wrt. match ergonomics.
-            MirSpan::BindingId(binding) => matches!(
-                body.bindings[binding].mode,
-                BindingAnnotation::Ref | BindingAnnotation::RefMut
-            ),
+            MirSpan::BindingId(binding) => {
+                matches!(body[binding].mode, BindingAnnotation::Ref | BindingAnnotation::RefMut)
+            }
             MirSpan::PatId(_) | MirSpan::SelfParam | MirSpan::Unknown => false,
         }
     }