Only use the cache for the slice path so PGO can leave it outlined

Author: saethlin

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -10,7 +10,6 @@ use rustc_hir::{self as hir, def_id::DefId, definitions::DefPathData};
 use rustc_index::IndexVec;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::{ErrorHandled, InvalidMetaKind, ReportedErrorInfo};
-use rustc_middle::mir::ConstValue;
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::{
     self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOf, LayoutOfHelpers,
@@ -23,9 +22,9 @@ use rustc_span::Span;
 use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout};
 
 use super::{
-    AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace,
-    MemPlaceMeta, Memory, MemoryKind, OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic,
-    Projectable, Provenance, Scalar, StackPopJump,
+    AllocId, ConstAllocation, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy,
+    Machine, MemPlace, MemPlaceMeta, Memory, MemoryKind, OpTy, Operand, Place, PlaceTy, Pointer,
+    PointerArithmetic, Projectable, Provenance, Scalar, StackPopJump,
 };
 use crate::errors;
 use crate::util;
@@ -51,8 +50,7 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
     /// The recursion limit (cached from `tcx.recursion_limit(())`)
     pub recursion_limit: Limit,
 
-    pub const_cache:
-        Lock<FxHashMap<(ConstValue<'tcx>, TyAndLayout<'tcx>, Span), OpTy<'tcx, M::Provenance>>>,
+    pub const_cache: Lock<FxHashMap<(ConstAllocation<'tcx>, u64, Span), Immediate<M::Provenance>>>,
 }
 
 // The Phantomdata exists to prevent this type from being `Send`. If it were sent across a thread

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -2,6 +2,7 @@
 //! All high-level functions to read from memory work on operands as sources.
 
 use std::assert_matches::assert_matches;
+use std::collections::hash_map::Entry;
 
 use either::{Either, Left, Right};
 
@@ -13,9 +14,9 @@ use rustc_middle::{mir, ty};
 use rustc_target::abi::{self, Abi, HasDataLayout, Size};
 
 use super::{
-    alloc_range, from_known_layout, mir_assign_valid_types, AllocId, Frame, InterpCx, InterpResult,
-    MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer, Projectable,
-    Provenance, Scalar,
+    alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstAllocation, Frame,
+    InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy,
+    Pointer, Projectable, Provenance, Scalar,
 };
 
 /// An `Immediate` represents a single immediate self-contained Rust value.
@@ -748,10 +749,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             })
         };
         let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
-        let span = self.cur_span();
-        if let Some(op) = self.const_cache.lock().get(&(val_val, layout, span)).cloned() {
-            return Ok(op);
-        }
         let imm = match val_val {
             mir::ConstValue::Indirect { alloc_id, offset } => {
                 // We rely on mutability being set correctly in that allocation to prevent writes
@@ -761,16 +758,29 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
             mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
             mir::ConstValue::ZeroSized => Immediate::Uninit,
-            mir::ConstValue::Slice { data, meta } => {
+            mir::ConstValue::Slice { data, meta } => self.const_slice_to_op(data, meta)?,
+        };
+        Ok(OpTy { op: Operand::Immediate(imm), layout })
+    }
+
+    fn const_slice_to_op(
+        &self,
+        data: ConstAllocation<'tcx>,
+        meta: u64,
+    ) -> InterpResult<'tcx, Immediate<M::Provenance>> {
+        let span = self.cur_span();
+        let imm = match self.const_cache.lock().entry((data, meta, span)) {
+            Entry::Occupied(e) => e.get().clone(),
+            Entry::Vacant(e) => {
                 // We rely on mutability being set correctly in `data` to prevent writes
                 // where none should happen.
                 let ptr = Pointer::new(self.tcx.reserve_and_set_memory_alloc(data), Size::ZERO);
-                Immediate::new_slice(self.global_base_pointer(ptr)?.into(), meta, self)
+                let imm = Immediate::new_slice(self.global_base_pointer(ptr)?.into(), meta, self);
+                e.insert(imm.clone());
+                imm
             }
         };
-        let res = OpTy { op: Operand::Immediate(imm), layout };
-        self.const_cache.lock().insert((val_val, layout, span), res.clone());
-        Ok(res)
+        Ok(imm)
     }
 }