Add inline.

Author: cjgillot

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -245,6 +245,7 @@ impl DepNodeExt for DepNode {
     /// Construct a DepNode from the given DepKind and DefPathHash. This
     /// method will assert that the given DepKind actually requires a
     /// single DefId/DefPathHash parameter.
+    #[inline]
     fn from_def_path_hash(tcx: TyCtxt<'_>, def_path_hash: DefPathHash, kind: DepKind) -> DepNode {
         debug_assert!(tcx.query_fingerprint_style(kind) == FingerprintStyle::DefPathHash);
         DepNode { kind, hash: def_path_hash.0.into() }
@@ -260,6 +261,7 @@ impl DepNodeExt for DepNode {
     /// DepNode. Condition (2) might not be fulfilled if a DepNode
     /// refers to something from the previous compilation session that
     /// has been removed.
+    #[inline]
     fn extract_def_id<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Option<DefId> {
         if tcx.query_fingerprint_style(self.kind) == FingerprintStyle::DefPathHash {
             Some(tcx.def_path_hash_to_def_id(DefPathHash(self.hash.into()), &mut || {
@@ -288,6 +290,7 @@ impl DepNodeExt for DepNode {
     }
 
     /// Used in testing
+    #[inline]
     fn has_label_string(label: &str) -> bool {
         dep_kind_from_label_string(label).is_ok()
     }

compiler/rustc_query_system/src/dep_graph/dep_node.rs

@@ -77,6 +77,7 @@ impl DepNode {
     /// Creates a new, parameterless DepNode. This method will assert
     /// that the DepNode corresponding to the given DepKind actually
     /// does not require any parameters.
+    #[inline]
     pub fn new_no_params<Ctxt>(tcx: Ctxt, kind: DepKind) -> DepNode
     where
         Ctxt: super::DepContext,
@@ -121,6 +122,7 @@ fn default_kind_debug(kind: &DepKind, f: &mut std::fmt::Formatter<'_>) -> std::f
 }
 
 impl fmt::Debug for DepNode {
+    #[inline]
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         (*NODE_DEBUG)(self, f)
     }
@@ -210,6 +212,7 @@ pub struct WorkProductId {
 }
 
 impl WorkProductId {
+    #[inline]
     pub fn from_cgu_name(cgu_name: &str) -> WorkProductId {
         let mut hasher = StableHasher::new();
         cgu_name.len().hash(&mut hasher);

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -147,6 +147,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn new_disabled() -> DepGraph {
         DepGraph { data: None, virtual_dep_node_index: Lrc::new(AtomicU32::new(0)) }
     }
@@ -163,6 +164,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn assert_ignored(&self) {
         if let Some(..) = self.data {
             crate::tls::read_deps(|task_deps| {
@@ -497,18 +499,21 @@ impl DepGraph {
         self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some()
     }
 
+    #[inline]
     pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
         self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
     }
 
     /// Checks whether a previous work product exists for `v` and, if
     /// so, return the path that leads to it. Used to skip doing work.
+    #[inline]
     pub fn previous_work_product(&self, v: &WorkProductId) -> Option<WorkProduct> {
         self.data.as_ref().and_then(|data| data.previous_work_products.get(v).cloned())
     }
 
     /// Access the map of work-products created during the cached run. Only
     /// used during saving of the dep-graph.
+    #[inline]
     pub fn previous_work_products(&self) -> &FxHashMap<WorkProductId, WorkProduct> {
         &self.data.as_ref().unwrap().previous_work_products
     }
@@ -535,10 +540,12 @@ impl DepGraph {
         dep_node_debug.borrow_mut().insert(dep_node, debug_str);
     }
 
+    #[inline]
     pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option<String> {
         self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned()
     }
 
+    #[inline]
     fn node_color(&self, dep_node: &DepNode) -> Option<DepNodeColor> {
         if let Some(ref data) = self.data {
             if let Some(prev_index) = data.previous.node_to_index_opt(dep_node) {
@@ -792,12 +799,14 @@ impl DepGraph {
 
     // Returns true if the given node has been marked as red during the
     // current compilation session. Used in various assertions
+    #[inline]
     pub fn is_red(&self, dep_node: &DepNode) -> bool {
         self.node_color(dep_node) == Some(DepNodeColor::Red)
     }
 
     // Returns true if the given node has been marked as green during the
     // current compilation session. Used in various assertions
+    #[inline]
     pub fn is_green(&self, dep_node: &DepNode) -> bool {
         self.node_color(dep_node).map_or(false, |c| c.is_green())
     }
@@ -838,6 +847,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         if let Some(data) = &self.data {
             data.current.encoder.steal().finish(profiler)
@@ -846,6 +856,7 @@ impl DepGraph {
         }
     }
 
+    #[inline]
     pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
         let index = self.virtual_dep_node_index.fetch_add(1, Relaxed);
         DepNodeIndex::from_u32(index)
@@ -1012,6 +1023,7 @@ impl CurrentDepGraph {
     }
 
     #[cfg(debug_assertions)]
+    #[inline]
     fn record_edge(&self, dep_node_index: DepNodeIndex, key: DepNode) {
         if let Some(forbidden_edge) = &self.forbidden_edge {
             forbidden_edge.index_to_node.lock().insert(dep_node_index, key);
@@ -1020,6 +1032,7 @@ impl CurrentDepGraph {
 
     /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it.
     /// Assumes that this is a node that has no equivalent in the previous dep-graph.
+    #[inline]
     fn intern_new_node(
         &self,
         profiler: &SelfProfilerRef,
@@ -1040,6 +1053,7 @@ impl CurrentDepGraph {
         }
     }
 
+    #[inline]
     fn intern_node(
         &self,
         profiler: &SelfProfilerRef,
@@ -1142,6 +1156,7 @@ impl CurrentDepGraph {
         }
     }
 
+    #[inline]
     fn promote_node_and_deps_to_current(
         &self,
         profiler: &SelfProfilerRef,
@@ -1237,6 +1252,7 @@ const COMPRESSED_RED: u32 = 1;
 const COMPRESSED_FIRST_GREEN: u32 = 2;
 
 impl DepNodeColorMap {
+    #[inline]
     fn new(size: usize) -> DepNodeColorMap {
         DepNodeColorMap { values: (0..size).map(|_| AtomicU32::new(COMPRESSED_NONE)).collect() }
     }
@@ -1252,6 +1268,7 @@ impl DepNodeColorMap {
         }
     }
 
+    #[inline]
     fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) {
         self.values[index].store(
             match color {

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -53,6 +53,7 @@ pub struct SerializedDepGraph {
 }
 
 impl Default for SerializedDepGraph {
+    #[inline]
     fn default() -> Self {
         SerializedDepGraph {
             nodes: Default::default(),
@@ -91,6 +92,7 @@ impl SerializedDepGraph {
         self.fingerprints[dep_node_index]
     }
 
+    #[inline]
     pub fn node_count(&self) -> usize {
         self.index.len()
     }
@@ -147,7 +149,7 @@ impl<'a> Decodable<opaque::Decoder<'a>> for SerializedDepGraph {
     }
 }
 
-#[derive(Debug, Encodable, Decodable)]
+#[derive(Debug, Encodable)]
 pub struct NodeInfo {
     node: DepNode,
     fingerprint: Fingerprint,
@@ -169,6 +171,7 @@ struct EncoderState {
 }
 
 impl EncoderState {
+    #[inline]
     fn new(encoder: FileEncoder, record_stats: bool) -> Self {
         Self {
             encoder,
@@ -239,6 +242,7 @@ pub struct GraphEncoder {
 }
 
 impl GraphEncoder {
+    #[inline]
     pub fn new(
         encoder: FileEncoder,
         prev_node_count: usize,
@@ -312,6 +316,7 @@ impl GraphEncoder {
         }
     }
 
+    #[inline]
     pub(crate) fn send(
         &self,
         profiler: &SelfProfilerRef,
@@ -324,6 +329,7 @@ impl GraphEncoder {
         self.status.lock().encode_node(&node, &self.record_graph)
     }
 
+    #[inline]
     pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph");
         self.status.into_inner().finish(profiler)

compiler/rustc_query_system/src/query/job.rs

@@ -41,21 +41,25 @@ pub type QueryMap = FxHashMap<QueryJobId, QueryJobInfo>;
 pub struct QueryJobId(pub NonZeroU64);
 
 impl QueryJobId {
+    #[inline]
     fn query(self, map: &QueryMap) -> QueryStackFrame {
         map.get(&self).unwrap().query.clone()
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn span(self, map: &QueryMap) -> Span {
         map.get(&self).unwrap().job.span
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn parent(self, map: &QueryMap) -> Option<QueryJobId> {
         map.get(&self).unwrap().job.parent
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     fn latch<'a>(self, map: &'a QueryMap) -> Option<&'a QueryLatch> {
         map.get(&self).unwrap().job.latch.as_ref()
     }
@@ -84,6 +88,7 @@ pub struct QueryJob {
 
 impl QueryJob {
     /// Creates a new query job.
+    #[inline]
     pub fn new(id: QueryJobId, span: Span, parent: Option<QueryJobId>) -> Self {
         QueryJob {
             id,
@@ -95,6 +100,7 @@ impl QueryJob {
     }
 
     #[cfg(parallel_compiler)]
+    #[inline]
     pub(super) fn latch(&mut self) -> QueryLatch {
         if self.latch.is_none() {
             self.latch = Some(QueryLatch::new());
@@ -106,6 +112,7 @@ impl QueryJob {
     ///
     /// This does nothing for single threaded rustc,
     /// as there are no concurrent jobs which could be waiting on us
+    #[inline]
     pub fn signal_complete(self) {
         #[cfg(parallel_compiler)]
         {
@@ -188,13 +195,15 @@ pub(super) struct QueryLatch {
 
 #[cfg(parallel_compiler)]
 impl QueryLatch {
+    #[inline]
     fn new() -> Self {
         QueryLatch {
             info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
         }
     }
 
     /// Awaits for the query job to complete.
+    #[inline]
     pub(super) fn wait_on(&self, query: Option<QueryJobId>, span: Span) -> Result<(), CycleError> {
         let waiter =
             Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() });
@@ -210,6 +219,7 @@ impl QueryLatch {
     }
 
     /// Awaits the caller on this latch by blocking the current thread.
+    #[inline]
     fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter>) {
         let mut info = self.info.lock();
         if !info.complete {
@@ -232,6 +242,7 @@ impl QueryLatch {
     }
 
     /// Sets the latch and resumes all waiters on it
+    #[inline]
     fn set(&self) {
         let mut info = self.info.lock();
         debug_assert!(!info.complete);
@@ -244,6 +255,7 @@ impl QueryLatch {
 
     /// Removes a single waiter from the list of waiters.
     /// This is used to break query cycles.
+    #[inline]
     fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter> {
         let mut info = self.info.lock();
         debug_assert!(!info.complete);