Even more ObligationForest improvements

src/librustc/infer/mod.rs

@@ -1600,8 +1600,8 @@ impl<'a, 'tcx> ShallowResolver<'a, 'tcx> {
 
     // `resolver.shallow_resolve_changed(ty)` is equivalent to
     // `resolver.shallow_resolve(ty) != ty`, but more efficient. It's always
-    // inlined, despite being large, because it has a single call site that is
-    // extremely hot.
+    // inlined, despite being large, because it has only two call sites that
+    // are extremely hot.
     #[inline(always)]
     pub fn shallow_resolve_changed(&mut self, typ: Ty<'tcx>) -> bool {
         match typ.sty {

src/librustc/traits/fulfill.rs

@@ -256,29 +256,46 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> {
         &mut self,
         pending_obligation: &mut Self::Obligation,
     ) -> ProcessResult<Self::Obligation, Self::Error> {
-        // If we were stalled on some unresolved variables, first check
-        // whether any of them have been resolved; if not, don't bother
-        // doing more work yet
-        if !pending_obligation.stalled_on.is_empty() {
-            let mut changed = false;
-            // This `for` loop was once a call to `all()`, but this lower-level
-            // form was a perf win. See #64545 for details.
-            for &ty in &pending_obligation.stalled_on {
-                if ShallowResolver::new(self.selcx.infcx()).shallow_resolve_changed(ty) {
-                    changed = true;
-                    break;
-                }
+        // If we were stalled on some unresolved variables, first check whether
+        // any of them have been resolved; if not, don't bother doing more work
+        // yet.
+        let change = match pending_obligation.stalled_on.len() {
+            // Match arms are in order of frequency, which matters because this
+            // code is so hot. 1 and 0 dominate; 2+ is fairly rare.
+            1 => {
+                let ty = pending_obligation.stalled_on[0];
+                ShallowResolver::new(self.selcx.infcx()).shallow_resolve_changed(ty)
+            }
+            0 => {
+                // In this case we haven't changed, but wish to make a change.
+                true
             }
-            if !changed {
-                debug!("process_predicate: pending obligation {:?} still stalled on {:?}",
-                       self.selcx.infcx()
-                           .resolve_vars_if_possible(&pending_obligation.obligation),
-                       pending_obligation.stalled_on);
-                return ProcessResult::Unchanged;
+            _ => {
+                // This `for` loop was once a call to `all()`, but this lower-level
+                // form was a perf win. See #64545 for details.
+                (|| {
+                    for &ty in &pending_obligation.stalled_on {
+                        if ShallowResolver::new(self.selcx.infcx()).shallow_resolve_changed(ty) {
+                            return true;
+                        }
+                    }
+                    false
+                })()
             }
-            pending_obligation.stalled_on = vec![];
+        };
+
+        if !change {
+            debug!("process_predicate: pending obligation {:?} still stalled on {:?}",
+                   self.selcx.infcx()
+                       .resolve_vars_if_possible(&pending_obligation.obligation),
+                   pending_obligation.stalled_on);
+            return ProcessResult::Unchanged;
         }
 
+        // This part of the code is much colder.
+
+        pending_obligation.stalled_on.truncate(0);
+
         let obligation = &mut pending_obligation.obligation;
 
         if obligation.predicate.has_infer_types() {

src/librustc_data_structures/obligation_forest/mod.rs

@@ -484,13 +484,16 @@ impl<O: ForestObligation> ObligationForest<O> {
         debug!("process_cycles()");
 
         for (index, node) in self.nodes.iter().enumerate() {
-            // For rustc-benchmarks/inflate-0.1.0 this state test is extremely
-            // hot and the state is almost always `Pending` or `Waiting`. It's
-            // a win to handle the no-op cases immediately to avoid the cost of
-            // the function call.
+            // For some benchmarks this state test is extremely
+            // hot. It's a win to handle the no-op cases immediately to avoid
+            // the cost of the function call.
             match node.state.get() {
-                NodeState::Waiting | NodeState::Pending | NodeState::Done | NodeState::Error => {},
-                _ => self.find_cycles_from_node(&mut stack, processor, index),
+                // Match arms are in order of frequency. Pending, Success and
+                // Waiting dominate; the others are rare.
+                NodeState::Pending => {},
+                NodeState::Success => self.find_cycles_from_node(&mut stack, processor, index),
+                NodeState::Waiting | NodeState::Done | NodeState::Error => {},
+                NodeState::OnDfsStack => self.find_cycles_from_node(&mut stack, processor, index),
             }
         }