Only store the LocalDefId instead of the whole instance.

Author: cjgillot

compiler/rustc_middle/src/query/mod.rs

@@ -1284,7 +1284,7 @@ rustc_queries! {
     }
 
     /// Return the set of (transitive) callees that may result in a recursive call to `key`.
-    query mir_callgraph_cyclic(key: LocalDefId) -> &'tcx UnordSet<ty::Instance<'tcx>> {
+    query mir_callgraph_cyclic(key: LocalDefId) -> &'tcx UnordSet<LocalDefId> {
         fatal_cycle
         arena_cache
         desc { |tcx|

compiler/rustc_mir_transform/src/inline.rs

@@ -770,14 +770,15 @@ fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
         return Ok(());
     }
 
-    if callee_def_id.is_local()
+    if let Some(callee_def_id) = callee_def_id.as_local()
         && !inliner
             .tcx()
             .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce)
     {
         // If we know for sure that the function we're calling will itself try to
         // call us, then we avoid inlining that function.
-        if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee) {
+        if inliner.tcx().mir_callgraph_cyclic(caller_def_id.expect_local()).contains(&callee_def_id)
+        {
             debug!("query cycle avoidance");
             return Err("caller might be reachable from callee");
         }

compiler/rustc_mir_transform/src/inline/cycle.rs

@@ -8,6 +8,7 @@ use rustc_session::Limit;
 use rustc_span::sym;
 use tracing::{instrument, trace};
 
+#[instrument(level = "debug", skip(tcx), ret)]
 fn should_recurse<'tcx>(tcx: TyCtxt<'tcx>, callee: ty::Instance<'tcx>) -> bool {
     match callee.def {
         // If there is no MIR available (either because it was not in metadata or
@@ -64,7 +65,7 @@ fn process<'tcx>(
     caller: ty::Instance<'tcx>,
     target: LocalDefId,
     seen: &mut FxHashSet<ty::Instance<'tcx>>,
-    involved: &mut FxHashSet<ty::Instance<'tcx>>,
+    involved: &mut FxHashSet<LocalDefId>,
     recursion_limiter: &mut FxHashMap<DefId, usize>,
     recursion_limit: Limit,
 ) -> bool {
@@ -122,7 +123,10 @@ fn process<'tcx>(
                 true
             };
             if found_recursion {
-                involved.insert(callee);
+                if let Some(callee) = callee.def_id().as_local() {
+                    // Calling `optimized_mir` of a non-local definition cannot cycle.
+                    involved.insert(callee);
+                }
                 cycle_found = true;
             }
         }
@@ -135,7 +139,7 @@ fn process<'tcx>(
 pub(crate) fn mir_callgraph_cyclic<'tcx>(
     tcx: TyCtxt<'tcx>,
     root: LocalDefId,
-) -> UnordSet<ty::Instance<'tcx>> {
+) -> UnordSet<LocalDefId> {
     assert!(
         !tcx.is_constructor(root.to_def_id()),
         "you should not call `mir_callgraph_reachable` on enum/struct constructor functions"