Only check the DefId for the recursion check in MIR inliner.

compiler/rustc_mir_transform/src/inline.rs

@@ -10,6 +10,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::subst::Subst;
 use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
+use rustc_span::def_id::DefId;
 use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span};
 use rustc_target::spec::abi::Abi;
 
@@ -103,8 +104,12 @@ struct Inliner<'tcx> {
     param_env: ParamEnv<'tcx>,
     /// Caller codegen attributes.
     codegen_fn_attrs: &'tcx CodegenFnAttrs,
-    /// Stack of inlined Instances.
-    history: Vec<ty::Instance<'tcx>>,
+    /// Stack of inlined instances.
+    /// We only check the `DefId` and not the substs because we want to
+    /// avoid inlining cases of polymorphic recursion.
+    /// The number of `DefId`s is finite, so checking history is enough
+    /// to ensure that we do not loop endlessly while inlining.
+    history: Vec<DefId>,
     /// Indicates that the caller body has been modified.
     changed: bool,
 }
@@ -132,7 +137,7 @@ impl<'tcx> Inliner<'tcx> {
                 Ok(new_blocks) => {
                     debug!("inlined {}", callsite.callee);
                     self.changed = true;
-                    self.history.push(callsite.callee);
+                    self.history.push(callsite.callee.def_id());
                     self.process_blocks(caller_body, new_blocks);
                     self.history.pop();
                 }
@@ -308,7 +313,7 @@ impl<'tcx> Inliner<'tcx> {
                     return None;
                 }
 
-                if self.history.contains(&callee) {
+                if self.history.contains(&callee.def_id()) {
                     return None;
                 }
 

src/test/mir-opt/inline/polymorphic-recursion.rs

@@ -0,0 +1,25 @@
+// Make sure that the MIR inliner does not loop indefinitely on polymorphic recursion.
+// compile-flags: --crate-type lib
+
+// Randomize `def_path_hash` by defining them under a module with different names
+macro_rules! emit {
+    ($($m:ident)*) => {$(
+        pub mod $m {
+            pub trait Tr { type Next: Tr; }
+
+            pub fn hoge<const N: usize, T: Tr>() {
+                inner::<N, T>();
+            }
+
+            #[inline(always)]
+            fn inner<const N: usize, T: Tr>()
+            {
+                inner::<N, T::Next>();
+                inner::<N, T::Next>();
+            }
+        }
+    )*};
+}
+
+// Increase the chance of triggering the bug
+emit!(m00 m01 m02 m03 m04 m05 m06 m07 m08 m09 m10 m11 m12 m13 m14 m15 m16 m17 m18 m19);