8337142: StackOverflowError in Types.containsTypeRecursive with deeply nested type hierarchy

Reviewed-by: mcimadamore

Author: Vicente Romero

src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java

@@ -1346,7 +1346,7 @@ public boolean isSameType(Type t, Type s) {
          * Type-equality relation - type variables are considered
          * equals if they share the same object identity.
          */
-        TypeRelation isSameTypeVisitor = new TypeRelation() {
+        abstract class TypeEqualityVisitor extends TypeRelation {
 
             public Boolean visitType(Type t, Type s) {
                 if (t.equalsIgnoreMetadata(s))
@@ -1385,10 +1385,12 @@ public Boolean visitWildcardType(WildcardType t, Type s) {
                 } else {
                     WildcardType t2 = (WildcardType)s;
                     return (t.kind == t2.kind || (t.isExtendsBound() && s.isExtendsBound())) &&
-                            isSameType(t.type, t2.type);
+                            sameTypeComparator(t.type, t2.type);
                 }
             }
 
+            abstract boolean sameTypeComparator(Type t, Type s);
+
             @Override
             public Boolean visitClassType(ClassType t, Type s) {
                 if (t == s)
@@ -1419,9 +1421,11 @@ public Boolean visitClassType(ClassType t, Type s) {
                 }
                 return t.tsym == s.tsym
                     && visit(t.getEnclosingType(), s.getEnclosingType())
-                    && containsTypeEquivalent(t.getTypeArguments(), s.getTypeArguments());
+                    && sameTypeArguments(t.getTypeArguments(), s.getTypeArguments());
             }
 
+            abstract boolean sameTypeArguments(List<Type> ts, List<Type> ss);
+
             @Override
             public Boolean visitArrayType(ArrayType t, Type s) {
                 if (t == s)
@@ -1477,6 +1481,16 @@ public Boolean visitUndetVar(UndetVar t, Type s) {
             public Boolean visitErrorType(ErrorType t, Type s) {
                 return true;
             }
+        }
+
+        TypeEqualityVisitor isSameTypeVisitor = new TypeEqualityVisitor() {
+            boolean sameTypeComparator(Type t, Type s) {
+                return isSameType(t, s);
+            }
+
+            boolean sameTypeArguments(List<Type> ts, List<Type> ss) {
+                return containsTypeEquivalent(ts, ss);
+            }
         };
 
     // </editor-fold>
@@ -3862,7 +3876,7 @@ public List<Type> intersect(List<Type> cl1, List<Type> cl2) {
     // where
         class TypePair {
             final Type t1;
-            final Type t2;;
+            final Type t2;
 
             TypePair(Type t1, Type t2) {
                 this.t1 = t1;
@@ -3875,10 +3889,28 @@ public int hashCode() {
             @Override
             public boolean equals(Object obj) {
                 return (obj instanceof TypePair typePair)
-                        && isSameType(t1, typePair.t1)
-                        && isSameType(t2, typePair.t2);
+                        && exactTypeVisitor.visit(t1, typePair.t1)
+                        && exactTypeVisitor.visit(t2, typePair.t2);
             }
         }
+
+        TypeEqualityVisitor exactTypeVisitor = new TypeEqualityVisitor() {
+            @Override
+            boolean sameTypeArguments(List<Type> ts, List<Type> ss) {
+                while (ts.nonEmpty() && ss.nonEmpty()
+                        && sameTypeComparator(ts.head, ss.head)) {
+                    ts = ts.tail;
+                    ss = ss.tail;
+                }
+                return ts.isEmpty() && ss.isEmpty();
+            }
+
+            @Override
+            boolean sameTypeComparator(Type t, Type s) {
+                return exactTypeVisitor.visit(t, s);
+            }
+        };
+
         Set<TypePair> mergeCache = new HashSet<>();
         private Type merge(Type c1, Type c2) {
             ClassType class1 = (ClassType) c1;

test/langtools/tools/javac/types/SOEForDeeplyNestedTypeTest.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8337142
+ * @summary StackOverflowError in Types.containsTypeRecursive with deeply nested type hierarchy
+ * @compile SOEForDeeplyNestedTypeTest.java
+ */
+
+import java.util.List;
+
+public class SOEForDeeplyNestedTypeTest {
+    class T {
+        List<M<?, ?>> xs = List.of(new M<>(One.class), new M<>(Two.class), new M<>(Two.class));
+    }
+
+    class M<R extends Eight, I extends Nine<? extends R>> {
+        M(Class<? extends I> c) {}
+    }
+
+    class One implements Three<Five>, Six<One> {}
+    class Two implements Four<Seven>, Six<Two> {}
+    interface Three<R extends Eight> extends Nine<R> {}
+    interface Four<R extends Eight> extends Nine<R> {}
+    class Five extends Ten<Five> implements Eleven<Twelve>, Thirteen {}
+    interface Six<FullKeyT extends TwentyTwo> extends TwentyTwo {}
+    class Seven extends Ten<Seven> implements Eleven<Fourteen>, Thirteen {}
+    interface Eight {}
+    interface Nine<R extends Eight> extends TwentyTwo {}
+    class Ten<K extends TwentyTwo> implements TwentyTwo {}
+    interface Eleven<PkT extends Twenty> extends Twenty {}
+    class Twelve extends Ten<Twelve> implements Eleven<Sixteen>, Thirteen {}
+    interface Thirteen extends Seventeen {}
+    class Fourteen extends Ten<Fourteen> implements Eleven<Fifteen>, Thirteen {}
+    class Fifteen extends Ten<Fifteen> implements Eleven<Twelve>, Thirteen {}
+    class Sixteen extends Nineteen<Sixteen> implements Eighteen, Thirteen {}
+    interface Seventeen extends Twenty {}
+    interface Eighteen extends Twenty {}
+    class Nineteen<K extends Twenty> extends Ten<K> implements Twenty {}
+    interface Twenty extends TwentyTwo {}
+    class TwentyOne<R extends Eight> {}
+    interface TwentyTwo extends Eight {}
+}