Fix StackOverflowError

(cherry picked from commit c32e146)

Author: marcphilipp

junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/discovery/predicates/TestClassPredicates.java

@@ -20,6 +20,8 @@
 import static org.junit.platform.commons.util.ReflectionUtils.isNestedClassPresent;
 
 import java.lang.reflect.Method;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.function.Predicate;
 
 import org.apiguardian.api.API;
@@ -65,9 +67,16 @@ public TestClassPredicates(DiscoveryIssueReporter issueReporter) {
 	}
 
 	public boolean looksLikeIntendedTestClass(Class<?> candidate) {
-		return this.isAnnotatedWithClassTemplate.test(candidate) //
-				|| hasTestOrTestFactoryOrTestTemplateMethods(candidate) //
-				|| hasNestedTests(candidate);
+		return looksLikeIntendedTestClass(candidate, new HashSet<>());
+	}
+
+	private boolean looksLikeIntendedTestClass(Class<?> candidate, Set<Class<?>> seen) {
+		if (seen.add(candidate)) {
+			return this.isAnnotatedWithClassTemplate.test(candidate) //
+					|| hasTestOrTestFactoryOrTestTemplateMethods(candidate) //
+					|| hasNestedTests(candidate, seen);
+		}
+		return false;
 	}
 
 	public boolean isValidNestedTestClass(Class<?> candidate) {
@@ -84,11 +93,11 @@ private boolean hasTestOrTestFactoryOrTestTemplateMethods(Class<?> candidate) {
 		return isMethodPresent(candidate, this.isTestOrTestFactoryOrTestTemplateMethod);
 	}
 
-	private boolean hasNestedTests(Class<?> candidate) {
+	private boolean hasNestedTests(Class<?> candidate, Set<Class<?>> seen) {
 		return isNestedClassPresent( //
 			candidate, //
 			isNotSame(candidate).and(
-				this.isAnnotatedWithNested.or(it -> isInnerClass(it) && looksLikeIntendedTestClass(it))));
+				this.isAnnotatedWithNested.or(it -> isInnerClass(it) && looksLikeIntendedTestClass(it, seen))));
 	}
 
 	private static Predicate<Class<?>> isNotSame(Class<?> candidate) {

jupiter-tests/src/test/java/org/junit/jupiter/engine/discovery/DiscoveryTests.java

@@ -288,6 +288,13 @@ void reportsWarningForTestClassWithPotentialNestedTestClasses() {
 					InvalidTestCases.InvalidTestClassTestCase.class.getName());
 	}
 
+	@Test
+	void ignoresUnrelatedClassDefinitionCycles() {
+		var results = discoverTestsForClass(UnrelatedRecursiveHierarchyTestCase.class);
+
+		assertThat(results.getDiscoveryIssues()).isEmpty();
+	}
+
 	@Test
 	void reportsWarningsForInvalidTags() throws NoSuchMethodException {
 
@@ -441,6 +448,20 @@ private class InvalidTestClassSubclassTestCase extends InvalidTestClassTestCase
 
 	}
 
+	@SuppressWarnings("JUnitMalformedDeclaration")
+	static class UnrelatedRecursiveHierarchyTestCase {
+
+		@Test
+		void test() {
+		}
+
+		@SuppressWarnings({ "InnerClassMayBeStatic", "unused" })
+		class Inner {
+			class Recursive extends Inner {
+			}
+		}
+	}
+
 	@SuppressWarnings("JUnitMalformedDeclaration")
 	@Tag("")
 	static class InvalidTagsTestCase {