8284944: assert(cnt++ < 40) failed: infinite cycle in loop optimization

Reviewed-by: mdoerr
Backport-of: 796494d0fecfb9587e8b68ff1d5c09411cb82f89

src/hotspot/share/opto/compile.cpp

@@ -2181,10 +2181,8 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) {
 
 bool Compile::optimize_loops(int& loop_opts_cnt, PhaseIterGVN& igvn, LoopOptsMode mode) {
   if(loop_opts_cnt > 0) {
-    debug_only( int cnt = 0; );
-    while(major_progress() && (loop_opts_cnt > 0)) {
+    while (major_progress() && (loop_opts_cnt > 0)) {
       TracePhase tp("idealLoop", &timers[_t_idealLoop]);
-      assert( cnt++ < 40, "infinite cycle in loop optimization" );
       PhaseIdealLoop ideal_loop(igvn, mode);
       loop_opts_cnt--;
       if (failing())  return false;

test/hotspot/jtreg/compiler/loopopts/TestMaxLoopOptsCountReached.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8284944
+ * @requires vm.compiler2.enabled
+ * @summary triggers the loop optimization phase `LoopOptsCount` many times
+ * @run main/othervm -Xcomp -XX:-PartialPeelLoop -XX:CompileCommand=compileonly,TestMaxLoopOptsCountReached::test TestMaxLoopOptsCountReached
+ */
+
+import java.lang.System.Logger.Level;
+
+class A {
+
+   static String e(long f, boolean b, String g, Level h, String s,
+                   Object... i) {
+       return "message" + s + new String() + g;
+   }
+}
+
+public class TestMaxLoopOptsCountReached {
+
+    static Long a = Long.valueOf(42);
+
+
+    public static void main(String[] args) {
+        test(null, "", null, null);
+        test(null, "", null, null);
+    }
+
+    static void test(Integer o, String g, String name, Object obj) {
+        for (Level q : Level.values())
+            for (Level r : Level.values())
+                A.e(a.longValue(), q != Level.OFF, g, null, null);
+        for (Level q : Level.values())
+            for (Level r : Level.values())
+                A.e(a.longValue(), q != Level.OFF, g, null, null);
+        for (Level q : Level.values()) {
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+        }
+        for (Level q : Level.values()) {
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+        }
+        for (Level q : Level.values()) {
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+            for (Level r : Level.values())
+                ;
+        }
+        for (Level q : Level.values()) {
+            for (Level r : Level.values())
+                ;
+            for (Level r : Level.values())
+                ;
+        }
+        for (Level q : Level.values()) {
+            for (Level r : Level.values()) {
+                String msg = q + "message";
+                String val =
+                        (q != Level.OFF || name != msg)
+                                ? A.e(a.longValue(), q != Level.OFF, g, null, null, "foo")
+                                : null;
+            }
+            for (Level r : Level.values())
+                ;
+        }
+        for (Level q : Level.values()) {
+            for (Level r : Level.values())
+                ;
+            for (Level r : Level.values())
+                ;
+        }
+    }
+}