diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 88057e59b5f..8819db3f035 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1472,6 +1472,15 @@ void PhaseIdealLoop::insert_pre_post_loops(IdealLoopTree *loop, Node_List &old_n set_idom(new_pre_exit, pre_end, dd_main_head); set_loop(new_pre_exit, outer_loop->_parent); + if (peel_only) { + // Nodes in the peeled iteration that were marked as reductions within the + // original loop might not be reductions within their new outer loop. + for (uint i = 0; i < loop->_body.size(); i++) { + Node* n = old_new[loop->_body[i]->_idx]; + n->remove_flag(Node::Flag_is_reduction); + } + } + // Step B2: Build a zero-trip guard for the main-loop. After leaving the // pre-loop, the main-loop may not execute at all. Later in life this // zero-trip guard will become the minimum-trip guard when we unroll diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index d80f911513f..1a3fcb63051 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -2555,6 +2555,17 @@ uint IdealLoopTree::est_loop_flow_merge_sz() const { return 0; } +#ifdef ASSERT +bool IdealLoopTree::has_reduction_nodes() const { + for (uint i = 0; i < _body.size(); i++) { + if (_body[i]->is_reduction()) { + return true; + } + } + return false; +} +#endif // ASSERT + #ifndef PRODUCT //------------------------------dump_head-------------------------------------- // Dump 1 liner for loop header info diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index 06c1237e919..ef2454e9c34 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -658,6 +658,11 @@ class IdealLoopTree : public ResourceObj { void remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop *phase); +#ifdef ASSERT + // Tell whether the body contains nodes marked as reductions. + bool has_reduction_nodes() const; +#endif // ASSERT + #ifndef PRODUCT void dump_head() const; // Dump loop head only void dump() const; // Dump this loop recursively diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp index 393d271b57a..3e24960f436 100644 --- a/src/hotspot/share/opto/superword.cpp +++ b/src/hotspot/share/opto/superword.cpp @@ -104,6 +104,8 @@ void SuperWord::transform_loop(IdealLoopTree* lpt, bool do_optimization) { if (!cl->is_valid_counted_loop()) return; // skip malformed counted loop + assert(!lpt->has_reduction_nodes() || cl->is_reduction_loop(), + "non-reduction loop contains reduction nodes"); bool post_loop_allowed = (PostLoopMultiversioning && Matcher::has_predicated_vectors() && cl->is_post_loop()); if (post_loop_allowed) { if (cl->is_reduction_loop()) return; // no predication mapping diff --git a/test/hotspot/jtreg/compiler/loopopts/superword/TestPeeledReductionNode.java b/test/hotspot/jtreg/compiler/loopopts/superword/TestPeeledReductionNode.java new file mode 100644 index 00000000000..99e1e2465f8 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/superword/TestPeeledReductionNode.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8279622 + * @summary Test that reduction nodes peeled out of an inner loop are not + * vectorized as reductions within the outer loop. + * @library /test/lib + * @comment The test is run with -XX:LoopUnrollLimit=32 to prevent unrolling + * from fully replacing vectorization. + * @run main/othervm -Xbatch -XX:LoopUnrollLimit=32 + * compiler.loopopts.superword.TestPeeledReductionNode + */ +package compiler.loopopts.superword; + +import jdk.test.lib.Asserts; + +public class TestPeeledReductionNode { + static final int N = 32; + static final int M = 65; // Must be odd and >= 65 to trigger the failure. + static final int INPUT = 0b0000_0000_0000_0000_0000_0000_0000_0001; + static final int MASK = 0b0000_0000_1000_0000_0000_0000_0000_0000; + static final int EXPECTED = (M % 2 == 0 ? INPUT : INPUT ^ MASK); + static int mask = 0; + public static void main(String[] args) { + int r[] = new int[N]; + for (int i = 0; i < N; i++) { + r[i] = INPUT; + } + // Trigger the relevant OSR compilation and set + // TestPeeledReductionNode.mask to MASK. + for (int k = 0; k < MASK; k++) { + TestPeeledReductionNode.mask++; + } + for (int i = 0; i < N; i++) { + for (int j = 0; j < M; j++) { + // Before the fix, this reduction is peeled out of its loop and + // wrongly remains marked as a reduction within the outer loop. + r[i] ^= TestPeeledReductionNode.mask; + } + } + for (int i = 0; i < N; i++) { + Asserts.assertEquals(r[i], EXPECTED); + } + return; + } +}