8342043: Split Opaque4Node into OpaqueTemplateAssertionPredicateNode and OpaqueNotNullNode

Reviewed-by: kvn, roland

Author: chhagedorn

src/hotspot/share/opto/classes.hpp

@@ -270,8 +270,9 @@ macro(Opaque1)
 macro(OpaqueLoopInit)
 macro(OpaqueLoopStride)
 macro(OpaqueZeroTripGuard)
-macro(Opaque4)
+macro(OpaqueNotNull)
 macro(OpaqueInitializedAssertionPredicate)
+macro(OpaqueTemplateAssertionPredicate)
 macro(ProfileBoolean)
 macro(OrI)
 macro(OrL)

src/hotspot/share/opto/compile.cpp

@@ -393,7 +393,7 @@ void Compile::remove_useless_node(Node* dead) {
   if (dead->is_expensive()) {
     remove_expensive_node(dead);
   }
-  if (dead->Opcode() == Op_Opaque4) {
+  if (dead->is_OpaqueTemplateAssertionPredicate()) {
     remove_template_assertion_predicate_opaq(dead);
   }
   if (dead->is_ParsePredicate()) {

src/hotspot/share/opto/compile.hpp

@@ -370,7 +370,8 @@ class Compile : public Phase {
   GrowableArray<CallGenerator*> _intrinsics;    // List of intrinsics.
   GrowableArray<Node*>  _macro_nodes;           // List of nodes which need to be expanded before matching.
   GrowableArray<ParsePredicateNode*> _parse_predicates; // List of Parse Predicates.
-  GrowableArray<Node*>  _template_assertion_predicate_opaqs; // List of Opaque4 nodes for Template Assertion Predicates.
+  // List of OpaqueTemplateAssertionPredicateNode nodes for Template Assertion Predicates.
+  GrowableArray<Node*>  _template_assertion_predicate_opaqs;
   GrowableArray<Node*>  _expensive_nodes;       // List of nodes that are expensive to compute and that we'd better not let the GVN freely common
   GrowableArray<Node*>  _for_post_loop_igvn;    // List of nodes for IGVN after loop opts are over
   GrowableArray<UnstableIfTrap*> _unstable_if_traps;        // List of ifnodes after IGVN
@@ -770,7 +771,7 @@ class Compile : public Phase {
 
   void add_template_assertion_predicate_opaq(Node* n) {
     assert(!_template_assertion_predicate_opaqs.contains(n),
-           "duplicate entry in template assertion predicate opaque4 list");
+           "Duplicate entry in Template Assertion Predicate OpaqueTemplateAssertionPredicate list");
     _template_assertion_predicate_opaqs.append(n);
   }
 

src/hotspot/share/opto/escape.cpp

@@ -574,14 +574,14 @@ bool ConnectionGraph::can_reduce_check_users(Node* n, uint nesting) const {
         // CmpP/N used by the If controlling the cast.
         if (use->in(0)->is_IfTrue() || use->in(0)->is_IfFalse()) {
           Node* iff = use->in(0)->in(0);
-          // We may have Opaque4 node between If and Bool nodes.
-          // Bail out in such case - we need to preserve Opaque4 for correct
-          // processing predicates after loop opts.
+          // We may have an OpaqueNotNull node between If and Bool nodes. Bail out in such case.
           bool can_reduce = (iff->Opcode() == Op_If) && iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp();
           if (can_reduce) {
             Node* iff_cmp = iff->in(1)->in(1);
             int opc = iff_cmp->Opcode();
             can_reduce = (opc == Op_CmpP || opc == Op_CmpN) && can_reduce_cmp(n, iff_cmp);
+          } else {
+            assert(iff->in(1)->is_OpaqueNotNull(), "must be OpaqueNotNull");
           }
           if (!can_reduce) {
 #ifndef PRODUCT

src/hotspot/share/opto/graphKit.cpp

@@ -1458,17 +1458,17 @@ Node* GraphKit::cast_not_null(Node* obj, bool do_replace_in_map) {
 // In that case that data path will die and we need the control path
 // to become dead as well to keep the graph consistent. So we have to
 // add a check for null for which one branch can't be taken. It uses
-// an Opaque4 node that will cause the check to be removed after loop
+// an OpaqueNotNull node that will cause the check to be removed after loop
 // opts so the test goes away and the compiled code doesn't execute a
 // useless check.
 Node* GraphKit::must_be_not_null(Node* value, bool do_replace_in_map) {
   if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(value))) {
     return value;
   }
   Node* chk = _gvn.transform(new CmpPNode(value, null()));
-  Node *tst = _gvn.transform(new BoolNode(chk, BoolTest::ne));
-  Node* opaq = _gvn.transform(new Opaque4Node(C, tst, intcon(1)));
-  IfNode *iff = new IfNode(control(), opaq, PROB_MAX, COUNT_UNKNOWN);
+  Node* tst = _gvn.transform(new BoolNode(chk, BoolTest::ne));
+  Node* opaq = _gvn.transform(new OpaqueNotNullNode(C, tst));
+  IfNode* iff = new IfNode(control(), opaq, PROB_MAX, COUNT_UNKNOWN);
   _gvn.set_type(iff, iff->Value(&_gvn));
   if (!tst->is_Con()) {
     record_for_igvn(iff);

src/hotspot/share/opto/loopPredicate.cpp

@@ -338,7 +338,8 @@ void PhaseIdealLoop::clone_assertion_predicates_to_unswitched_loop(IdealLoopTree
 }
 
 // Put all Assertion Predicate projections on a list, starting at 'predicate' and going up in the tree. If 'get_opaque'
-// is set, then the Opaque4 nodes of the Assertion Predicates are put on the list instead of the projections.
+// is set, then the OpaqueTemplateAssertionPredicate nodes of the Assertion Predicates are put on the list instead of
+// the projections.
 void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque) {
   ParsePredicateNode* parse_predicate = predicate->in(0)->as_ParsePredicate();
   ProjNode* uncommon_proj = parse_predicate->proj_out(1 - predicate->as_Proj()->_con);
@@ -353,10 +354,10 @@ void PhaseIdealLoop::get_assertion_predicates(Node* predicate, Unique_Node_List&
     }
     Node* bol = iff->in(1);
     assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate");
-    if (bol->is_Opaque4()) {
+    if (bol->is_OpaqueTemplateAssertionPredicate()) {
       assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes");
       if (get_opaque) {
-        // Collect the predicate Opaque4 node.
+        // Collect the OpaqueTemplateAssertionPredicateNode.
         list.push(bol);
       } else {
         // Collect the predicate projection.
@@ -374,11 +375,11 @@ IfProjNode* PhaseIdealLoop::clone_assertion_predicate_for_unswitched_loops(IfNod
                                                                            IfProjNode* predicate,
                                                                            Deoptimization::DeoptReason reason,
                                                                            ParsePredicateSuccessProj* parse_predicate_proj) {
-  TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_Opaque4());
-  Opaque4Node* cloned_opaque4_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this);
+  TemplateAssertionExpression template_assertion_expression(template_assertion_predicate->in(1)->as_OpaqueTemplateAssertionPredicate());
+  OpaqueTemplateAssertionPredicateNode* cloned_opaque_node = template_assertion_expression.clone(parse_predicate_proj->in(0)->in(0), this);
   IfProjNode* if_proj = create_new_if_for_predicate(parse_predicate_proj, nullptr, reason,
                                                     template_assertion_predicate->Opcode(), false);
-  _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque4_node);
+  _igvn.replace_input_of(if_proj->in(0), 1, cloned_opaque_node);
   _igvn.replace_input_of(parse_predicate_proj->in(0), 0, if_proj);
   set_idom(parse_predicate_proj->in(0), if_proj, dom_depth(if_proj));
   return if_proj;

src/hotspot/share/opto/loopTransform.cpp

@@ -1189,8 +1189,10 @@ bool IdealLoopTree::policy_range_check(PhaseIdealLoop* phase, bool provisional,
         continue;
       }
       if (!bol->is_Bool()) {
-        assert(bol->is_Opaque4() || bol->is_OpaqueInitializedAssertionPredicate(),
-               "Opaque node of non-null-check or of Initialized Assertion Predicate");
+        assert(bol->is_OpaqueNotNull() ||
+               bol->is_OpaqueTemplateAssertionPredicate() ||
+               bol->is_OpaqueInitializedAssertionPredicate(),
+               "Opaque node of a non-null-check or an Assertion Predicate");
         continue;
       }
       if (bol->as_Bool()->_test._test == BoolTest::ne) {
@@ -1359,7 +1361,7 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica
         break;
       Node* bol = iff->in(1);
       assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate");
-      if (bol->is_Opaque4()) {
+      if (bol->is_OpaqueTemplateAssertionPredicate()) {
         // Clone the Assertion Predicate twice and initialize one with the initial
         // value of the loop induction variable. Leave the other predicate
         // to be initialized when increasing the stride during loop unrolling.
@@ -1399,11 +1401,11 @@ void PhaseIdealLoop::copy_assertion_predicates_to_main_loop_helper(const Predica
   }
 }
 
+#ifdef ASSERT
 bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
   uint init;
   uint stride;
   count_opaque_loop_nodes(iff->in(1)->in(1), init, stride);
-#ifdef ASSERT
   ResourceMark rm;
   Unique_Node_List wq;
   wq.clear();
@@ -1429,10 +1431,10 @@ bool PhaseIdealLoop::assertion_predicate_has_loop_opaque_node(IfNode* iff) {
     }
   }
   assert(init == verif_init && stride == verif_stride, "missed opaque node");
-#endif
   assert(stride == 0 || init != 0, "init should be there every time stride is");
   return init != 0;
 }
+#endif // ASSERT
 
 void PhaseIdealLoop::count_opaque_loop_nodes(Node* n, uint& init, uint& stride) {
   init = 0;
@@ -1475,14 +1477,15 @@ IfTrueNode* PhaseIdealLoop::create_initialized_assertion_predicate(IfNode* templ
 
 // Clone the Template Assertion Predicate and set a new OpaqueLoopInitNode to create a new Template Assertion Predicate.
 // This is done when creating a new Template Assertion Predicate for the main loop which requires a new init node.
-// We keep the Opaque4 node since it's still a template. Since the templates are eventually removed after loop opts,
-// these are never executed. We therefore insert a Halt node instead of an uncommon trap.
+// We keep the OpaqueTemplateAssertionPredicate node since it's still a template. Since the templates are eventually
+// removed after loop opts, these are never executed. We therefore insert a Halt node instead of an uncommon trap.
 Node* PhaseIdealLoop::clone_template_assertion_predicate(IfNode* iff, Node* new_init, Node* predicate, Node* uncommon_proj,
                                                          Node* control, IdealLoopTree* outer_loop, Node* new_control) {
   assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes for Template Assertion Predicate");
-  TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_Opaque4());
+  TemplateAssertionExpression template_assertion_expression(iff->in(1)->as_OpaqueTemplateAssertionPredicate());
   assert(new_init->is_OpaqueLoopInit(), "only for creating new Template Assertion Predicates");
-  Opaque4Node* new_opaque_node = template_assertion_expression.clone_and_replace_init(new_init, control, this);
+  OpaqueTemplateAssertionPredicateNode* new_opaque_node =
+      template_assertion_expression.clone_and_replace_init(new_init, control, this);
   AssertionPredicateIfCreator assertion_predicate_if_creator(this);
   IfTrueNode* success_proj =
       assertion_predicate_if_creator.create_for_template(new_control, iff->Opcode(), new_opaque_node
@@ -1922,25 +1925,21 @@ void PhaseIdealLoop::update_main_loop_assertion_predicates(Node* ctrl, CountedLo
       break;
     }
     Node* bol = iff->in(1);
-    if (bol->is_Opaque4()) {
-      if (assertion_predicate_has_loop_opaque_node(iff)) {
-        // This is a Template Assertion Predicate for the initial or last access.
-        // Create an Initialized Assertion Predicates for it accordingly:
-        // - For the initial access a[init] (same as before)
-        // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride)
-        prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj);
-      } else {
-        // Ignore Opaque4 from a non-null-check for an intrinsic or unsafe access. This could happen when we maximally
-        // unroll a non-main loop with such an If with an Opaque4 node directly above the loop entry.
-        assert(!loop_head->is_main_loop(), "Opaque4 node from a non-null check - should not be at main loop");
-      }
+    if (bol->is_OpaqueTemplateAssertionPredicate()) {
+      assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes");
+      // This is a Template Assertion Predicate for the initial or last access.
+      // Create an Initialized Assertion Predicates for it accordingly:
+      // - For the initial access a[init] (same as before)
+      // - For the last access a[init+new_stride-orig_stride] (with the new unroll stride)
+      prev_proj = create_initialized_assertion_predicate(iff, init, max_value, prev_proj);
     } else if (bol->is_OpaqueInitializedAssertionPredicate()) {
       // This is one of the two Initialized Assertion Predicates:
       // - For the initial access a[init]
       // - For the last access a[init+old_stride-orig_stride]
       // We could keep the one for the initial access but we do not know which one we currently have here. Just kill both.
       _igvn.replace_input_of(iff, 1, _igvn.intcon(1));
     }
+    assert(!bol->is_OpaqueNotNull() || !loop_head->is_main_loop(), "OpaqueNotNull should not be at main loop");
     entry = entry->in(0)->in(0);
   }
   if (prev_proj != ctrl) {
@@ -1967,7 +1966,7 @@ void PhaseIdealLoop::copy_assertion_predicates_to_post_loop(LoopNode* main_loop_
     if (!proj->unique_ctrl_out()->is_Halt()) {
       break;
     }
-    if (iff->in(1)->is_Opaque4()) {
+    if (iff->in(1)->is_OpaqueTemplateAssertionPredicate()) {
       // Initialize from Template Assertion Predicate.
       prev_proj = create_initialized_assertion_predicate(iff, init, stride, prev_proj);
     }
@@ -2000,7 +1999,7 @@ void PhaseIdealLoop::initialize_assertion_predicates_for_peeled_loop(const Predi
     }
     Node* bol = iff->in(1);
     assert(!bol->is_OpaqueInitializedAssertionPredicate(), "should not find an Initialized Assertion Predicate");
-    if (bol->is_Opaque4()) {
+    if (bol->is_OpaqueTemplateAssertionPredicate()) {
       // Initialize from Template Assertion Predicate.
       input_proj = create_initialized_assertion_predicate(iff, init, stride, input_proj);
 

src/hotspot/share/opto/loopnode.cpp

@@ -4443,7 +4443,8 @@ void PhaseIdealLoop::add_useless_parse_predicates_to_igvn_worklist() {
 
 
 // Eliminate all Template Assertion Predicates that do not belong to their originally associated loop anymore by
-// replacing the Opaque4 node of the If node with true. These nodes will be removed during the next round of IGVN.
+// replacing the OpaqueTemplateAssertionPredicate node of the If node with true. These nodes will be removed during the
+// next round of IGVN.
 void PhaseIdealLoop::eliminate_useless_template_assertion_predicates() {
   Unique_Node_List useful_predicates;
   if (C->has_loops()) {
@@ -4484,9 +4485,10 @@ void PhaseIdealLoop::collect_useful_template_assertion_predicates_for_loop(Ideal
 
 void PhaseIdealLoop::eliminate_useless_template_assertion_predicates(Unique_Node_List& useful_predicates) {
   for (int i = C->template_assertion_predicate_count(); i > 0; i--) {
-    Opaque4Node* opaque4_node = C->template_assertion_predicate_opaq_node(i - 1)->as_Opaque4();
-    if (!useful_predicates.member(opaque4_node)) { // not in the useful list
-      _igvn.replace_node(opaque4_node, opaque4_node->in(2));
+    OpaqueTemplateAssertionPredicateNode* opaque_node =
+        C->template_assertion_predicate_opaq_node(i - 1)->as_OpaqueTemplateAssertionPredicate();
+    if (!useful_predicates.member(opaque_node)) { // not in the useful list
+      _igvn.replace_node(opaque_node, _igvn.intcon(1));
     }
   }
 }

src/hotspot/share/opto/loopnode.hpp

@@ -955,7 +955,7 @@ class PhaseIdealLoop : public PhaseTransform {
   IfTrueNode* create_initialized_assertion_predicate(IfNode* template_assertion_predicate, Node* new_init,
                                                      Node* new_stride, Node* control);
   static void count_opaque_loop_nodes(Node* n, uint& init, uint& stride);
-  static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);
+  DEBUG_ONLY(static bool assertion_predicate_has_loop_opaque_node(IfNode* iff);)
   static void get_assertion_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false);
   void update_main_loop_assertion_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con);
   void copy_assertion_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head,

src/hotspot/share/opto/loopopts.cpp

@@ -787,10 +787,9 @@ Node *PhaseIdealLoop::conditional_move( Node *region ) {
   }//for
   Node* bol = iff->in(1);
   assert(!bol->is_OpaqueInitializedAssertionPredicate(), "Initialized Assertion Predicates cannot form a diamond with Halt");
-  if (bol->is_Opaque4()) {
-    // Ignore Template Assertion Predicates with Opaque4 nodes.
-    assert(assertion_predicate_has_loop_opaque_node(iff),
-           "must be Template Assertion Predicate, non-null-check with Opaque4 cannot form a diamond with Halt");
+  if (bol->is_OpaqueTemplateAssertionPredicate()) {
+    // Ignore Template Assertion Predicates with OpaqueTemplateAssertionPredicate nodes.
+    assert(assertion_predicate_has_loop_opaque_node(iff), "must find OpaqueLoop* nodes");
     return nullptr;
   }
   assert(bol->Opcode() == Op_Bool, "Unexpected node");
@@ -1702,8 +1701,9 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
       !n->is_Proj() &&
       !n->is_MergeMem() &&
       !n->is_CMove() &&
-      !n->is_Opaque4() &&
+      !n->is_OpaqueNotNull() &&
       !n->is_OpaqueInitializedAssertionPredicate() &&
+      !n->is_OpaqueTemplateAssertionPredicate() &&
       !n->is_Type()) {
     Node *n_ctrl = get_ctrl(n);
     IdealLoopTree *n_loop = get_loop(n_ctrl);
@@ -2021,14 +2021,14 @@ Node* PhaseIdealLoop::clone_iff(PhiNode* phi) {
     if (b->is_Phi()) {
       _igvn.replace_input_of(phi, i, clone_iff(b->as_Phi()));
     } else {
-      assert(b->is_Bool() || b->is_Opaque4() || b->is_OpaqueInitializedAssertionPredicate(),
-             "bool, non-null check with Opaque4 node or Initialized Assertion Predicate with its Opaque node");
+      assert(b->is_Bool() || b->is_OpaqueNotNull() || b->is_OpaqueInitializedAssertionPredicate(),
+             "bool, non-null check with OpaqueNotNull or Initialized Assertion Predicate with its Opaque node");
     }
   }
   Node* n = phi->in(1);
   Node* sample_opaque = nullptr;
   Node *sample_bool = nullptr;
-  if (n->is_Opaque4() || n->is_OpaqueInitializedAssertionPredicate()) {
+  if (n->is_OpaqueNotNull() || n->is_OpaqueInitializedAssertionPredicate()) {
     sample_opaque = n;
     sample_bool = n->in(1);
     assert(sample_bool->is_Bool(), "wrong type");
@@ -2202,7 +2202,9 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new,
       // For example, it is unexpected that there is a Phi between an
       // AllocateArray node and its ValidLengthTest input that could cause
       // split if to break.
-      if (use->is_If() || use->is_CMove() || use->is_Opaque4() || use->is_OpaqueInitializedAssertionPredicate() ||
+      assert(!use->is_OpaqueTemplateAssertionPredicate(),
+             "should not clone a Template Assertion Predicate which should be removed once it's useless");
+      if (use->is_If() || use->is_CMove() || use->is_OpaqueNotNull() || use->is_OpaqueInitializedAssertionPredicate() ||
           (use->Opcode() == Op_AllocateArray && use->in(AllocateNode::ValidLengthTest) == old)) {
         // Since this code is highly unlikely, we lazily build the worklist
         // of such Nodes to go split.