8334228: C2 SuperWord: fix JDK-24 regression in VPointer::cmp_for_sor…

…t after JDK-8325155

Reviewed-by: chagedorn, kvn

src/hotspot/share/opto/vectorization.cpp

@@ -807,24 +807,26 @@ void VPointer::maybe_add_to_invar(Node* new_invar, bool negate) {
   _invar = register_if_new(add);
 }
 
+// We use two comparisons, because a subtraction could underflow.
+#define RETURN_CMP_VALUE_IF_NOT_EQUAL(a, b) \
+  if (a < b) { return -1; }                 \
+  if (a > b) { return  1; }
+
 // To be in the same group, two VPointers must be the same,
 // except for the offset.
 int VPointer::cmp_for_sort_by_group(const VPointer** p1, const VPointer** p2) {
   const VPointer* a = *p1;
   const VPointer* b = *p2;
 
-  int cmp_base = a->base()->_idx - b->base()->_idx;
-  if (cmp_base != 0) { return cmp_base; }
-
-  int cmp_opcode = a->mem()->Opcode() - b->mem()->Opcode();
-  if (cmp_opcode != 0) { return cmp_opcode; }
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a->base()->_idx,     b->base()->_idx);
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a->mem()->Opcode(),  b->mem()->Opcode());
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a->scale_in_bytes(), b->scale_in_bytes());
 
-  int cmp_scale = a->scale_in_bytes() - b->scale_in_bytes();
-  if (cmp_scale != 0) { return cmp_scale; }
+  int a_inva_idx = a->invar() == nullptr ? 0 : a->invar()->_idx;
+  int b_inva_idx = b->invar() == nullptr ? 0 : b->invar()->_idx;
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a_inva_idx,          b_inva_idx);
 
-  int cmp_invar = (a->invar() == nullptr ? 0 : a->invar()->_idx) -
-                  (b->invar() == nullptr ? 0 : b->invar()->_idx);
-  return cmp_invar;
+  return 0; // equal
 }
 
 // We compare by group, then by offset, and finally by node idx.
@@ -835,10 +837,9 @@ int VPointer::cmp_for_sort(const VPointer** p1, const VPointer** p2) {
   const VPointer* a = *p1;
   const VPointer* b = *p2;
 
-  int cmp_offset = a->offset_in_bytes() - b->offset_in_bytes();
-  if (cmp_offset != 0) { return cmp_offset; }
-
-  return a->mem()->_idx - b->mem()->_idx;
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a->offset_in_bytes(), b->offset_in_bytes());
+  RETURN_CMP_VALUE_IF_NOT_EQUAL(a->mem()->_idx,       b->mem()->_idx);
+  return 0; // equal
 }
 
 #ifndef PRODUCT

test/hotspot/jtreg/compiler/vectorization/TestOffsetSorting.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8334228
+ * @summary Test sorting of VPointer by offset, when subtraction of two offsets can overflow.
+ * @run main/othervm -XX:CompileCommand=compileonly,compiler.vectorization.TestOffsetSorting::test -Xcomp compiler.vectorization.TestOffsetSorting
+ * @run main compiler.vectorization.TestOffsetSorting
+ */
+
+package compiler.vectorization;
+
+public class TestOffsetSorting {
+    static int RANGE = 10_000;
+
+    public static void main(String[] args) {
+        int[] a = new int[RANGE];
+        for (int i = 0; i < 10_000; i++) {
+            try {
+                test(a, 0);
+                throw new RuntimeException("test should go out-of-bounds");
+            } catch (ArrayIndexOutOfBoundsException e) {
+            }
+        }
+    }
+
+    static void test(int[] a, int invar) {
+        int large = (1 << 28) + (1 << 20);
+        for (int i = 0; i < 1_000; i++) {
+            a[i + invar - large] = 42;
+            a[i + invar + large] = 42;
+        }
+    }
+}