Skip to content

Commit

Permalink
8320999: RISC-V: C2 RotateLeftV
Browse files Browse the repository at this point in the history 
8321000: RISC-V: C2 RotateRightV

Reviewed-by: luhenry, fyang
  • Loading branch information
Hamlin Li committed May 29, 2024
1 parent 6cda4c5 commit fed2b56
Show file tree
Hide file tree
Showing 4 changed files with 248 additions and 8 deletions.
44 changes: 41 additions & 3 deletions src/hotspot/cpu/riscv/assembler_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1863,14 +1863,52 @@ enum Nf {
patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6); \
}

// Vector Bit-manipulation used in Cryptography (Zvkb) Extension
// Vector Bit-manipulation used in Cryptography (Zvbb) Extension
INSN(vandn_vv, 0b1010111, 0b000, 0b000001);
INSN(vclmul_vv, 0b1010111, 0b010, 0b001100);
INSN(vclmulh_vv, 0b1010111, 0b010, 0b001101);
INSN(vror_vv, 0b1010111, 0b000, 0b010100);
INSN(vrol_vv, 0b1010111, 0b000, 0b010101);

// Vector Bit-manipulation used in Cryptography (Zvbc) Extension
INSN(vclmul_vv, 0b1010111, 0b010, 0b001100);
INSN(vclmulh_vv, 0b1010111, 0b010, 0b001101);

#undef INSN

#define INSN(NAME, op, funct3, funct6) \
void NAME(VectorRegister Vd, VectorRegister Vs2, Register Rs1, VectorMask vm = unmasked) { \
patch_VArith(op, Vd, funct3, Rs1->raw_encoding(), Vs2, vm, funct6); \
}

// Vector Bit-manipulation used in Cryptography (Zvbb) Extension
INSN(vrol_vx, 0b1010111, 0b100, 0b010101);
INSN(vror_vx, 0b1010111, 0b100, 0b010100);

#undef INSN

#define patch_VArith_imm6(op, Reg, funct3, Reg_or_Imm5, I5, Vs2, vm, funct6) \
unsigned insn = 0; \
patch((address)&insn, 6, 0, op); \
patch((address)&insn, 14, 12, funct3); \
patch((address)&insn, 19, 15, Reg_or_Imm5); \
patch((address)&insn, 25, vm); \
patch((address)&insn, 26, I5); \
patch((address)&insn, 31, 27, funct6); \
patch_reg((address)&insn, 7, Reg); \
patch_reg((address)&insn, 20, Vs2); \
emit(insn)

#define INSN(NAME, op, funct3, funct6) \
void NAME(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) { \
guarantee(is_uimm6(imm), "uimm is invalid"); \
patch_VArith_imm6(op, Vd, funct3, (uint32_t)(imm & 0x1f), (uint32_t)((imm >> 5) & 0x1), Vs2, vm, funct6); \
}

// Vector Bit-manipulation used in Cryptography (Zvbb) Extension
// NOTE: there is no corresponding vrol.vi supplied by the extension, but it can be emulated with vror.vi easily.
INSN(vror_vi, 0b1010111, 0b011, 0b01010);

#undef INSN
#undef patch_VArith_imm6

#define INSN(NAME, op, funct3, Vs1, funct6) \
void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
Expand Down
8 changes: 4 additions & 4 deletions src/hotspot/cpu/riscv/matcher_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,13 @@
}

// Does the CPU supports vector variable rotate instructions?
static constexpr bool supports_vector_variable_rotates(void) {
return false;
static bool supports_vector_variable_rotates(void) {
return UseZvbb;
}

// Does the CPU supports vector constant rotate instructions?
static constexpr bool supports_vector_constant_rotates(int shift) {
return false;
static bool supports_vector_constant_rotates(int shift) {
return UseZvbb;
}

// Does the CPU supports vector unsigned comparison instructions?
Expand Down
198 changes: 197 additions & 1 deletion src/hotspot/cpu/riscv/riscv_v.ad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,11 @@ source %{
break;
case Op_CountTrailingZerosV:
case Op_CountLeadingZerosV:
case Op_ReverseBytesV:
case Op_PopCountVL:
case Op_PopCountVI:
case Op_ReverseBytesV:
case Op_RotateLeftV:
case Op_RotateRightV:
return UseZvbb;
case Op_LoadVectorGather:
case Op_LoadVectorGatherMasked:
Expand Down Expand Up @@ -3057,6 +3059,200 @@ instruct vshiftcnt(vReg dst, iRegIorL2I cnt) %{
ins_pipe(pipe_slow);
%}

// --------------------------------- Vector Rotation ----------------------------------
//
// Following rotate instruct's are shared by vectorization (in SLP, superword.cpp) and Vector API.
//
// Rotate behaviour in vectorization is defined by java API, which includes:
// 1. Integer.rorateRight, Integer.rorateLeft.
// `rotation by any multiple of 32 is a no-op, so all but the last five bits of the rotation distance can be ignored`.
// 2. Long.rorateRight, Long.rorateLeft.
// `rotation by any multiple of 64 is a no-op, so all but the last six bits of the rotation distance can be ignored`.
//
// Rotate behaviour in Vector API is defined as below, e.g.
// 1. For Byte ROR, `a ROR b` is: (byte)(((((byte)a) & 0xFF) >>> (b & 7)) | ((((byte)a) & 0xFF) << (8 - (b & 7))))
// 2. For Short ROR, `a ROR b` is: (short)(((((short)a) & 0xFFFF) >>> (b & 15)) | ((((short)a) & 0xFFFF) << (16 - (b & 15))))
// 3. For Integer ROR, `a ROR b` is: Integer.rotateRight(a, ((int)b))
// 4. For Long ROR, `a ROR b` is: Long.rotateRight(a, ((int)b))
//
// Basically, the behaviour between vectorization and Vector API is the same for Long and Integer, except that Vector API
// also supports Byte and Short rotation. But we can still share the intrinsics between vectorization and Vector API.
//
// NOTE: As vror.vi encodes 6-bits immediate rotate amount, which is different from other vector-immediate instructions,
// implementation of vector rotation for long and other types can be unified.

// Rotate right

instruct vrotate_right(vReg dst, vReg src, vReg shift) %{
match(Set dst (RotateRightV src shift));
format %{ "vrotate_right $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
as_VectorRegister($shift$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_right_reg(vReg dst, vReg src, iRegIorL2I shift) %{
match(Set dst (RotateRightV src (Replicate shift)));
format %{ "vrotate_right_reg $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
as_Register($shift$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_right_imm(vReg dst, vReg src, immI shift) %{
match(Set dst (RotateRightV src shift));
format %{ "vrotate_right_imm $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint32_t bits = type2aelembytes(bt) * 8;
uint32_t con = (unsigned)$shift$$constant & (bits - 1);
if (con == 0) {
return;
}
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
%}
ins_pipe(pipe_slow);
%}

// Rotate right - masked

instruct vrotate_right_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateRightV (Binary dst_src shift) v0));
format %{ "vrotate_right_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
as_VectorRegister($shift$$reg), Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_right_reg_masked(vReg dst_src, iRegIorL2I shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateRightV (Binary dst_src (Replicate shift)) v0));
format %{ "vrotate_right_reg_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vx(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
as_Register($shift$$reg), Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_right_imm_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateRightV (Binary dst_src shift) v0));
format %{ "vrotate_right_imm_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint32_t bits = type2aelembytes(bt) * 8;
uint32_t con = (unsigned)$shift$$constant & (bits - 1);
if (con == 0) {
return;
}
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vror_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
con, Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

// Rotate left

instruct vrotate_left(vReg dst, vReg src, vReg shift) %{
match(Set dst (RotateLeftV src shift));
format %{ "vrotate_left $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vrol_vv(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
as_VectorRegister($shift$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_left_reg(vReg dst, vReg src, iRegIorL2I shift) %{
match(Set dst (RotateLeftV src (Replicate shift)));
format %{ "vrotate_left_reg $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vrol_vx(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg),
as_Register($shift$$reg));
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_left_imm(vReg dst, vReg src, immI shift) %{
match(Set dst (RotateLeftV src shift));
format %{ "vrotate_left_imm $dst, $src, $shift\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint32_t bits = type2aelembytes(bt) * 8;
uint32_t con = (unsigned)$shift$$constant & (bits - 1);
if (con == 0) {
return;
}
__ vsetvli_helper(bt, Matcher::vector_length(this));
con = bits - con;
__ vror_vi(as_VectorRegister($dst$$reg), as_VectorRegister($src$$reg), con);
%}
ins_pipe(pipe_slow);
%}

// Rotate left - masked

instruct vrotate_left_masked(vReg dst_src, vReg shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateLeftV (Binary dst_src shift) v0));
format %{ "vrotate_left_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vrol_vv(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
as_VectorRegister($shift$$reg), Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_left_reg_masked(vReg dst_src, iRegIorL2I shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateLeftV (Binary dst_src (Replicate shift)) v0));
format %{ "vrotate_left_reg_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
__ vsetvli_helper(bt, Matcher::vector_length(this));
__ vrol_vx(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
as_Register($shift$$reg), Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

instruct vrotate_left_imm_masked(vReg dst_src, immI shift, vRegMask_V0 v0) %{
match(Set dst_src (RotateLeftV (Binary dst_src shift) v0));
format %{ "vrotate_left_imm_masked $dst_src, $dst_src, $shift, v0.t\t" %}
ins_encode %{
BasicType bt = Matcher::vector_element_basic_type(this);
uint32_t bits = type2aelembytes(bt) * 8;
uint32_t con = (unsigned)$shift$$constant & (bits - 1);
if (con == 0) {
return;
}
__ vsetvli_helper(bt, Matcher::vector_length(this));
con = bits - con;
__ vror_vi(as_VectorRegister($dst_src$$reg), as_VectorRegister($dst_src$$reg),
con, Assembler::v0_t);
%}
ins_pipe(pipe_slow);
%}

// vector sqrt

instruct vsqrt_fp(vReg dst, vReg src) %{
Expand Down
6 changes: 6 additions & 0 deletions test/hotspot/jtreg/compiler/vectorization/runner/ArrayShiftOpTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ public ArrayShiftOpTest() {
counts = {IRNode.STORE_VECTOR, ">0"})
@IR(applyIfCPUFeature = {"avx512f", "true"},
counts = {IRNode.ROTATE_RIGHT_V, ">0"})
@IR(applyIfPlatform = {"riscv64", "true"},
applyIfCPUFeature = {"zvbb", "true"},
counts = {IRNode.ROTATE_RIGHT_V, ">0"})
public int[] intCombinedRotateShift() {
int[] res = new int[SIZE];
for (int i = 0; i < SIZE; i++) {
Expand All @@ -88,6 +91,9 @@ public int[] intCombinedRotateShift() {
counts = {IRNode.STORE_VECTOR, ">0"})
@IR(applyIfCPUFeature = {"avx512f", "true"},
counts = {IRNode.ROTATE_RIGHT_V, ">0"})
@IR(applyIfPlatform = {"riscv64", "true"},
applyIfCPUFeature = {"zvbb", "true"},
counts = {IRNode.ROTATE_RIGHT_V, ">0"})
public long[] longCombinedRotateShift() {
long[] res = new long[SIZE];
for (int i = 0; i < SIZE; i++) {
Expand Down

1 comment on commit fed2b56

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.