Skip to content

Commit

Permalink
JIT: Fix over-shifting logic when constant-folding `AdvSimd.ShiftRigh…
Browse files Browse the repository at this point in the history 
…t*` and `AdvSimd.ShiftLeft*` (#105847)
  • Loading branch information
@amanasifkhalid
amanasifkhalid authored Aug 5, 2024
1 parent c2d0589 commit 6931b3b
Show file tree
Hide file tree
Showing 7 changed files with 335 additions and 23 deletions.
9 changes: 6 additions & 3 deletions src/coreclr/jit/hwintrinsic.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1597,12 +1597,15 @@ bool Compiler::CheckHWIntrinsicImmRange(NamedIntrinsic intrinsic,
#ifdef TARGET_ARM64
switch (intrinsic)
{
case NI_AdvSimd_ShiftLeftLogical:
case NI_AdvSimd_ShiftLeftLogicalScalar:
case NI_AdvSimd_ShiftRightLogical:
case NI_AdvSimd_ShiftRightLogicalScalar:
case NI_AdvSimd_ShiftRightArithmetic:
case NI_AdvSimd_ShiftRightArithmeticScalar:
*useFallback = true;
break;

// TODO: Implement more AdvSimd fallbacks in Compiler::impNonConstFallback

default:
assert(*useFallback == false);
break;
Expand Down Expand Up @@ -1633,7 +1636,7 @@ bool Compiler::CheckHWIntrinsicImmRange(NamedIntrinsic intrinsic,
}
}
else
#endif // TARGET_XARCH
#endif // TARGET_X86
{
*useFallback = true;
return false;
Expand Down
49 changes: 44 additions & 5 deletions src/coreclr/jit/hwintrinsicarm64.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,20 +564,59 @@ void HWIntrinsicInfo::lookupImmBounds(
//
GenTree* Compiler::impNonConstFallback(NamedIntrinsic intrinsic, var_types simdType, CorInfoType simdBaseJitType)
{
bool isRightShift = true;

switch (intrinsic)
{
case NI_AdvSimd_ShiftLeftLogical:
case NI_AdvSimd_ShiftLeftLogicalScalar:
isRightShift = false;
FALLTHROUGH;

case NI_AdvSimd_ShiftRightLogical:
case NI_AdvSimd_ShiftRightLogicalScalar:
case NI_AdvSimd_ShiftRightArithmetic:
case NI_AdvSimd_ShiftRightArithmeticScalar:
{
// AdvSimd.ShiftRightLogical be replaced with AdvSimd.ShiftLogical, which takes op2 in a simd register
// AdvSimd.ShiftLeft* and AdvSimd.ShiftRight* can be replaced with AdvSimd.Shift*, which takes op2 in a simd
// register

GenTree* op2 = impPopStack().val;
GenTree* op1 = impSIMDPopStack();

// AdvSimd.ShiftLogical does right-shifts with negative immediates, hence the negation
GenTree* tmpOp =
gtNewSimdCreateBroadcastNode(simdType, gtNewOperNode(GT_NEG, genActualType(op2->TypeGet()), op2),
simdBaseJitType, genTypeSize(simdType));
return gtNewSimdHWIntrinsicNode(simdType, op1, tmpOp, NI_AdvSimd_ShiftLogical, simdBaseJitType,
if (isRightShift)
{
op2 = gtNewOperNode(GT_NEG, genActualType(op2->TypeGet()), op2);
}

NamedIntrinsic fallbackIntrinsic;
switch (intrinsic)
{
case NI_AdvSimd_ShiftLeftLogical:
case NI_AdvSimd_ShiftRightLogical:
fallbackIntrinsic = NI_AdvSimd_ShiftLogical;
break;

case NI_AdvSimd_ShiftLeftLogicalScalar:
case NI_AdvSimd_ShiftRightLogicalScalar:
fallbackIntrinsic = NI_AdvSimd_ShiftLogicalScalar;
break;

case NI_AdvSimd_ShiftRightArithmetic:
fallbackIntrinsic = NI_AdvSimd_ShiftArithmetic;
break;

case NI_AdvSimd_ShiftRightArithmeticScalar:
fallbackIntrinsic = NI_AdvSimd_ShiftArithmeticScalar;
break;

default:
unreached();
}

GenTree* tmpOp = gtNewSimdCreateBroadcastNode(simdType, op2, simdBaseJitType, genTypeSize(simdType));
return gtNewSimdHWIntrinsicNode(simdType, op1, tmpOp, fallbackIntrinsic, simdBaseJitType,
genTypeSize(simdType));
}

Expand Down
12 changes: 6 additions & 6 deletions src/coreclr/jit/hwintrinsiclistarm64.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,12 +433,12 @@ HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticSaturateScalar,
HARDWARE_INTRINSIC(AdvSimd, ShiftArithmeticScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshl, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsert, -1, 3, {INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli, INS_sli}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftAndInsertScalar, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sli, INS_sli, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, {INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogical, -1, 2, {INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_shl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturate, -1, 2, {INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshl, INS_uqshl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsigned, -1, 2, {INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalSaturateUnsignedScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sqshlu, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_shl, INS_shl, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningLower, 8, 2, {INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_sshll, INS_ushll, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftLeftLogicalWideningUpper, 16, 2, {INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_sshll2, INS_ushll2, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftLeftByImmediate, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftLogical, -1, 2, {INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_ushl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_NoFlag)
Expand All @@ -451,7 +451,7 @@ HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalSaturateScalar,
HARDWARE_INTRINSIC(AdvSimd, ShiftLogicalScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushl, INS_ushl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsert, -1, 3, {INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri, INS_sri}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightAndInsertScalar, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sri, INS_sri, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, {INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmetic, -1, 2, {INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAdd, -1, 3, {INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticAddScalar, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ssra, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticNarrowingSaturateLower, 8, 2, {INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_sqshrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand)
Expand All @@ -466,8 +466,8 @@ HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUn
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, 16, 3, {INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_sqrshrun2, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedNarrowingSaturateUpper, 16, 3, {INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_sqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticRoundedScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_srshr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, {INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightArithmeticScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sshr, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogical, -1, 2, {INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_ushr, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAdd, -1, 3, {INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_usra, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalAddScalar, 8, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_usra, INS_usra, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalNarrowingLower, 8, 2, {INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_shrn, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand)
Expand All @@ -482,7 +482,7 @@ HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateLower
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingSaturateUpper, 16, 3, {INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_uqrshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedNarrowingUpper, 16, 3, {INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_rshrn2, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalRoundedScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_urshr, INS_urshr, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar)
HARDWARE_INTRINSIC(AdvSimd, ShiftRightLogicalScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_ushr, INS_ushr, INS_invalid, INS_invalid}, HW_Category_ShiftRightByImmediate, HW_Flag_HasImmediateOperand|HW_Flag_SIMDScalar|HW_Flag_NoJmpTableIMM)
HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningLower, 8, 1, {INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_sxtl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(AdvSimd, SignExtendWideningUpper, 16, 1, {INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_sxtl2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(AdvSimd, SqrtScalar, 8, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_fsqrt, INS_fsqrt}, HW_Category_SIMD, HW_Flag_SIMDScalar)
Expand Down
Loading

0 comments on commit 6931b3b

Please sign in to comment.