Skip to content

Commit 39d6396

Browse files
echesakovechesakov
authored
[Arm64] Implement MultiplyHigh (#47362)
1 parent 974e96b commit 39d6396

File tree

17 files changed

+614
-0
lines changed

17 files changed

+614
-0
lines changed

‎src/coreclr/jit/hwintrinsic.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1062,6 +1062,18 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic,
10621062
retNode->AsHWIntrinsic()->SetAuxiliaryType(getBaseTypeOfSIMDType(sigReader.op2ClsHnd));
10631063
break;
10641064

1065+
case NI_ArmBase_Arm64_MultiplyHigh:
1066+
if (sig->retType == CORINFO_TYPE_ULONG)
1067+
{
1068+
retNode->AsHWIntrinsic()->gtSIMDBaseType = TYP_ULONG;
1069+
}
1070+
else
1071+
{
1072+
assert(sig->retType == CORINFO_TYPE_LONG);
1073+
retNode->AsHWIntrinsic()->gtSIMDBaseType = TYP_LONG;
1074+
}
1075+
break;
1076+
10651077
default:
10661078
break;
10671079
}

‎src/coreclr/jit/hwintrinsiclistarm64.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -530,6 +530,7 @@ HARDWARE_INTRINSIC(ArmBase, ReverseElementBits,
530530
// Base 64-bit only Intrinsics
531531
HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingSignCount, 0, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_cls, INS_invalid, INS_cls, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed)
532532
HARDWARE_INTRINSIC(ArmBase_Arm64, LeadingZeroCount, 0, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_clz, INS_clz, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_BaseTypeFromFirstArg|HW_Flag_NoFloatingPointUsed)
533+
HARDWARE_INTRINSIC(ArmBase_Arm64, MultiplyHigh, 0, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_smulh, INS_umulh, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed)
533534
HARDWARE_INTRINSIC(ArmBase_Arm64, ReverseElementBits, 0, 1, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_rbit, INS_rbit, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_NoFloatingPointUsed)
534535

535536
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************

‎src/libraries/System.Private.CoreLib/src/System/Math.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,11 @@ public static unsafe ulong BigMul(ulong a, ulong b, out ulong low)
157157
low = tmp;
158158
return high;
159159
}
160+
else if (ArmBase.Arm64.IsSupported)
161+
{
162+
low = a * b;
163+
return ArmBase.Arm64.MultiplyHigh(a, b);
164+
}
160165

161166
return SoftwareFallback(a, b, out low);
162167

@@ -191,6 +196,12 @@ static ulong SoftwareFallback(ulong a, ulong b, out ulong low)
191196
/// <returns>The high 64-bit of the product of the specied numbers.</returns>
192197
public static long BigMul(long a, long b, out long low)
193198
{
199+
if (ArmBase.Arm64.IsSupported)
200+
{
201+
low = a * b;
202+
return ArmBase.Arm64.MultiplyHigh(a, b);
203+
}
204+
194205
ulong high = BigMul((ulong)a, (ulong)b, out ulong ulow);
195206
low = (long)ulow;
196207
return (long)high - ((a >> 63) & b) - ((b >> 63) & a);

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/ArmBase.PlatformNotSupported.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,16 @@ internal Arm64() { }
4747
/// </summary>
4848
public static int LeadingZeroCount(ulong value) { throw new PlatformNotSupportedException(); }
4949

50+
/// <summary>
51+
/// A64: SMULH Xd, Xn, Xm
52+
/// </summary>
53+
public static long MultiplyHigh(long left, long right) { throw new PlatformNotSupportedException(); }
54+
55+
/// <summary>
56+
/// A64: UMULH Xd, Xn, Xm
57+
/// </summary>
58+
public static ulong MultiplyHigh(ulong left, ulong right) { throw new PlatformNotSupportedException(); }
59+
5060
/// <summary>
5161
/// A64: RBIT Xd, Xn
5262
/// </summary>

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/ArmBase.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ internal Arm64() { }
4343
/// </summary>
4444
public static int LeadingZeroCount(ulong value) => LeadingZeroCount(value);
4545

46+
/// <summary>
47+
/// A64: SMULH Xd, Xn, Xm
48+
/// </summary>
49+
public static long MultiplyHigh(long left, long right) => MultiplyHigh(left, right);
50+
51+
/// <summary>
52+
/// A64: UMULH Xd, Xn, Xm
53+
/// </summary>
54+
public static ulong MultiplyHigh(ulong left, ulong right) => MultiplyHigh(left, right);
55+
4656
/// <summary>
4757
/// A64: RBIT Xd, Xn
4858
/// </summary>

‎src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2563,6 +2563,8 @@ internal Arm64() { }
25632563
public static int LeadingSignCount(long value) { throw null; }
25642564
public static int LeadingZeroCount(long value) { throw null; }
25652565
public static int LeadingZeroCount(ulong value) { throw null; }
2566+
public static long MultiplyHigh(long left, long right) { throw null; }
2567+
public static ulong MultiplyHigh(ulong left, ulong right) { throw null; }
25662568
public static long ReverseElementBits(long value) { throw null; }
25672569
public static ulong ReverseElementBits(ulong value) { throw null; }
25682570
}

‎src/mono/mono/mini/mini-llvm.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9134,6 +9134,22 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
91349134
values [ins->dreg] = LLVMBuildCall (builder, get_intrins (ctx, ins->opcode == OP_LSCNT32 ? INTRINS_CTLZ_I32 : INTRINS_CTLZ_I64), args, 2, "");
91359135
break;
91369136
}
9137+
case OP_ARM64_SMULH:
9138+
case OP_ARM64_UMULH: {
9139+
LLVMValueRef op1, op2;
9140+
if (ins->opcode == OP_ARM64_SMULH) {
9141+
op1 = LLVMBuildSExt (builder, lhs, LLVMInt128Type (), "");
9142+
op2 = LLVMBuildSExt (builder, rhs, LLVMInt128Type (), "");
9143+
} else {
9144+
op1 = LLVMBuildZExt (builder, lhs, LLVMInt128Type (), "");
9145+
op2 = LLVMBuildZExt (builder, rhs, LLVMInt128Type (), "");
9146+
}
9147+
LLVMValueRef mul = LLVMBuildMul (builder, op1, op2, "");
9148+
LLVMValueRef hi64 = LLVMBuildLShr (builder, mul,
9149+
LLVMConstInt (LLVMInt128Type (), 64, FALSE), "");
9150+
values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), "");
9151+
break;
9152+
}
91379153
#endif
91389154

91399155
case OP_DUMMY_USE:

‎src/mono/mono/mini/mini-ops.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1579,4 +1579,6 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE)
15791579
#ifdef TARGET_ARM64
15801580
MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE)
15811581
MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE)
1582+
MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG)
1583+
MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG)
15821584
#endif // TARGET_ARM64

‎src/mono/mono/mini/simd-intrinsics-netcore.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -807,6 +807,7 @@ emit_invalid_operation (MonoCompile *cfg, const char* message)
807807
static SimdIntrinsic armbase_methods [] = {
808808
{SN_LeadingSignCount},
809809
{SN_LeadingZeroCount},
810+
{SN_MultiplyHigh},
810811
{SN_ReverseElementBits},
811812
{SN_get_IsSupported}
812813
};
@@ -874,6 +875,9 @@ emit_arm64_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignatur
874875
return emit_simd_ins_for_sig (cfg, klass, arg0_i32 ? OP_LZCNT32 : OP_LZCNT64, 0, arg0_type, fsig, args);
875876
case SN_LeadingSignCount:
876877
return emit_simd_ins_for_sig (cfg, klass, arg0_i32 ? OP_LSCNT32 : OP_LSCNT64, 0, arg0_type, fsig, args);
878+
case SN_MultiplyHigh:
879+
return emit_simd_ins_for_sig (cfg, klass,
880+
(arg0_type == MONO_TYPE_I8 ? OP_ARM64_SMULH : OP_ARM64_UMULH), 0, arg0_type, fsig, args);
877881
case SN_ReverseElementBits:
878882
return emit_simd_ins_for_sig (cfg, klass,
879883
(is_64bit ? OP_XOP_I8_I8 : OP_XOP_I4_I4),

‎src/tests/JIT/HardwareIntrinsics/Arm/ArmBase.Arm64/ArmBase.Arm64_r.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
<Compile Include="LeadingSignCount.Int64.cs" />
1313
<Compile Include="LeadingZeroCount.Int64.cs" />
1414
<Compile Include="LeadingZeroCount.UInt64.cs" />
15+
<Compile Include="MultiplyHigh.Int64.cs" />
16+
<Compile Include="MultiplyHigh.UInt64.cs" />
1517
<Compile Include="ReverseElementBits.Int64.cs" />
1618
<Compile Include="ReverseElementBits.UInt64.cs" />
1719
<Compile Include="Program.ArmBase.Arm64.cs" />

0 commit comments

Comments
 (0)