Skip to content

Commit 95ec7de

Browse files
jacob-crawleyjacob-crawley
committed
SVE2 API for AddCarryWideningUpper
Change-Id: Ia7160cba8655f9f38e608f25d5c412f5a44b533a
1 parent 6fac66d commit 95ec7de

File tree

6 files changed

+81
-1
lines changed

6 files changed

+81
-1
lines changed

‎src/coreclr/jit/hwintrinsiclistarm64sve.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAddWideningUpper,
319319
HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLower, -1, 2, {INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
320320
HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningUpper, -1, 2, {INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
321321
HARDWARE_INTRINSIC(Sve2, AddCarryWideningLower, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
322+
HARDWARE_INTRINSIC(Sve2, AddCarryWideningUpper, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
322323
HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, {INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
323324
HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, {INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
324325
HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, {INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,20 @@ internal Arm64() { }
244244
/// </summary>
245245
public static unsafe Vector<ulong> AddCarryWideningLower(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) { throw new PlatformNotSupportedException(); }
246246

247+
// Add with carry long (top)
248+
249+
/// <summary>
250+
/// svuint32_t svadclt[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
251+
/// ADCLT Ztied1.S, Zop2.S, Zop3.S
252+
/// </summary>
253+
public static unsafe Vector<uint> AddCarryWideningUpper(Vector<uint> op1, Vector<uint> op2, Vector<uint> op3) { throw new PlatformNotSupportedException(); }
254+
255+
/// <summary>
256+
/// svuint64_t svadclt[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
257+
/// ADCLT Ztied1.D, Zop2.D, Zop3.D
258+
/// </summary>
259+
public static unsafe Vector<ulong> AddCarryWideningUpper(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) { throw new PlatformNotSupportedException(); }
260+
247261
// Bitwise clear and exclusive OR
248262

249263
/// <summary>

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,20 @@ internal Arm64() { }
244244
/// </summary>
245245
public static unsafe Vector<ulong> AddCarryWideningLower(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) => AddCarryWideningLower(op1, op2, op3);
246246

247+
// Add with carry long (top)
248+
249+
/// <summary>
250+
/// svuint32_t svadclt[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3)
251+
/// ADCLT Ztied1.S, Zop2.S, Zop3.S
252+
/// </summary>
253+
public static unsafe Vector<uint> AddCarryWideningUpper(Vector<uint> op1, Vector<uint> op2, Vector<uint> op3) => AddCarryWideningUpper(op1, op2, op3);
254+
255+
/// <summary>
256+
/// svuint64_t svadclt[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3)
257+
/// ADCLT Ztied1.D, Zop2.D, Zop3.D
258+
/// </summary>
259+
public static unsafe Vector<ulong> AddCarryWideningUpper(Vector<ulong> op1, Vector<ulong> op2, Vector<ulong> op3) => AddCarryWideningUpper(op1, op2, op3);
260+
247261
// Bitwise clear and exclusive OR
248262

249263
/// <summary>

‎src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6118,7 +6118,8 @@ internal Arm64() { }
61186118

61196119
public static System.Numerics.Vector<uint> AddCarryWideningLower(System.Numerics.Vector<uint> op1, System.Numerics.Vector<uint> op2, System.Numerics.Vector<uint> op3) { throw null; }
61206120
public static System.Numerics.Vector<ulong> AddCarryWideningLower(System.Numerics.Vector<ulong> op1, System.Numerics.Vector<ulong> op2, System.Numerics.Vector<ulong> op3) { throw null; }
6121-
6121+
public static System.Numerics.Vector<uint> AddCarryWideningUpper(System.Numerics.Vector<uint> op1, System.Numerics.Vector<uint> op2, System.Numerics.Vector<uint> op3) { throw null; }
6122+
public static System.Numerics.Vector<ulong> AddCarryWideningUpper(System.Numerics.Vector<ulong> op1, System.Numerics.Vector<ulong> op2, System.Numerics.Vector<ulong> op3) { throw null; }
61226123
public static System.Numerics.Vector<byte> BitwiseClearXor(System.Numerics.Vector<byte> xor, System.Numerics.Vector<byte> value, System.Numerics.Vector<byte> mask) { throw null; }
61236124
public static System.Numerics.Vector<short> BitwiseClearXor(System.Numerics.Vector<short> xor, System.Numerics.Vector<short> value, System.Numerics.Vector<short> mask) { throw null; }
61246125
public static System.Numerics.Vector<int> BitwiseClearXor(System.Numerics.Vector<int> xor, System.Numerics.Vector<int> value, System.Numerics.Vector<int> mask) { throw null; }

‎src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4819,6 +4819,8 @@
48194819

48204820
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningLower_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningLower", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AddCarryWideningEven(first, second, third, i)"}),
48214821
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningLower_ulong", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningLower", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AddCarryWideningEven(first, second, third, i)"}),
4822+
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningUpper_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningOdd(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AddCarryWideningOdd(first, second, third, i)"}),
4823+
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_AddCarryWideningUpper_ulong", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningOdd(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AddCarryWideningOdd(first, second, third, i)"}),
48224824

48234825
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_BitwiseClearXor_sbyte", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "BitwiseClearXor", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))", ["GetIterResult"] = "(SByte) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}),
48244826
("SveVecTernOpTest.template", new Dictionary<string, string> { ["TestName"] = "Sve2_BitwiseClearXor_short", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "BitwiseClearXor", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt16()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))", ["GetIterResult"] = "(Int16) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}),

‎src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2314,6 +2314,28 @@ public static uint AddCarryWideningEven(uint[] op1, uint[] op2, uint[] op3, int
23142314
}
23152315
}
23162316

2317+
public static uint AddCarryWideningOdd(uint[] op1, uint[] op2, uint[] op3, int i)
2318+
{
2319+
uint lsb;
2320+
ulong res;
2321+
2322+
if (i % 2 == 0)
2323+
{
2324+
lsb = op3[i + 1] & 1u;
2325+
res = (ulong)op1[i] + op2[i + 1] + lsb;
2326+
return (uint)res;
2327+
}
2328+
else
2329+
{
2330+
lsb = op3[i] & 1u;
2331+
res = (ulong)op1[i - 1] + op2[i] + lsb;
2332+
2333+
// Shift result to get the carry bit
2334+
return (uint)(res >> 32);
2335+
}
2336+
}
2337+
2338+
23172339
private static short HighNarrowing(int op1, bool round)
23182340
{
23192341
uint roundConst = 0;
@@ -2462,6 +2484,32 @@ public static ulong AddCarryWideningEven(ulong[] op1, ulong[] op2, ulong[] op3,
24622484
}
24632485
}
24642486

2487+
public static ulong AddCarryWideningOdd(ulong[] op1, ulong[] op2, ulong[] op3, int i)
2488+
{
2489+
ulong lsb;
2490+
ulong res;
2491+
2492+
if (i % 2 == 0)
2493+
{
2494+
lsb = op3[i + 1] & 1UL;
2495+
res = op1[i] + op2[i + 1] + lsb;
2496+
return res;
2497+
}
2498+
else
2499+
{
2500+
lsb = op3[i] & 1UL;
2501+
2502+
// Look for an overflow in the addition to get the carry bit
2503+
ulong sum1 = op1[i - 1] + op2[i];
2504+
bool overflow1 = sum1 < op1[i - 1];
2505+
2506+
ulong sum2 = sum1 + lsb;
2507+
bool overflow2 = sum2 < sum1;
2508+
2509+
return (overflow1 || overflow2) ? 1UL : 0UL;
2510+
}
2511+
}
2512+
24652513
private static int HighNarrowing(long op1, bool round)
24662514
{
24672515
ulong roundConst = 0;

0 commit comments

Comments
 (0)