Skip to content

Commit 813a9ef

Browse files
ebephorzikm
ebepho
authored and
rzikm
committed
Arm64/SVE: Implemented ReciprocalEstimate, ReciprocalExponent, ReciprocalSqrtEstimate, ReciprocalSqrtStep, and ReciprocalStep (dotnet#103673)
* Added ConverToInt32 and ConvertToUInt32 for float inputs. * Added flags to handle only low predicate registers. * Fix whitespace * Remove special codegen flag * Added new test template for operations with different return types. * Add new test template. * Added api for ConvertToInt32 and ConvertToUInt 32 for double. * Finished 4/5 Round intrinsics for sve fp. * Completed ReciprocalExponent SVE api. * Revert "Finished 4/5 Round intrinsics for sve fp." This reverts commit aab2067. * Reciprocal SVE apis - note: ReciprocalExponent still has issues. * Completed ReciprocalExponent SVE api. * fixed naming. * Fixed some naming. * fixed more affect function names. * Fixed some more naming and retested. All stress tests pass. * fixed more helper function naming.
1 parent ecf7165 commit 813a9ef

File tree

6 files changed

+242
-24
lines changed

6 files changed

+242
-24
lines changed

‎src/coreclr/jit/hwintrinsiclistarm64sve.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,11 @@ HARDWARE_INTRINSIC(Sve, PrefetchBytes,
168168
HARDWARE_INTRINSIC(Sve, PrefetchInt16, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
169169
HARDWARE_INTRINSIC(Sve, PrefetchInt32, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
170170
HARDWARE_INTRINSIC(Sve, PrefetchInt64, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
171+
HARDWARE_INTRINSIC(Sve, ReciprocalEstimate, -1, 1, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpe, INS_sve_frecpe}, HW_Category_SIMD, HW_Flag_Scalable)
172+
HARDWARE_INTRINSIC(Sve, ReciprocalExponent, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecpx, INS_sve_frecpx}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
173+
HARDWARE_INTRINSIC(Sve, ReciprocalSqrtEstimate, -1, 1, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrte, INS_sve_frsqrte}, HW_Category_SIMD, HW_Flag_Scalable)
174+
HARDWARE_INTRINSIC(Sve, ReciprocalSqrtStep, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frsqrts, INS_sve_frsqrts}, HW_Category_SIMD, HW_Flag_Scalable)
175+
HARDWARE_INTRINSIC(Sve, ReciprocalStep, -1, 2, true, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_frecps, INS_sve_frecps}, HW_Category_SIMD, HW_Flag_Scalable)
171176
HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, false, {INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
172177
HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, true, {INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen)
173178
HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4445,6 +4445,85 @@ internal Arm64() { }
44454445
public static unsafe void PrefetchInt64(Vector<ulong> mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); }
44464446

44474447

4448+
/// ReciprocalEstimate : Reciprocal estimate
4449+
4450+
/// <summary>
4451+
/// svfloat64_t svrecpe[_f64](svfloat64_t op)
4452+
/// FRECPE Zresult.D, Zop.D
4453+
/// </summary>
4454+
public static unsafe Vector<double> ReciprocalEstimate(Vector<double> value) { throw new PlatformNotSupportedException(); }
4455+
4456+
/// <summary>
4457+
/// svfloat32_t svrecpe[_f32](svfloat32_t op)
4458+
/// FRECPE Zresult.S, Zop.S
4459+
/// </summary>
4460+
public static unsafe Vector<float> ReciprocalEstimate(Vector<float> value) { throw new PlatformNotSupportedException(); }
4461+
4462+
4463+
/// ReciprocalExponent : Reciprocal exponent
4464+
4465+
/// <summary>
4466+
/// svfloat64_t svrecpx[_f64]_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op)
4467+
/// FRECPX Ztied.D, Pg/M, Zop.D
4468+
/// svfloat64_t svrecpx[_f64]_x(svbool_t pg, svfloat64_t op)
4469+
/// FRECPX Ztied.D, Pg/M, Ztied.D
4470+
/// svfloat64_t svrecpx[_f64]_z(svbool_t pg, svfloat64_t op)
4471+
/// </summary>
4472+
public static unsafe Vector<double> ReciprocalExponent(Vector<double> value) { throw new PlatformNotSupportedException(); }
4473+
4474+
/// <summary>
4475+
/// svfloat32_t svrecpx[_f32]_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op)
4476+
/// FRECPX Ztied.S, Pg/M, Zop.S
4477+
/// svfloat32_t svrecpx[_f32]_x(svbool_t pg, svfloat32_t op)
4478+
/// FRECPX Ztied.S, Pg/M, Ztied.S
4479+
/// svfloat32_t svrecpx[_f32]_z(svbool_t pg, svfloat32_t op)
4480+
/// </summary>
4481+
public static unsafe Vector<float> ReciprocalExponent(Vector<float> value) { throw new PlatformNotSupportedException(); }
4482+
4483+
4484+
/// ReciprocalSqrtEstimate : Reciprocal square root estimate
4485+
4486+
/// <summary>
4487+
/// svfloat64_t svrsqrte[_f64](svfloat64_t op)
4488+
/// FRSQRTE Zresult.D, Zop.D
4489+
/// </summary>
4490+
public static unsafe Vector<double> ReciprocalSqrtEstimate(Vector<double> value) { throw new PlatformNotSupportedException(); }
4491+
4492+
/// <summary>
4493+
/// svfloat32_t svrsqrte[_f32](svfloat32_t op)
4494+
/// FRSQRTE Zresult.S, Zop.S
4495+
/// </summary>
4496+
public static unsafe Vector<float> ReciprocalSqrtEstimate(Vector<float> value) { throw new PlatformNotSupportedException(); }
4497+
4498+
4499+
/// ReciprocalSqrtStep : Reciprocal square root step
4500+
4501+
/// <summary>
4502+
/// svfloat64_t svrsqrts[_f64](svfloat64_t op1, svfloat64_t op2)
4503+
/// FRSQRTS Zresult.D, Zop1.D, Zop2.D
4504+
/// </summary>
4505+
public static unsafe Vector<double> ReciprocalSqrtStep(Vector<double> left, Vector<double> right) { throw new PlatformNotSupportedException(); }
4506+
4507+
/// <summary>
4508+
/// svfloat32_t svrsqrts[_f32](svfloat32_t op1, svfloat32_t op2)
4509+
/// FRSQRTS Zresult.S, Zop1.S, Zop2.S
4510+
/// </summary>
4511+
public static unsafe Vector<float> ReciprocalSqrtStep(Vector<float> left, Vector<float> right) { throw new PlatformNotSupportedException(); }
4512+
4513+
4514+
/// ReciprocalStep : Reciprocal step
4515+
4516+
/// <summary>
4517+
/// svfloat64_t svrecps[_f64](svfloat64_t op1, svfloat64_t op2)
4518+
/// FRECPS Zresult.D, Zop1.D, Zop2.D
4519+
/// </summary>
4520+
public static unsafe Vector<double> ReciprocalStep(Vector<double> left, Vector<double> right) { throw new PlatformNotSupportedException(); }
4521+
4522+
/// <summary>
4523+
/// svfloat32_t svrecps[_f32](svfloat32_t op1, svfloat32_t op2)
4524+
/// FRECPS Zresult.S, Zop1.S, Zop2.S
4525+
/// </summary>
4526+
public static unsafe Vector<float> ReciprocalStep(Vector<float> left, Vector<float> right) { throw new PlatformNotSupportedException(); }
44484527
/// Reverse bits
44494528

44504529
/// <summary>

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4499,6 +4499,86 @@ internal Arm64() { }
44994499
/// </summary>
45004500
public static unsafe void PrefetchInt64(Vector<ulong> mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchInt64(mask, address, prefetchType);
45014501

4502+
4503+
/// ReciprocalEstimate : Reciprocal estimate
4504+
4505+
/// <summary>
4506+
/// svfloat64_t svrecpe[_f64](svfloat64_t op)
4507+
/// FRECPE Zresult.D, Zop.D
4508+
/// </summary>
4509+
public static unsafe Vector<double> ReciprocalEstimate(Vector<double> value) => ReciprocalEstimate(value);
4510+
4511+
/// <summary>
4512+
/// svfloat32_t svrecpe[_f32](svfloat32_t op)
4513+
/// FRECPE Zresult.S, Zop.S
4514+
/// </summary>
4515+
public static unsafe Vector<float> ReciprocalEstimate(Vector<float> value) => ReciprocalEstimate(value);
4516+
4517+
4518+
/// ReciprocalExponent : Reciprocal exponent
4519+
4520+
/// <summary>
4521+
/// svfloat64_t svrecpx[_f64]_m(svfloat64_t inactive, svbool_t pg, svfloat64_t op)
4522+
/// FRECPX Ztied.D, Pg/M, Zop.D
4523+
/// svfloat64_t svrecpx[_f64]_x(svbool_t pg, svfloat64_t op)
4524+
/// FRECPX Ztied.D, Pg/M, Ztied.D
4525+
/// svfloat64_t svrecpx[_f64]_z(svbool_t pg, svfloat64_t op)
4526+
/// </summary>
4527+
public static unsafe Vector<double> ReciprocalExponent(Vector<double> value) => ReciprocalExponent(value);
4528+
4529+
/// <summary>
4530+
/// svfloat32_t svrecpx[_f32]_m(svfloat32_t inactive, svbool_t pg, svfloat32_t op)
4531+
/// FRECPX Ztied.S, Pg/M, Zop.S
4532+
/// svfloat32_t svrecpx[_f32]_x(svbool_t pg, svfloat32_t op)
4533+
/// FRECPX Ztied.S, Pg/M, Ztied.S
4534+
/// svfloat32_t svrecpx[_f32]_z(svbool_t pg, svfloat32_t op)
4535+
/// </summary>
4536+
public static unsafe Vector<float> ReciprocalExponent(Vector<float> value) => ReciprocalExponent(value);
4537+
4538+
4539+
/// ReciprocalSqrtEstimate : Reciprocal square root estimate
4540+
4541+
/// <summary>
4542+
/// svfloat64_t svrsqrte[_f64](svfloat64_t op)
4543+
/// FRSQRTE Zresult.D, Zop.D
4544+
/// </summary>
4545+
public static unsafe Vector<double> ReciprocalSqrtEstimate(Vector<double> value) => ReciprocalSqrtEstimate(value);
4546+
4547+
/// <summary>
4548+
/// svfloat32_t svrsqrte[_f32](svfloat32_t op)
4549+
/// FRSQRTE Zresult.S, Zop.S
4550+
/// </summary>
4551+
public static unsafe Vector<float> ReciprocalSqrtEstimate(Vector<float> value) => ReciprocalSqrtEstimate(value);
4552+
4553+
4554+
/// ReciprocalSqrtStep : Reciprocal square root step
4555+
4556+
/// <summary>
4557+
/// svfloat64_t svrsqrts[_f64](svfloat64_t op1, svfloat64_t op2)
4558+
/// FRSQRTS Zresult.D, Zop1.D, Zop2.D
4559+
/// </summary>
4560+
public static unsafe Vector<double> ReciprocalSqrtStep(Vector<double> left, Vector<double> right) => ReciprocalSqrtStep(left, right);
4561+
4562+
/// <summary>
4563+
/// svfloat32_t svrsqrts[_f32](svfloat32_t op1, svfloat32_t op2)
4564+
/// FRSQRTS Zresult.S, Zop1.S, Zop2.S
4565+
/// </summary>
4566+
public static unsafe Vector<float> ReciprocalSqrtStep(Vector<float> left, Vector<float> right) => ReciprocalSqrtStep(left, right);
4567+
4568+
4569+
/// ReciprocalStep : Reciprocal step
4570+
4571+
/// <summary>
4572+
/// svfloat64_t svrecps[_f64](svfloat64_t op1, svfloat64_t op2)
4573+
/// FRECPS Zresult.D, Zop1.D, Zop2.D
4574+
/// </summary>
4575+
public static unsafe Vector<double> ReciprocalStep(Vector<double> left, Vector<double> right) => ReciprocalStep(left, right);
4576+
4577+
/// <summary>
4578+
/// svfloat32_t svrecps[_f32](svfloat32_t op1, svfloat32_t op2)
4579+
/// FRECPS Zresult.S, Zop1.S, Zop2.S
4580+
/// </summary>
4581+
public static unsafe Vector<float> ReciprocalStep(Vector<float> left, Vector<float> right) => ReciprocalStep(left, right);
45024582
/// Reverse bits
45034583

45044584
/// <summary>

‎src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4826,6 +4826,17 @@ internal Arm64() { }
48264826
public static unsafe void PrefetchInt32(System.Numerics.Vector<uint> mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; }
48274827
public static unsafe void PrefetchInt64(System.Numerics.Vector<ulong> mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; }
48284828

4829+
public static System.Numerics.Vector<double> ReciprocalEstimate(System.Numerics.Vector<double> value) { throw null; }
4830+
public static System.Numerics.Vector<float> ReciprocalEstimate(System.Numerics.Vector<float> value) { throw null; }
4831+
public static System.Numerics.Vector<double> ReciprocalExponent(System.Numerics.Vector<double> value) { throw null; }
4832+
public static System.Numerics.Vector<float> ReciprocalExponent(System.Numerics.Vector<float> value) { throw null; }
4833+
public static System.Numerics.Vector<double> ReciprocalSqrtEstimate(System.Numerics.Vector<double> value) { throw null; }
4834+
public static System.Numerics.Vector<float> ReciprocalSqrtEstimate(System.Numerics.Vector<float> value) { throw null; }
4835+
public static System.Numerics.Vector<double> ReciprocalSqrtStep(System.Numerics.Vector<double> left, System.Numerics.Vector<double> right) { throw null; }
4836+
public static System.Numerics.Vector<float> ReciprocalSqrtStep(System.Numerics.Vector<float> left, System.Numerics.Vector<float> right) { throw null; }
4837+
public static System.Numerics.Vector<double> ReciprocalStep(System.Numerics.Vector<double> left, System.Numerics.Vector<double> right) { throw null; }
4838+
public static System.Numerics.Vector<float> ReciprocalStep(System.Numerics.Vector<float> left, System.Numerics.Vector<float> right) { throw null; }
4839+
48294840
public static System.Numerics.Vector<byte> ReverseBits(System.Numerics.Vector<byte> value) { throw null; }
48304841
public static System.Numerics.Vector<short> ReverseBits(System.Numerics.Vector<short> value) { throw null; }
48314842
public static System.Numerics.Vector<int> ReverseBits(System.Numerics.Vector<int> value) { throw null; }

0 commit comments

Comments
 (0)