Skip to content

Commit e954b73

Browse files
tannergoodingtannergooding
committed
Move the implementation of ReciprocalEstimate and ReciprocalSqrtEstimate to the JIT so R2R works
1 parent eb950fe commit e954b73

File tree

7 files changed

+166
-62
lines changed

7 files changed

+166
-62
lines changed

‎src/coreclr/jit/compiler.h

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1594,7 +1594,7 @@ enum class ProfileChecks : unsigned int
15941594
{
15951595
CHECK_NONE = 0,
15961596
CHECK_HASLIKELIHOOD = 1 << 0, // check all FlowEdges for hasLikelihood
1597-
CHECK_LIKELIHOODSUM = 1 << 1, // check block succesor likelihoods sum to 1
1597+
CHECK_LIKELIHOODSUM = 1 << 1, // check block succesor likelihoods sum to 1
15981598
CHECK_LIKELY = 1 << 2, // fully check likelihood based weights
15991599
RAISE_ASSERT = 1 << 3, // assert on check failure
16001600
CHECK_ALL_BLOCKS = 1 << 4, // check blocks even if bbHasProfileWeight is false
@@ -4525,6 +4525,11 @@ class Compiler
45254525
CORINFO_THIS_TRANSFORM constraintCallThisTransform,
45264526
NamedIntrinsic* pIntrinsicName,
45274527
bool* isSpecialIntrinsic = nullptr);
4528+
GenTree* impEstimateIntrinsic(CORINFO_METHOD_HANDLE method,
4529+
CORINFO_SIG_INFO* sig,
4530+
CorInfoType callJitType,
4531+
NamedIntrinsic intrinsicName,
4532+
bool tailCall);
45284533
GenTree* impMathIntrinsic(CORINFO_METHOD_HANDLE method,
45294534
CORINFO_SIG_INFO* sig,
45304535
var_types callType,

‎src/coreclr/jit/importercalls.cpp

Lines changed: 146 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4146,6 +4146,13 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
41464146
break;
41474147
}
41484148

4149+
case NI_System_Math_ReciprocalEstimate:
4150+
case NI_System_Math_ReciprocalSqrtEstimate:
4151+
{
4152+
retNode = impEstimateIntrinsic(method, sig, callJitType, ni, tailCall);
4153+
break;
4154+
}
4155+
41494156
case NI_System_Array_Clone:
41504157
case NI_System_Collections_Generic_Comparer_get_Default:
41514158
case NI_System_Collections_Generic_EqualityComparer_get_Default:
@@ -7413,13 +7420,15 @@ bool Compiler::IsTargetIntrinsic(NamedIntrinsic intrinsicName)
74137420
// instructions to directly compute round/ceiling/floor/truncate.
74147421

74157422
case NI_System_Math_Abs:
7423+
case NI_System_Math_ReciprocalEstimate:
7424+
case NI_System_Math_ReciprocalSqrtEstimate:
74167425
case NI_System_Math_Sqrt:
74177426
return true;
74187427

74197428
case NI_System_Math_Ceiling:
74207429
case NI_System_Math_Floor:
7421-
case NI_System_Math_Truncate:
74227430
case NI_System_Math_Round:
7431+
case NI_System_Math_Truncate:
74237432
return compOpportunisticallyDependsOn(InstructionSet_SSE41);
74247433

74257434
case NI_System_Math_FusedMultiplyAdd:
@@ -7434,11 +7443,13 @@ bool Compiler::IsTargetIntrinsic(NamedIntrinsic intrinsicName)
74347443
case NI_System_Math_Abs:
74357444
case NI_System_Math_Ceiling:
74367445
case NI_System_Math_Floor:
7437-
case NI_System_Math_Truncate:
7438-
case NI_System_Math_Round:
7439-
case NI_System_Math_Sqrt:
74407446
case NI_System_Math_Max:
74417447
case NI_System_Math_Min:
7448+
case NI_System_Math_ReciprocalEstimate:
7449+
case NI_System_Math_ReciprocalSqrtEstimate:
7450+
case NI_System_Math_Round:
7451+
case NI_System_Math_Sqrt:
7452+
case NI_System_Math_Truncate:
74427453
return true;
74437454

74447455
case NI_System_Math_FusedMultiplyAdd:
@@ -7513,6 +7524,8 @@ bool Compiler::IsMathIntrinsic(NamedIntrinsic intrinsicName)
75137524
case NI_System_Math_MinMagnitudeNumber:
75147525
case NI_System_Math_MinNumber:
75157526
case NI_System_Math_Pow:
7527+
case NI_System_Math_ReciprocalEstimate:
7528+
case NI_System_Math_ReciprocalSqrtEstimate:
75167529
case NI_System_Math_Round:
75177530
case NI_System_Math_Sin:
75187531
case NI_System_Math_Sinh:
@@ -8730,6 +8743,121 @@ void Compiler::impCheckCanInline(GenTreeCall* call,
87308743
}
87318744
}
87328745

8746+
//------------------------------------------------------------------------
8747+
// impMinMaxIntrinsic: Imports a min or max intrinsic
8748+
//
8749+
// Arguments:
8750+
// method - The handle of the method being imported
8751+
// callType - The underlying type for the call
8752+
// intrinsicName - The intrinsic being imported
8753+
// tailCall - true if the method is a tail call; otherwise false
8754+
// isMax - true if the intrinsic computes the max; false for the min
8755+
// isMagnitude - true if the intrinsic compares using the absolute value of the inputs
8756+
// isNumber - true if the intrinsic propagates the number; false for NaN
8757+
//
8758+
GenTree* Compiler::impEstimateIntrinsic(CORINFO_METHOD_HANDLE method,
8759+
CORINFO_SIG_INFO* sig,
8760+
CorInfoType callJitType,
8761+
NamedIntrinsic intrinsicName,
8762+
bool tailCall)
8763+
{
8764+
var_types callType = JITtype2varType(callJitType);
8765+
8766+
assert(varTypeIsFloating(callType));
8767+
assert(sig->numArgs == 1);
8768+
8769+
#if defined(FEATURE_HW_INTRINSICS)
8770+
// We use compExactlyDependsOn since these are estimate APIs where
8771+
// the behavior is explicitly allowed to differ across machines and
8772+
// we want to ensure that it gets marked as such in R2R.
8773+
8774+
var_types simdType = TYP_UNKNOWN;
8775+
NamedIntrinsic intrinsicId = NI_Illegal;
8776+
8777+
switch (intrinsicName)
8778+
{
8779+
case NI_System_Math_ReciprocalEstimate:
8780+
{
8781+
#if defined(TARGET_XARCH)
8782+
if (compExactlyDependsOn(InstructionSet_AVX512F))
8783+
{
8784+
simdType = TYP_SIMD16;
8785+
intrinsicId = NI_AVX512F_Reciprocal14Scalar;
8786+
}
8787+
else if ((callType == TYP_FLOAT) && compExactlyDependsOn(InstructionSet_SSE))
8788+
{
8789+
simdType = TYP_SIMD16;
8790+
intrinsicId = NI_SSE_ReciprocalScalar;
8791+
}
8792+
#elif defined(TARGET_ARM64)
8793+
if (compExactlyDependsOn(InstructionSet_AdvSimd_Arm64))
8794+
{
8795+
simdType = TYP_SIMD8;
8796+
intrinsicId = NI_AdvSimd_Arm64_ReciprocalEstimateScalar;
8797+
}
8798+
#endif // TARGET_ARM64
8799+
break;
8800+
}
8801+
8802+
case NI_System_Math_ReciprocalSqrtEstimate:
8803+
{
8804+
#if defined(TARGET_XARCH)
8805+
if (compExactlyDependsOn(InstructionSet_AVX512F))
8806+
{
8807+
simdType = TYP_SIMD16;
8808+
intrinsicId = NI_AVX512F_ReciprocalSqrt14Scalar;
8809+
}
8810+
else if ((callType == TYP_FLOAT) && compExactlyDependsOn(InstructionSet_SSE))
8811+
{
8812+
simdType = TYP_SIMD16;
8813+
intrinsicId = NI_SSE_ReciprocalSqrtScalar;
8814+
}
8815+
#elif defined(TARGET_ARM64)
8816+
if (compExactlyDependsOn(InstructionSet_AdvSimd_Arm64))
8817+
{
8818+
simdType = TYP_SIMD8;
8819+
intrinsicId = NI_AdvSimd_Arm64_ReciprocalSquareRootEstimateScalar;
8820+
}
8821+
#endif // TARGET_ARM64
8822+
break;
8823+
}
8824+
8825+
default:
8826+
{
8827+
unreached();
8828+
}
8829+
}
8830+
8831+
if (intrinsicId != NI_Illegal)
8832+
{
8833+
unsigned simdSize = 0;
8834+
8835+
if (simdType == TYP_SIMD8)
8836+
{
8837+
simdSize = 8;
8838+
}
8839+
else
8840+
{
8841+
assert(simdType == TYP_SIMD16);
8842+
simdSize = 16;
8843+
}
8844+
8845+
GenTree* op1 = impPopStack().val;
8846+
8847+
op1 = gtNewSimdCreateScalarUnsafeNode(simdType, op1, callJitType, simdSize);
8848+
op1 = gtNewSimdHWIntrinsicNode(simdType, op1, intrinsicId, callJitType, simdSize);
8849+
8850+
return gtNewSimdToScalarNode(callType, op1, callJitType, simdSize);
8851+
}
8852+
#endif // FEATURE_HW_INTRINSICS
8853+
8854+
// TODO-CQ: Returning this as an intrinsic blocks inlining and is undesirable
8855+
// return impMathIntrinsic(method, sig, callType, intrinsicName, tailCall);
8856+
8857+
return nullptr;
8858+
}
8859+
8860+
87338861
GenTree* Compiler::impMathIntrinsic(CORINFO_METHOD_HANDLE method,
87348862
CORINFO_SIG_INFO* sig,
87358863
var_types callType,
@@ -10339,7 +10467,20 @@ NamedIntrinsic Compiler::lookupPrimitiveFloatNamedIntrinsic(CORINFO_METHOD_HANDL
1033910467

1034010468
case 'R':
1034110469
{
10342-
if (strcmp(methodName, "Round") == 0)
10470+
if (strncmp(methodName, "Reciprocal", 10) == 0)
10471+
{
10472+
methodName += 10;
10473+
10474+
if (strcmp(methodName, "Estimate") == 0)
10475+
{
10476+
result = NI_System_Math_ReciprocalEstimate;
10477+
}
10478+
else if (strcmp(methodName, "SqrtEstimate") == 0)
10479+
{
10480+
result = NI_System_Math_ReciprocalSqrtEstimate;
10481+
}
10482+
}
10483+
else if (strcmp(methodName, "Round") == 0)
1034310484
{
1034410485
result = NI_System_Math_Round;
1034510486
}

‎src/coreclr/jit/namedintrinsiclist.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ enum NamedIntrinsic : unsigned short
5151
NI_System_Math_MinMagnitudeNumber,
5252
NI_System_Math_MinNumber,
5353
NI_System_Math_Pow,
54+
NI_System_Math_ReciprocalEstimate,
55+
NI_System_Math_ReciprocalSqrtEstimate,
5456
NI_System_Math_Round,
5557
NI_System_Math_Sin,
5658
NI_System_Math_Sinh,

‎src/libraries/System.Private.CoreLib/src/System/Double.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -865,9 +865,11 @@ bool IFloatingPoint<double>.TryWriteSignificandLittleEndian(Span<byte> destinati
865865
public static double Lerp(double value1, double value2, double amount) => (value1 * (1.0 - amount)) + (value2 * amount);
866866

867867
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ReciprocalEstimate(TSelf)" />
868+
[Intrinsic]
868869
public static double ReciprocalEstimate(double x) => Math.ReciprocalEstimate(x);
869870

870871
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ReciprocalSqrtEstimate(TSelf)" />
872+
[Intrinsic]
871873
public static double ReciprocalSqrtEstimate(double x) => Math.ReciprocalSqrtEstimate(x);
872874

873875
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ScaleB(TSelf, int)" />

‎src/libraries/System.Private.CoreLib/src/System/Math.cs

Lines changed: 4 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1195,21 +1195,11 @@ public static double MinMagnitude(double x, double y)
11951195
/// <para>On ARM64 hardware this may use the <c>FRECPE</c> instruction which performs a single Newton-Raphson iteration.</para>
11961196
/// <para>On hardware without specialized support, this may just return <c>1.0 / d</c>.</para>
11971197
/// </remarks>
1198+
[Intrinsic]
11981199
[MethodImpl(MethodImplOptions.AggressiveInlining)]
11991200
public static double ReciprocalEstimate(double d)
12001201
{
1201-
if (Avx512F.IsSupported)
1202-
{
1203-
return Avx512F.Reciprocal14Scalar(Vector128.CreateScalarUnsafe(d)).ToScalar();
1204-
}
1205-
else if (AdvSimd.Arm64.IsSupported)
1206-
{
1207-
return AdvSimd.Arm64.ReciprocalEstimateScalar(Vector64.CreateScalar(d)).ToScalar();
1208-
}
1209-
else
1210-
{
1211-
return 1.0 / d;
1212-
}
1202+
return 1.0 / d;
12131203
}
12141204

12151205
/// <summary>Returns an estimate of the reciprocal square root of a specified number.</summary>
@@ -1219,21 +1209,11 @@ public static double ReciprocalEstimate(double d)
12191209
/// <para>On ARM64 hardware this may use the <c>FRSQRTE</c> instruction which performs a single Newton-Raphson iteration.</para>
12201210
/// <para>On hardware without specialized support, this may just return <c>1.0 / Sqrt(d)</c>.</para>
12211211
/// </remarks>
1212+
[Intrinsic]
12221213
[MethodImpl(MethodImplOptions.AggressiveInlining)]
12231214
public static double ReciprocalSqrtEstimate(double d)
12241215
{
1225-
if (Avx512F.IsSupported)
1226-
{
1227-
return Avx512F.ReciprocalSqrt14Scalar(Vector128.CreateScalarUnsafe(d)).ToScalar();
1228-
}
1229-
else if (AdvSimd.Arm64.IsSupported)
1230-
{
1231-
return AdvSimd.Arm64.ReciprocalSquareRootEstimateScalar(Vector64.CreateScalar(d)).ToScalar();
1232-
}
1233-
else
1234-
{
1235-
return 1.0 / Sqrt(d);
1236-
}
1216+
return 1.0 / Sqrt(d);
12371217
}
12381218

12391219
[MethodImpl(MethodImplOptions.AggressiveInlining)]

‎src/libraries/System.Private.CoreLib/src/System/MathF.cs

Lines changed: 4 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -313,25 +313,11 @@ public static float MinMagnitude(float x, float y)
313313
/// <para>On ARM64 hardware this may use the <c>FRECPE</c> instruction which performs a single Newton-Raphson iteration.</para>
314314
/// <para>On hardware without specialized support, this may just return <c>1.0 / x</c>.</para>
315315
/// </remarks>
316+
[Intrinsic]
316317
[MethodImpl(MethodImplOptions.AggressiveInlining)]
317318
public static float ReciprocalEstimate(float x)
318319
{
319-
if (Avx512F.IsSupported)
320-
{
321-
return Avx512F.Reciprocal14Scalar(Vector128.CreateScalarUnsafe(x)).ToScalar();
322-
}
323-
else if (Sse.IsSupported)
324-
{
325-
return Sse.ReciprocalScalar(Vector128.CreateScalarUnsafe(x)).ToScalar();
326-
}
327-
else if (AdvSimd.Arm64.IsSupported)
328-
{
329-
return AdvSimd.Arm64.ReciprocalEstimateScalar(Vector64.CreateScalarUnsafe(x)).ToScalar();
330-
}
331-
else
332-
{
333-
return 1.0f / x;
334-
}
320+
return 1.0f / x;
335321
}
336322

337323
/// <summary>Returns an estimate of the reciprocal square root of a specified number.</summary>
@@ -342,25 +328,11 @@ public static float ReciprocalEstimate(float x)
342328
/// <para>On ARM64 hardware this may use the <c>FRSQRTE</c> instruction which performs a single Newton-Raphson iteration.</para>
343329
/// <para>On hardware without specialized support, this may just return <c>1.0 / Sqrt(x)</c>.</para>
344330
/// </remarks>
331+
[Intrinsic]
345332
[MethodImpl(MethodImplOptions.AggressiveInlining)]
346333
public static float ReciprocalSqrtEstimate(float x)
347334
{
348-
if (Avx512F.IsSupported)
349-
{
350-
return Avx512F.ReciprocalSqrt14Scalar(Vector128.CreateScalarUnsafe(x)).ToScalar();
351-
}
352-
else if (Sse.IsSupported)
353-
{
354-
return Sse.ReciprocalSqrtScalar(Vector128.CreateScalarUnsafe(x)).ToScalar();
355-
}
356-
else if (AdvSimd.Arm64.IsSupported)
357-
{
358-
return AdvSimd.Arm64.ReciprocalSquareRootEstimateScalar(Vector64.CreateScalarUnsafe(x)).ToScalar();
359-
}
360-
else
361-
{
362-
return 1.0f / Sqrt(x);
363-
}
335+
return 1.0f / Sqrt(x);
364336
}
365337

366338
[Intrinsic]

‎src/libraries/System.Private.CoreLib/src/System/Single.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -848,9 +848,11 @@ bool IFloatingPoint<float>.TryWriteSignificandLittleEndian(Span<byte> destinatio
848848
public static float Lerp(float value1, float value2, float amount) => (value1 * (1.0f - amount)) + (value2 * amount);
849849

850850
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ReciprocalEstimate(TSelf)" />
851+
[Intrinsic]
851852
public static float ReciprocalEstimate(float x) => MathF.ReciprocalEstimate(x);
852853

853854
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ReciprocalSqrtEstimate(TSelf)" />
855+
[Intrinsic]
854856
public static float ReciprocalSqrtEstimate(float x) => MathF.ReciprocalSqrtEstimate(x);
855857

856858
/// <inheritdoc cref="IFloatingPointIeee754{TSelf}.ScaleB(TSelf, int)" />

0 commit comments

Comments
 (0)