Skip to content

Commit 39b5b8e

Browse files
DeepakRajendrakumaranDeepakRajendrakumaran
authored
Adding Shuffle(). (#85129)
* Adding Shuffle(). * Fixing Vector256 bug + Simplifying logic by switching to AND instead of ConditionalSelect * Adding regression test.
1 parent fd9be9f commit 39b5b8e

7 files changed

Lines changed: 139 additions & 8 deletions

File tree

‎src/coreclr/jit/emitxarch.cpp‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18066,8 +18066,10 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins
1806618066
case INS_psadbw:
1806718067
case INS_vpermps:
1806818068
case INS_vpermpd:
18069+
case INS_vpermpd_reg:
1806918070
case INS_vpermd:
1807018071
case INS_vpermq:
18072+
case INS_vpermq_reg:
1807118073
case INS_vperm2i128:
1807218074
case INS_vperm2f128:
1807318075
case INS_vextractf128:
@@ -18086,6 +18088,11 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins
1808618088
result.insLatency += PERFSCORE_LATENCY_3C;
1808718089
break;
1808818090

18091+
case INS_vpermw:
18092+
result.insThroughput = PERFSCORE_THROUGHPUT_2C;
18093+
result.insLatency += PERFSCORE_LATENCY_6C;
18094+
break;
18095+
1808918096
case INS_pextrb:
1809018097
case INS_pextrd:
1809118098
case INS_pextrw:

‎src/coreclr/jit/gentree.cpp‎

Lines changed: 60 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23217,7 +23217,7 @@ GenTree* Compiler::gtNewSimdShuffleNode(
2321723217
#if defined(TARGET_XARCH)
2321823218
uint8_t control = 0;
2321923219
bool crossLane = false;
23220-
bool needsZero = varTypeIsSmallInt(simdBaseType);
23220+
bool needsZero = varTypeIsSmallInt(simdBaseType) && (simdSize != 64);
2322123221
uint64_t value = 0;
2322223222
simd_t vecCns = {};
2322323223
simd_t mskCns = {};
@@ -23351,6 +23351,61 @@ GenTree* Compiler::gtNewSimdShuffleNode(
2335123351
retNode = gtNewSimdHWIntrinsicNode(type, op1, cnsNode, NI_AVX2_Permute4x64, simdBaseJitType, simdSize);
2335223352
}
2335323353
}
23354+
else if (simdSize == 64)
23355+
{
23356+
if (elementSize == 4)
23357+
{
23358+
for (uint32_t i = 0; i < elementCount; i++)
23359+
{
23360+
vecCns.u32[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize);
23361+
}
23362+
23363+
op2 = gtNewVconNode(type);
23364+
op2->AsVecCon()->gtSimdVal = vecCns;
23365+
23366+
// swap the operands to match the encoding requirements
23367+
retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512F_PermuteVar16x32, simdBaseJitType, simdSize);
23368+
}
23369+
else if (elementSize == 2)
23370+
{
23371+
for (uint32_t i = 0; i < elementCount; i++)
23372+
{
23373+
vecCns.u16[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize);
23374+
}
23375+
23376+
op2 = gtNewVconNode(type);
23377+
op2->AsVecCon()->gtSimdVal = vecCns;
23378+
23379+
// swap the operands to match the encoding requirements
23380+
retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512BW_PermuteVar32x16, simdBaseJitType, simdSize);
23381+
}
23382+
else
23383+
{
23384+
assert(elementSize == 8);
23385+
23386+
for (uint32_t i = 0; i < elementCount; i++)
23387+
{
23388+
vecCns.u64[i] = (uint8_t)(vecCns.u8[i * elementSize] / elementSize);
23389+
}
23390+
23391+
op2 = gtNewVconNode(type);
23392+
op2->AsVecCon()->gtSimdVal = vecCns;
23393+
23394+
// swap the operands to match the encoding requirements
23395+
retNode = gtNewSimdHWIntrinsicNode(type, op2, op1, NI_AVX512F_Permute8x64, simdBaseJitType, simdSize);
23396+
}
23397+
assert(retNode != nullptr);
23398+
23399+
// TODO-XArch-AVX512: Switch to VPERMI2*
23400+
if (needsZero)
23401+
{
23402+
op2 = gtNewVconNode(type);
23403+
op2->AsVecCon()->gtSimdVal = mskCns;
23404+
retNode = gtNewSimdBinOpNode(GT_AND, type, op2, retNode, simdBaseJitType, simdSize);
23405+
}
23406+
23407+
return retNode;
23408+
}
2335423409
else
2335523410
{
2335623411
if (needsZero && compOpportunisticallyDependsOn(InstructionSet_SSSE3))
@@ -23397,13 +23452,11 @@ GenTree* Compiler::gtNewSimdShuffleNode(
2339723452

2339823453
if (needsZero)
2339923454
{
23400-
assert(!compIsaSupportedDebugOnly(InstructionSet_SSSE3));
23401-
23402-
op2 = gtNewVconNode(type);
23403-
op2->AsVecCon()->gtSimd16Val = mskCns.v128[0];
23455+
assert((simdSize == 32) || !compIsaSupportedDebugOnly(InstructionSet_SSSE3));
2340423456

23405-
GenTree* zero = gtNewZeroConNode(type);
23406-
retNode = gtNewSimdCndSelNode(type, op2, retNode, zero, simdBaseJitType, simdSize);
23457+
op2 = gtNewVconNode(type);
23458+
op2->AsVecCon()->gtSimdVal = mskCns;
23459+
retNode = gtNewSimdBinOpNode(GT_AND, type, op2, retNode, simdBaseJitType, simdSize);
2340723460
}
2340823461

2340923462
return retNode;

‎src/coreclr/jit/hwintrinsiclistxarch.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -864,6 +864,8 @@ HARDWARE_INTRINSIC(AVX512F, Min,
864864
HARDWARE_INTRINSIC(AVX512F, Multiply, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmuldq, INS_pmuludq, INS_mulps, INS_mulpd}, HW_Category_SimpleSIMD, HW_Flag_Commutative)
865865
HARDWARE_INTRINSIC(AVX512F, MultiplyLow, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pmulld, INS_pmulld, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_Commutative)
866866
HARDWARE_INTRINSIC(AVX512F, Or, 64, 2, {INS_por, INS_por, INS_por, INS_por, INS_por, INS_por, INS_vporq, INS_vporq, INS_orps, INS_orpd}, HW_Category_SimpleSIMD, HW_Flag_Commutative)
867+
HARDWARE_INTRINSIC(AVX512F, Permute8x64, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermq_reg, INS_vpermq_reg, INS_invalid, INS_vpermpd_reg}, HW_Category_SimpleSIMD, HW_Flag_SpecialImport)
868+
HARDWARE_INTRINSIC(AVX512F, PermuteVar16x32, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpermd, INS_vpermd, INS_invalid, INS_invalid, INS_vpermps, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SpecialImport)
867869
HARDWARE_INTRINSIC(AVX512F, ShiftLeftLogical, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_pslld, INS_pslld, INS_psllq, INS_psllq, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM)
868870
HARDWARE_INTRINSIC(AVX512F, ShiftRightArithmetic, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrad, INS_invalid, INS_vpsraq, INS_invalid, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM)
869871
HARDWARE_INTRINSIC(AVX512F, ShiftRightLogical, 64, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_psrld, INS_psrld, INS_psrlq, INS_psrlq, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM)
@@ -913,6 +915,7 @@ HARDWARE_INTRINSIC(AVX512BW, MultiplyHighRoundScale,
913915
HARDWARE_INTRINSIC(AVX512BW, MultiplyLow, 64, 2, {INS_invalid, INS_invalid, INS_pmullw, INS_pmullw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_Commutative)
914916
HARDWARE_INTRINSIC(AVX512BW, PackSignedSaturate, 64, 2, {INS_packsswb, INS_invalid, INS_packssdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
915917
HARDWARE_INTRINSIC(AVX512BW, PackUnsignedSaturate, 64, 2, {INS_invalid, INS_packuswb, INS_invalid, INS_packusdw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
918+
HARDWARE_INTRINSIC(AVX512BW, PermuteVar32x16, 64, 2, {INS_invalid, INS_invalid, INS_vpermw, INS_vpermw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_SpecialImport)
916919
HARDWARE_INTRINSIC(AVX512BW, ShiftLeftLogical, 64, 2, {INS_invalid, INS_invalid, INS_psllw, INS_psllw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM)
917920
HARDWARE_INTRINSIC(AVX512BW, ShiftLeftLogical128BitLane, 64, 2, {INS_pslldq, INS_pslldq, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_FullRangeIMM)
918921
HARDWARE_INTRINSIC(AVX512BW, ShiftRightArithmetic, 64, 2, {INS_invalid, INS_invalid, INS_psraw, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_IMM, HW_Flag_MaybeIMM|HW_Flag_NoJmpTableIMM|HW_Flag_FullRangeIMM)

‎src/coreclr/jit/hwintrinsicxarch.cpp‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2219,9 +2219,10 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
22192219

22202220
case NI_Vector128_Shuffle:
22212221
case NI_Vector256_Shuffle:
2222+
case NI_Vector512_Shuffle:
22222223
{
22232224
assert((sig->numArgs == 2) || (sig->numArgs == 3));
2224-
assert((simdSize == 16) || (simdSize == 32));
2225+
assert((simdSize == 16) || (simdSize == 32) || (simdSize == 64));
22252226

22262227
GenTree* indices = impStackTop(0).val;
22272228

@@ -2277,6 +2278,14 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic,
22772278
}
22782279
}
22792280
}
2281+
else if (simdSize == 64)
2282+
{
2283+
if (varTypeIsByte(simdBaseType))
2284+
{
2285+
// TYP_BYTE, TYP_UBYTE need AVX512_VBMI.
2286+
break;
2287+
}
2288+
}
22802289
else
22812290
{
22822291
assert(simdSize == 16);

‎src/coreclr/jit/instrsxarch.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -620,6 +620,8 @@ INST3(vmovdqu64, "movdqu64", IUM_WR, SSEFLT(0x7F), BAD_
620620
INST3(vpabsq, "pabsq", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x1F), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX) // Packed absolute value of 64-bit integers
621621
INST3(vpandq, "pandq", IUM_WR, BAD_CODE, BAD_CODE, PCKDBL(0xDB), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Packed bit-wise AND of two xmm regs
622622
INST3(vpandnq, "pandnq", IUM_WR, BAD_CODE, BAD_CODE, PCKDBL(0xDF), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // Packed bit-wise AND NOT of two xmm regs
623+
INST3(vpermq_reg, "permq", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x36), INS_TT_FULL, Input_64Bit | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Permute 64-bit of input register
624+
INST3(vpermpd_reg, "permpd", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x16), INS_TT_FULL, Input_64Bit | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Permute 64-bit of input register
623625
INST3(vpmaxsq, "pmaxsq", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x3D), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // packed maximum 64-bit signed integers
624626
INST3(vpmaxuq, "pmaxuq", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x3F), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // packed maximum 64-bit unsigned integers
625627
INST3(vpminsq, "pminsq", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x39), INS_TT_FULL, Input_64Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsDstDstSrcAVXInstruction) // packed minimum 64-bit signed integers
@@ -640,6 +642,7 @@ INST3(kortestd, "kortestd", IUM_WR, BAD_CODE, BAD_
640642
INST3(kortestq, "kortestq", IUM_WR, BAD_CODE, BAD_CODE, PCKFLT(0x98), INS_TT_NONE, REX_W1 | Encoding_VEX | Resets_OF | Resets_SF | Writes_ZF | Resets_AF | Resets_PF | Writes_CF | KInstruction)
641643
INST3(vmovdqu8, "movdqu8", IUM_WR, SSEFLT(0x7F), BAD_CODE, SSEFLT(0x6F), INS_TT_FULL_MEM, Input_8Bit | REX_W0_EVEX | Encoding_EVEX)
642644
INST3(vmovdqu16, "movdqu16", IUM_WR, SSEFLT(0x7F), BAD_CODE, SSEFLT(0x6F), INS_TT_FULL_MEM, Input_16Bit | REX_W1_EVEX | Encoding_EVEX)
645+
INST3(vpermw, "permw", IUM_WR, BAD_CODE, BAD_CODE, SSE38(0x8D), INS_TT_FULL_MEM, Input_16Bit | REX_W1 | Encoding_EVEX | INS_Flags_IsDstSrcSrcAVXInstruction) // Permute Packed Doublewords Elements
643646
INST3(vpcmpb, "pcmpb", IUM_WR, BAD_CODE, BAD_CODE, SSE3A(0x3F), INS_TT_FULL_MEM, Input_8Bit | REX_W0_EVEX | Encoding_EVEX | INS_Flags_IsMskSrcSrcEvexInstruction)
644647
INST3(vpcmpw, "pcmpw", IUM_WR, BAD_CODE, BAD_CODE, SSE3A(0x3F), INS_TT_FULL, Input_16Bit | REX_W1_EVEX | Encoding_EVEX | INS_Flags_IsMskSrcSrcEvexInstruction)
645648
INST3(vpcmpub, "pcmpub", IUM_WR, BAD_CODE, BAD_CODE, SSE3A(0x3E), INS_TT_FULL_MEM, Input_8Bit | REX_W0_EVEX | Encoding_EVEX | INS_Flags_IsMskSrcSrcEvexInstruction)

‎src/tests/JIT/Regression/JitBlue/GitHub_85129/GitHub_85129.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
//
4+
5+
using System;
6+
using System.Runtime.CompilerServices;
7+
using System.Runtime.InteropServices;
8+
using System.Runtime.Intrinsics;
9+
using Xunit;
10+
11+
public class Program
12+
{
13+
[Fact]
14+
public static int TestEntryPoint()
15+
{
16+
17+
Vector256<int> v256Shuffle = Vector256.Create(100, 101, 102, 103, 104, 105, 106, 107);
18+
Vector256<int> v256ShuffleExpectedResult = Vector256.Create(107, 105, 0, 101, 106, 104, 0, 100);
19+
Vector256<int> v256ShuffleActualResult = Vector256Shuffle(v256Shuffle);
20+
if(v256ShuffleExpectedResult != v256ShuffleActualResult)
21+
{
22+
return 1;
23+
}
24+
25+
Vector512<int> v512Shuffle = Vector512.Create(100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115);
26+
Vector512<int> v512ShuffleExpectedResult = Vector512.Create(115, 113, 111, 0, 107, 105, 103, 101, 114, 112, 110, 108, 0, 104, 102, 100);
27+
Vector512<int> v512ShuffleActualResult = Vector512Shuffle(v512Shuffle);
28+
if (v512ShuffleExpectedResult != v512ShuffleActualResult)
29+
{
30+
return 1;
31+
}
32+
return 100;
33+
}
34+
35+
36+
[MethodImpl(MethodImplOptions.NoInlining)]
37+
public static Vector256<int> Vector256Shuffle(Vector256<int> v1)
38+
{
39+
return Vector256.Shuffle(v1, Vector256.Create(7, 5, 132, 1, 6, 4, -3, 0));
40+
}
41+
42+
[MethodImpl(MethodImplOptions.NoInlining)]
43+
public static Vector512<int> Vector512Shuffle(Vector512<int> v1)
44+
{
45+
return Vector512.Shuffle(v1, Vector512.Create(15, 13, 11, 99, 7, 5, 3, 1, 14, 12, 10, 8, -11, 4, 2, 0));
46+
}
47+
}

‎src/tests/JIT/Regression/JitBlue/GitHub_85129/GitHub_85129.csproj‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<DebugType>None</DebugType>
4+
<Optimize>True</Optimize>
5+
</PropertyGroup>
6+
<ItemGroup>
7+
<Compile Include="$(MSBuildProjectName).cs" />
8+
</ItemGroup>
9+
</Project>

0 commit comments

Comments
 (0)