Skip to content

Commit 295bcdc

Browse files
bbartelsgfoidl
bbartels
and
gfoidl
authored
Vectorized common String.Split() paths (dotnet#38001)
* Vectorized String.Split() * Fixed variable name * Update src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs Co-authored-by: Günther Foidl <gue@korporal.at> * Update src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs Co-authored-by: Günther Foidl <gue@korporal.at> * Applied Review Feedback * Update src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs Co-authored-by: Günther Foidl <gue@korporal.at> * Applied Review Feedback * Built branchless version with help of @gfoidl * Update src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs Co-authored-by: Günther Foidl <gue@korporal.at> * Removed nullable separator parameters * Refactored MakeSeparatorList * Fixed mistakenly removed comments * Removed dependency on BMI2 PEXT instruction * Fixed mistaken use of Vector<ushort>.Count * Lowered string.Split() vectorization dependency from Avx2 to SSE41 * Added Sse.IsSupported check * Updated IsSupported check to match highest used ISA * Fixed possible cause for failing tests Co-authored-by: Günther Foidl <gue@korporal.at>
1 parent b4e1102 commit 295bcdc

File tree

1 file changed

+115
-54
lines changed

1 file changed

+115
-54
lines changed

‎src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs

Lines changed: 115 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
using System.Globalization;
77
using System.Numerics;
88
using System.Runtime.InteropServices;
9+
using System.Runtime.Intrinsics;
10+
using System.Runtime.Intrinsics.X86;
911
using System.Text;
1012
using Internal.Runtime.CompilerServices;
1113

@@ -1491,78 +1493,137 @@ private string[] SplitWithPostProcessing(ReadOnlySpan<int> sepList, ReadOnlySpan
14911493
/// <param name="sepListBuilder"><see cref="ValueListBuilder{T}"/> to store indexes</param>
14921494
private void MakeSeparatorList(ReadOnlySpan<char> separators, ref ValueListBuilder<int> sepListBuilder)
14931495
{
1494-
char sep0, sep1, sep2;
1495-
1496-
switch (separators.Length)
1496+
// Special-case no separators to mean any whitespace is a separator.
1497+
if (separators.Length == 0)
14971498
{
1498-
// Special-case no separators to mean any whitespace is a separator.
1499-
case 0:
1500-
for (int i = 0; i < Length; i++)
1499+
for (int i = 0; i < Length; i++)
1500+
{
1501+
if (char.IsWhiteSpace(this[i]))
15011502
{
1502-
if (char.IsWhiteSpace(this[i]))
1503-
{
1504-
sepListBuilder.Append(i);
1505-
}
1503+
sepListBuilder.Append(i);
15061504
}
1507-
break;
1505+
}
1506+
}
15081507

1509-
// Special-case the common cases of 1, 2, and 3 separators, with manual comparisons against each separator.
1510-
case 1:
1511-
sep0 = separators[0];
1512-
for (int i = 0; i < Length; i++)
1508+
// Special-case the common cases of 1, 2, and 3 separators, with manual comparisons against each separator.
1509+
else if (separators.Length <= 3)
1510+
{
1511+
char sep0, sep1, sep2;
1512+
sep0 = separators[0];
1513+
sep1 = separators.Length > 1 ? separators[1] : sep0;
1514+
sep2 = separators.Length > 2 ? separators[2] : sep1;
1515+
1516+
if (Length >= 16 && Sse41.IsSupported)
1517+
{
1518+
MakeSeparatorListVectorized(ref sepListBuilder, sep0, sep1, sep2);
1519+
return;
1520+
}
1521+
1522+
for (int i = 0; i < Length; i++)
1523+
{
1524+
char c = this[i];
1525+
if (c == sep0 || c == sep1 || c == sep2)
15131526
{
1514-
if (this[i] == sep0)
1515-
{
1516-
sepListBuilder.Append(i);
1517-
}
1527+
sepListBuilder.Append(i);
15181528
}
1519-
break;
1520-
case 2:
1521-
sep0 = separators[0];
1522-
sep1 = separators[1];
1529+
}
1530+
}
1531+
1532+
// Handle > 3 separators with a probabilistic map, ala IndexOfAny.
1533+
// This optimizes for chars being unlikely to match a separator.
1534+
else
1535+
{
1536+
unsafe
1537+
{
1538+
ProbabilisticMap map = default;
1539+
uint* charMap = (uint*)&map;
1540+
InitializeProbabilisticMap(charMap, separators);
1541+
15231542
for (int i = 0; i < Length; i++)
15241543
{
15251544
char c = this[i];
1526-
if (c == sep0 || c == sep1)
1545+
if (IsCharBitSet(charMap, (byte)c) && IsCharBitSet(charMap, (byte)(c >> 8)) &&
1546+
separators.Contains(c))
15271547
{
15281548
sepListBuilder.Append(i);
15291549
}
15301550
}
1531-
break;
1532-
case 3:
1533-
sep0 = separators[0];
1534-
sep1 = separators[1];
1535-
sep2 = separators[2];
1536-
for (int i = 0; i < Length; i++)
1551+
}
1552+
}
1553+
}
1554+
1555+
private void MakeSeparatorListVectorized(ref ValueListBuilder<int> sepListBuilder, char c, char c2, char c3)
1556+
{
1557+
// Redundant test so we won't prejit remainder of this method
1558+
// on platforms without SSE.
1559+
if (!Sse41.IsSupported)
1560+
{
1561+
throw new PlatformNotSupportedException();
1562+
}
1563+
1564+
// Constant that allows for the truncation of 16-bit (FFFF/0000) values within a register to 4-bit (F/0)
1565+
Vector128<byte> shuffleConstant = Vector128.Create(0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF);
1566+
1567+
Vector128<ushort> v1 = Vector128.Create(c);
1568+
Vector128<ushort> v2 = Vector128.Create(c2);
1569+
Vector128<ushort> v3 = Vector128.Create(c3);
1570+
1571+
ref char c0 = ref MemoryMarshal.GetReference(this.AsSpan());
1572+
int cond = Length & -Vector128<ushort>.Count;
1573+
int i = 0;
1574+
1575+
for (; i < cond; i += Vector128<ushort>.Count)
1576+
{
1577+
Vector128<ushort> charVector = ReadVector(ref c0, i);
1578+
Vector128<ushort> cmp = Sse2.CompareEqual(charVector, v1);
1579+
1580+
cmp = Sse2.Or(Sse2.CompareEqual(charVector, v2), cmp);
1581+
cmp = Sse2.Or(Sse2.CompareEqual(charVector, v3), cmp);
1582+
1583+
if (Sse41.TestZ(cmp, cmp)) { continue; }
1584+
1585+
Vector128<byte> mask = Sse2.ShiftRightLogical(cmp.AsUInt64(), 4).AsByte();
1586+
mask = Ssse3.Shuffle(mask, shuffleConstant);
1587+
1588+
uint lowBits = Sse2.ConvertToUInt32(mask.AsUInt32());
1589+
mask = Sse2.ShiftRightLogical(mask.AsUInt64(), 32).AsByte();
1590+
uint highBits = Sse2.ConvertToUInt32(mask.AsUInt32());
1591+
1592+
for (int idx = i; lowBits != 0; idx++)
1593+
{
1594+
if ((lowBits & 0xF) != 0)
15371595
{
1538-
char c = this[i];
1539-
if (c == sep0 || c == sep1 || c == sep2)
1540-
{
1541-
sepListBuilder.Append(i);
1542-
}
1596+
sepListBuilder.Append(idx);
15431597
}
1544-
break;
15451598

1546-
// Handle > 3 separators with a probabilistic map, ala IndexOfAny.
1547-
// This optimizes for chars being unlikely to match a separator.
1548-
default:
1549-
unsafe
1550-
{
1551-
ProbabilisticMap map = default;
1552-
uint* charMap = (uint*)&map;
1553-
InitializeProbabilisticMap(charMap, separators);
1599+
lowBits >>= 8;
1600+
}
15541601

1555-
for (int i = 0; i < Length; i++)
1556-
{
1557-
char c = this[i];
1558-
if (IsCharBitSet(charMap, (byte)c) && IsCharBitSet(charMap, (byte)(c >> 8)) &&
1559-
separators.Contains(c))
1560-
{
1561-
sepListBuilder.Append(i);
1562-
}
1563-
}
1602+
for (int idx = i + 4; highBits != 0; idx++)
1603+
{
1604+
if ((highBits & 0xF) != 0)
1605+
{
1606+
sepListBuilder.Append(idx);
15641607
}
1565-
break;
1608+
1609+
highBits >>= 8;
1610+
}
1611+
}
1612+
1613+
for (; i < Length; i++)
1614+
{
1615+
char curr = Unsafe.Add(ref c0, (IntPtr)(uint)i);
1616+
if (curr == c || curr == c2 || curr == c3)
1617+
{
1618+
sepListBuilder.Append(i);
1619+
}
1620+
}
1621+
1622+
static Vector128<ushort> ReadVector(ref char c0, int offset)
1623+
{
1624+
ref char ci = ref Unsafe.Add(ref c0, (IntPtr)(uint)offset);
1625+
ref byte b = ref Unsafe.As<char, byte>(ref ci);
1626+
return Unsafe.ReadUnaligned<Vector128<ushort>>(ref b);
15661627
}
15671628
}
15681629

0 commit comments

Comments
 (0)