Use NarrowNative for narrowing operations

src/libraries/Common/src/System/HexConverter.cs

@@ -354,7 +354,7 @@ public static bool TryDecodeFrom_Vector128<TChar>(ReadOnlySpan<TChar> source, Sp
                     Vector128<ushort> vec1 = Vector128.LoadUnsafe(ref srcRef, offset).AsUInt16();
                     Vector128<ushort> vec2 = Vector128.LoadUnsafe(ref srcRef, offset + (nuint)Vector128<ushort>.Count).AsUInt16();
 
-                    vec = Ascii.ExtractAsciiVector(vec1, vec2);
+                    vec = Vector128.NarrowNative(vec1, vec2);
 
                     if (!Utf16Utility.AllCharsInVectorAreAscii(vec1 | vec2))
                     {

src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlDecoder.cs

@@ -675,7 +675,9 @@ public unsafe bool TryLoadVector512(ushort* src, ushort* srcStart, int sourceLen
                     return false;
                 }
 
-#if NET9_0_OR_GREATER
+#if NET10_0_OR_GREATER
+                str = Vector512.NarrowNative(utf16VectorLower, utf16VectorUpper).AsSByte();
+#elif NET9_0_OR_GREATER
                 str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper).AsSByte();
 #else
                 str = Vector512.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte();
@@ -703,7 +705,9 @@ public unsafe bool TryLoadAvxVector256(ushort* src, ushort* srcStart, int source
                     return false;
                 }
 
-#if NET9_0_OR_GREATER
+#if NET10_0_OR_GREATER
+                str = Vector256.NarrowNative(utf16VectorLower, utf16VectorUpper).AsSByte();
+#elif NET9_0_OR_GREATER
                 str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper).AsSByte();
 #else
                 str = Vector256.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte();
@@ -719,19 +723,19 @@ public unsafe bool TryLoadVector128(ushort* src, ushort* srcStart, int sourceLen
                 Vector128<ushort> utf16VectorUpper = Vector128.LoadUnsafe(ref *src, 8);
 #if NET9_0_OR_GREATER
                 if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper))
-                {
-                    str = default;
-                    return false;
-                }
-
-                str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper);
 #else
                 if (Base64Helper.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper))
+#endif
                 {
                     str = default;
                     return false;
                 }
 
+#if NET10_0_OR_GREATER
+                str = Vector128.NarrowNative(utf16VectorLower, utf16VectorUpper);
+#elif NET9_0_OR_GREATER
+                str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper);
+#else
                 str = Base64Helper.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper);
 #endif
                 return true;
@@ -752,10 +756,17 @@ public unsafe bool TryLoadArmVector128x4(ushort* src, ushort* srcStart, int sour
                     return false;
                 }
 
+#if NET10_0_OR_GREATER
+                str1 = Vector128.NarrowNative(s11, s31);
+                str2 = Vector128.NarrowNative(s12, s32);
+                str3 = Vector128.NarrowNative(s21, s41);
+                str4 = Vector128.NarrowNative(s22, s42);
+#else
                 str1 = Ascii.ExtractAsciiVector(s11, s31);
                 str2 = Ascii.ExtractAsciiVector(s12, s32);
                 str3 = Ascii.ExtractAsciiVector(s21, s41);
                 str4 = Ascii.ExtractAsciiVector(s22, s42);
+#endif
 
                 return true;
             }

src/libraries/System.Private.CoreLib/src/System/Numerics/Vector.cs

@@ -2457,6 +2457,24 @@ public static Vector<ushort> Narrow(Vector<uint> low, Vector<uint> high)
         public static Vector<uint> Narrow(Vector<ulong> low, Vector<ulong> high)
             => Narrow<ulong, uint>(low, high);
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static Vector<byte> NarrowNative(Vector<ushort> low, Vector<ushort> high)
+        {
+            if (Vector<byte>.Count == Vector512<byte>.Count)
+            {
+                return Vector512.NarrowNative(low.AsVector512(), high.AsVector512()).AsVector();
+            }
+            else if (Vector<byte>.Count == Vector256<byte>.Count)
+            {
+                return Vector256.NarrowNative(low.AsVector256(), high.AsVector256()).AsVector();
+            }
+            else
+            {
+                Debug.Assert(Vector<byte>.Count == Vector128<byte>.Count);
+                return Vector128.NarrowNative(low.AsVector128(), high.AsVector128()).AsVector();
+            }
+        }
+
         [Intrinsic]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static Vector<TResult> NarrowWithSaturation<TSource, TResult>(Vector<TSource> low, Vector<TSource> high)

src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs

@@ -2794,6 +2794,31 @@ public static Vector128<ushort> Narrow(Vector128<uint> lower, Vector128<uint> up
         public static Vector128<uint> Narrow(Vector128<ulong> lower, Vector128<ulong> upper)
             => Narrow<ulong, uint>(lower, upper);
 
+        /// <summary>Narrows two vector of <see cref="ushort" /> instances into one vector of <see cref="byte" /> using platform specific behavior.</summary>
+        /// <param name="lower">The vector that will be narrowed to the lower half of the result vector.</param>
+        /// <param name="upper">The vector that will be narrowed to the upper half of the result vector.</param>
+        /// <returns>A vector of <see cref="byte" /> containing elements narrowed from <paramref name="lower" /> and <paramref name="upper" />.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static Vector128<byte> NarrowNative(Vector128<ushort> lower, Vector128<ushort> upper)
+        {
+            if (Sse2.IsSupported)
+            {
+                return Sse2.PackUnsignedSaturate(lower.AsInt16(), upper.AsInt16());
+            }
+            else if (AdvSimd.Arm64.IsSupported)
+            {
+                return AdvSimd.Arm64.UnzipEven(lower.AsByte(), upper.AsByte());
+            }
+            else if (PackedSimd.IsSupported)
+            {
+                return PackedSimd.ConvertNarrowingSaturateUnsigned(lower.AsInt16(), upper.AsInt16());
+            }
+            else
+            {
+                return Narrow(lower, upper);
+            }
+        }
+
         [Intrinsic]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static Vector128<TResult> NarrowWithSaturation<TSource, TResult>(Vector128<TSource> lower, Vector128<TSource> upper)

src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs

@@ -2850,6 +2850,21 @@ public static Vector256<ushort> Narrow(Vector256<uint> lower, Vector256<uint> up
         public static Vector256<uint> Narrow(Vector256<ulong> lower, Vector256<ulong> upper)
             => Narrow<ulong, uint>(lower, upper);
 
+        /// <inheritdoc cref="Vector128.NarrowNative(Vector128{ushort}, Vector128{ushort})"/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static Vector256<byte> NarrowNative(Vector256<ushort> lower, Vector256<ushort> upper)
+        {
+            if (Avx2.IsSupported)
+            {
+                const byte control = 0b_11_01_10_00;
+                return Avx2.Permute4x64(Avx2.PackUnsignedSaturate(lower.AsInt16(), upper.AsInt16()).AsInt64(), control).AsByte();
+            }
+            else
+            {
+                return Narrow(lower, upper);
+            }
+        }
+
         [Intrinsic]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static Vector256<TResult> NarrowWithSaturation<TSource, TResult>(Vector256<TSource> lower, Vector256<TSource> upper)

src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs

@@ -5,6 +5,7 @@
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
 
 namespace System.Runtime.Intrinsics
 {
@@ -2875,6 +2876,21 @@ public static Vector512<ushort> Narrow(Vector512<uint> lower, Vector512<uint> up
         public static Vector512<uint> Narrow(Vector512<ulong> lower, Vector512<ulong> upper)
             => Narrow<ulong, uint>(lower, upper);
 
+        /// <inheritdoc cref="Vector128.NarrowNative(Vector128{ushort}, Vector128{ushort})"/>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static Vector512<byte> NarrowNative(Vector512<ushort> lower, Vector512<ushort> upper)
+        {
+            if (Avx512BW.IsSupported)
+            {
+                Vector512<long> control = Create(0, 2, 4, 6, 1, 3, 5, 7);
+                return Avx512F.PermuteVar8x64(Avx512BW.PackUnsignedSaturate(lower.AsInt16(), upper.AsInt16()).AsInt64(), control).AsByte();
+            }
+            else
+            {
+                return Narrow(lower, upper);
+            }
+        }
+
         [Intrinsic]
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static Vector512<TResult> NarrowWithSaturation<TSource, TResult>(Vector512<TSource> lower, Vector512<TSource> upper)

src/libraries/System.Private.CoreLib/src/System/Text/Ascii.CaseConversion.cs

@@ -352,8 +352,10 @@ private static unsafe nuint ChangeCase<TFrom, TTo, TCasing>(TFrom* pSrc, TTo* pD
                     }
                     else
                     {
+                        // Narrowing operation required, platform-dependent narrowing is safe as data is all-ASCII.
+
                         Vector128<ushort> blockAsVectorOfUInt16 = blockAsVectorOfUInt64.AsUInt16();
-                        Vector128<uint> narrowedBlock = Vector128.Narrow(blockAsVectorOfUInt16, blockAsVectorOfUInt16).AsUInt32();
+                        Vector128<uint> narrowedBlock = Vector128.NarrowNative(blockAsVectorOfUInt16, blockAsVectorOfUInt16).AsUInt32();
                         Unsafe.WriteUnaligned(&pDest[i], narrowedBlock.ToScalar());
                     }
                 }
@@ -412,8 +414,10 @@ private static unsafe nuint ChangeCase<TFrom, TTo, TCasing>(TFrom* pSrc, TTo* pD
                     }
                     else
                     {
+                        // Narrowing operation required, platform-dependent narrowing is safe as data is all-ASCII.
+
                         Vector128<ushort> blockAsVectorOfUInt16 = blockAsVectorOfUInt32.AsUInt16();
-                        Vector128<ushort> narrowedBlock = Vector128.Narrow(blockAsVectorOfUInt16, blockAsVectorOfUInt16).AsUInt16();
+                        Vector128<ushort> narrowedBlock = Vector128.NarrowNative(blockAsVectorOfUInt16, blockAsVectorOfUInt16).AsUInt16();
                         Unsafe.WriteUnaligned(&pDest[i], narrowedBlock.ToScalar());
                     }
                 }
@@ -489,8 +493,9 @@ private static unsafe void ChangeWidthAndWriteTo<TFrom, TTo>(Vector128<TFrom> ve
             }
             else if (sizeof(TFrom) == 2 && sizeof(TTo) == 1)
             {
-                // narrowing operation required, we know data is all-ASCII so use extract helper
-                Vector128<byte> narrow = ExtractAsciiVector(vector.AsUInt16(), vector.AsUInt16());
+                // Narrowing operation required, platform-dependent narrowing is safe as data is all-ASCII.
+
+                Vector128<byte> narrow = Vector128.NarrowNative(vector.AsUInt16(), vector.AsUInt16());
                 narrow.StoreLowerUnsafe(ref *(byte*)pDest, elementOffset);
             }
             else

src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs

@@ -1662,6 +1662,7 @@ private static bool AllCharsInVectorAreAscii<T>(Vector512<T> vector)
             }
         }
 
+#if !NET10_0_OR_GREATER
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static Vector128<byte> ExtractAsciiVector(Vector128<ushort> vectorFirst, Vector128<ushort> vectorSecond)
         {
@@ -1702,6 +1703,7 @@ internal static Vector512<byte> ExtractAsciiVector(Vector512<ushort> vectorFirst
                 ? PackedSpanHelpers.FixUpPackedVector512Result(Avx512BW.PackUnsignedSaturate(vectorFirst.AsInt16(), vectorSecond.AsInt16()))
                 : Vector512.Narrow(vectorFirst, vectorSecond);
         }
+#endif
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer, byte* pAsciiBuffer, nuint elementCount)
@@ -1735,7 +1737,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer,
             // Turn the 8 ASCII chars we just read into 8 ASCII bytes, then copy it to the destination.
 
             ref byte asciiBuffer = ref *pAsciiBuffer;
+#if NET10_0_OR_GREATER
+            Vector128<byte> asciiVector = Vector128.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             Vector128<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.StoreLowerUnsafe(ref asciiBuffer, 0);
             nuint currentOffsetInElements = SizeOfVector128 / 2; // we processed 8 elements so far
 
@@ -1762,7 +1768,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer,
                 }
 
                 // Turn the 8 ASCII chars we just read into 8 ASCII bytes, then copy it to the destination.
+#if NET10_0_OR_GREATER
+                asciiVector = Vector128.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
                 asciiVector.StoreLowerUnsafe(ref asciiBuffer, currentOffsetInElements);
             }
 
@@ -1792,7 +1802,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer,
                 // Build up the ASCII vector and perform the store.
 
                 Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % SizeOfVector128 == 0, "Write should be aligned.");
+#if NET10_0_OR_GREATER
+                asciiVector = Vector128.NarrowNative(utf16VectorFirst, utf16VectorSecond);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
+#endif
                 asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 
                 currentOffsetInElements += SizeOfVector128;
@@ -1815,7 +1829,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified(char* pUtf16Buffer,
             // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector.
 
             Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % sizeof(ulong) == 0, "Destination should be ulong-aligned.");
+#if NET10_0_OR_GREATER
+            asciiVector = Vector128.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.StoreLowerUnsafe(ref asciiBuffer, currentOffsetInElements);
             currentOffsetInElements += SizeOfVector128 / 2;
 
@@ -1853,7 +1871,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_256(char* pUtf16Buff
             // Turn the 16 ASCII chars we just read into 16 ASCII bytes, then copy it to the destination.
 
             ref byte asciiBuffer = ref *pAsciiBuffer;
+#if NET10_0_OR_GREATER
+            Vector256<byte> asciiVector = Vector256.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             Vector256<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, 0);
             nuint currentOffsetInElements = Vector256.Size / 2; // we processed 16 elements so far
 
@@ -1879,7 +1901,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_256(char* pUtf16Buff
                 }
 
                 // Turn the 16 ASCII chars we just read into 16 ASCII bytes, then copy it to the destination.
+#if NET10_0_OR_GREATER
+                asciiVector = Vector256.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
                 asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
             }
 
@@ -1909,7 +1935,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_256(char* pUtf16Buff
                 // Build up the ASCII vector and perform the store.
 
                 Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector256.Size == 0, "Write should be aligned.");
+#if NET10_0_OR_GREATER
+                asciiVector = Vector256.NarrowNative(utf16VectorFirst, utf16VectorSecond);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
+#endif
                 asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 
                 currentOffsetInElements += Vector256.Size;
@@ -1932,7 +1962,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_256(char* pUtf16Buff
             // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector.
 
             Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector128.Size == 0, "Destination should be 128-bit-aligned.");
+#if NET10_0_OR_GREATER
+            asciiVector = Vector256.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
             currentOffsetInElements += Vector256.Size / 2;
 
@@ -1970,7 +2004,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_512(char* pUtf16Buff
             // Turn the 32 ASCII chars we just read into 32 ASCII bytes, then copy it to the destination.
 
             ref byte asciiBuffer = ref *pAsciiBuffer;
+#if NET10_0_OR_GREATER
+            Vector512<byte> asciiVector = Vector512.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             Vector512<byte> asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, 0); // how to store the lower part of a avx512
             nuint currentOffsetInElements = Vector512.Size / 2; // we processed 32 elements so far
 
@@ -1997,7 +2035,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_512(char* pUtf16Buff
                 }
 
                 // Turn the 32 ASCII chars we just read into 32 ASCII bytes, then copy it to the destination.
+#if NET10_0_OR_GREATER
+                asciiVector = Vector512.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
                 asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
             }
 
@@ -2027,7 +2069,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_512(char* pUtf16Buff
                 // Build up the ASCII vector and perform the store.
 
                 Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector512.Size == 0, "Write should be aligned.");
+#if NET10_0_OR_GREATER
+                asciiVector = Vector512.NarrowNative(utf16VectorFirst, utf16VectorSecond);
+#else
                 asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorSecond);
+#endif
                 asciiVector.StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
 
                 currentOffsetInElements += Vector512.Size;
@@ -2050,7 +2096,11 @@ private static unsafe nuint NarrowUtf16ToAscii_Intrinsified_512(char* pUtf16Buff
             // First part was all ASCII, narrow and aligned write. Note we're only filling in the low half of the vector.
 
             Debug.Assert(((nuint)pAsciiBuffer + currentOffsetInElements) % Vector256.Size == 0, "Destination should be 256-bit-aligned.");
+#if NET10_0_OR_GREATER
+            asciiVector = Vector512.NarrowNative(utf16VectorFirst, utf16VectorFirst);
+#else
             asciiVector = ExtractAsciiVector(utf16VectorFirst, utf16VectorFirst);
+#endif
             asciiVector.GetLower().StoreUnsafe(ref asciiBuffer, currentOffsetInElements);
             currentOffsetInElements += Vector512.Size / 2;