diff --git a/src/System.Memory/ref/System.Memory.cs b/src/System.Memory/ref/System.Memory.cs index fe9a1bd86b2a..2e6ab083f590 100644 --- a/src/System.Memory/ref/System.Memory.cs +++ b/src/System.Memory/ref/System.Memory.cs @@ -200,6 +200,14 @@ public abstract class OwnedMemory : IDisposable, IRetainable public abstract void Retain(); protected internal abstract bool TryGetArray(out ArraySegment arraySegment); } + + public enum OperationStatus + { + Done, + DestinationTooSmall, + NeedMoreData, + InvalidData, + } } namespace System.Buffers.Binary @@ -277,4 +285,17 @@ public static class BinaryPrimitives public static bool TryWriteUInt32BigEndian(Span buffer, uint value) { throw null; } public static bool TryWriteUInt64BigEndian(Span buffer, ulong value) { throw null; } } +} + +namespace System.Buffers.Text +{ + public static class Base64 + { + public static OperationStatus EncodeToUtf8(ReadOnlySpan bytes, Span utf8, out int consumed, out int written, bool isFinalBlock = true) { throw null; } + public static OperationStatus EncodeToUtf8InPlace(Span buffer, int dataLength, out int written) { throw null; } + public static int GetMaxEncodedToUtf8Length(int length) { throw null; } + public static OperationStatus DecodeFromUtf8(ReadOnlySpan utf8, Span bytes, out int consumed, out int written, bool isFinalBlock = true) { throw null; } + public static OperationStatus DecodeFromUtf8InPlace(Span buffer, out int written) { throw null; } + public static int GetMaxDecodedFromUtf8Length(int length) { throw null; } + } } \ No newline at end of file diff --git a/src/System.Memory/src/System.Memory.csproj b/src/System.Memory/src/System.Memory.csproj index f0e48299b7f2..f5336070eb18 100644 --- a/src/System.Memory/src/System.Memory.csproj +++ b/src/System.Memory/src/System.Memory.csproj @@ -25,12 +25,16 @@ + + + + @@ -39,7 +43,6 @@ - diff --git a/src/System.Memory/src/System/Buffers/Binary/Reader.cs b/src/System.Memory/src/System/Buffers/Binary/Reader.cs index 9c7982aad95c..f669f7402fc2 100644 --- a/src/System.Memory/src/System/Buffers/Binary/Reader.cs +++ b/src/System.Memory/src/System/Buffers/Binary/Reader.cs @@ -106,12 +106,12 @@ public static T ReadMachineEndian(ReadOnlySpan buffer) #else if (RuntimeHelpers.IsReferenceOrContainsReferences()) { - throw new ArgumentException(SR.Format(SR.Argument_InvalidTypeWithPointersNotSupported, typeof(T))); + ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T)); } #endif if (Unsafe.SizeOf() > buffer.Length) { - throw new ArgumentOutOfRangeException(); + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length); } return Unsafe.ReadUnaligned(ref buffer.DangerousGetPinnableReference()); } @@ -132,7 +132,7 @@ public static bool TryReadMachineEndian(ReadOnlySpan buffer, out T valu #else if (RuntimeHelpers.IsReferenceOrContainsReferences()) { - throw new ArgumentException(SR.Format(SR.Argument_InvalidTypeWithPointersNotSupported, typeof(T))); + ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T)); } #endif if (Unsafe.SizeOf() > (uint)buffer.Length) diff --git a/src/System.Memory/src/System/Buffers/Binary/Writer.cs b/src/System.Memory/src/System/Buffers/Binary/Writer.cs index 9834986f4a69..2f0ea14caf41 100644 --- a/src/System.Memory/src/System/Buffers/Binary/Writer.cs +++ b/src/System.Memory/src/System/Buffers/Binary/Writer.cs @@ -24,12 +24,12 @@ public static void WriteMachineEndian(Span buffer, ref T value) #else if (RuntimeHelpers.IsReferenceOrContainsReferences()) { - throw new ArgumentException(SR.Format(SR.Argument_InvalidTypeWithPointersNotSupported, typeof(T))); + ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T)); } #endif if ((uint)Unsafe.SizeOf() > (uint)buffer.Length) { - throw new ArgumentOutOfRangeException(); + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length); } Unsafe.WriteUnaligned(ref buffer.DangerousGetPinnableReference(), value); } @@ -50,7 +50,7 @@ public static bool TryWriteMachineEndian(Span buffer, ref T value) #else if (RuntimeHelpers.IsReferenceOrContainsReferences()) { - throw new ArgumentException(SR.Format(SR.Argument_InvalidTypeWithPointersNotSupported, typeof(T))); + ThrowHelper.ThrowArgumentException_InvalidTypeWithPointersNotSupported(typeof(T)); } #endif if (Unsafe.SizeOf() > (uint)buffer.Length) diff --git a/src/System.Memory/src/System/Buffers/OperationStatus.cs b/src/System.Memory/src/System/Buffers/OperationStatus.cs new file mode 100644 index 000000000000..e9ddcab57148 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/OperationStatus.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace System.Buffers +{ + /// + /// This enum defines the various potential status that can be returned from Span-based operations + /// that support processing of input contained in multiple discontiguous buffers. + /// + public enum OperationStatus + { + /// + /// The entire input buffer has been processed and the operation is complete. + /// + Done, + /// + /// The input is partially processed, up to what could fit into the destination buffer. + /// The caller can enlarge the destination buffer, slice the buffers appropriately, and retry. + /// + DestinationTooSmall, + /// + /// The input is partially processed, up to the last valid chunk of the input that could be consumed. + /// The caller can stitch the remaining unprocessed input with more data, slice the buffers appropriately, and retry. + /// + NeedMoreData, + /// + /// The input contained invalid bytes which could not be processed. If the input is partially processed, + /// the destination contains the partial result. This guarantees that no additional data appended to the input + /// will make the invalid sequence valid. + /// + InvalidData, + } +} diff --git a/src/System.Memory/src/System/Buffers/Text/Base64Decoder.cs b/src/System.Memory/src/System/Buffers/Text/Base64Decoder.cs new file mode 100644 index 000000000000..c04ce0b4abe3 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/Text/Base64Decoder.cs @@ -0,0 +1,319 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers.Text +{ + public static partial class Base64 + { + /// + /// Decode the span of UTF-8 encoded text represented as base 64 into binary data. + /// If the input is not a multiple of 4, it will decode as much as it can, to the closest multiple of 4. + /// + /// The input span which contains UTF-8 encoded text in base 64 that needs to be decoded. + /// The output span which contains the result of the operation, i.e. the decoded binary data. + /// The number of input bytes consumed during the operation. This can be used to slice the input for subsequent calls, if necessary. + /// The number of bytes written into the output span. This can be used to slice the output for subsequent calls, if necessary. + /// True (default) when the input span contains the entire data to decode. + /// Set to false only if it is known that the input span contains partial data with more data to follow. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire input span + /// - DestinationTooSmall - if there is not enough space in the output span to fit the decoded input + /// - NeedMoreData - only if isFinalBlock is false and the input is not a multiple of 4, otherwise the partial input would be considered as InvalidData + /// - InvalidData - if the input contains bytes outside of the expected base 64 range, or if it contains invalid/more than two padding characters, + /// or if the input is incomplete (i.e. not a multiple of 4) and isFinalBlock is true. + /// + public static OperationStatus DecodeFromUtf8(ReadOnlySpan utf8, Span bytes, out int consumed, out int written, bool isFinalBlock = true) + { + ref byte srcBytes = ref utf8.DangerousGetPinnableReference(); + ref byte destBytes = ref bytes.DangerousGetPinnableReference(); + + int srcLength = utf8.Length & ~0x3; // only decode input up to the closest multiple of 4. + int destLength = bytes.Length; + + int sourceIndex = 0; + int destIndex = 0; + + if (utf8.Length == 0) goto DoneExit; + + ref sbyte decodingMap = ref s_decodingMap[0]; + + // Last bytes could have padding characters, so process them separately and treat them as valid only if isFinalBlock is true + // if isFinalBlock is false, padding characters are considered invalid + int skipLastChunk = isFinalBlock ? 4 : 0; + + int maxSrcLength = 0; + if (destLength >= GetMaxDecodedFromUtf8Length(srcLength)) + { + maxSrcLength = srcLength - skipLastChunk; + } + else + { + // This should never overflow since destLength here is less than int.MaxValue / 4 * 3 (i.e. 1610612733) + // Therefore, (destLength / 3) * 4 will always be less than 2147483641 + maxSrcLength = (destLength / 3) * 4; + } + + while (sourceIndex < maxSrcLength) + { + int result = Decode(ref Unsafe.Add(ref srcBytes, sourceIndex), ref decodingMap); + if (result < 0) goto InvalidExit; + WriteThreeLowOrderBytes(ref Unsafe.Add(ref destBytes, destIndex), result); + destIndex += 3; + sourceIndex += 4; + } + + if (maxSrcLength != srcLength - skipLastChunk) goto DestinationSmallExit; + + // If input is less than 4 bytes, srcLength == sourceIndex == 0 + // If input is not a multiple of 4, sourceIndex == srcLength != 0 + if (sourceIndex == srcLength) + { + if (isFinalBlock) goto InvalidExit; + goto NeedMoreExit; + } + + // if isFinalBlock is false, we will never reach this point + + int i0 = Unsafe.Add(ref srcBytes, srcLength - 4); + int i1 = Unsafe.Add(ref srcBytes, srcLength - 3); + int i2 = Unsafe.Add(ref srcBytes, srcLength - 2); + int i3 = Unsafe.Add(ref srcBytes, srcLength - 1); + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + + i0 <<= 18; + i1 <<= 12; + + i0 |= i1; + + if (i3 != EncodingPad) + { + i2 = Unsafe.Add(ref decodingMap, i2); + i3 = Unsafe.Add(ref decodingMap, i3); + + i2 <<= 6; + + i0 |= i3; + i0 |= i2; + + if (i0 < 0) goto InvalidExit; + if (destIndex > destLength - 3) goto DestinationSmallExit; + WriteThreeLowOrderBytes(ref Unsafe.Add(ref destBytes, destIndex), i0); + destIndex += 3; + } + else if (i2 != EncodingPad) + { + i2 = Unsafe.Add(ref decodingMap, i2); + + i2 <<= 6; + + i0 |= i2; + + if (i0 < 0) goto InvalidExit; + if (destIndex > destLength - 2) goto DestinationSmallExit; + Unsafe.Add(ref destBytes, destIndex) = (byte)(i0 >> 16); + Unsafe.Add(ref destBytes, destIndex + 1) = (byte)(i0 >> 8); + destIndex += 2; + } + else + { + if (i0 < 0) goto InvalidExit; + if (destIndex > destLength - 1) goto DestinationSmallExit; + Unsafe.Add(ref destBytes, destIndex) = (byte)(i0 >> 16); + destIndex += 1; + } + + sourceIndex += 4; + + if (srcLength != utf8.Length) goto InvalidExit; + + DoneExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.Done; + + DestinationSmallExit: + if (srcLength != utf8.Length && isFinalBlock) goto InvalidExit; // if input is not a multiple of 4, and there is no more data, return invalid data instead + consumed = sourceIndex; + written = destIndex; + return OperationStatus.DestinationTooSmall; + + NeedMoreExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.NeedMoreData; + + InvalidExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.InvalidData; + } + + /// + /// Returns the maximum length (in bytes) of the result if you were to deocde base 64 encoded text within a byte span of size "length". + /// + /// + /// Thrown when the specified is less than 0. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetMaxDecodedFromUtf8Length(int length) + { + if (length < 0) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length); + + return (length >> 2) * 3; + } + + /// + /// Decode the span of UTF-8 encoded text in base 64 (in-place) into binary data. + /// The decoded binary output is smaller than the text data contained in the input (the operation deflates the data). + /// If the input is not a multiple of 4, it will not decode any. + /// + /// The input span which contains the base 64 text data that needs to be decoded. + /// The number of bytes written into the buffer. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire input span + /// - InvalidData - if the input contains bytes outside of the expected base 64 range, or if it contains invalid/more than two padding characters, + /// or if the input is incomplete (i.e. not a multiple of 4). + /// It does not return DestinationTooSmall since that is not possible for base 64 decoding. + /// It does not return NeedMoreData since this method tramples the data in the buffer and + /// hence can only be called once with all the data in the buffer. + /// + public static OperationStatus DecodeFromUtf8InPlace(Span buffer, out int written) + { + int bufferLength = buffer.Length; + int sourceIndex = 0; + int destIndex = 0; + + // only decode input if it is a multiple of 4 + if (bufferLength != ((bufferLength >> 2) * 4)) goto InvalidExit; + if (bufferLength == 0) goto DoneExit; + + ref byte bufferBytes = ref buffer.DangerousGetPinnableReference(); + + ref sbyte decodingMap = ref s_decodingMap[0]; + + while (sourceIndex < bufferLength - 4) + { + int result = Decode(ref Unsafe.Add(ref bufferBytes, sourceIndex), ref decodingMap); + if (result < 0) goto InvalidExit; + WriteThreeLowOrderBytes(ref Unsafe.Add(ref bufferBytes, destIndex), result); + destIndex += 3; + sourceIndex += 4; + } + + int i0 = Unsafe.Add(ref bufferBytes, bufferLength - 4); + int i1 = Unsafe.Add(ref bufferBytes, bufferLength - 3); + int i2 = Unsafe.Add(ref bufferBytes, bufferLength - 2); + int i3 = Unsafe.Add(ref bufferBytes, bufferLength - 1); + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + + i0 <<= 18; + i1 <<= 12; + + i0 |= i1; + + if (i3 != EncodingPad) + { + i2 = Unsafe.Add(ref decodingMap, i2); + i3 = Unsafe.Add(ref decodingMap, i3); + + i2 <<= 6; + + i0 |= i3; + i0 |= i2; + + if (i0 < 0) goto InvalidExit; + WriteThreeLowOrderBytes(ref Unsafe.Add(ref bufferBytes, destIndex), i0); + destIndex += 3; + } + else if (i2 != EncodingPad) + { + i2 = Unsafe.Add(ref decodingMap, i2); + + i2 <<= 6; + + i0 |= i2; + + if (i0 < 0) goto InvalidExit; + Unsafe.Add(ref bufferBytes, destIndex) = (byte)(i0 >> 16); + Unsafe.Add(ref bufferBytes, destIndex + 1) = (byte)(i0 >> 8); + destIndex += 2; + } + else + { + if (i0 < 0) goto InvalidExit; + Unsafe.Add(ref bufferBytes, destIndex) = (byte)(i0 >> 16); + destIndex += 1; + } + + DoneExit: + written = destIndex; + return OperationStatus.Done; + + InvalidExit: + written = destIndex; + return OperationStatus.InvalidData; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Decode(ref byte encodedBytes, ref sbyte decodingMap) + { + int i0 = encodedBytes; + int i1 = Unsafe.Add(ref encodedBytes, 1); + int i2 = Unsafe.Add(ref encodedBytes, 2); + int i3 = Unsafe.Add(ref encodedBytes, 3); + + i0 = Unsafe.Add(ref decodingMap, i0); + i1 = Unsafe.Add(ref decodingMap, i1); + i2 = Unsafe.Add(ref decodingMap, i2); + i3 = Unsafe.Add(ref decodingMap, i3); + + i0 <<= 18; + i1 <<= 12; + i2 <<= 6; + + i0 |= i3; + i1 |= i2; + + i0 |= i1; + return i0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void WriteThreeLowOrderBytes(ref byte destination, int value) + { + destination = (byte)(value >> 16); + Unsafe.Add(ref destination, 1) = (byte)(value >> 8); + Unsafe.Add(ref destination, 2) = (byte)value; + } + + // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in tests) + private static readonly sbyte[] s_decodingMap = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, //62 is placed at index 43 (for +), 63 at index 47 (for /) + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, //52-61 are placed at index 48-57 (for 0-9), 64 at index 61 (for =) + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, //0-25 are placed at index 65-90 (for A-Z) + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, //26-51 are placed at index 97-122 (for a-z) + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Bytes over 122 ('z') are invalid and cannot be decoded + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Hence, padding the map with 255, which indicates invalid input + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + } +} diff --git a/src/System.Memory/src/System/Buffers/Text/Base64Encoder.cs b/src/System.Memory/src/System/Buffers/Text/Base64Encoder.cs new file mode 100644 index 000000000000..050719017db3 --- /dev/null +++ b/src/System.Memory/src/System/Buffers/Text/Base64Encoder.cs @@ -0,0 +1,227 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace System.Buffers.Text +{ + /// + /// Convert between binary data and UTF-8 encoded text that is represented in base 64. + /// + public static partial class Base64 + { + /// + /// Encode the span of binary data into UTF-8 encoded text represented as base 64. + /// + /// The input span which contains binary data that needs to be encoded. + /// The output span which contains the result of the operation, i.e. the UTF-8 encoded text in base 64. + /// The number of input bytes consumed during the operation. This can be used to slice the input for subsequent calls, if necessary. + /// The number of bytes written into the output span. This can be used to slice the output for subsequent calls, if necessary. + /// True (default) when the input span contains the entire data to decode. + /// Set to false only if it is known that the input span contains partial data with more data to follow. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire input span + /// - DestinationTooSmall - if there is not enough space in the output span to fit the encoded input + /// - NeedMoreData - only if isFinalBlock is false, otherwise the output is padded if the input is not a multiple of 3 + /// It does not return InvalidData since that is not possible for base 64 encoding. + /// + public static OperationStatus EncodeToUtf8(ReadOnlySpan bytes, Span utf8, out int consumed, out int written, bool isFinalBlock = true) + { + ref byte srcBytes = ref bytes.DangerousGetPinnableReference(); + ref byte destBytes = ref utf8.DangerousGetPinnableReference(); + + int srcLength = bytes.Length; + int destLength = utf8.Length; + + int maxSrcLength = 0; + if (srcLength <= MaximumEncodeLength && destLength >= GetMaxEncodedToUtf8Length(srcLength)) + { + maxSrcLength = srcLength - 2; + } + else + { + maxSrcLength = (destLength >> 2) * 3 - 2; + } + + int sourceIndex = 0; + int destIndex = 0; + int result = 0; + + ref byte encodingMap = ref s_encodingMap[0]; + + while (sourceIndex < maxSrcLength) + { + result = Encode(ref Unsafe.Add(ref srcBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destBytes, destIndex), result); + destIndex += 4; + sourceIndex += 3; + } + + if (maxSrcLength != srcLength - 2) goto DestinationSmallExit; + + if (isFinalBlock != true) goto NeedMoreDataExit; + + if (sourceIndex == srcLength - 1) + { + result = EncodeAndPadTwo(ref Unsafe.Add(ref srcBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destBytes, destIndex), result); + destIndex += 4; + sourceIndex += 1; + } + else if (sourceIndex == srcLength - 2) + { + result = EncodeAndPadOne(ref Unsafe.Add(ref srcBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref destBytes, destIndex), result); + destIndex += 4; + sourceIndex += 2; + } + + consumed = sourceIndex; + written = destIndex; + return OperationStatus.Done; + + NeedMoreDataExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.NeedMoreData; + + DestinationSmallExit: + consumed = sourceIndex; + written = destIndex; + return OperationStatus.DestinationTooSmall; + } + + /// + /// Returns the maximum length (in bytes) of the result if you were to encode binary data within a byte span of size "length". + /// + /// + /// Thrown when the specified is less than 0 or larger than 1610612733 (since encode inflates the data by 4/3). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetMaxEncodedToUtf8Length(int length) + { + if (length < 0 || length > MaximumEncodeLength) + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.length); + + return (((length + 2) / 3) * 4); + } + + /// + /// Encode the span of binary data (in-place) into UTF-8 encoded text represented as base 64. + /// The encoded text output is larger than the binary data contained in the input (the operation inflates the data). + /// + /// The input span which contains binary data that needs to be encoded. + /// It needs to be large enough to fit the result of the operation. + /// The amount of binary data contained within the buffer that needs to be encoded + /// (and needs to be smaller than the buffer length). + /// The number of bytes written into the buffer. + /// It returns the OperationStatus enum values: + /// - Done - on successful processing of the entire buffer + /// - DestinationTooSmall - if there is not enough space in the buffer beyond dataLength to fit the result of encoding the input + /// It does not return NeedMoreData since this method tramples the data in the buffer and hence can only be called once with all the data in the buffer. + /// It does not return InvalidData since that is not possible for base 64 encoding. + /// + public static OperationStatus EncodeToUtf8InPlace(Span buffer, int dataLength, out int written) + { + int encodedLength = GetMaxEncodedToUtf8Length(dataLength); + if (buffer.Length < encodedLength) goto FalseExit; + + int leftover = dataLength - dataLength / 3 * 3; // how many bytes after packs of 3 + + int destinationIndex = encodedLength - 4; + int sourceIndex = dataLength - leftover; + int result = 0; + + ref byte encodingMap = ref s_encodingMap[0]; + ref byte bufferBytes = ref buffer.DangerousGetPinnableReference(); + + // encode last pack to avoid conditional in the main loop + if (leftover != 0) + { + if (leftover == 1) + { + result = EncodeAndPadTwo(ref Unsafe.Add(ref bufferBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref bufferBytes, destinationIndex), result); + destinationIndex -= 4; + } + else + { + result = EncodeAndPadOne(ref Unsafe.Add(ref bufferBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref bufferBytes, destinationIndex), result); + destinationIndex -= 4; + } + } + + sourceIndex -= 3; + while (sourceIndex >= 0) + { + result = Encode(ref Unsafe.Add(ref bufferBytes, sourceIndex), ref encodingMap); + Unsafe.WriteUnaligned(ref Unsafe.Add(ref bufferBytes, destinationIndex), result); + destinationIndex -= 4; + sourceIndex -= 3; + } + + written = encodedLength; + return OperationStatus.Done; + + FalseExit: + written = 0; + return OperationStatus.DestinationTooSmall; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Encode(ref byte threeBytes, ref byte encodingMap) + { + int i = (threeBytes << 16) | (Unsafe.Add(ref threeBytes, 1) << 8) | Unsafe.Add(ref threeBytes, 2); + + int i0 = Unsafe.Add(ref encodingMap, i >> 18); + int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + int i3 = Unsafe.Add(ref encodingMap, i & 0x3F); + + return i0 | (i1 << 8) | (i2 << 16) | (i3 << 24); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int EncodeAndPadOne(ref byte twoBytes, ref byte encodingMap) + { + int i = (twoBytes << 16) | (Unsafe.Add(ref twoBytes, 1) << 8); + + int i0 = Unsafe.Add(ref encodingMap, i >> 18); + int i1 = Unsafe.Add(ref encodingMap, (i >> 12) & 0x3F); + int i2 = Unsafe.Add(ref encodingMap, (i >> 6) & 0x3F); + + return i0 | (i1 << 8) | (i2 << 16) | (EncodingPad << 24); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int EncodeAndPadTwo(ref byte oneByte, ref byte encodingMap) + { + int i = (oneByte << 8); + + int i0 = Unsafe.Add(ref encodingMap, i >> 10); + int i1 = Unsafe.Add(ref encodingMap, (i >> 4) & 0x3F); + + return i0 | (i1 << 8) | (EncodingPad << 16) | (EncodingPad << 24); + } + + // Pre-computing this table using a custom string(s_characters) and GenerateEncodingMapAndVerify (found in tests) + private static readonly byte[] s_encodingMap = { + 65, 66, 67, 68, 69, 70, 71, 72, //A..H + 73, 74, 75, 76, 77, 78, 79, 80, //I..P + 81, 82, 83, 84, 85, 86, 87, 88, //Q..X + 89, 90, 97, 98, 99, 100, 101, 102, //Y..Z, a..f + 103, 104, 105, 106, 107, 108, 109, 110, //g..n + 111, 112, 113, 114, 115, 116, 117, 118, //o..v + 119, 120, 121, 122, 48, 49, 50, 51, //w..z, 0..3 + 52, 53, 54, 55, 56, 57, 43, 47 //4..9, +, / + }; + + private const byte EncodingPad = (byte)'='; // '=', for padding + + private const int MaximumEncodeLength = (int.MaxValue >> 2) * 3; // 1610612733 + } +} diff --git a/src/System.Memory/tests/Base64/Base64DecoderUnitTests.cs b/src/System.Memory/tests/Base64/Base64DecoderUnitTests.cs new file mode 100644 index 000000000000..2b0232f713f1 --- /dev/null +++ b/src/System.Memory/tests/Base64/Base64DecoderUnitTests.cs @@ -0,0 +1,540 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; +using Xunit; + +namespace System.Buffers.Text.Tests +{ + public class Base64DecoderUnitTests + { + [Fact] + public void BasicDecoding() + { + var rnd = new Random(42); + for (int i = 0; i < 10; i++) + { + int numBytes = rnd.Next(100, 1000 * 1000); + while (numBytes % 4 != 0) + { + numBytes = rnd.Next(100, 1000 * 1000); + } + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeDecodableBytes(source, numBytes); + + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + Assert.Equal(OperationStatus.Done, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + Assert.Equal(source.Length, consumed); + Assert.Equal(decodedBytes.Length, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(source.Length, decodedBytes.Length, source, decodedBytes)); + } + } + + [Fact] + public void DecodeEmptySpan() + { + Span source = Span.Empty; + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.Done, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + Assert.Equal(source.Length, consumed); + Assert.Equal(decodedBytes.Length, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(source.Length, decodedBytes.Length, source, decodedBytes)); + } + + [Fact] + public void BasicDecodingWithFinalBlockFalse() + { + var rnd = new Random(42); + for (int i = 0; i < 10; i++) + { + int numBytes = rnd.Next(100, 1000 * 1000); + while (numBytes % 4 != 0) + { + numBytes = rnd.Next(100, 1000 * 1000); + } + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeDecodableBytes(source, numBytes); + + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + int expectedConsumed = source.Length / 4 * 4; // only consume closest multiple of four since isFinalBlock is false + + Assert.Equal(OperationStatus.NeedMoreData, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount, isFinalBlock: false)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(decodedBytes.Length, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + } + + [Theory] + [InlineData("A", 0, 0)] + [InlineData("AQ", 0, 0)] + [InlineData("AQI", 0, 0)] + [InlineData("AQIDBA", 4, 3)] + [InlineData("AQIDBAU", 4, 3)] + [InlineData("AQID", 4, 3)] + [InlineData("AQIDBAUG", 8, 6)] + public void BasicDecodingWithFinalBlockFalseKnownInputNeedMoreData(string inputString, int expectedConsumed, int expectedWritten) + { + Span source = Encoding.ASCII.GetBytes(inputString); + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.NeedMoreData, Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount, isFinalBlock: false)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, decodedByteCount); // expectedWritten == decodedBytes.Length + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + + [Theory] + [InlineData("AQ==", 0, 0)] + [InlineData("AQI=", 0, 0)] + [InlineData("AQIDBA==", 4, 3)] + [InlineData("AQIDBAU=", 4, 3)] + public void BasicDecodingWithFinalBlockFalseKnownInputInvalid(string inputString, int expectedConsumed, int expectedWritten) + { + Span source = Encoding.ASCII.GetBytes(inputString); + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount, isFinalBlock: false)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes)); + } + + [Theory] + [InlineData("A", 0, 0)] + [InlineData("AQ", 0, 0)] + [InlineData("AQI", 0, 0)] + [InlineData("AQIDBA", 4, 3)] + [InlineData("AQIDBAU", 4, 3)] + public void BasicDecodingWithFinalBlockTrueKnownInputInvalid(string inputString, int expectedConsumed, int expectedWritten) + { + Span source = Encoding.ASCII.GetBytes(inputString); + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, decodedByteCount); // expectedWritten == decodedBytes.Length + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + + [Theory] + [InlineData("AQ==", 4, 1)] + [InlineData("AQI=", 4, 2)] + [InlineData("AQID", 4, 3)] + [InlineData("AQIDBA==", 8, 4)] + [InlineData("AQIDBAU=", 8, 5)] + [InlineData("AQIDBAUG", 8, 6)] + public void BasicDecodingWithFinalBlockTrueKnownInputDone(string inputString, int expectedConsumed, int expectedWritten) + { + Span source = Encoding.ASCII.GetBytes(inputString); + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes)); + } + + [Fact] + public void DecodingInvalidBytes() + { + // Invalid Bytes: + // 0-42 + // 44-46 + // 58-64 + // 91-96 + // 123-255 + byte[] invalidBytes = Base64TestHelper.InvalidBytes; + Assert.Equal(byte.MaxValue + 1 - 64, invalidBytes.Length); // 192 + + for (int j = 0; j < 8; j++) + { + Span source = new byte[8] { 50, 50, 50, 50, 80, 80, 80, 80 }; // valid input - "2222PPPP" + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + + for (int i = 0; i < invalidBytes.Length; i++) + { + // Don't test padding (byte 61 i.e. '='), which is tested in DecodingInvalidBytesPadding + if (invalidBytes[i] == Base64TestHelper.s_encodingPad) continue; + + // replace one byte with an invalid input + source[j] = invalidBytes[i]; + + Assert.Equal(OperationStatus.InvalidData, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + + if (j < 4) + { + Assert.Equal(0, consumed); + Assert.Equal(0, decodedByteCount); + } + else + { + Assert.Equal(4, consumed); + Assert.Equal(3, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(4, 3, source, decodedBytes)); + } + } + } + + // Input that is not a multiple of 4 is considered invalid + { + Span source = new byte[7] { 50, 50, 50, 50, 80, 80, 80 }; // incomplete input - "2222PPP" + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + Assert.Equal(OperationStatus.InvalidData, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + Assert.Equal(4, consumed); + Assert.Equal(3, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(4, 3, source, decodedBytes)); + } + } + + [Fact] + public void DecodingInvalidBytesPadding() + { + // Only last 2 bytes can be padding, all other occurrence of padding is invalid + for (int j = 0; j < 7; j++) + { + Span source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; // valid input - "2222PPPP" + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + source[j] = Base64TestHelper.s_encodingPad; + Assert.Equal(OperationStatus.InvalidData, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + + if (j < 4) + { + Assert.Equal(0, consumed); + Assert.Equal(0, decodedByteCount); + } + else + { + Assert.Equal(4, consumed); + Assert.Equal(3, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(4, 3, source, decodedBytes)); + } + } + + // Invalid input with valid padding + { + Span source = new byte[] { 50, 50, 50, 50, 80, 42, 42, 42 }; + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + source[6] = Base64TestHelper.s_encodingPad; + source[7] = Base64TestHelper.s_encodingPad; // invalid input - "2222P*==" + Assert.Equal(OperationStatus.InvalidData, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + + Assert.Equal(4, consumed); + Assert.Equal(3, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(4, 3, source, decodedBytes)); + + source = new byte[] { 50, 50, 50, 50, 80, 42, 42, 42 }; + decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + source[7] = Base64TestHelper.s_encodingPad; // invalid input - "2222PP**=" + Assert.Equal(OperationStatus.InvalidData, + Base64.DecodeFromUtf8(source, decodedBytes, out consumed, out decodedByteCount)); + + Assert.Equal(4, consumed); + Assert.Equal(3, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(4, 3, source, decodedBytes)); + } + + // The last byte or the last 2 bytes being the padding character is valid + { + Span source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + source[6] = Base64TestHelper.s_encodingPad; + source[7] = Base64TestHelper.s_encodingPad; // valid input - "2222PP==" + Assert.Equal(OperationStatus.Done, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + + Assert.Equal(source.Length, consumed); + Assert.Equal(4, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(source.Length, 4, source, decodedBytes)); + + source = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; + decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(source.Length)]; + source[7] = Base64TestHelper.s_encodingPad; // valid input - "2222PPP=" + Assert.Equal(OperationStatus.Done, + Base64.DecodeFromUtf8(source, decodedBytes, out consumed, out decodedByteCount)); + + Assert.Equal(source.Length, consumed); + Assert.Equal(5, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(source.Length, 5, source, decodedBytes)); + } + } + + [Fact] + public void DecodingOutputTooSmall() + { + for (int numBytes = 5; numBytes < 20; numBytes++) + { + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeDecodableBytes(source, numBytes); + + Span decodedBytes = new byte[3]; + int consumed, written; + if (numBytes % 4 != 0) + { + Assert.True(OperationStatus.InvalidData == + Base64.DecodeFromUtf8(source, decodedBytes, out consumed, out written), "Number of Input Bytes: " + numBytes); + } + else + { + Assert.True(OperationStatus.DestinationTooSmall == + Base64.DecodeFromUtf8(source, decodedBytes, out consumed, out written), "Number of Input Bytes: " + numBytes); + } + int expectedConsumed = 4; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(decodedBytes.Length, written); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + + // Output too small even with padding characters in the input + { + Span source = new byte[12]; + Base64TestHelper.InitalizeDecodableBytes(source); + source[10] = Base64TestHelper.s_encodingPad; + source[11] = Base64TestHelper.s_encodingPad; + + Span decodedBytes = new byte[6]; + Assert.Equal(OperationStatus.DestinationTooSmall, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int written)); + int expectedConsumed = 8; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(decodedBytes.Length, written); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + + { + Span source = new byte[12]; + Base64TestHelper.InitalizeDecodableBytes(source); + source[11] = Base64TestHelper.s_encodingPad; + + Span decodedBytes = new byte[7]; + Assert.Equal(OperationStatus.DestinationTooSmall, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int written)); + int expectedConsumed = 8; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(6, written); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, 6, source, decodedBytes)); + } + } + + [Fact] + public void DecodingOutputTooSmallRetry() + { + Span source = new byte[1000]; + Base64TestHelper.InitalizeDecodableBytes(source); + + int outputSize = 240; + int requiredSize = Base64.GetMaxDecodedFromUtf8Length(source.Length); + + Span decodedBytes = new byte[outputSize]; + Assert.Equal(OperationStatus.DestinationTooSmall, + Base64.DecodeFromUtf8(source, decodedBytes, out int consumed, out int decodedByteCount)); + int expectedConsumed = decodedBytes.Length / 3 * 4; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(decodedBytes.Length, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + + decodedBytes = new byte[requiredSize - outputSize]; + source = source.Slice(consumed); + Assert.Equal(OperationStatus.Done, + Base64.DecodeFromUtf8(source, decodedBytes, out consumed, out decodedByteCount)); + expectedConsumed = decodedBytes.Length / 3 * 4; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(decodedBytes.Length, decodedByteCount); + Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, decodedBytes.Length, source, decodedBytes)); + } + + [Fact] + public void GetMaxDecodedLength() + { + Span sourceEmpty = Span.Empty; + Assert.Equal(0, Base64.GetMaxDecodedFromUtf8Length(0)); + + // int.MaxValue - (int.MaxValue % 4) => 2147483644, largest multiple of 4 less than int.MaxValue + int[] input = { 0, 4, 8, 12, 16, 20, 2000000000, 2147483640, 2147483644 }; + int[] expected = { 0, 3, 6, 9, 12, 15, 1500000000, 1610612730, 1610612733 }; + + for (int i = 0; i < input.Length; i++) + { + Assert.Equal(expected[i], Base64.GetMaxDecodedFromUtf8Length(input[i])); + } + + // Lengths that are not a multiple of 4. + int[] lengthsNotMultipleOfFour = { 1, 2, 3, 5, 6, 7, 9, 10, 11, 13, 14, 15, 1001, 1002, 1003, 2147483645, 2147483646, 2147483647 }; + int[] expectedOutput = { 0, 0, 0, 3, 3, 3, 6, 6, 6, 9, 9, 9, 750, 750, 750, 1610612733, 1610612733, 1610612733 }; + for (int i = 0; i < lengthsNotMultipleOfFour.Length; i++) + { + Assert.Equal(expectedOutput[i], Base64.GetMaxDecodedFromUtf8Length(lengthsNotMultipleOfFour[i])); + } + + // negative input + Assert.Throws(() => Base64.GetMaxDecodedFromUtf8Length(-1)); + Assert.Throws(() => Base64.GetMaxDecodedFromUtf8Length(int.MinValue)); + } + + + [Fact] + public void DecodeInPlace() + { + const int numberOfBytes = 15; + + for (int numberOfBytesToTest = 0; numberOfBytesToTest <= numberOfBytes; numberOfBytesToTest += 4) + { + Span testBytes = new byte[numberOfBytes]; + Base64TestHelper.InitalizeDecodableBytes(testBytes); + string sourceString = Encoding.ASCII.GetString(testBytes.Slice(0, numberOfBytesToTest).ToArray()); + Span expectedBytes = Convert.FromBase64String(sourceString); + + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(testBytes.Slice(0, numberOfBytesToTest), out int bytesWritten)); + Assert.Equal(Base64.GetMaxDecodedFromUtf8Length(numberOfBytesToTest), bytesWritten); + Assert.True(expectedBytes.SequenceEqual(testBytes.Slice(0, bytesWritten))); + } + } + + [Fact] + public void EncodeAndDecodeInPlace() + { + byte[] testBytes = new byte[256]; + for (int i = 0; i < 256; i++) + { + testBytes[i] = (byte)i; + } + + for (int value = 0; value < 256; value++) + { + Span sourceBytes = testBytes.AsSpan().Slice(0, value + 1); + Span buffer = new byte[Base64.GetMaxEncodedToUtf8Length(sourceBytes.Length)]; + + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8(sourceBytes, buffer, out int consumed, out int written)); + + var encodedText = Encoding.ASCII.GetString(buffer.ToArray()); + var expectedText = Convert.ToBase64String(testBytes, 0, value + 1); + Assert.Equal(expectedText, encodedText); + + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(sourceBytes.Length, bytesWritten); + Assert.True(sourceBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + } + + [Fact] + public void DecodeInPlaceInvalidBytes() + { + byte[] invalidBytes = Base64TestHelper.InvalidBytes; + + for (int j = 0; j < 8; j++) + { + for (int i = 0; i < invalidBytes.Length; i++) + { + Span buffer = new byte[8] { 50, 50, 50, 50, 80, 80, 80, 80 }; // valid input - "2222PPPP" + + // Don't test padding (byte 61 i.e. '='), which is tested in DecodeInPlaceInvalidBytesPadding + if (invalidBytes[i] == Base64TestHelper.s_encodingPad) continue; + + // replace one byte with an invalid input + buffer[j] = invalidBytes[i]; + string sourceString = Encoding.ASCII.GetString(buffer.Slice(0, 4).ToArray()); + + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + + if (j < 4) + { + Assert.Equal(0, bytesWritten); + } + else + { + Assert.Equal(3, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + } + } + + // Input that is not a multiple of 4 is considered invalid + { + Span buffer = new byte[7] { 50, 50, 50, 50, 80, 80, 80 }; // incomplete input - "2222PPP" + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(0, bytesWritten); + } + } + + [Fact] + public void DecodeInPlaceInvalidBytesPadding() + { + // Only last 2 bytes can be padding, all other occurrence of padding is invalid + for (int j = 0; j < 7; j++) + { + Span buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; // valid input - "2222PPPP" + buffer[j] = Base64TestHelper.s_encodingPad; + string sourceString = Encoding.ASCII.GetString(buffer.Slice(0, 4).ToArray()); + + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + + if (j < 4) + { + Assert.Equal(0, bytesWritten); + } + else + { + Assert.Equal(3, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + } + + // Invalid input with valid padding + { + Span buffer = new byte[] { 50, 50, 50, 50, 80, 42, 42, 42 }; + buffer[6] = Base64TestHelper.s_encodingPad; + buffer[7] = Base64TestHelper.s_encodingPad; // invalid input - "2222P*==" + string sourceString = Encoding.ASCII.GetString(buffer.Slice(0, 4).ToArray()); + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(3, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + + { + Span buffer = new byte[] { 50, 50, 50, 50, 80, 42, 42, 42 }; + buffer[7] = Base64TestHelper.s_encodingPad; // invalid input - "2222P**=" + string sourceString = Encoding.ASCII.GetString(buffer.Slice(0, 4).ToArray()); + Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(3, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + + // The last byte or the last 2 bytes being the padding character is valid + { + Span buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; + buffer[6] = Base64TestHelper.s_encodingPad; + buffer[7] = Base64TestHelper.s_encodingPad; // valid input - "2222PP==" + string sourceString = Encoding.ASCII.GetString(buffer.ToArray()); + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(4, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + + { + Span buffer = new byte[] { 50, 50, 50, 50, 80, 80, 80, 80 }; + buffer[7] = Base64TestHelper.s_encodingPad; // valid input - "2222PPP=" + string sourceString = Encoding.ASCII.GetString(buffer.ToArray()); + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8InPlace(buffer, out int bytesWritten)); + Assert.Equal(5, bytesWritten); + Span expectedBytes = Convert.FromBase64String(sourceString); + Assert.True(expectedBytes.SequenceEqual(buffer.Slice(0, bytesWritten))); + } + } + + } +} diff --git a/src/System.Memory/tests/Base64/Base64EncoderUnitTests.cs b/src/System.Memory/tests/Base64/Base64EncoderUnitTests.cs new file mode 100644 index 000000000000..26fe6a9cb431 --- /dev/null +++ b/src/System.Memory/tests/Base64/Base64EncoderUnitTests.cs @@ -0,0 +1,270 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Buffers; +using System.Text; +using Xunit; + +namespace System.Buffers.Text.Tests +{ + public class Base64EncoderUnitTests + { + [Fact] + public void BasicEncodingAndDecoding() + { + var bytes = new byte[byte.MaxValue + 1]; + for (int i = 0; i < byte.MaxValue + 1; i++) + { + bytes[i] = (byte)i; + } + + for (int value = 0; value < 256; value++) + { + Span sourceBytes = bytes.AsSpan().Slice(0, value + 1); + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(sourceBytes.Length)]; + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8(sourceBytes, encodedBytes, out int consumed, out int encodedBytesCount)); + Assert.Equal(sourceBytes.Length, consumed); + Assert.Equal(encodedBytes.Length, encodedBytesCount); + + string encodedText = Encoding.ASCII.GetString(encodedBytes.ToArray()); + string expectedText = Convert.ToBase64String(bytes, 0, value + 1); + Assert.Equal(expectedText, encodedText); + + if (encodedBytes.Length % 4 == 0) + { + Span decodedBytes = new byte[Base64.GetMaxDecodedFromUtf8Length(encodedBytes.Length)]; + Assert.Equal(OperationStatus.Done, Base64.DecodeFromUtf8(encodedBytes, decodedBytes, out consumed, out int decodedByteCount)); + Assert.Equal(encodedBytes.Length, consumed); + Assert.Equal(sourceBytes.Length, decodedByteCount); + Assert.True(sourceBytes.SequenceEqual(decodedBytes.Slice(0, decodedByteCount))); + } + } + } + + [Fact] + public void BasicEncoding() + { + var rnd = new Random(42); + for (int i = 0; i < 10; i++) + { + int numBytes = rnd.Next(100, 1000 * 1000); + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeBytes(source, numBytes); + + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(source.Length)]; + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount)); + Assert.Equal(source.Length, consumed); + Assert.Equal(encodedBytes.Length, encodedBytesCount); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(source.Length, encodedBytes.Length, source, encodedBytes)); + } + } + + [Fact] + public void EncodeEmptySpan() + { + Span source = Span.Empty; + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount)); + Assert.Equal(source.Length, consumed); + Assert.Equal(encodedBytes.Length, encodedBytesCount); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(source.Length, encodedBytes.Length, source, encodedBytes)); + } + + [Fact] + [OuterLoop] + public void EncodeTooLargeSpan() + { + // int.MaxValue - (int.MaxValue % 4) => 2147483644, largest multiple of 4 less than int.MaxValue + // CLR default limit of 2 gigabytes (GB). + try + { + // 1610612734, larger than MaximumEncodeLength, requires output buffer of size 2147483648 (which is > int.MaxValue) + Span source = new byte[(int.MaxValue >> 2) * 3 + 1]; + Span encodedBytes = new byte[2000000000]; + Assert.Equal(OperationStatus.DestinationTooSmall, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount)); + Assert.Equal((encodedBytes.Length >> 2) * 3, consumed); // encoding 1500000000 bytes fits into buffer of 2000000000 bytes + Assert.Equal(encodedBytes.Length, encodedBytesCount); + } + catch (OutOfMemoryException) + { + // do nothing + } + } + + [Fact] + public void BasicEncodingWithFinalBlockFalse() + { + var rnd = new Random(42); + for (int i = 0; i < 10; i++) + { + int numBytes = rnd.Next(100, 1000 * 1000); + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeBytes(source, numBytes); + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(source.Length)]; + int expectedConsumed = source.Length / 3 * 3; // only consume closest multiple of three since isFinalBlock is false + int expectedWritten = source.Length / 3 * 4; + + Assert.Equal(OperationStatus.NeedMoreData, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount, isFinalBlock: false)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, encodedBytesCount); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(expectedConsumed, expectedWritten, source, encodedBytes)); + } + } + + [Theory] + [InlineData(1, "", 0, 0)] + [InlineData(2, "", 0, 0)] + [InlineData(3, "AQID", 3, 4)] + [InlineData(4, "AQID", 3, 4)] + [InlineData(5, "AQID", 3, 4)] + [InlineData(6, "AQIDBAUG", 6, 8)] + [InlineData(7, "AQIDBAUG", 6, 8)] + public void BasicEncodingWithFinalBlockFalseKnownInput(int numBytes, string expectedText, int expectedConsumed, int expectedWritten) + { + Span source = new byte[numBytes]; + for (int i = 0; i < numBytes; i++) + { + source[i] = (byte)(i + 1); + } + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.NeedMoreData, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount, isFinalBlock: false)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, encodedBytesCount); + + string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray()); + Assert.Equal(expectedText, encodedText); + } + + [Theory] + [InlineData(1, "AQ==", 1, 4)] + [InlineData(2, "AQI=", 2, 4)] + [InlineData(3, "AQID", 3, 4)] + [InlineData(4, "AQIDBA==", 4, 8)] + [InlineData(5, "AQIDBAU=", 5, 8)] + [InlineData(6, "AQIDBAUG", 6, 8)] + [InlineData(7, "AQIDBAUGBw==", 7, 12)] + public void BasicEncodingWithFinalBlockTrueKnownInput(int numBytes, string expectedText, int expectedConsumed, int expectedWritten) + { + Span source = new byte[numBytes]; + for (int i = 0; i < numBytes; i++) + { + source[i] = (byte)(i + 1); + } + Span encodedBytes = new byte[Base64.GetMaxEncodedToUtf8Length(source.Length)]; + + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int encodedBytesCount, isFinalBlock: true)); + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(expectedWritten, encodedBytesCount); + + string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray()); + Assert.Equal(expectedText, encodedText); + } + + [Fact] + public void EncodingOutputTooSmall() + { + for (int numBytes = 4; numBytes < 20; numBytes++) + { + Span source = new byte[numBytes]; + Base64TestHelper.InitalizeBytes(source, numBytes); + + Span encodedBytes = new byte[4]; + Assert.Equal(OperationStatus.DestinationTooSmall, + Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int written)); + int expectedConsumed = 3; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(encodedBytes.Length, written); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(expectedConsumed, encodedBytes.Length, source, encodedBytes)); + } + } + + [Fact] + public void EncodingOutputTooSmallRetry() + { + Span source = new byte[750]; + Base64TestHelper.InitalizeBytes(source); + + int outputSize = 320; + int requiredSize = Base64.GetMaxEncodedToUtf8Length(source.Length); + + Span encodedBytes = new byte[outputSize]; + Assert.Equal(OperationStatus.DestinationTooSmall, + Base64.EncodeToUtf8(source, encodedBytes, out int consumed, out int written)); + int expectedConsumed = encodedBytes.Length / 4 * 3; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(encodedBytes.Length, written); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(expectedConsumed, encodedBytes.Length, source, encodedBytes)); + + encodedBytes = new byte[requiredSize - outputSize]; + source = source.Slice(consumed); + Assert.Equal(OperationStatus.Done, + Base64.EncodeToUtf8(source, encodedBytes, out consumed, out written)); + expectedConsumed = encodedBytes.Length / 4 * 3; + Assert.Equal(expectedConsumed, consumed); + Assert.Equal(encodedBytes.Length, written); + Assert.True(Base64TestHelper.VerifyEncodingCorrectness(expectedConsumed, encodedBytes.Length, source, encodedBytes)); + } + + [Fact] + public void GetMaxEncodedLength() + { + // (int.MaxValue - 4)/(4/3) => 1610612733, otherwise integer overflow + int[] input = { 0, 1, 2, 3, 4, 5, 6, 1610612728, 1610612729, 1610612730, 1610612731, 1610612732, 1610612733 }; + int[] expected = { 0, 4, 4, 4, 8, 8, 8, 2147483640, 2147483640, 2147483640, 2147483644, 2147483644, 2147483644 }; + for (int i = 0; i < input.Length; i++) + { + Assert.Equal(expected[i], Base64.GetMaxEncodedToUtf8Length(input[i])); + } + + // integer overflow + Assert.Throws(() => Base64.GetMaxEncodedToUtf8Length(1610612734)); + Assert.Throws(() => Base64.GetMaxEncodedToUtf8Length(int.MaxValue)); + + // negative input + Assert.Throws(() => Base64.GetMaxEncodedToUtf8Length(-1)); + Assert.Throws(() => Base64.GetMaxEncodedToUtf8Length(int.MinValue)); + } + + [Fact] + public void EncodeInPlace() + { + const int numberOfBytes = 15; + Span testBytes = new byte[numberOfBytes / 3 * 4]; // slack since encoding inflates the data + Base64TestHelper.InitalizeBytes(testBytes); + + for (int numberOfBytesToTest = 0; numberOfBytesToTest <= numberOfBytes; numberOfBytesToTest++) + { + var expectedText = Convert.ToBase64String(testBytes.Slice(0, numberOfBytesToTest).ToArray()); + + Assert.Equal(OperationStatus.Done, Base64.EncodeToUtf8InPlace(testBytes, numberOfBytesToTest, out int bytesWritten)); + Assert.Equal(Base64.GetMaxEncodedToUtf8Length(numberOfBytesToTest), bytesWritten); + + var encodedText = Encoding.ASCII.GetString(testBytes.Slice(0, bytesWritten).ToArray()); + Assert.Equal(expectedText, encodedText); + } + } + + [Fact] + public void EncodeInPlaceOutputTooSmall() + { + byte[] testBytes = {1, 2, 3}; + + for (int numberOfBytesToTest = 1; numberOfBytesToTest <= testBytes.Length; numberOfBytesToTest++) + { + Assert.Equal(OperationStatus.DestinationTooSmall, Base64.EncodeToUtf8InPlace(testBytes, numberOfBytesToTest, out int bytesWritten)); + Assert.Equal(0, bytesWritten); + } + } + + [Fact] + public void EncodeInPlaceDataLengthTooLarge() + { + byte[] testBytes = {1, 2, 3}; + Assert.Equal(OperationStatus.DestinationTooSmall, Base64.EncodeToUtf8InPlace(testBytes, testBytes.Length + 1, out int bytesWritten)); + Assert.Equal(0, bytesWritten); + } + } +} diff --git a/src/System.Memory/tests/Base64/Base64TestHelper.cs b/src/System.Memory/tests/Base64/Base64TestHelper.cs new file mode 100644 index 000000000000..1026a97683eb --- /dev/null +++ b/src/System.Memory/tests/Base64/Base64TestHelper.cs @@ -0,0 +1,133 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Xunit; + +namespace System.Buffers.Text.Tests +{ + public static class Base64TestHelper + { + public static string s_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + // Pre-computing this table using a custom string(s_characters) and GenerateEncodingMapAndVerify (found in tests) + public static readonly byte[] s_encodingMap = { + 65, 66, 67, 68, 69, 70, 71, 72, //A..H + 73, 74, 75, 76, 77, 78, 79, 80, //I..P + 81, 82, 83, 84, 85, 86, 87, 88, //Q..X + 89, 90, 97, 98, 99, 100, 101, 102, //Y..Z, a..f + 103, 104, 105, 106, 107, 108, 109, 110, //g..n + 111, 112, 113, 114, 115, 116, 117, 118, //o..v + 119, 120, 121, 122, 48, 49, 50, 51, //w..z, 0..3 + 52, 53, 54, 55, 56, 57, 43, 47 //4..9, +, / + }; + + // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in tests) + public static readonly sbyte[] s_decodingMap = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, //62 is placed at index 43 (for +), 63 at index 47 (for /) + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, //52-61 are placed at index 48-57 (for 0-9), 64 at index 61 (for =) + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, //0-25 are placed at index 65-90 (for A-Z) + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, //26-51 are placed at index 97-122 (for a-z) + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Bytes over 122 ('z') are invalid and cannot be decoded + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Hence, padding the map with 255, which indicates invalid input + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + public static readonly byte s_encodingPad = (byte)'='; // '=', for padding + + public static readonly sbyte s_invalidByte = -1; // Designating -1 for invalid bytes in the decoding map + + public static byte[] InvalidBytes + { + get + { + int[] indices = s_decodingMap.FindAllIndexOf(s_invalidByte); + // Workaroudn for indices.Cast().ToArray() since it throws + // InvalidCastException: Unable to cast object of type 'System.Int32' to type 'System.Byte' + byte[] bytes = new byte[indices.Length]; + for(int i = 0; i < indices.Length; i++) + { + bytes[i] = (byte)indices[i]; + } + return bytes; + } + } + + public static void InitalizeBytes(Span bytes, int seed = 100) + { + var rnd = new Random(seed); + for (int i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)rnd.Next(0, byte.MaxValue + 1); + } + } + + public static void InitalizeDecodableBytes(Span bytes, int seed = 100) + { + var rnd = new Random(seed); + for (int i = 0; i < bytes.Length; i++) + { + int index = (byte)rnd.Next(0, s_encodingMap.Length - 1); // Do not pick '=' + bytes[i] = s_encodingMap[index]; + } + } + + [Fact] + public static void GenerateEncodingMapAndVerify() + { + var data = new byte[64]; // Base64 + for (int i = 0; i < s_characters.Length; i++) + { + data[i] = (byte)s_characters[i]; + } + Assert.True(s_encodingMap.AsSpan().SequenceEqual(data)); + } + + [Fact] + public static void GenerateDecodingMapAndVerify() + { + var data = new sbyte[256]; // 0 to byte.MaxValue (255) + for (int i = 0; i < data.Length; i++) + { + data[i] = s_invalidByte; + } + for (int i = 0; i < s_characters.Length; i++) + { + data[s_characters[i]] = (sbyte)i; + } + Assert.True(s_decodingMap.AsSpan().SequenceEqual(data)); + } + + public static int[] FindAllIndexOf(this IEnumerable values, T valueToFind) + { + return values.Select((element, index) => Equals(element, valueToFind) ? index : -1).Where(index => index != -1).ToArray(); + } + + public static bool VerifyEncodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span encodedBytes) + { + string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed).ToArray()); + string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray()); + return expectedText.Equals(encodedText); + } + + public static bool VerifyDecodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span decodedBytes) + { + string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed).ToArray()); + byte[] expectedBytes = Convert.FromBase64String(sourceString); + return expectedBytes.AsSpan().SequenceEqual(decodedBytes.Slice(0, expectedWritten)); + } + } +} + diff --git a/src/System.Memory/tests/Performance/Perf.Base64EncodeDecode.cs b/src/System.Memory/tests/Performance/Perf.Base64EncodeDecode.cs new file mode 100644 index 000000000000..97c5c73a0b91 --- /dev/null +++ b/src/System.Memory/tests/Performance/Perf.Base64EncodeDecode.cs @@ -0,0 +1,252 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Xunit.Performance; +using System.Text; +using Xunit; + +namespace System.Buffers.Text.Tests +{ + public class Base64EncodeDecodeTests + { + private const int InnerCount = 1000; + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64Encode(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + Span destination = new byte[Base64.GetMaxEncodedToUtf8Length(numberOfBytes)]; + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Base64.EncodeToUtf8(source, destination, out int consumed, out int written); + } + } + + Span backToSource = new byte[numberOfBytes]; + Base64.DecodeFromUtf8(destination, backToSource, out _, out _); + Assert.True(source.SequenceEqual(backToSource)); + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64EncodeDestinationTooSmall(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + Span destination = new byte[Base64.GetMaxEncodedToUtf8Length(numberOfBytes) - 1]; + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Base64.EncodeToUtf8(source, destination, out int consumed, out int written); + } + } + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64EncodeBaseline(int numberOfBytes) + { + var source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source.AsSpan()); + var destination = new char[Base64.GetMaxEncodedToUtf8Length(numberOfBytes)]; + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Convert.ToBase64CharArray(source, 0, source.Length, destination, 0); + } + } + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64Decode(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + Span encoded = new byte[Base64.GetMaxEncodedToUtf8Length(numberOfBytes)]; + Base64.EncodeToUtf8(source, encoded, out _, out _); + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Base64.DecodeFromUtf8(encoded, source, out int bytesConsumed, out int bytesWritten); + } + } + + Span backToEncoded = encoded.ToArray(); + Base64.EncodeToUtf8(source, encoded, out _, out _); + Assert.True(backToEncoded.SequenceEqual(encoded)); + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64DecodeDetinationTooSmall(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + Span encoded = new byte[Base64.GetMaxEncodedToUtf8Length(numberOfBytes)]; + Base64.EncodeToUtf8(source, encoded, out _, out _); + + source = source.Slice(0, source.Length - 1); + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Base64.DecodeFromUtf8(encoded, source, out int bytesConsumed, out int bytesWritten); + } + } + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64DecodeBaseline(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + ReadOnlySpan encoded = Convert.ToBase64String(source.ToArray()).ToCharArray(); + + foreach (var iteration in Benchmark.Iterations) { + using (iteration.StartMeasurement()) { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + Convert.TryFromBase64Chars(encoded, source, out int bytesWritten); + } + } + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64EncodeInPlace(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + int length = Base64.GetMaxEncodedToUtf8Length(numberOfBytes); + Span decodedSpan = new byte[length]; + source.CopyTo(decodedSpan); + Span backupSpan = decodedSpan.ToArray(); + + int bytesWritten = 0; + foreach (var iteration in Benchmark.Iterations) + { + using (iteration.StartMeasurement()) + { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + { + backupSpan.CopyTo(decodedSpan); + Base64.EncodeToUtf8InPlace(decodedSpan, numberOfBytes, out bytesWritten); + } + } + } + + Span backToSource = new byte[numberOfBytes]; + Base64.DecodeFromUtf8(decodedSpan, backToSource, out _, out _); + Assert.True(backupSpan.Slice(0, numberOfBytes).SequenceEqual(backToSource)); + } + + [Benchmark] + [InlineData(1000 * 1000)] + private static void Base64EncodeInPlaceOnce(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + int length = Base64.GetMaxEncodedToUtf8Length(numberOfBytes); + Span decodedSpan = new byte[length]; + source.CopyTo(decodedSpan); + Span backupSpan = decodedSpan.ToArray(); + + int bytesWritten = 0; + foreach (var iteration in Benchmark.Iterations) + { + backupSpan.CopyTo(decodedSpan); + using (iteration.StartMeasurement()) + { + Base64.EncodeToUtf8InPlace(decodedSpan, numberOfBytes, out bytesWritten); + } + } + + Span backToSource = new byte[numberOfBytes]; + Base64.DecodeFromUtf8(decodedSpan, backToSource, out _, out _); + Assert.True(backupSpan.Slice(0, numberOfBytes).SequenceEqual(backToSource)); + } + + [Benchmark(InnerIterationCount = InnerCount)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(1000 * 1000)] + private static void Base64DecodeInPlace(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + int length = Base64.GetMaxEncodedToUtf8Length(numberOfBytes); + Span encodedSpan = new byte[length]; + Base64.EncodeToUtf8(source, encodedSpan, out _, out _); + + Span backupSpan = encodedSpan.ToArray(); + + int bytesWritten = 0; + foreach (var iteration in Benchmark.Iterations) + { + using (iteration.StartMeasurement()) + { + for (int i = 0; i < Benchmark.InnerIterationCount; i++) + { + backupSpan.CopyTo(encodedSpan); + Base64.DecodeFromUtf8InPlace(encodedSpan, out bytesWritten); + } + } + } + + Assert.True(source.SequenceEqual(encodedSpan.Slice(0, bytesWritten))); + } + + [Benchmark] + [InlineData(1000 * 1000)] + private static void Base64DecodeInPlaceOnce(int numberOfBytes) + { + Span source = new byte[numberOfBytes]; + Base64TestHelper.InitalizeBytes(source); + int length = Base64.GetMaxEncodedToUtf8Length(numberOfBytes); + Span encodedSpan = new byte[length]; + + int bytesWritten = 0; + foreach (var iteration in Benchmark.Iterations) + { + Base64.EncodeToUtf8(source, encodedSpan, out _, out _); + using (iteration.StartMeasurement()) + { + Base64.DecodeFromUtf8InPlace(encodedSpan, out bytesWritten); + } + } + + Assert.True(source.SequenceEqual(encodedSpan.Slice(0, bytesWritten))); + } + } +} diff --git a/src/System.Memory/tests/Performance/System.Memory.Performance.Tests.csproj b/src/System.Memory/tests/Performance/System.Memory.Performance.Tests.csproj index 493b7421ae67..02ed25869a80 100644 --- a/src/System.Memory/tests/Performance/System.Memory.Performance.Tests.csproj +++ b/src/System.Memory/tests/Performance/System.Memory.Performance.Tests.csproj @@ -8,12 +8,14 @@ + + Common\System\PerfUtils.cs diff --git a/src/System.Memory/tests/System.Memory.Tests.csproj b/src/System.Memory/tests/System.Memory.Tests.csproj index 714ca05d9569..4dfffdb03db8 100644 --- a/src/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/System.Memory/tests/System.Memory.Tests.csproj @@ -103,6 +103,11 @@ + + + + +