Add Base64 parity APIs with Base64Url (#123151)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: stephentoub <2642209+stephentoub@users.noreply.github.com>
Co-authored-by: MihaZupan <25307628+MihaZupan@users.noreply.github.com>
Co-authored-by: Miha Zupan <mihazupan.zupan1@gmail.com>

Author: 3 people

src/libraries/Fuzzing/DotnetFuzzing/Fuzzers/Base64Fuzzer.cs

@@ -24,8 +24,9 @@ public void FuzzTarget(ReadOnlySpan<byte> bytes)
         }
 
         private void TestCases(Span<byte> input, PoisonPagePlacement poison)
-        { 
+        {
             TestBase64(input, poison);
+            TestBase64Chars(input, poison);
             TestToStringToCharArray(input, Base64FormattingOptions.None);
             TestToStringToCharArray(input, Base64FormattingOptions.InsertLineBreaks);
         }
@@ -129,6 +130,167 @@ private void TestBase64(Span<byte> input, PoisonPagePlacement poison)
                     Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromUtf8InPlace(input, out int inPlaceDecoded));
                 }
             }
+
+            { // Test new simplified UTF-8 APIs
+                // Test EncodeToUtf8 returning byte[]
+                byte[] encodedArray = Base64.EncodeToUtf8(input);
+                Assert.Equal(true, maxEncodedLength >= encodedArray.Length && maxEncodedLength - 2 <= encodedArray.Length);
+
+                // Test EncodeToUtf8 returning int
+                encoderDest.Clear();
+                int charsWritten = Base64.EncodeToUtf8(input, encoderDest);
+                Assert.SequenceEqual(encodedArray.AsSpan(), encoderDest.Slice(0, charsWritten));
+
+                // Test TryEncodeToUtf8
+                encoderDest.Clear();
+                Assert.Equal(true, Base64.TryEncodeToUtf8(input, encoderDest, out int tryCharsWritten));
+                Assert.Equal(charsWritten, tryCharsWritten);
+                Assert.SequenceEqual(encodedArray.AsSpan(), encoderDest.Slice(0, tryCharsWritten));
+
+                // Test DecodeFromUtf8 returning byte[]
+                byte[] decodedArray = Base64.DecodeFromUtf8(encodedArray);
+                Assert.SequenceEqual(input, decodedArray.AsSpan());
+
+                // Test DecodeFromUtf8 returning int
+                decoderDest.Clear();
+                int bytesWritten = Base64.DecodeFromUtf8(encodedArray, decoderDest);
+                Assert.Equal(input.Length, bytesWritten);
+                Assert.SequenceEqual(input, decoderDest.Slice(0, bytesWritten));
+
+                // Test TryDecodeFromUtf8
+                decoderDest.Clear();
+                Assert.Equal(true, Base64.TryDecodeFromUtf8(encodedArray, decoderDest, out int tryBytesWritten));
+                Assert.Equal(input.Length, tryBytesWritten);
+                Assert.SequenceEqual(input, decoderDest.Slice(0, tryBytesWritten));
+
+                // Test TryEncodeToUtf8InPlace
+                using PooledBoundedMemory<byte> inPlaceBuffer = PooledBoundedMemory<byte>.Rent(maxEncodedLength, poison);
+                Span<byte> inPlaceDest = inPlaceBuffer.Span;
+                input.CopyTo(inPlaceDest);
+                Assert.Equal(true, Base64.TryEncodeToUtf8InPlace(inPlaceDest, input.Length, out int inPlaceWritten));
+                Assert.SequenceEqual(encodedArray.AsSpan(), inPlaceDest.Slice(0, inPlaceWritten));
+
+                // Test GetEncodedLength matches GetMaxEncodedToUtf8Length
+                Assert.Equal(Base64.GetMaxEncodedToUtf8Length(input.Length), Base64.GetEncodedLength(input.Length));
+
+                // Test GetMaxDecodedLength matches GetMaxDecodedFromUtf8Length
+                Assert.Equal(Base64.GetMaxDecodedFromUtf8Length(maxEncodedLength), Base64.GetMaxDecodedLength(maxEncodedLength));
+            }
+        }
+
+        private static void TestBase64Chars(Span<byte> input, PoisonPagePlacement poison)
+        {
+            int encodedLength = Base64.GetEncodedLength(input.Length);
+            int maxDecodedLength = Base64.GetMaxDecodedLength(encodedLength);
+
+            using PooledBoundedMemory<char> destPoisoned = PooledBoundedMemory<char>.Rent(encodedLength, poison);
+            using PooledBoundedMemory<byte> decoderDestPoisoned = PooledBoundedMemory<byte>.Rent(maxDecodedLength, poison);
+
+            Span<char> encoderDest = destPoisoned.Span;
+            Span<byte> decoderDest = decoderDestPoisoned.Span;
+
+            { // IsFinalBlock = true
+                OperationStatus status = Base64.EncodeToChars(input, encoderDest, out int bytesConsumed, out int charsEncoded);
+
+                Assert.Equal(OperationStatus.Done, status);
+                Assert.Equal(input.Length, bytesConsumed);
+                Assert.Equal(encodedLength, charsEncoded);
+
+                string encodedString = Base64.EncodeToString(input);
+                Assert.Equal(encodedString, new string(encoderDest.Slice(0, charsEncoded)));
+
+                status = Base64.DecodeFromChars(encoderDest.Slice(0, charsEncoded), decoderDest, out int charsRead, out int bytesDecoded);
+
+                Assert.Equal(OperationStatus.Done, status);
+                Assert.Equal(input.Length, bytesDecoded);
+                Assert.Equal(charsEncoded, charsRead);
+                Assert.SequenceEqual(input, decoderDest.Slice(0, bytesDecoded));
+            }
+
+            { // IsFinalBlock = false
+                encoderDest.Clear();
+                decoderDest.Clear();
+                OperationStatus status = Base64.EncodeToChars(input, encoderDest, out int bytesConsumed, out int charsEncoded, isFinalBlock: false);
+                Span<char> decodeInput = encoderDest.Slice(0, charsEncoded);
+
+                if (input.Length % 3 == 0)
+                {
+                    Assert.Equal(OperationStatus.Done, status);
+                    Assert.Equal(input.Length, bytesConsumed);
+                    Assert.Equal(encodedLength, charsEncoded);
+
+                    status = Base64.DecodeFromChars(decodeInput, decoderDest, out int charsRead, out int bytesDecoded, isFinalBlock: false);
+
+                    Assert.Equal(OperationStatus.Done, status);
+                    Assert.Equal(input.Length, bytesDecoded);
+                    Assert.Equal(charsEncoded, charsRead);
+                    Assert.SequenceEqual(input, decoderDest.Slice(0, bytesDecoded));
+                }
+                else
+                {
+                    Assert.Equal(OperationStatus.NeedMoreData, status);
+                    Assert.Equal(true, input.Length / 3 * 4 == charsEncoded);
+
+                    status = Base64.DecodeFromChars(decodeInput, decoderDest, out int charsRead, out int bytesDecoded, isFinalBlock: false);
+
+                    if (decodeInput.Length % 4 == 0)
+                    {
+                        Assert.Equal(OperationStatus.Done, status);
+                        Assert.Equal(bytesConsumed, bytesDecoded);
+                        Assert.Equal(charsEncoded, charsRead);
+                    }
+                    else
+                    {
+                        Assert.Equal(OperationStatus.NeedMoreData, status);
+                    }
+
+                    Assert.SequenceEqual(input.Slice(0, bytesDecoded), decoderDest.Slice(0, bytesDecoded));
+                }
+            }
+
+            { // Test array-returning and int-returning overloads
+                char[] encodedChars = Base64.EncodeToChars(input);
+                Assert.Equal(encodedLength, encodedChars.Length);
+
+                encoderDest.Clear();
+                int charsWritten = Base64.EncodeToChars(input, encoderDest);
+                Assert.Equal(encodedLength, charsWritten);
+                Assert.SequenceEqual(encodedChars.AsSpan(), encoderDest.Slice(0, charsWritten));
+
+                byte[] decodedBytes = Base64.DecodeFromChars(encodedChars);
+                Assert.SequenceEqual(input, decodedBytes.AsSpan());
+
+                decoderDest.Clear();
+                int bytesWritten = Base64.DecodeFromChars(encodedChars, decoderDest);
+                Assert.Equal(input.Length, bytesWritten);
+                Assert.SequenceEqual(input, decoderDest.Slice(0, bytesWritten));
+            }
+
+            { // Test Try* variants
+                encoderDest.Clear();
+                Assert.Equal(true, Base64.TryEncodeToChars(input, encoderDest, out int charsWritten));
+                Assert.Equal(encodedLength, charsWritten);
+
+                decoderDest.Clear();
+                Assert.Equal(true, Base64.TryDecodeFromChars(encoderDest.Slice(0, charsWritten), decoderDest, out int bytesWritten));
+                Assert.Equal(input.Length, bytesWritten);
+                Assert.SequenceEqual(input, decoderDest.Slice(0, bytesWritten));
+            }
+
+            { // Decode the random chars directly (as chars, from the input bytes interpreted as UTF-16)
+                // Create a char span from the input bytes for testing decode with random data
+                if (input.Length >= 2)
+                {
+                    ReadOnlySpan<char> inputChars = System.Runtime.InteropServices.MemoryMarshal.Cast<byte, char>(input);
+                    decoderDest.Clear();
+
+                    // Try decoding - may succeed or fail depending on if input is valid base64
+                    OperationStatus status = Base64.DecodeFromChars(inputChars, decoderDest, out int charsConsumed, out int bytesDecoded);
+                    // Just verify we don't crash - the result depends on input validity
+                    Assert.Equal(true, status == OperationStatus.Done || status == OperationStatus.InvalidData ||
+                                       status == OperationStatus.NeedMoreData || status == OperationStatus.DestinationTooSmall);
+                }
+            }
         }
 
         private static void TestToStringToCharArray(Span<byte> input, Base64FormattingOptions options)

src/libraries/System.Memory/tests/Base64/Base64DecoderUnitTests.cs

@@ -771,6 +771,134 @@ public void BasicDecodingWithExtraWhitespaceShouldBeCountedInConsumedBytes(strin
             Assert.True(Base64TestHelper.VerifyDecodingCorrectness(expectedConsumed, expectedWritten, source, decodedBytes));
         }
 
+        [Fact]
+        public void DecodeFromCharsWithLargeSpan()
+        {
+            var rnd = new Random(42);
+            for (int i = 0; i < 5; i++)
+            {
+                int numBytes = rnd.Next(100, 1000 * 1000);
+                // Ensure we have a valid length (multiple of 4 for standard Base64)
+                numBytes = (numBytes / 4) * 4;
+
+                Span<char> source = new char[numBytes];
+                Base64TestHelper.InitializeDecodableChars(source, numBytes);
+
+                Span<byte> decodedBytes = new byte[Base64.GetMaxDecodedLength(source.Length)];
+                Assert.Equal(OperationStatus.Done, Base64.DecodeFromChars(source, decodedBytes, out int consumed, out int decodedByteCount));
+                Assert.Equal(source.Length, consumed);
+
+                string sourceString = source.ToString();
+                byte[] expectedBytes = Convert.FromBase64String(sourceString);
+                Assert.True(expectedBytes.AsSpan().SequenceEqual(decodedBytes.Slice(0, decodedByteCount)));
+            }
+        }
+
+        [Theory]
+        [InlineData("\u5948cz/T", 0, 0)] // tests the scalar code-path with non-ASCII
+        [InlineData("z/Ta123\u5948", 4, 3)]
+        public void DecodeFromCharsNonAsciiInputInvalid(string inputString, int expectedConsumed, int expectedWritten)
+        {
+            Span<char> source = inputString.ToArray();
+            Span<byte> decodedBytes = new byte[Base64.GetMaxDecodedLength(source.Length)];
+
+            Assert.Equal(OperationStatus.InvalidData, Base64.DecodeFromChars(source, decodedBytes, out int consumed, out int decodedByteCount));
+            Assert.Equal(expectedConsumed, consumed);
+            Assert.Equal(expectedWritten, decodedByteCount);
+        }
+
+        [Fact]
+        public void DecodeFromUtf8_ArrayOverload()
+        {
+            byte[] utf8Input = Encoding.UTF8.GetBytes("dGVzdA=="); // "test" encoded
+            byte[] result = Base64.DecodeFromUtf8(utf8Input);
+            Assert.Equal(4, result.Length);
+            Assert.Equal("test", Encoding.UTF8.GetString(result));
+        }
+
+        [Fact]
+        public void DecodeFromUtf8_SpanOverload()
+        {
+            byte[] utf8Input = Encoding.UTF8.GetBytes("dGVzdA=="); // "test" encoded
+            Span<byte> destination = new byte[10];
+            int bytesWritten = Base64.DecodeFromUtf8(utf8Input, destination);
+            Assert.Equal(4, bytesWritten);
+            Assert.Equal("test", Encoding.UTF8.GetString(destination.Slice(0, bytesWritten)));
+        }
+
+        [Fact]
+        public void TryDecodeFromUtf8_Success()
+        {
+            byte[] utf8Input = Encoding.UTF8.GetBytes("dGVzdA==");
+            Span<byte> destination = new byte[10];
+            Assert.True(Base64.TryDecodeFromUtf8(utf8Input, destination, out int bytesWritten));
+            Assert.Equal(4, bytesWritten);
+            Assert.Equal("test", Encoding.UTF8.GetString(destination.Slice(0, bytesWritten)));
+        }
+
+        [Fact]
+        public void TryDecodeFromUtf8_DestinationTooSmall()
+        {
+            byte[] utf8Input = Encoding.UTF8.GetBytes("dGVzdA==");
+            Span<byte> destination = new byte[2]; // Too small
+            Assert.False(Base64.TryDecodeFromUtf8(utf8Input, destination, out int bytesWritten));
+            Assert.Equal(0, bytesWritten);
+        }
+
+        [Fact]
+        public void DecodeFromChars_InvalidData()
+        {
+            string invalidInput = "@#$%";
+            byte[] destination = new byte[10];
+            Assert.Throws<FormatException>(() => Base64.DecodeFromChars(invalidInput, destination));
+            Assert.Throws<FormatException>(() => Base64.DecodeFromChars(invalidInput.AsSpan()));
+        }
+
+        [Fact]
+        public void DecodeFromChars_DestinationTooSmall()
+        {
+            string validInput = "dGVzdA=="; // "test" encoded
+            byte[] destination = new byte[2]; // Too small
+            Assert.Throws<ArgumentException>("destination", () => Base64.DecodeFromChars(validInput, destination));
+        }
+
+        [Fact]
+        public void TryDecodeFromChars_DestinationTooSmall()
+        {
+            string validInput = "dGVzdA=="; // "test" encoded
+            Span<byte> destination = new byte[2]; // Too small
+            Assert.False(Base64.TryDecodeFromChars(validInput, destination, out int bytesWritten));
+        }
+
+        [Fact]
+        public void DecodeFromChars_OperationStatus_DistinguishesBetweenInvalidAndDestinationTooSmall()
+        {
+            // This is the key use case from the issue - distinguishing between invalid data and destination too small
+            string validInput = "dGVzdA=="; // "test" encoded - produces 4 bytes
+            string invalidInput = "@#$%";
+            Span<byte> smallDestination = new byte[2];
+
+            // With destination too small, we should get DestinationTooSmall
+            OperationStatus status1 = Base64.DecodeFromChars(validInput, smallDestination, out int consumed1, out int written1);
+            Assert.Equal(OperationStatus.DestinationTooSmall, status1);
+            Assert.True(consumed1 > 0 || written1 >= 0); // Some progress was made or at least we know why it failed
+
+            // With invalid data, we should get InvalidData
+            OperationStatus status2 = Base64.DecodeFromChars(invalidInput, smallDestination, out int consumed2, out int written2);
+            Assert.Equal(OperationStatus.InvalidData, status2);
+            Assert.Equal(0, consumed2);
+            Assert.Equal(0, written2);
+        }
+
+        [Fact]
+        public void GetMaxDecodedLength_Matches_GetMaxDecodedFromUtf8Length()
+        {
+            for (int i = 0; i < 100; i++)
+            {
+                Assert.Equal(Base64.GetMaxDecodedFromUtf8Length(i), Base64.GetMaxDecodedLength(i));
+            }
+        }
+
         [Fact]
         public void DecodingWithWhiteSpaceIntoSmallDestination()
         {