Skip to content

Commit 3bad19f

Browse files
eiriktsarpalisstephentoub
eiriktsarpalis
and
stephentoub
authored
Add Utf8JsonReader.AllowMultipleValues and related APIs. (#104328)
* Add Utf8JsonReader.AllowMultipleValues and related APIs. * Update src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs Co-authored-by: Stephen Toub <stoub@microsoft.com> * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs Co-authored-by: Stephen Toub <stoub@microsoft.com> * Address feedback * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs * Update src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Read.Stream.cs * Update DOM tests to demonstrate chained root-value parsing. * Add another assertion --------- Co-authored-by: Stephen Toub <stoub@microsoft.com>
1 parent aa9e4d3 commit 3bad19f

24 files changed

+956
-143
lines changed

‎src/libraries/System.Text.Json/ref/System.Text.Json.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ public partial struct JsonReaderOptions
187187
{
188188
private int _dummyPrimitive;
189189
public bool AllowTrailingCommas { readonly get { throw null; } set { } }
190+
public bool AllowMultipleValues { readonly get { throw null; } set { } }
190191
public System.Text.Json.JsonCommentHandling CommentHandling { readonly get { throw null; } set { } }
191192
public int MaxDepth { readonly get { throw null; } set { } }
192193
}
@@ -247,7 +248,11 @@ public static partial class JsonSerializer
247248
public static System.Threading.Tasks.ValueTask<object?> DeserializeAsync(System.IO.Stream utf8Json, System.Type returnType, System.Text.Json.Serialization.JsonSerializerContext context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
248249
[System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
249250
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]
251+
public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, bool topLevelValues, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
252+
[System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
253+
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]
250254
public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
255+
public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.Serialization.Metadata.JsonTypeInfo<TValue> jsonTypeInfo, bool topLevelValues, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
251256
public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.Serialization.Metadata.JsonTypeInfo<TValue> jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
252257
[System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
253258
[System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]

‎src/libraries/System.Text.Json/src/System.Text.Json.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET
124124
<Compile Include="System\Text\Json\Serialization\Converters\Collection\ImmutableEnumerableOfTConverterWithReflection.cs" />
125125
<Compile Include="System\Text\Json\Serialization\Converters\Collection\ReadOnlyMemoryConverter.cs" />
126126
<Compile Include="System\Text\Json\Serialization\Converters\Collection\MemoryConverterFactory.cs" />
127+
<Compile Include="System\Text\Json\Serialization\Converters\Collection\RootLevelListConverter.cs" />
127128
<Compile Include="System\Text\Json\Serialization\Converters\Collection\StackOrQueueConverterWithReflection.cs" />
128129
<Compile Include="System\Text\Json\Serialization\Converters\JsonMetadataServicesConverter.cs" />
129130
<Compile Include="System\Text\Json\Serialization\Converters\Object\ObjectWithParameterizedConstructorConverter.Large.Reflection.cs" />

‎src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs

Lines changed: 58 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -37,43 +37,72 @@ public static bool TryAdvanceWithOptionalReadAhead(this scoped ref Utf8JsonReade
3737
// No read-ahead necessary if we're at the final block of JSON data.
3838
bool readAhead = requiresReadAhead && !reader.IsFinalBlock;
3939
return readAhead ? TryAdvanceWithReadAhead(ref reader) : reader.Read();
40+
}
4041

41-
// The read-ahead method is not inlined
42-
static bool TryAdvanceWithReadAhead(scoped ref Utf8JsonReader reader)
42+
/// <summary>
43+
/// Attempts to read ahead to the next root-level JSON value, if it exists.
44+
/// </summary>
45+
public static bool TryAdvanceToNextRootLevelValueWithOptionalReadAhead(this scoped ref Utf8JsonReader reader, bool requiresReadAhead, out bool isAtEndOfStream)
46+
{
47+
Debug.Assert(reader.AllowMultipleValues, "only supported by readers that support multiple values.");
48+
Debug.Assert(reader.CurrentDepth == 0, "should only invoked for top-level values.");
49+
50+
Utf8JsonReader checkpoint = reader;
51+
if (!reader.Read())
4352
{
44-
// When we're reading ahead we always have to save the state
45-
// as we don't know if the next token is a start object or array.
46-
Utf8JsonReader restore = reader;
53+
// If the reader didn't return any tokens and it's the final block,
54+
// then there are no other JSON values to be read.
55+
isAtEndOfStream = reader.IsFinalBlock;
56+
reader = checkpoint;
57+
return false;
58+
}
4759

48-
if (!reader.Read())
49-
{
50-
return false;
51-
}
60+
// We found another JSON value, read ahead accordingly.
61+
isAtEndOfStream = false;
62+
if (requiresReadAhead && !reader.IsFinalBlock)
63+
{
64+
// Perform full read-ahead to ensure the full JSON value has been buffered.
65+
reader = checkpoint;
66+
return TryAdvanceWithReadAhead(ref reader);
67+
}
5268

53-
// Perform the actual read-ahead.
54-
JsonTokenType tokenType = reader.TokenType;
55-
if (tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray)
69+
return true;
70+
}
71+
72+
private static bool TryAdvanceWithReadAhead(scoped ref Utf8JsonReader reader)
73+
{
74+
// When we're reading ahead we always have to save the state
75+
// as we don't know if the next token is a start object or array.
76+
Utf8JsonReader restore = reader;
77+
78+
if (!reader.Read())
79+
{
80+
return false;
81+
}
82+
83+
// Perform the actual read-ahead.
84+
JsonTokenType tokenType = reader.TokenType;
85+
if (tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray)
86+
{
87+
// Attempt to skip to make sure we have all the data we need.
88+
bool complete = reader.TrySkipPartial();
89+
90+
// We need to restore the state in all cases as we need to be positioned back before
91+
// the current token to either attempt to skip again or to actually read the value.
92+
reader = restore;
93+
94+
if (!complete)
5695
{
57-
// Attempt to skip to make sure we have all the data we need.
58-
bool complete = reader.TrySkipPartial();
59-
60-
// We need to restore the state in all cases as we need to be positioned back before
61-
// the current token to either attempt to skip again or to actually read the value.
62-
reader = restore;
63-
64-
if (!complete)
65-
{
66-
// Couldn't read to the end of the object, exit out to get more data in the buffer.
67-
return false;
68-
}
69-
70-
// Success, requeue the reader to the start token.
71-
reader.ReadWithVerify();
72-
Debug.Assert(tokenType == reader.TokenType);
96+
// Couldn't read to the end of the object, exit out to get more data in the buffer.
97+
return false;
7398
}
7499

75-
return true;
100+
// Success, requeue the reader to the start token.
101+
reader.ReadWithVerify();
102+
Debug.Assert(tokenType == reader.TokenType);
76103
}
104+
105+
return true;
77106
}
78107

79108
#if !NET

‎src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,14 @@ public int MaxDepth
7070
/// By default, it's set to false, and <exception cref="JsonException"/> is thrown if a trailing comma is encountered.
7171
/// </remarks>
7272
public bool AllowTrailingCommas { get; set; }
73+
74+
/// <summary>
75+
/// Defines whether the <see cref="Utf8JsonReader"/> should tolerate
76+
/// zero or more top-level JSON values that are whitespace separated.
77+
/// </summary>
78+
/// <remarks>
79+
/// By default, it's set to false, and <exception cref="JsonException"/> is thrown if trailing content is encountered after the first top-level JSON value.
80+
/// </remarks>
81+
public bool AllowMultipleValues { get; set; }
7382
}
7483
}

‎src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -363,17 +363,13 @@ private bool ReadFirstTokenMultiSegment(byte first)
363363
}
364364
_tokenType = JsonTokenType.Number;
365365
_consumed += numberOfBytes;
366-
return true;
367366
}
368367
else if (!ConsumeValueMultiSegment(first))
369368
{
370369
return false;
371370
}
372371

373-
if (_tokenType == JsonTokenType.StartObject || _tokenType == JsonTokenType.StartArray)
374-
{
375-
_isNotPrimitive = true;
376-
}
372+
_isNotPrimitive = _tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray;
377373
// Intentionally fall out of the if-block to return true
378374
}
379375
return true;
@@ -1580,6 +1576,11 @@ private ConsumeTokenResult ConsumeNextTokenMultiSegment(byte marker)
15801576

15811577
if (_bitStack.CurrentDepth == 0)
15821578
{
1579+
if (_readerOptions.AllowMultipleValues)
1580+
{
1581+
return ReadFirstTokenMultiSegment(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
1582+
}
1583+
15831584
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
15841585
}
15851586

@@ -1711,6 +1712,11 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentTokenMultiSegment()
17111712

17121713
if (_bitStack.CurrentDepth == 0 && _tokenType != JsonTokenType.None)
17131714
{
1715+
if (_readerOptions.AllowMultipleValues)
1716+
{
1717+
return ReadFirstTokenMultiSegment(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
1718+
}
1719+
17141720
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, first);
17151721
}
17161722

@@ -2064,6 +2070,11 @@ private ConsumeTokenResult ConsumeNextTokenUntilAfterAllCommentsAreSkippedMultiS
20642070
}
20652071
else if (_bitStack.CurrentDepth == 0)
20662072
{
2073+
if (_readerOptions.AllowMultipleValues)
2074+
{
2075+
return ReadFirstTokenMultiSegment(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
2076+
}
2077+
20672078
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
20682079
}
20692080
else if (marker == JsonConstants.ListSeparator)

‎src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
using System.Buffers;
55
using System.Diagnostics;
66
using System.Runtime.CompilerServices;
7-
using System.Runtime.InteropServices;
87

98
namespace System.Text.Json
109
{
@@ -55,6 +54,8 @@ public ref partial struct Utf8JsonReader
5554

5655
internal readonly int ValueLength => HasValueSequence ? checked((int)ValueSequence.Length) : ValueSpan.Length;
5756

57+
internal readonly bool AllowMultipleValues => _readerOptions.AllowMultipleValues;
58+
5859
/// <summary>
5960
/// Gets the value of the last processed token as a ReadOnlySpan&lt;byte&gt; slice
6061
/// of the input payload. If the JSON is provided within a ReadOnlySequence&lt;byte&gt;
@@ -280,7 +281,7 @@ public bool Read()
280281

281282
if (!retVal)
282283
{
283-
if (_isFinalBlock && TokenType == JsonTokenType.None)
284+
if (_isFinalBlock && TokenType is JsonTokenType.None && !_readerOptions.AllowMultipleValues)
284285
{
285286
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedJsonTokens);
286287
}
@@ -929,7 +930,7 @@ private bool HasMoreData()
929930
return false;
930931
}
931932

932-
if (_tokenType != JsonTokenType.EndArray && _tokenType != JsonTokenType.EndObject)
933+
if (_tokenType is not JsonTokenType.EndArray and not JsonTokenType.EndObject)
933934
{
934935
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.InvalidEndOfJsonNonPrimitive);
935936
}
@@ -991,17 +992,13 @@ private bool ReadFirstToken(byte first)
991992
_tokenType = JsonTokenType.Number;
992993
_consumed += numberOfBytes;
993994
_bytePositionInLine += numberOfBytes;
994-
return true;
995995
}
996996
else if (!ConsumeValue(first))
997997
{
998998
return false;
999999
}
10001000

1001-
if (_tokenType == JsonTokenType.StartObject || _tokenType == JsonTokenType.StartArray)
1002-
{
1003-
_isNotPrimitive = true;
1004-
}
1001+
_isNotPrimitive = _tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray;
10051002
// Intentionally fall out of the if-block to return true
10061003
}
10071004
return true;
@@ -1016,10 +1013,10 @@ private void SkipWhiteSpace()
10161013
byte val = localBuffer[_consumed];
10171014

10181015
// JSON RFC 8259 section 2 says only these 4 characters count, not all of the Unicode definitions of whitespace.
1019-
if (val != JsonConstants.Space &&
1020-
val != JsonConstants.CarriageReturn &&
1021-
val != JsonConstants.LineFeed &&
1022-
val != JsonConstants.Tab)
1016+
if (val is not JsonConstants.Space and
1017+
not JsonConstants.CarriageReturn and
1018+
not JsonConstants.LineFeed and
1019+
not JsonConstants.Tab)
10231020
{
10241021
break;
10251022
}
@@ -1747,6 +1744,11 @@ private ConsumeTokenResult ConsumeNextToken(byte marker)
17471744

17481745
if (_bitStack.CurrentDepth == 0)
17491746
{
1747+
if (_readerOptions.AllowMultipleValues)
1748+
{
1749+
return ReadFirstToken(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
1750+
}
1751+
17501752
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
17511753
}
17521754

@@ -1869,6 +1871,11 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentToken()
18691871

18701872
if (_bitStack.CurrentDepth == 0 && _tokenType != JsonTokenType.None)
18711873
{
1874+
if (_readerOptions.AllowMultipleValues)
1875+
{
1876+
return ReadFirstToken(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
1877+
}
1878+
18721879
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, first);
18731880
}
18741881

@@ -2033,7 +2040,7 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentToken()
20332040
}
20342041
else
20352042
{
2036-
Debug.Assert(_tokenType == JsonTokenType.EndArray || _tokenType == JsonTokenType.EndObject);
2043+
Debug.Assert(_tokenType is JsonTokenType.EndArray or JsonTokenType.EndObject);
20372044
if (_inObject)
20382045
{
20392046
Debug.Assert(first != JsonConstants.CloseBrace);
@@ -2207,6 +2214,11 @@ private ConsumeTokenResult ConsumeNextTokenUntilAfterAllCommentsAreSkipped(byte
22072214
}
22082215
else if (_bitStack.CurrentDepth == 0)
22092216
{
2217+
if (_readerOptions.AllowMultipleValues)
2218+
{
2219+
return ReadFirstToken(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
2220+
}
2221+
22102222
ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
22112223
}
22122224
else if (marker == JsonConstants.ListSeparator)

0 commit comments

Comments
 (0)