Rewrite and restructure JSON parser

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/PrimitiveValuesBenchmark.kt

@@ -59,11 +59,3 @@ open class PrimitiveValuesBenchmark {
     @Benchmark
     fun encodeLong(): LongHolder = Json.decodeFromString(LongHolder.serializer(), longValue)
 }
-
-
-private val booleanHolder = PrimitiveValuesBenchmark.BooleanHolder(true, false, true, false, true, true, false, false)
-private val booleanValue = Json.encodeToString(booleanHolder)
-
-fun main() {
-    println(Json.decodeFromString(PrimitiveValuesBenchmark.BooleanHolder.serializer(), booleanValue))
-}

docs/basic-serialization.md

@@ -493,7 +493,8 @@ Even though the `language` property has a default value, it is still an error to
 the `null` value to it.
 
 ```text
-Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected quotation mark '"', but had 'n' instead
+Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found.
+Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values.
 ```                    
 
 <!--- TEST LINES_START -->

formats/json/commonMain/src/kotlinx/serialization/json/Json.kt

@@ -83,7 +83,7 @@ public sealed class Json(internal val configuration: JsonConf) : StringFormat {
         val reader = JsonReader(string)
         val input = StreamingJsonDecoder(this, WriteMode.OBJ, reader)
         val result = input.decodeSerializableValue(deserializer)
-        if (!reader.isDone) { error("Reader has not consumed the whole input: $reader") }
+        reader.expectEof()
         return result
     }
     /**

formats/json/commonMain/src/kotlinx/serialization/json/JsonElementSerializers.kt

@@ -97,7 +97,7 @@ internal object JsonNullSerializer : KSerializer<JsonNull> {
     override fun deserialize(decoder: Decoder): JsonNull {
         verify(decoder)
         if (decoder.decodeNotNullMark()) {
-
+            throw JsonDecodingException("Expected 'null' literal")
         }
         decoder.decodeNull()
         return JsonNull

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonReader.kt

@@ -123,7 +123,12 @@ internal class JsonReader(private val source: String) {
 
     @JvmField
     var currentPosition: Int = 0 // position in source
-    val isDone: Boolean get() = consumeNextToken() == TC_EOF
+
+    fun expectEof() {
+        val nextToken = consumeNextToken()
+        if (nextToken != TC_EOF)
+            fail("Expected EOF, but had ${source[currentPosition - 1]} instead")
+    }
 
     fun tryConsumeComma(): Boolean {
         val current = skipWhitespaces()
@@ -139,6 +144,7 @@ internal class JsonReader(private val source: String) {
         var current = currentPosition
         while (current < source.length) {
             val c = source[current]
+            // Inlined skipWhitespaces without field spill and nested loop. Also faster then char2TokenClass
             if (c == ' ' || c == '\n' || c == '\r' || c == '\t') {
                 ++current
                 continue
@@ -179,9 +185,17 @@ internal class JsonReader(private val source: String) {
             val c = source[currentPosition++]
             if (c == ' ' || c == '\n' || c == '\r' || c == '\t') continue
             if (c == expected) return
-            fail(charToTokenClass(expected))
+            unexpectedToken(expected)
         }
-        fail(charToTokenClass(expected)) // EOF
+        unexpectedToken(expected) // EOF
+    }
+
+    private fun unexpectedToken(expected: Char) {
+        --currentPosition // To properly handle null
+        if (expected == STRING && consumeStringLenient() == NULL) {
+            fail("Expected string literal but 'null' literal was found.\n$coerceInputValuesHint", currentPosition - 4)
+        }
+        fail(charToTokenClass(expected))
     }
 
     private fun fail(expectedToken: Byte) {
@@ -197,21 +211,19 @@ internal class JsonReader(private val source: String) {
             TC_END_LIST -> "end of the array ']'"
             else -> "valid token" // should never happen
         }
-        val s = if (currentPosition == source.length) "EOF" else source[currentPosition - 1].toString()
+        val s = if (currentPosition == source.length || currentPosition == 0) "EOF" else source[currentPosition - 1].toString()
         fail("Expected $expected, but had '$s' instead", currentPosition - 1)
     }
 
     fun peekNextToken(): Byte {
         val source = source
         while (currentPosition < source.length) {
             val ch = source[currentPosition]
-            return when (val tc = charToTokenClass(ch)) {
-                TC_WHITESPACE -> {
-                    ++currentPosition
-                    continue
-                }
-                else -> tc
+            if (ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') {
+                ++currentPosition
+                continue
             }
+            return charToTokenClass(ch)
         }
         return TC_EOF
     }
@@ -466,14 +478,19 @@ internal class JsonReader(private val source: String) {
     }
 
     fun consumeNumericLiteral(): Long {
+        /*
+         * This is an optimized (~40% for numbers) version of consumeString().toLong()
+         * that doesn't allocate and also doesn't support any radix but 10
+         */
         var current = skipWhitespaces()
+        if (current == source.length) fail("EOF")
         val hasQuotation = if (source[current] == STRING) {
-            ++current
+            // Check it again
+            if (++current == source.length) fail("EOF")
             true
         } else {
             false
         }
-        if (current == source.length) fail("EOF")
         var accumulator = 0L
         var isNegative = false
         val start = current
@@ -495,7 +512,7 @@ internal class JsonReader(private val source: String) {
             accumulator = accumulator * 10 - digit
             if (accumulator > 0) fail("Numeric value overflow")
         }
-        if (start == current) {
+        if (start == current || (isNegative && start == current - 1)) {
             fail("Expected numeric literal")
         }
         if (hasQuotation) {
@@ -549,7 +566,7 @@ internal class JsonReader(private val source: String) {
                 false
             }
             else -> {
-                fail("Expected valid boolean literal prefix, but had ${source[current - 1]}")
+                fail("Expected valid boolean literal prefix, but had '${consumeStringLenient()}'")
             }
         }
     }
@@ -563,7 +580,7 @@ internal class JsonReader(private val source: String) {
             val expected = literalSuffix[i]
             val actual = source[current + i]
             if (expected.toInt() != actual.toInt() or asciiCaseMask) {
-                fail("Expected valid boolean literal prefix, but had ${source.substring(current - 1, current - 1 + i)}")
+                fail("Expected valid boolean literal prefix, but had '${consumeStringLenient()}'")
             }
         }
 

formats/json/commonMain/src/kotlinx/serialization/json/internal/StreamingJsonDecoder.kt

@@ -177,7 +177,7 @@ internal open class StreamingJsonDecoder(
         return if (configuration.isLenient) {
             reader.consumeBooleanLenient()
         } else {
-            return reader.consumeBoolean()
+            reader.consumeBoolean()
         }
     }
 
@@ -225,7 +225,7 @@ internal open class StreamingJsonDecoder(
     }
 
     override fun decodeChar(): Char {
-        val string= reader.consumeStringLenient()
+        val string = reader.consumeStringLenient()
         if (string.length != 1) reader.fail("Expected single char, but got '$string'")
         return string[0]
     }

formats/json/commonTest/src/kotlinx/serialization/json/JsonParserFailureModesTest.kt

@@ -125,4 +125,16 @@ class JsonParserFailureModesTest : JsonTestBase() {
         default.decodeFromString<Holder>("""{"id": ${Long.MIN_VALUE}}""", it)
         default.decodeFromString<Holder>("""{"id": ${Long.MAX_VALUE}}""", it)
     }
+
+    @Test
+    fun testInvalidNumber() = parametrizedTest {
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":-}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":+}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":--}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":1-1}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":0-1}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":0-}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":a}""", it) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString<Holder>("""{"id":-a}""", it) }
+    }
 }

formats/json/commonTest/src/kotlinx/serialization/json/JsonTestBase.kt

@@ -37,10 +37,10 @@ abstract class JsonTestBase {
         return if (useStreaming) {
             decodeFromString(deserializer, source)
         } else {
-            val parser = JsonReader(source)
-            val input = StreamingJsonDecoder(this, WriteMode.OBJ, parser)
+            val reader = JsonReader(source)
+            val input = StreamingJsonDecoder(this, WriteMode.OBJ, reader)
             val tree = input.decodeJsonElement()
-            if (!input.reader.isDone) { error("Reader has not consumed the whole input: ${input.reader}") }
+            reader.expectEof()
             readJson(tree, deserializer)
         }
     }

formats/json/commonTest/src/kotlinx/serialization/json/serializers/JsonNullSerializerTest.kt

@@ -5,6 +5,7 @@
 package kotlinx.serialization.json.serializers
 
 import kotlinx.serialization.json.*
+import kotlinx.serialization.json.internal.*
 import kotlinx.serialization.test.assertStringFormAndRestored
 import kotlin.test.*
 
@@ -15,6 +16,12 @@ class JsonNullSerializerTest : JsonTestBase() {
         assertStringFormAndRestored("{\"element\":null}", JsonNullWrapper(JsonNull), JsonNullWrapper.serializer())
     }
 
+    @Test
+    fun testJsonNullFailure() = parametrizedTest(default) {
+        val t = assertFailsWith<JsonException> { default.decodeFromString(JsonNullWrapper.serializer(), "{\"element\":\"foo\"}", true) }
+        assertTrue { t.message!!.contains("'null' literal") }
+    }
+
     @Test
     fun testJsonNullAsElement() = parametrizedTest(default) {
         assertStringFormAndRestored("{\"element\":null}", JsonElementWrapper(JsonNull), JsonElementWrapper.serializer())

formats/json/commonTest/src/kotlinx/serialization/json/serializers/JsonObjectSerializerTest.kt

@@ -66,7 +66,7 @@ class JsonObjectSerializerTest : JsonTestBase() {
     fun testInvalidObject() = parametrizedTest { useStreaming ->
         assertFailsWith<JsonDecodingException> { default.decodeFromString(JsonObjectSerializer, "{\"a\":\"b\"]", false) }
         assertFailsWith<JsonDecodingException> { default.decodeFromString(JsonObjectSerializer, "{", useStreaming) }
-        assertFailsWith<IllegalStateException> { default.decodeFromString(JsonObjectSerializer, "{}}", useStreaming) }
+        assertFailsWith<JsonDecodingException> { default.decodeFromString(JsonObjectSerializer, "{}}", useStreaming) }
         assertFailsWith<JsonDecodingException> { default.decodeFromString(JsonObjectSerializer, "{]", useStreaming) }
     }
 

guide/test/BasicSerializationTest.kt

@@ -101,7 +101,8 @@ class BasicSerializationTest {
     @Test
     fun testExampleClasses11() {
         captureOutput("ExampleClasses11") { example.exampleClasses11.main() }.verifyOutputLinesStart(
-            "Exception in thread \"main\" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected quotation mark '\"', but had 'n' instead"
+            "Exception in thread \"main\" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 52: Expected string literal but 'null' literal was found.",
+            "Use 'coerceInputValues = true' in 'Json {}` builder to coerce nulls to default values."
         )
     }