Rewrite and restructure JSON parser (#1343)

* Get rid of spontaneous lookahead
* Spill fewer variables into object state
* Optimize token and whitespaces reading
* Separate fast and slow paths
* Add optimistic key consumption optimization to leverage indexOf intrinsic
* Parse numbers and booleans without allocations
* Improve exception messages in few places
* Optimize JsonReader further -- don't make conditional array lookups where they are not really necessary
* Make consumeString inliner-friendly

* Optimize strings handling

    * Replace handrolled char-array with StringBuilder, tweak it here and there
    * Always use optimistic path for strings
    * Properly handle slow-path from the middle of a string

* Remove no longer used WriteMode fields
* Rename JsonReader to JsonLExer and JsonParser to JsonTreeReader

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/CoerceInputValuesBenchmark.kt

@@ -16,8 +16,6 @@ import java.util.concurrent.*
 @Fork(2)
 open class CoerceInputValuesBenchmark {
 
-    // Specific benchmark to isolate effect on #1156. Remove after release of 1.0.1
-
     @Serializable
     class Holder(
         val i1: Int,

benchmark/src/jmh/kotlin/kotlinx/benchmarks/json/PrimitiveValuesBenchmark.kt

@@ -59,4 +59,3 @@ open class PrimitiveValuesBenchmark {
     @Benchmark
     fun encodeLong(): LongHolder = Json.decodeFromString(LongHolder.serializer(), longValue)
 }
-

core/commonMain/src/kotlinx/serialization/encoding/Decoding.kt

@@ -137,6 +137,9 @@ public interface Decoder {
 
     /**
      * Decodes the `null` value and returns it.
+     *
+     * It is expected that `decodeNotNullMark` was called
+     * prior to `decodeNull` invocation and the case when it returned `true` was handled.
      */
     @ExperimentalSerializationApi
     public fun decodeNull(): Nothing?

core/commonMain/src/kotlinx/serialization/internal/PluginGeneratedSerialDescriptor.kt

@@ -31,10 +31,9 @@ internal open class PluginGeneratedSerialDescriptor(
     private val elementsOptionality = BooleanArray(elementsCount)
     public override val serialNames: Set<String> get() = indices.keys
 
-    // don't change lazy mode: KT-32871, KT-32872
-    private val indices: Map<String, Int> by lazy { buildIndices() }
+    private var indices: Map<String, Int> = emptyMap()
     // Cache child serializers, they are not cached by the implementation for nullable types
-    private val childSerializers by lazy { generatedSerializer?.childSerializers() ?: emptyArray() }
+    private val childSerializers: Array<KSerializer<*>> by lazy { generatedSerializer?.childSerializers() ?: emptyArray() }
 
     // Lazy because of JS specific initialization order (#789)
     internal val typeParameterDescriptors: Array<SerialDescriptor> by lazy {
@@ -48,6 +47,9 @@ internal open class PluginGeneratedSerialDescriptor(
         names[++added] = name
         elementsOptionality[added] = isOptional
         propertiesAnnotations[added] = null
+        if (added == elementsCount - 1) {
+            indices = buildIndices()
+        }
     }
 
     public fun pushAnnotation(annotation: Annotation) {

docs/basic-serialization.md

@@ -411,7 +411,7 @@ Attempts to explicitly specify its value in the serial format, even if the speci
 value is equal to the default one, produces the following exception.
 
 ```text
-Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 60: Encountered an unknown key 'language'.
+Exception in thread "main" kotlinx.serialization.json.internal.JsonDecodingException: Unexpected JSON token at offset 42: Encountered an unknown key 'language'.
 Use 'ignoreUnknownKeys = true' in 'Json {}' builder to ignore unknown keys.
 ```   
 

formats/json/commonMain/src/kotlinx/serialization/json/Json.kt

@@ -83,10 +83,10 @@ public sealed class Json(internal val configuration: JsonConf) : StringFormat {
      * @throws [SerializationException] if the given JSON string cannot be deserialized to the value of type [T].
      */
     public final override fun <T> decodeFromString(deserializer: DeserializationStrategy<T>, string: String): T {
-        val reader = JsonReader(string)
-        val input = StreamingJsonDecoder(this, WriteMode.OBJ, reader)
+        val lexer = JsonLexer(string)
+        val input = StreamingJsonDecoder(this, WriteMode.OBJ, lexer)
         val result = input.decodeSerializableValue(deserializer)
-        if (!reader.isDone) { error("Reader has not consumed the whole input: $reader") }
+        lexer.expectEof()
         return result
     }
     /**

formats/json/commonMain/src/kotlinx/serialization/json/JsonElementSerializers.kt

@@ -96,6 +96,9 @@ internal object JsonNullSerializer : KSerializer<JsonNull> {
 
     override fun deserialize(decoder: Decoder): JsonNull {
         verify(decoder)
+        if (decoder.decodeNotNullMark()) {
+            throw JsonDecodingException("Expected 'null' literal")
+        }
         decoder.decodeNull()
         return JsonNull
     }

formats/json/commonMain/src/kotlinx/serialization/json/internal/JsonExceptions.kt

@@ -45,7 +45,7 @@ internal fun InvalidFloatingPointDecoded(value: Number, key: String, output: Str
     JsonDecodingException(-1, unexpectedFpErrorMessage(value, key, output))
 
 // Extension on JSON reader and fail immediately
-internal fun JsonReader.throwInvalidFloatingPointDecoded(result: Number): Nothing {
+internal fun JsonLexer.throwInvalidFloatingPointDecoded(result: Number): Nothing {
     fail("Unexpected special floating-point value $result. By default, " +
             "non-finite floating point values are prohibited because they do not conform JSON specification. " +
             specialFlowingValuesHint