Skip to content

Commit

Permalink
Fix performance problems when compiling generated code with Kotlin 1.4
Browse files Browse the repository at this point in the history 
When a protobuf message definition contains a lot of fields/oneofs, the
generated Kotlin code cannot be compiled by a Kotlin 1.4 compiler. The
compiler runs out of memory when compiling the long list of
`FieldDescriptor`s that gets passed to the `MessageDescriptor`. This
change implements a workaround that avoids the Kotlin compiler bug by
initializing the list of `FieldDescriptor`s outside of the
`MessageDescriptor` constructor.

Also add a test proto file to catch regressions with large protobuf
messages.

Fixes #94
  • Loading branch information
@garyp
garyp committed Oct 23, 2020
1 parent 9f6e491 commit 1b1bf61
Show file tree
Hide file tree
Showing 3 changed files with 3,092 additions and 35 deletions.
69 changes: 44 additions & 25 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,51 +105,62 @@ data class Person(
override fun decodeWith(u: pbandk.MessageDecoder) = Person.decodeWithImpl(u)

override val descriptors: pbandk.MessageDescriptor<Person> by lazy {
pbandk.MessageDescriptor(
messageClass = Person::class,
messageCompanion = this,
fields = listOf(
val fieldsList = ArrayList<pbandk.FieldDescriptor<Person, *>>(5).apply {
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "name",
number = 1,
type = pbandk.FieldDescriptor.Type.Primitive.String(),
jsonName = "name",
value = Person::name
),
)
)
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "id",
number = 2,
type = pbandk.FieldDescriptor.Type.Primitive.Int32(),
jsonName = "id",
value = Person::id
),
)
)
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "email",
number = 3,
type = pbandk.FieldDescriptor.Type.Primitive.String(),
jsonName = "email",
value = Person::email
),
)
)
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "phones",
number = 4,
type = pbandk.FieldDescriptor.Type.Repeated<tutorial.Person.PhoneNumber>(valueType = pbandk.FieldDescriptor.Type.Message(messageCompanion = tutorial.Person.PhoneNumber.Companion)),
jsonName = "phones",
value = Person::phones
),
)
)
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "last_updated",
number = 5,
type = pbandk.FieldDescriptor.Type.Message(messageCompanion = pbandk.wkt.Timestamp.Companion),
jsonName = "lastUpdated",
value = Person::lastUpdated
)
)
}
pbandk.MessageDescriptor(
messageClass = Person::class,
messageCompanion = this,
fields = fieldsList
)
}
}
Expand Down Expand Up @@ -184,27 +195,32 @@ data class Person(
override fun decodeWith(u: pbandk.MessageDecoder) = Person.PhoneNumber.decodeWithImpl(u)

override val descriptor: pbandk.MessageDescriptor<PhoneNumber> by lazy {
pbandk.MessageDescriptor(
messageClass = Person.PhoneNumber::class,
messageCompanion = this,
fields = listOf(
val fieldsList = ArrayList<pbandk.FieldDescriptor<PhoneNumber, *>>(2).apply {
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "number",
number = 1,
type = pbandk.FieldDescriptor.Type.Primitive.String(),
jsonName = "number",
value = PhoneNumber::number
),
)
)
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "type",
number = 2,
type = pbandk.FieldDescriptor.Type.Enum(enumCompanion = tutorial.Person.PhoneType.Companion),
jsonName = "type",
value = PhoneNumber::type
)
)
}
pbandk.MessageDescriptor(
messageClass = Person.PhoneNumber::class,
messageCompanion = this,
fields = fieldsList
)
}
}
Expand All @@ -223,19 +239,22 @@ data class AddressBook(
override fun decodeWith(u: pbandk.MessageDecoder) = AddressBook.decodeWithImpl(u)

override val descriptor: pbandk.MessageDescriptor<AddressBook> by lazy {
pbandk.MessageDescriptor(
messageClass = AddressBook::class,
messageCompanion = this,
fields = listOf(
val fieldsList = ArrayList<pbandk.FieldDescriptor<AddressBook, *>>(1).apply {
add(
pbandk.FieldDescriptor(
messageDescriptor = this::descriptor,
messageDescriptor = this@Companion::descriptor,
name = "people",
number = 1,
type = pbandk.FieldDescriptor.Type.Repeated<tutorial.Person>(valueType = pbandk.FieldDescriptor.Type.Message(messageCompanion = tutorial.Person.Companion)),
jsonName = "people",
value = AddressBook::people
)
)
}
pbandk.MessageDescriptor(
messageClass = AddressBook::class,
messageCompanion = this,
fields = fieldsList
)
}
}
Expand Down
29 changes: 19 additions & 10 deletions protoc-gen-kotlin/lib/src/commonMain/kotlin/pbandk/gen/CodeGenerator.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,23 +122,32 @@ open class CodeGenerator(
// https://kotlinlang.org/docs/reference/native/concurrency.html#global-variables-and-singletons. In order to
// break the circular references, `descriptor` needs to be a `lazy` field.
line("override val descriptor: pbandk.MessageDescriptor<$fullTypeName> by lazy {").indented {
line("pbandk.MessageDescriptor(").indented {
line("messageClass = $fullTypeName::class,")
line("messageCompanion = this,")
line("fields = listOf(").indented {
val allFields = type.sortedStandardFieldsWithOneOfs()
allFields.forEachIndexed { i, (field, oneof) ->
// XXX: When a message has lots of fields (e.g. `TestAllTypesProto3`), declaring the list of field
// descriptors directly in the [MessageDescriptor] constructor can cause a
// `java.lang.OutOfMemoryError: Java heap space` error in the Kotlin compiler (as of Kotlin 1.4.10).
// As a workaround, we declare the list of fields as a separate variable rather than inline in the
// constructor. We also use a list builder rather than `listOf()` in order to have lots of small statements
// instead of a giant expression (which the compiler struggles with).
val allFields = type.sortedStandardFieldsWithOneOfs()
line("val fieldsList = ArrayList<pbandk.FieldDescriptor<$fullTypeName, *>>(${allFields.size}).apply {").indented {
allFields.forEach { (field, oneof) ->
line("add(").indented {
line("pbandk.FieldDescriptor(").indented {
line("messageDescriptor = this::descriptor,")
line("messageDescriptor = this@Companion::descriptor,")
line("name = \"${field.name}\",")
line("number = ${field.number},")
line("type = ${field.fieldDescriptorType(oneof != null)},")
oneof?.let { line("oneofMember = true,") }
field.jsonName?.let { line("jsonName = \"$it\",") }
line("value = $fullTypeName::${field.kotlinFieldName}")
}.line(if (i == allFields.lastIndex) ")" else "),")
}
}.line(")")
}.line(")")
}.line(")")
}
}.line("}")
line("pbandk.MessageDescriptor(").indented {
line("messageClass = $fullTypeName::class,")
line("messageCompanion = this,")
line("fields = fieldsList")
}.line(")")
}.line("}")
}
Expand Down
Loading

0 comments on commit 1b1bf61

Please sign in to comment.