Skip to content

Commit

Permalink
Merge pull request #233 from Chuckame/feat/duration-logical-type
Browse files Browse the repository at this point in the history
feat: Add duration logical type
  • Loading branch information
Chuckame authored Jul 10, 2024
2 parents 976dee7 + 5df37fb commit ad17bb3
Show file tree
Hide file tree
Showing 38 changed files with 1,300 additions and 692 deletions.
142 changes: 122 additions & 20 deletions api/avro4k-core.api

Large diffs are not rendered by default.

72 changes: 29 additions & 43 deletions src/main/kotlin/com/github/avrokotlin/avro4k/Annotations.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,18 @@

package com.github.avrokotlin.avro4k

import com.github.avrokotlin.avro4k.internal.asAvroLogicalType
import com.github.avrokotlin.avro4k.internal.nonNullSerialName
import com.github.avrokotlin.avro4k.serializer.BigDecimalSerializer
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerialInfo
import kotlinx.serialization.descriptors.SerialDescriptor
import org.apache.avro.LogicalType
import org.intellij.lang.annotations.Language

/**
* Adds a property to the Avro schema or field. Its value could be any valid JSON or just a string.
*
* When annotated on a value class or its underlying field, the props are applied to the underlying type.
*
* Only works with classes (data, enum & object types) and class properties (not enum values).
* Fails at runtime when used in value classes wrapping a named schema (fixed, enum or record).
*/
@SerialInfo
@Repeatable
Expand All @@ -24,26 +23,13 @@ public annotation class AvroProp(
@Language("JSON") val value: String,
)

/**
* To be used with [BigDecimalSerializer] to specify the scale, precision, type and rounding mode of the decimal value.
*
* Can be used with [AvroFixed] to serialize value as a fixed type.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroDecimal(
val scale: Int = 2,
val precision: Int = 8,
)

/**
* Adds documentation to:
* - a record's field
* - a record
* - an enum
* - a record's field when annotated on a data class property
* - a record when annotated on a data class or object
* - an enum type when annotated on an enum class
*
* Ignored in inline classes.
* Only works with classes (data, enum & object types) and class properties (not enum values). Ignored in value classes.
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
Expand All @@ -52,16 +38,34 @@ public annotation class AvroDoc(val value: String)
/**
* Adds aliases to a field of a record. It helps to allow having different names for the same field for better compatibility when changing a schema.
*
* Ignored in inline classes.
* Only works with classes (data, enum & object types) and class properties (not enum values). Ignored in value classes.
*
* @param value The aliases for the annotated property. Note that the given aliases won't be changed by the configured [AvroConfiguration.fieldNamingStrategy].
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
public annotation class AvroAlias(vararg val value: String)

/**
* To be used with [BigDecimalSerializer] to specify the scale, precision, type and rounding mode of the decimal value.
*
* Can be used with [AvroFixed] to serialize value as a fixed type.
*
* Only works with [java.math.BigDecimal] property type.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroDecimal(
val scale: Int = 2,
val precision: Int = 8,
)

/**
* Indicates that the annotated property should be encoded as an Avro fixed type.
*
* Only works with [ByteArray], [String] and [java.math.BigDecimal] property types.
*
* @param size The number of bytes of the fixed type. Note that smaller values will be padded with 0s during encoding, but not unpadded when decoding.
*/
@SerialInfo
Expand All @@ -76,7 +80,7 @@ public annotation class AvroFixed(val size: Int)
* - Nulls have to be represented as a json `null`. To set the string `"null"`, don't forget to quote the string, example: `""""null""""` or `"\"null\""`.
* - Any non json content will be treated as a string
*
* Ignored in inline classes.
* Only works with data class properties (not enum values). Ignored in value classes.
*/
@SerialInfo
@Target(AnnotationTarget.PROPERTY)
Expand All @@ -87,27 +91,9 @@ public annotation class AvroDefault(
/**
* Sets the enum default value when decoded an unknown enum value.
*
* It must be annotated on an enum value. Otherwise, it will be ignored.
* Only works with enum classes.
*/
@SerialInfo
@ExperimentalSerializationApi
@Target(AnnotationTarget.PROPERTY)
public annotation class AvroEnumDefault

/**
* Adds a logical type to the given serializer, where the logical type name is the descriptor's name.
*
* To use it:
* ```kotlin
* object YourCustomLogicalTypeSerializer : KSerializer<YourType> {
* override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("YourType", PrimitiveKind.STRING)
* .asAvroLogicalType()
* }
* ```
*
* For more complex needs, please file an feature request [here](https://github.com/avro-kotlin/avro4k/issues).
*/
@ExperimentalSerializationApi
public fun SerialDescriptor.asAvroLogicalType(): SerialDescriptor {
return asAvroLogicalType { LogicalType(nonNullSerialName) }
}
public annotation class AvroEnumDefault
24 changes: 5 additions & 19 deletions src/main/kotlin/com/github/avrokotlin/avro4k/Avro.kt
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,8 @@ import com.github.avrokotlin.avro4k.internal.EnumResolver
import com.github.avrokotlin.avro4k.internal.PolymorphicResolver
import com.github.avrokotlin.avro4k.internal.RecordResolver
import com.github.avrokotlin.avro4k.internal.schema.ValueVisitor
import com.github.avrokotlin.avro4k.serializer.BigDecimalSerializer
import com.github.avrokotlin.avro4k.serializer.BigIntegerSerializer
import com.github.avrokotlin.avro4k.serializer.InstantSerializer
import com.github.avrokotlin.avro4k.serializer.LocalDateSerializer
import com.github.avrokotlin.avro4k.serializer.LocalDateTimeSerializer
import com.github.avrokotlin.avro4k.serializer.LocalTimeSerializer
import com.github.avrokotlin.avro4k.serializer.URLSerializer
import com.github.avrokotlin.avro4k.serializer.UUIDSerializer
import com.github.avrokotlin.avro4k.serializer.JavaStdLibSerializersModule
import com.github.avrokotlin.avro4k.serializer.JavaTimeSerializersModule
import kotlinx.serialization.BinaryFormat
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.ExperimentalSerializationApi
Expand All @@ -21,8 +15,8 @@ import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.modules.EmptySerializersModule
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.contextual
import kotlinx.serialization.modules.overwriteWith
import kotlinx.serialization.modules.plus
import kotlinx.serialization.serializer
import okio.Buffer
import org.apache.avro.Schema
Expand All @@ -47,16 +41,8 @@ public sealed class Avro(

public companion object Default : Avro(
AvroConfiguration(),
SerializersModule {
contextual(UUIDSerializer)
contextual(URLSerializer)
contextual(BigIntegerSerializer)
contextual(BigDecimalSerializer)
contextual(InstantSerializer)
contextual(LocalDateSerializer)
contextual(LocalTimeSerializer)
contextual(LocalDateTimeSerializer)
}
JavaStdLibSerializersModule +
JavaTimeSerializersModule
)

public fun schema(descriptor: SerialDescriptor): Schema {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
package com.github.avrokotlin.avro4k.internal

import com.github.avrokotlin.avro4k.AvroDecoder
import com.github.avrokotlin.avro4k.AvroEncoder
import com.github.avrokotlin.avro4k.internal.decoder.direct.AbstractAvroDirectDecoder
import com.github.avrokotlin.avro4k.serializer.AvroDuration
import com.github.avrokotlin.avro4k.serializer.AvroDurationSerializer
import com.github.avrokotlin.avro4k.serializer.AvroSerializer
import com.github.avrokotlin.avro4k.serializer.SchemaSupplierContext
import com.github.avrokotlin.avro4k.serializer.SerialDescriptorWithAvroSchemaDelegate
import com.github.avrokotlin.avro4k.serializer.createSchema
import com.github.avrokotlin.avro4k.serializer.fixed
import kotlinx.serialization.DeserializationStrategy
import kotlinx.serialization.InternalSerializationApi
import kotlinx.serialization.KSerializer
import kotlinx.serialization.SerializationException
import kotlinx.serialization.SerializationStrategy
import kotlinx.serialization.builtins.ByteArraySerializer
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.descriptors.StructureKind
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.internal.AbstractCollectionSerializer
import org.apache.avro.Schema
import kotlin.io.encoding.Base64
import kotlin.io.encoding.ExperimentalEncodingApi
import kotlin.time.Duration
import kotlin.time.Duration.Companion.days
import kotlin.time.Duration.Companion.milliseconds

/**
* This middleware is here to intercept some native types like kotlin Duration or ByteArray as we want to apply some
* specific rules on them for generating custom schemas or having specific serialization strategies.
*/
@Suppress("UNCHECKED_CAST")
internal object SerializerLocatorMiddleware {
fun <T> apply(serializer: SerializationStrategy<T>): SerializationStrategy<T> {
return when {
serializer === ByteArraySerializer() -> AvroByteArraySerializer
serializer === Duration.serializer() -> KotlinDurationSerializer
else -> serializer
} as SerializationStrategy<T>
}

@OptIn(InternalSerializationApi::class)
fun <T> apply(deserializer: DeserializationStrategy<T>): DeserializationStrategy<T> {
return when {
deserializer === ByteArraySerializer() -> AvroByteArraySerializer
deserializer === Duration.serializer() -> KotlinDurationSerializer
deserializer is AbstractCollectionSerializer<*, T, *> -> AvroCollectionSerializer(deserializer)
else -> deserializer
} as DeserializationStrategy<T>
}

fun apply(descriptor: SerialDescriptor): SerialDescriptor {
return when {
descriptor.isCollectionOfBytes() -> SerialDescriptorWithAvroSchemaDelegate(descriptor, AvroByteArraySerializer)
descriptor == String.serializer().descriptor -> AvroStringSerialDescriptor
descriptor == Duration.serializer().descriptor -> KotlinDurationSerializer.descriptor
else -> descriptor
}
}

private fun SerialDescriptor.isCollectionOfBytes() = kind === StructureKind.LIST && elementsCount == 1 && getElementDescriptor(0).kind === PrimitiveKind.BYTE
}

private val AvroStringSerialDescriptor: SerialDescriptor =
SerialDescriptorWithAvroSchemaDelegate(String.serializer().descriptor) { context ->
context.fixed?.createSchema() ?: Schema.create(Schema.Type.STRING)
}

private object KotlinDurationSerializer : AvroSerializer<Duration>(Duration::class.qualifiedName!!) {
private const val MILLIS_PER_DAY = 1000 * 60 * 60 * 24

override fun getSchema(context: SchemaSupplierContext): Schema {
return AvroDurationSerializer.DURATION_SCHEMA
}

override fun serializeAvro(
encoder: AvroEncoder,
value: Duration,
) {
AvroDurationSerializer.serializeAvro(encoder, value.toAvroDuration())
}

override fun deserializeAvro(decoder: AvroDecoder): Duration {
return AvroDurationSerializer.deserializeAvro(decoder).toKotlinDuration()
}

override fun serializeGeneric(
encoder: Encoder,
value: Duration,
) {
encoder.encodeString(value.toString())
}

override fun deserializeGeneric(decoder: Decoder): Duration {
return Duration.parse(decoder.decodeString())
}

private fun AvroDuration.toKotlinDuration(): Duration {
if (months == UInt.MAX_VALUE && days == UInt.MAX_VALUE && millis == UInt.MAX_VALUE) {
return Duration.INFINITE
}
if (months != 0u) {
throw SerializationException("java.time.Duration cannot contains months")
}
return days.toLong().days + millis.toLong().milliseconds
}

private fun Duration.toAvroDuration(): AvroDuration {
if (isNegative()) {
throw SerializationException("${Duration::class.qualifiedName} cannot be converted to ${AvroDuration::class.qualifiedName} as it cannot be negative")
}
if (isInfinite()) {
return AvroDuration(
months = UInt.MAX_VALUE,
days = UInt.MAX_VALUE,
millis = UInt.MAX_VALUE
)
}
val millis = inWholeMilliseconds
return AvroDuration(
months = 0u,
days = (millis / MILLIS_PER_DAY).toUInt(),
millis = (millis % MILLIS_PER_DAY).toUInt()
)
}
}

private object AvroByteArraySerializer : AvroSerializer<ByteArray>(ByteArray::class.qualifiedName!!) {
override fun getSchema(context: SchemaSupplierContext): Schema {
return context.fixed?.createSchema() ?: Schema.create(Schema.Type.BYTES)
}

override fun serializeAvro(
encoder: AvroEncoder,
value: ByteArray,
) {
// encoding related to the type (fixed or bytes) is handled in AvroEncoder
encoder.encodeBytes(value)
}

override fun deserializeAvro(decoder: AvroDecoder): ByteArray {
// decoding related to the type (fixed or bytes) is handled in AvroDecoder
return decoder.decodeBytes()
}

@OptIn(ExperimentalEncodingApi::class)
override fun serializeGeneric(
encoder: Encoder,
value: ByteArray,
) {
encoder.encodeString(Base64.Mime.encode(value))
}

@OptIn(ExperimentalEncodingApi::class)
override fun deserializeGeneric(decoder: Decoder): ByteArray {
return Base64.Mime.decode(decoder.decodeString())
}
}

@OptIn(InternalSerializationApi::class)
internal class AvroCollectionSerializer<T>(private val original: AbstractCollectionSerializer<*, T, *>) : KSerializer<T> {
override val descriptor: SerialDescriptor
get() = original.descriptor

override fun deserialize(decoder: Decoder): T {
if (decoder is AbstractAvroDirectDecoder) {
var result: T? = null
decoder.decodedCollectionSize = -1
do {
result = original.merge(decoder, result)
} while (decoder.decodedCollectionSize > 0)
return result!!
}
return original.deserialize(decoder)
}

override fun serialize(
encoder: Encoder,
value: T,
) {
original.serialize(encoder, value)
}
}
Loading

0 comments on commit ad17bb3

Please sign in to comment.