Skip to content

Commit

Permalink
ArrowWriter.Companion.Mode -> ArrowWriter.Mode
Browse files Browse the repository at this point in the history
  • Loading branch information
Jolanrensen committed Dec 19, 2022
1 parent 407c60a commit 62adaff
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -41,25 +41,25 @@ public interface ArrowWriter : AutoCloseable {
dataFrame: AnyFrame,
targetSchema: Schema,
mode: Mode,
mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage
mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage,
): ArrowWriter = ArrowWriterImpl(dataFrame, targetSchema, mode, mismatchSubscriber)
}

/**
* If [restrictWidening] is true, [dataFrame] columns not described in [targetSchema] would not be saved (otherwise, would be saved as is).
* If [restrictNarrowing] is true, [targetSchema] fields that are not nullable and do not exist in [dataFrame] will produce exception (otherwise, would not be saved).
* If [strictType] is true, [dataFrame] columns described in [targetSchema] with non-compatible type will produce exception (otherwise, would be saved as is).
* If [strictNullable] is true, [targetSchema] fields that are not nullable and contain nulls in [dataFrame] will produce exception (otherwise, would be saved as is with nullable = true).
*/
public data class Mode(
public val restrictWidening: Boolean,
public val restrictNarrowing: Boolean,
public val strictType: Boolean,
public val strictNullable: Boolean
) {
public companion object {
public val STRICT: Mode = Mode(true, true, true, true)
public val LOYAL: Mode = Mode(false, false, false, false)
}
/**
* If [restrictWidening] is true, [dataFrame] columns not described in [targetSchema] would not be saved (otherwise, would be saved as is).
* If [restrictNarrowing] is true, [targetSchema] fields that are not nullable and do not exist in [dataFrame] will produce exception (otherwise, would not be saved).
* If [strictType] is true, [dataFrame] columns described in [targetSchema] with non-compatible type will produce exception (otherwise, would be saved as is).
* If [strictNullable] is true, [targetSchema] fields that are not nullable and contain nulls in [dataFrame] will produce exception (otherwise, would be saved as is with nullable = true).
*/
public data class Mode(
public val restrictWidening: Boolean,
public val restrictNarrowing: Boolean,
public val strictType: Boolean,
public val strictNullable: Boolean,
) {
public companion object {
public val STRICT: Mode = Mode(true, true, true, true)
public val LOYAL: Mode = Mode(false, false, false, false)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import org.apache.arrow.vector.types.pojo.Schema
import org.apache.arrow.vector.util.Text
import org.jetbrains.kotlinx.dataframe.AnyCol
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.name
import org.jetbrains.kotlinx.dataframe.api.convertToBigDecimal
import org.jetbrains.kotlinx.dataframe.api.convertToBoolean
import org.jetbrains.kotlinx.dataframe.api.convertToByte
Expand All @@ -53,6 +52,7 @@ import org.jetbrains.kotlinx.dataframe.api.forEachIndexed
import org.jetbrains.kotlinx.dataframe.api.map
import org.jetbrains.kotlinx.dataframe.exceptions.CellConversionException
import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
import org.jetbrains.kotlinx.dataframe.name

/**
* Save [dataFrame] content in Apache Arrow format (can be written to File, ByteArray, OutputStream or raw Channel) with [targetSchema].
Expand All @@ -61,8 +61,8 @@ import org.jetbrains.kotlinx.dataframe.exceptions.TypeConverterNotFoundException
internal class ArrowWriterImpl(
override val dataFrame: DataFrame<*>,
override val targetSchema: Schema,
override val mode: ArrowWriter.Companion.Mode,
override val mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage
override val mode: ArrowWriter.Mode,
override val mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage,
) : ArrowWriter {

private val allocator = RootAllocator()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ public fun AnyFrame.arrowWriter(): ArrowWriter = this.arrowWriter(this.columns()
*/
public fun AnyFrame.arrowWriter(
targetSchema: Schema,
mode: ArrowWriter.Companion.Mode = ArrowWriter.Companion.Mode.STRICT,
mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage
mode: ArrowWriter.Mode = ArrowWriter.Mode.STRICT,
mismatchSubscriber: (ConvertingMismatch) -> Unit = ignoreMismatchMessage,
): ArrowWriter = ArrowWriter.create(this, targetSchema, mode, mismatchSubscriber)

// IPC saving block with default parameters
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,14 +142,14 @@ internal class ArrowKtTest {
val warnings = ArrayList<ConvertingMismatch>()
val testRestrictWidening = citiesExampleFrame.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode.STRICT
ArrowWriter.Mode.STRICT
) { warning -> warnings.add(warning) }.use { it.saveArrowFeatherToByteArray() }
warnings.shouldContain(ConvertingMismatch.WideningMismatch.RejectedColumn("page_in_wiki"))
shouldThrow<IllegalArgumentException> { DataFrame.readArrowFeather(testRestrictWidening)["page_in_wiki"] }

val testAllowWidening = citiesExampleFrame.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode(
ArrowWriter.Mode(
restrictWidening = false,
restrictNarrowing = true,
strictType = true,
Expand All @@ -165,15 +165,15 @@ internal class ArrowKtTest {

frameWithoutRequiredField.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode.STRICT
ArrowWriter.Mode.STRICT
).use {
shouldThrow<ConvertingException> { it.saveArrowFeatherToByteArray() }
}

val warnings = ArrayList<ConvertingMismatch>()
val testAllowNarrowing = frameWithoutRequiredField.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode(
ArrowWriter.Mode(
restrictWidening = true,
restrictNarrowing = false,
strictType = true,
Expand All @@ -191,15 +191,15 @@ internal class ArrowKtTest {

frameWithIncompatibleField.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode.STRICT
ArrowWriter.Mode.STRICT
).use {
shouldThrow<ConvertingException> { it.saveArrowFeatherToByteArray() }
}

val warnings = ArrayList<ConvertingMismatch>()
val testLoyalType = frameWithIncompatibleField.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode(
ArrowWriter.Mode(
restrictWidening = true,
restrictNarrowing = true,
strictType = false,
Expand All @@ -219,15 +219,15 @@ internal class ArrowKtTest {

frameWithNulls.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode.STRICT
ArrowWriter.Mode.STRICT
).use {
shouldThrow<ConvertingException> { it.saveArrowFeatherToByteArray() }
}

val warnings = ArrayList<ConvertingMismatch>()
val testLoyalNullable = frameWithNulls.arrowWriter(
Schema.fromJSON(citiesExampleSchema),
ArrowWriter.Companion.Mode(
ArrowWriter.Mode(
restrictWidening = true,
restrictNarrowing = true,
strictType = true,
Expand Down
8 changes: 4 additions & 4 deletions docs/StardustDocs/topics/write.md
Original file line number Diff line number Diff line change
Expand Up @@ -176,16 +176,16 @@ df.arrowWriter(
// Specify your schema
schema,
// Specify desired behavior mode
ArrowWriter.Companion.Mode(
ArrowWriter.Mode(
restrictWidening = true,
restrictNarrowing = true,
strictType = true,
strictNullable = false
),
// Specify mismatch subscriber
writeMismatchMessage
writeMismatchMessage,
).use { writer ->
// Save to any format and sink, like in previous example
// Save to any format and sink, like in the previous example
writer.writeArrowFeather(file)
}
```
Expand All @@ -196,4 +196,4 @@ On executing you should get two warnings:
and

>Column "isHappy" is not described in target schema and was ignored
> Column "isHappy" is not described in the target schema and was ignored

0 comments on commit 62adaff

Please sign in to comment.