Enable Doc processor plugin

build.gradle.kts

@@ -14,6 +14,8 @@ plugins {
     id("org.jetbrains.dokka") version libs.versions.dokka
     id("org.jetbrains.kotlinx.kover") version libs.versions.kover
     id("org.jmailen.kotlinter") version libs.versions.ktlint
+    id("nl.jolanrensen.docProcessor") version libs.versions.docProcessor apply false
+    id("xyz.ronella.simple-git") version libs.versions.simpleGit apply false
 }
 
 val jupyterApiTCRepo: String by project

core/build.gradle.kts

@@ -1,3 +1,8 @@
+import nl.jolanrensen.docProcessor.defaultProcessors.*
+import nl.jolanrensen.docProcessor.gradle.creatingProcessDocTask
+import org.gradle.jvm.tasks.Jar
+import xyz.ronella.gradle.plugin.simple.git.task.GitTask
+
 @Suppress("DSL_SCOPE_VIOLATION", "UnstableApiUsage")
 plugins {
     kotlin("jvm")
@@ -10,6 +15,9 @@ plugins {
     id("org.jetbrains.kotlinx.kover")
     id("org.jmailen.kotlinter")
     id("org.jetbrains.kotlinx.dataframe")
+    id("nl.jolanrensen.docProcessor")
+    id("xyz.ronella.simple-git")
+    idea
 }
 
 group = "org.jetbrains.kotlinx"
@@ -52,6 +60,66 @@ kotlin.sourceSets {
     }
 }
 
+val generatedSourcesFolderName = "generated-sources"
+val addGeneratedSourcesToGit by tasks.creating(GitTask::class) {
+    directory.set(file("."))
+    command.set("add")
+    args.set(listOf("-A", generatedSourcesFolderName))
+}
+
+// Backup the kotlin source files location
+val kotlinMainSources = kotlin.sourceSets.main.get().kotlin.sourceDirectories
+
+// Task to generate the processed documentation
+val processKDocsMain by creatingProcessDocTask(
+    sources = kotlinMainSources.filterNot { "build/generated" in it.path }, // Exclude generated sources
+) {
+    target = file(generatedSourcesFolderName)
+    processors = listOf(
+        INCLUDE_DOC_PROCESSOR,
+        INCLUDE_FILE_DOC_PROCESSOR,
+        INCLUDE_ARG_DOC_PROCESSOR,
+        COMMENT_DOC_PROCESSOR,
+        SAMPLE_DOC_PROCESSOR,
+    )
+
+    task {
+        doLast {
+            // ensure generated sources are added to git
+            addGeneratedSourcesToGit.executeCommand()
+        }
+    }
+}
+
+// Exclude the generated/processed sources from the IDE
+idea {
+    module {
+        excludeDirs.add(file(generatedSourcesFolderName))
+    }
+}
+
+// Modify all Jar tasks such that before running the Kotlin sources are set to
+// the target of processKdocMain and they are returned back to normal afterwards.
+tasks.withType<Jar> {
+    dependsOn(processKDocsMain)
+    outputs.upToDateWhen { false }
+
+    doFirst {
+        kotlin.sourceSets.main {
+            kotlin.setSrcDirs(
+                processKDocsMain.targets +
+                    kotlinMainSources.filter { "build/generated" in it.path } // Include generated sources (which were excluded above)
+            )
+        }
+    }
+
+    doLast {
+        kotlin.sourceSets.main {
+            kotlin.setSrcDirs(kotlinMainSources)
+        }
+    }
+}
+
 korro {
     docs = fileTree(rootProject.rootDir) {
         include("docs/StardustDocs/topics/*.md")
@@ -169,3 +237,13 @@ dataframes {
         name = "org.jetbrains.kotlinx.dataframe.samples.api.Repository"
     }
 }
+
+// If we want to use Dokka, make sure to use the preprocessed sources
+tasks.withType<org.jetbrains.dokka.gradle.AbstractDokkaLeafTask> {
+    dependsOn(processKDocsMain)
+    dokkaSourceSets {
+        all {
+            sourceRoot(processKDocsMain.target.get())
+        }
+    }
+}

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/ColumnsContainer.kt

@@ -0,0 +1,66 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlinx.dataframe.api.ColumnSelectionDsl
+import org.jetbrains.kotlinx.dataframe.api.asColumnGroup
+import org.jetbrains.kotlinx.dataframe.api.cast
+import org.jetbrains.kotlinx.dataframe.api.castFrameColumn
+import org.jetbrains.kotlinx.dataframe.api.getColumn
+import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
+import org.jetbrains.kotlinx.dataframe.columns.ColumnPath
+import org.jetbrains.kotlinx.dataframe.columns.ColumnReference
+import org.jetbrains.kotlinx.dataframe.columns.FrameColumn
+import org.jetbrains.kotlinx.dataframe.impl.columnName
+import org.jetbrains.kotlinx.dataframe.impl.columns.asAnyFrameColumn
+import kotlin.reflect.KProperty
+
+/**
+ * Provides access to [columns][DataColumn].
+ *
+ * Base interface for [DataFrame] and [ColumnSelectionDsl]
+ *
+ * @param T Schema marker. Used to generate extension properties for typed column access.
+ */
+public interface ColumnsContainer<out T> {
+
+    // region columns
+
+    public fun columns(): List<AnyCol>
+    public fun columnsCount(): Int
+    public fun containsColumn(name: String): Boolean
+    public fun containsColumn(path: ColumnPath): Boolean
+    public fun getColumnIndex(name: String): Int
+
+    // endregion
+
+    // region getColumnOrNull
+
+    public fun getColumnOrNull(name: String): AnyCol?
+    public fun getColumnOrNull(index: Int): AnyCol?
+    public fun <R> getColumnOrNull(column: ColumnReference<R>): DataColumn<R>?
+    public fun getColumnOrNull(path: ColumnPath): AnyCol?
+    public fun <R> getColumnOrNull(column: ColumnSelector<T, R>): DataColumn<R>?
+
+    // endregion
+
+    // region get
+
+    public operator fun get(columnName: String): AnyCol = getColumn(columnName)
+    public operator fun get(columnPath: ColumnPath): AnyCol = getColumn(columnPath)
+
+    public operator fun <R> get(column: DataColumn<R>): DataColumn<R> = getColumn(column.name()).cast()
+    public operator fun <R> get(column: DataColumn<DataRow<R>>): ColumnGroup<R> = getColumn(column)
+    public operator fun <R> get(column: DataColumn<DataFrame<R>>): FrameColumn<R> = getColumn(column)
+
+    public operator fun <R> get(column: ColumnReference<R>): DataColumn<R> = getColumn(column)
+    public operator fun <R> get(column: ColumnReference<DataRow<R>>): ColumnGroup<R> = getColumn(column)
+    public operator fun <R> get(column: ColumnReference<DataFrame<R>>): FrameColumn<R> = getColumn(column)
+
+    public operator fun <R> get(column: KProperty<R>): DataColumn<R> = get(column.columnName).cast()
+    public operator fun <R> get(column: KProperty<DataRow<R>>): ColumnGroup<R> = get(column.columnName).asColumnGroup().cast()
+    public operator fun <R> get(column: KProperty<DataFrame<R>>): FrameColumn<R> = get(column.columnName).asAnyFrameColumn().castFrameColumn()
+
+    public operator fun <C> get(columns: ColumnsSelector<T, C>): List<DataColumn<C>>
+    public operator fun <C> get(column: ColumnSelector<T, C>): DataColumn<C> = get(column as ColumnsSelector<T, C>).single()
+
+    // endregion
+}

core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/DataColumn.kt

@@ -0,0 +1,138 @@
+package org.jetbrains.kotlinx.dataframe
+
+import org.jetbrains.kotlinx.dataframe.api.Infer
+import org.jetbrains.kotlinx.dataframe.api.asDataColumn
+import org.jetbrains.kotlinx.dataframe.api.cast
+import org.jetbrains.kotlinx.dataframe.api.concat
+import org.jetbrains.kotlinx.dataframe.api.filter
+import org.jetbrains.kotlinx.dataframe.api.schema
+import org.jetbrains.kotlinx.dataframe.api.take
+import org.jetbrains.kotlinx.dataframe.columns.BaseColumn
+import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
+import org.jetbrains.kotlinx.dataframe.columns.ColumnKind
+import org.jetbrains.kotlinx.dataframe.columns.ColumnPath
+import org.jetbrains.kotlinx.dataframe.columns.ColumnResolutionContext
+import org.jetbrains.kotlinx.dataframe.columns.ColumnWithPath
+import org.jetbrains.kotlinx.dataframe.columns.FrameColumn
+import org.jetbrains.kotlinx.dataframe.columns.ValueColumn
+import org.jetbrains.kotlinx.dataframe.impl.columns.ColumnGroupImpl
+import org.jetbrains.kotlinx.dataframe.impl.columns.FrameColumnImpl
+import org.jetbrains.kotlinx.dataframe.impl.columns.ValueColumnImpl
+import org.jetbrains.kotlinx.dataframe.impl.columns.addPath
+import org.jetbrains.kotlinx.dataframe.impl.columns.guessColumnType
+import org.jetbrains.kotlinx.dataframe.impl.columns.toColumnKind
+import org.jetbrains.kotlinx.dataframe.impl.getValuesType
+import org.jetbrains.kotlinx.dataframe.impl.splitByIndices
+import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
+import kotlin.reflect.KClass
+import kotlin.reflect.KProperty
+import kotlin.reflect.KType
+import kotlin.reflect.typeOf
+
+/**
+ * Column with [name] and [values] of specific [type].
+ *
+ * Base interface for [ValueColumn] and [FrameColumn], but not for [ColumnGroup]. However, implementations for all three [column kinds][ColumnKind] derive from DataColumn and can cast to it safely.
+ * Column operations that have signature clash with [DataFrame] API ([filter], [take], [map] etc.) are defined for [DataColumn] and not for [BaseColumn].
+ *
+ * @param T type of values in the column.
+ */
+public interface DataColumn<out T> : BaseColumn<T> {
+
+    public companion object {
+
+        /**
+         * Creates [ValueColumn] using given [name], [values] and [type].
+         *
+         * @param name name of the column
+         * @param values list of column values
+         * @param type type of the column
+         * @param infer column type inference mode
+         */
+        public fun <T> createValueColumn(
+            name: String,
+            values: List<T>,
+            type: KType,
+            infer: Infer = Infer.None,
+            defaultValue: T? = null
+        ): ValueColumn<T> = ValueColumnImpl(values, name, getValuesType(values, type, infer), defaultValue)
+
+        /**
+         * Creates [ValueColumn] using given [name], [values] and reified column [type].
+         *
+         * Note, that column [type] will be defined at compile-time using [T] argument
+         *
+         * @param T type of the column
+         * @param name name of the column
+         * @param values list of column values
+         * @param infer column type inference mode
+         */
+        public inline fun <reified T> createValueColumn(name: String, values: List<T>, infer: Infer = Infer.None): ValueColumn<T> = createValueColumn(
+            name, values,
+            getValuesType(
+                values,
+                typeOf<T>(),
+                infer
+            )
+        )
+
+        public fun <T> createColumnGroup(name: String, df: DataFrame<T>): ColumnGroup<T> = ColumnGroupImpl(name, df)
+
+        public fun <T> createFrameColumn(
+            name: String,
+            df: DataFrame<T>,
+            startIndices: Iterable<Int>
+        ): FrameColumn<T> =
+            FrameColumnImpl(name, df.splitByIndices(startIndices.asSequence()).toList(), lazy { df.schema() })
+
+        public fun <T> createFrameColumn(
+            name: String,
+            groups: List<DataFrame<T>>,
+            schema: Lazy<DataFrameSchema>? = null
+        ): FrameColumn<T> = FrameColumnImpl(name, groups, schema)
+
+        public fun <T> createWithTypeInference(name: String, values: List<T>, nullable: Boolean? = null): DataColumn<T> = guessColumnType(name, values, nullable = nullable)
+
+        public fun <T> create(name: String, values: List<T>, type: KType, infer: Infer = Infer.None): DataColumn<T> {
+            return when (type.toColumnKind()) {
+                ColumnKind.Value -> createValueColumn(name, values, type, infer)
+                ColumnKind.Group -> createColumnGroup(name, (values as List<AnyRow?>).concat()).asDataColumn().cast()
+                ColumnKind.Frame -> createFrameColumn(name, values as List<AnyFrame>).asDataColumn().cast()
+            }
+        }
+
+        public inline fun <reified T> create(name: String, values: List<T>, infer: Infer = Infer.None): DataColumn<T> = create(name, values, typeOf<T>(), infer)
+
+        public fun empty(name: String = ""): AnyCol = createValueColumn(name, emptyList<Unit>(), typeOf<Unit>())
+    }
+
+    public fun hasNulls(): Boolean = type().isMarkedNullable
+
+    override fun distinct(): DataColumn<T>
+
+    override fun get(indices: Iterable<Int>): DataColumn<T>
+
+    override fun rename(newName: String): DataColumn<T>
+
+    override fun resolveSingle(context: ColumnResolutionContext): ColumnWithPath<T>? = this.addPath()
+
+    override operator fun getValue(thisRef: Any?, property: KProperty<*>): DataColumn<T> = super.getValue(thisRef, property) as DataColumn<T>
+
+    public operator fun iterator(): Iterator<T> = values().iterator()
+
+    public override operator fun get(range: IntRange): DataColumn<T>
+}
+
+public val AnyCol.name: String get() = name()
+public val AnyCol.path: ColumnPath get() = path()
+
+public val <T> DataColumn<T>.values: Iterable<T> get() = values()
+public val AnyCol.hasNulls: Boolean get() = hasNulls()
+public val AnyCol.size: Int get() = size()
+public val AnyCol.indices: IntRange get() = indices()
+
+public val AnyCol.type: KType get() = type()
+public val AnyCol.kind: ColumnKind get() = kind()
+public val AnyCol.typeClass: KClass<*> get() = type.classifier as? KClass<*> ?: error("Cannot cast ${type.classifier?.javaClass} to a ${KClass::class}. Column $name: $type")
+
+public fun AnyBaseCol.indices(): IntRange = 0 until size()