Skip to content

Splitting off Jupyter yet again #1095

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Mar 16, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ plugins {
alias(kotlin.jvm)
alias(publisher)
alias(serialization) apply false
alias(jupyter.api) apply false
alias(dokka)
alias(kover)
alias(ktlint)
Expand Down Expand Up @@ -50,20 +49,21 @@ configurations {
}

dependencies {
api(project(":core"))
api(project(":dataframe-arrow"))
api(project(":dataframe-excel"))
api(project(":dataframe-openapi"))
api(project(":dataframe-jdbc"))
api(project(":dataframe-csv"))
api(projects.core)
api(projects.dataframeArrow)
api(projects.dataframeExcel)
api(projects.dataframeOpenapi)
api(projects.dataframeJdbc)
api(projects.dataframeCsv)

kover(project(":core"))
kover(project(":dataframe-arrow"))
kover(project(":dataframe-excel"))
kover(project(":dataframe-openapi"))
kover(project(":dataframe-jdbc"))
kover(project(":dataframe-csv"))
kover(project(":plugins:kotlin-dataframe"))
kover(projects.core)
kover(projects.dataframeArrow)
kover(projects.dataframeExcel)
kover(projects.dataframeOpenapi)
kover(projects.dataframeJdbc)
kover(projects.dataframeCsv)
kover(projects.plugins.kotlinDataframe)
kover(projects.dataframeJupyter)
}

enum class Version : Comparable<Version> {
Expand Down
9 changes: 4 additions & 5 deletions core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,9 @@ or [:dataframe-jdbc](../dataframe-jdbc), however, this is has not happened yet f

At the moment, these integrations are still part of the `:core` module:

- csv/tsv
- (deprecated) csv/tsv
- html
- json
- jupyter

### Korro code samples

Expand All @@ -36,10 +35,10 @@ We can also generate "normal" DataFrame samples for the website. This can be don
[OtherSamples class](./src/test/kotlin/org/jetbrains/kotlinx/dataframe/samples/api/OtherSamples.kt). Generated
HTML files will be stored in [docs/StardustDocs/snippets/manual](../docs/StardustDocs/snippets/manual).

### KDoc Preprocesor
### KoDEx

The code you're working on needs to be edited in [src](src), but the KDocs are processed by the
[KDoc Preprocessor](https://github.com/Jolanrensen/docProcessorGradlePlugin) when the project is published (or the task
The code you're working on needs to be edited in [src](src), but the KDocs are processed by
[KoDEx](https://github.com/Jolanrensen/kodex) when the project is published (or the task
is run manually). The generated sources with adjusted KDocs will be overwritten
in [generated-sources](generated-sources).
See the [KDoc Preprocessing Guide](../KDOC_PREPROCESSING.md) for more information.
Expand Down
393 changes: 194 additions & 199 deletions core/api/core.api

Large diffs are not rendered by default.

14 changes: 3 additions & 11 deletions core/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ plugins {
alias(kotlin.jvm)
alias(publisher)
alias(serialization)
alias(jupyter.api)
alias(korro)
alias(kover)
alias(ktlint)
Expand All @@ -35,13 +34,10 @@ plugins {

group = "org.jetbrains.kotlinx"

val jupyterApiTCRepo: String by project

repositories {
mavenLocal()
mavenCentral()
maven("https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven")
maven(jupyterApiTCRepo)
}

kotlin.sourceSets {
Expand All @@ -65,7 +61,7 @@ dependencies {

api(libs.kotlin.reflect)
implementation(libs.kotlin.stdlib)
kotlinCompilerPluginClasspathSamples(project(":plugins:expressions-converter"))
kotlinCompilerPluginClasspathSamples(projects.plugins.expressionsConverter)

api(libs.commonsCsv)
implementation(libs.commonsIo)
Expand All @@ -86,8 +82,8 @@ dependencies {
testImplementation(libs.jsoup)
testImplementation(libs.sl4jsimple)

// for JupyterCodegenTests and samples.api
testImplementation(project(":dataframe-csv"))
// for samples.api
testImplementation(projects.dataframeCsv)
}

val samplesImplementation by configurations.getting {
Expand Down Expand Up @@ -391,10 +387,6 @@ tasks.test {
}
}

tasks.processJupyterApiResources {
libraryProducers = listOf("org.jetbrains.kotlinx.dataframe.jupyter.Integration")
}

kotlinPublications {
publication {
publicationName = "core"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.jetbrains.kotlinx.dataframe.api

import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility
import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer
import org.jetbrains.kotlinx.dataframe.impl.codeGen.from
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,5 @@
package org.jetbrains.dataframe.impl.codeGen
package org.jetbrains.kotlinx.dataframe.codeGen

import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.codeGen.ExtensionsCodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.Marker
import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility
import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor
import org.jetbrains.kotlinx.dataframe.codeGen.NameNormalizer
import org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGeneratorImpl
import org.jetbrains.kotlinx.dataframe.impl.codeGen.FullyQualifiedNames
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ShortNames
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
package org.jetbrains.kotlinx.dataframe.codeGen

import org.jetbrains.kotlinx.jupyter.api.Code
import org.jetbrains.kotlinx.jupyter.api.VariableName

/**
* Class representing generated code declarations for a [Marker].
*
Expand All @@ -28,3 +25,6 @@ public data class CodeWithConverter(val declarations: Code, val converter: (Vari
else -> declarations + "\n" + converter(name)
}
}

public typealias Code = String
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it an option to not have these in public api?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think they're part of our codegen api. CodeWithConverter is public too after all

public typealias VariableName = String
Original file line number Diff line number Diff line change
Expand Up @@ -270,13 +270,16 @@ internal fun <C : Comparable<C>> Sequence<C?>.indexOfMax(): Int {
return maxIndex
}

internal fun KClass<*>.createStarProjectedType(nullable: Boolean): KType =
if (this == Nothing::class) {
internal fun createStarProjectedType(klass: KClass<*>, nullable: Boolean): KType =
if (klass == Nothing::class) {
nothingType(nullable) // would be Void otherwise
} else {
this.starProjectedType.let { if (nullable) it.withNullability(true) else it }
klass.starProjectedType.let { if (nullable) it.withNullability(true) else it }
}

@JvmName("createStarProjectedTypeExt")
internal fun KClass<*>.createStarProjectedType(nullable: Boolean): KType = createStarProjectedType(this, nullable)

internal fun KType.isSubtypeWithNullabilityOf(type: KType) =
this.isSubtypeOf(type) && (!this.isMarkedNullable || type.isMarkedNullable)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,16 +1,6 @@
package org.jetbrains.kotlinx.dataframe.impl.codeGen

import com.squareup.kotlinpoet.buildCodeBlock
import org.jetbrains.dataframe.impl.codeGen.CodeGenResult
import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode.Enum
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode.NoFields
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode.None
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode.TypeAlias
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode.WithFields
import org.jetbrains.dataframe.keywords.HardKeywords
import org.jetbrains.dataframe.keywords.ModifierKeywords
import org.jetbrains.kotlinx.dataframe.ColumnsContainer
import org.jetbrains.kotlinx.dataframe.ColumnsScope
import org.jetbrains.kotlinx.dataframe.DataColumn
Expand All @@ -20,10 +10,19 @@ import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.ColumnSelectionDsl
import org.jetbrains.kotlinx.dataframe.api.DataSchemaEnum
import org.jetbrains.kotlinx.dataframe.codeGen.BaseField
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenResult
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.codeGen.ExtensionsCodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.FieldType
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode.Enum
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode.NoFields
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode.None
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode.TypeAlias
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode.WithFields
import org.jetbrains.kotlinx.dataframe.codeGen.IsolatedMarker
import org.jetbrains.kotlinx.dataframe.codeGen.Marker
import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility
Expand All @@ -33,8 +32,9 @@ import org.jetbrains.kotlinx.dataframe.codeGen.ValidFieldName
import org.jetbrains.kotlinx.dataframe.codeGen.toNullable
import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
import org.jetbrains.kotlinx.dataframe.impl.toSnakeCase
import org.jetbrains.kotlinx.dataframe.keywords.HardKeywords
import org.jetbrains.kotlinx.dataframe.keywords.ModifierKeywords
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code

private fun renderNullability(nullable: Boolean) = if (nullable) "?" else ""

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
package org.jetbrains.dataframe.impl.codeGen
package org.jetbrains.kotlinx.dataframe.impl.codeGen

import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.AnyRow
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl
import org.jetbrains.kotlinx.jupyter.api.Code
import kotlin.reflect.KClass
import kotlin.reflect.KProperty

Expand Down
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
package org.jetbrains.kotlinx.dataframe.impl.codeGen

import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode
import org.jetbrains.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.AnyRow
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter
import org.jetbrains.kotlinx.dataframe.codeGen.InterfaceGenerationMode
import org.jetbrains.kotlinx.dataframe.codeGen.Marker
import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility
import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code
import kotlin.reflect.KClass
import kotlin.reflect.KProperty
import kotlin.reflect.KType
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package org.jetbrains.kotlinx.dataframe.impl.codeGen

import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.AnyFrame
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.CodeGenerator
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat
import org.jetbrains.kotlinx.dataframe.io.SupportedDataFrameFormat
Expand All @@ -12,7 +13,6 @@ import org.jetbrains.kotlinx.dataframe.io.guessFormat
import org.jetbrains.kotlinx.dataframe.io.read
import org.jetbrains.kotlinx.dataframe.io.readCodeForGeneration
import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema
import org.jetbrains.kotlinx.jupyter.api.Code
import java.net.URL

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@ import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.addAll
import kotlinx.serialization.json.buildJsonArray
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.encodeToJsonElement
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import org.jetbrains.kotlinx.dataframe.AnyCol
Expand Down Expand Up @@ -401,3 +403,15 @@ internal fun encodeDataFrameWithMetadata(
),
)
}

@OptIn(ExperimentalSerializationApi::class)
internal fun encodeFrameNoDynamicNestedTables(df: AnyFrame, limit: Int): JsonObject =
buildJsonObject {
put(NROW, df.rowsCount())
put(NCOL, df.columnsCount())
putJsonArray(COLUMNS) { addAll(df.columnNames()) }
put(
KOTLIN_DATAFRAME,
encodeFrame(df.take(limit)),
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema
import org.jetbrains.kotlinx.dataframe.api.single
import org.jetbrains.kotlinx.dataframe.codeGen.Code
import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod
import org.jetbrains.kotlinx.jupyter.api.Code
import java.io.BufferedInputStream
import java.io.File
import java.io.FileNotFoundException
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,12 +198,6 @@ internal fun renderValueToString(value: Any?, decimalFormat: RendererDecimalForm
?: value.toString()
}

internal fun internallyRenderable(value: Any?): Boolean =
when (value) {
is AnyFrame, is Double, is List<*>, null, "" -> true
else -> false
}

internal fun Double.format(decimalFormat: RendererDecimalFormat): String = decimalFormat.format.format(this)

internal fun Float.format(decimalFormat: RendererDecimalFormat): String = decimalFormat.format.format(this)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
package org.jetbrains.kotlinx.dataframe.codeGen

import io.kotest.matchers.shouldBe
import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.dataframe.impl.codeGen.InterfaceGenerationMode
import org.jetbrains.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.dataframe.impl.codeGen.generate
import org.jetbrains.kotlinx.dataframe.AnyRow
import org.jetbrains.kotlinx.dataframe.ColumnsScope
import org.jetbrains.kotlinx.dataframe.DataColumn
Expand All @@ -16,9 +12,9 @@ import org.jetbrains.kotlinx.dataframe.api.groupBy
import org.jetbrains.kotlinx.dataframe.api.move
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.api.toCodeString
import org.jetbrains.kotlinx.dataframe.api.toDataFrame
import org.jetbrains.kotlinx.dataframe.api.under
import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl
import org.jetbrains.kotlinx.dataframe.testSets.person.BaseTest
import org.jetbrains.kotlinx.dataframe.testSets.person.Person
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.jetbrains.kotlinx.dataframe.codeGen

import io.kotest.matchers.shouldBe
import org.jetbrains.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.kotlinx.dataframe.DataFrame
import org.jetbrains.kotlinx.dataframe.DataRow
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.add
import org.jetbrains.kotlinx.dataframe.api.cast
import org.jetbrains.kotlinx.dataframe.api.generateCode
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.kotlinx.dataframe.io.readJsonStr
import org.junit.Test

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
package org.jetbrains.kotlinx.dataframe.codeGen

import io.kotest.matchers.shouldBe
import org.jetbrains.dataframe.impl.codeGen.CodeGenerator
import org.jetbrains.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.dataframe.impl.codeGen.process
import org.jetbrains.kotlinx.dataframe.annotations.ColumnName
import org.jetbrains.kotlinx.dataframe.annotations.DataSchema
import org.jetbrains.kotlinx.dataframe.api.dataFrameOf
import org.jetbrains.kotlinx.dataframe.api.schema
import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGenerator
import org.jetbrains.kotlinx.dataframe.impl.codeGen.process
import org.junit.Test

class NameGenerationTests {
Expand Down
Loading