Skip to content

[jacodb-storage] Add ability to fine-tune Xodus K/V storage using #317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 30, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@

package org.jacodb.impl

import jetbrains.exodus.env.EnvironmentConfig
import org.jacodb.api.storage.ers.ErsSettings
import org.jacodb.impl.storage.kv.lmdb.LMDB_KEY_VALUE_STORAGE_SPI
import org.jacodb.impl.storage.kv.xodus.XODUS_KEY_VALUE_STORAGE_SPI

class RamErsSettings(
val immutableDumpsPath: String? = null
Expand All @@ -31,3 +33,6 @@ open class JcKvErsSettings(val kvId: String) : ErsSettings
// by default, mapSize is 1Gb
class JcLmdbErsSettings(val mapSize: Long = 0x40_00_00_00) : JcKvErsSettings(LMDB_KEY_VALUE_STORAGE_SPI)

class JcXodusErsSettings(val configurer: (EnvironmentConfig.() -> Unit)? = null) :
JcKvErsSettings(XODUS_KEY_VALUE_STORAGE_SPI)

Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package org.jacodb.impl.storage.kv.xodus

import jetbrains.exodus.env.Environment
import jetbrains.exodus.env.EnvironmentConfig
import jetbrains.exodus.env.Environments
import jetbrains.exodus.env.Store
import jetbrains.exodus.env.StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING
Expand All @@ -25,14 +26,20 @@ import jetbrains.exodus.env.TransactionBase
import org.jacodb.api.storage.kv.PluggableKeyValueStorage
import org.jacodb.api.storage.kv.Transaction

internal class XodusKeyValueStorage(location: String) : PluggableKeyValueStorage() {
internal class XodusKeyValueStorage(location: String, configurer: (EnvironmentConfig.() -> Unit)?) :
PluggableKeyValueStorage() {

private val env: Environment = Environments.newInstance(location,
private val env: Environment = Environments.newInstance(
location,
environmentConfig {
logFileSize = 32768
logCachePageSize = 65536 * 4
gcStartIn = 600_000
useVersion1Format = false // use v2 data format, as we use stores with prefixing, i.e., patricia trees

// If a configurer is set, apply it after default settings ^^^ are defined.
// This allows overriding them as well.
configurer?.let { it() }
}
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.jacodb.impl.storage.kv.xodus
import org.jacodb.api.storage.ers.ErsSettings
import org.jacodb.api.storage.kv.PluggableKeyValueStorage
import org.jacodb.api.storage.kv.PluggableKeyValueStorageSPI
import org.jacodb.impl.JcXodusErsSettings
import kotlin.io.path.createTempDirectory

const val XODUS_KEY_VALUE_STORAGE_SPI = "org.jacodb.impl.storage.kv.xodus.XodusKeyValueStorageSPI"
Expand All @@ -27,6 +28,11 @@ class XodusKeyValueStorageSPI : PluggableKeyValueStorageSPI {

override val id = XODUS_KEY_VALUE_STORAGE_SPI

override fun newStorage(location: String?, settings: ErsSettings): PluggableKeyValueStorage =
XodusKeyValueStorage(location ?: createTempDirectory(prefix = "xodusKeyValueStorage").toString())
override fun newStorage(location: String?, settings: ErsSettings): PluggableKeyValueStorage {
val configurer = (settings as? JcXodusErsSettings)?.configurer
return XodusKeyValueStorage(
location ?: createTempDirectory(prefix = "xodusKeyValueStorage").toString(),
configurer
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,36 @@

package org.jacodb.testing.storage.kv

import jetbrains.exodus.io.DataReaderWriterProvider
import org.jacodb.impl.JcXodusErsSettings
import org.jacodb.impl.storage.kv.xodus.XODUS_KEY_VALUE_STORAGE_SPI
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertNotNull
import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.lang.Long.getLong

class XodusKeyValueStorageTest : PluggableKeyValueStorageTest() {

override val kvStorageId = XODUS_KEY_VALUE_STORAGE_SPI

@Test
fun `test shared usage of the same db`() {
val settings = JcXodusErsSettings {
logDataReaderWriterProvider = DataReaderWriterProvider.WATCHING_READER_WRITER_PROVIDER
}
val roStorage = kvStorageSpi.newStorage(location = location, settings = settings)
roStorage.transactional { txn ->
assertTrue(txn.isReadonly)
assertNull(txn.get("a map", "key".asByteArray))
}
putGet()
Thread.sleep(getLong("jetbrains.exodus.io.watching.forceCheckEach", 3000L) + 500)
roStorage.transactional { txn ->
val got = txn.get("a map", "key".asByteArray)
assertNotNull(got)
assertEquals("value", got?.asString)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,16 @@ import org.junit.jupiter.api.Assertions.assertNull
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import kotlin.io.path.createTempDirectory

abstract class PluggableKeyValueStorageTest {

abstract val kvStorageId: String

private val kvStorageSpi by lazy(LazyThreadSafetyMode.NONE) {
protected val kvStorageSpi by lazy(LazyThreadSafetyMode.NONE) {
PluggableKeyValueStorageSPI.getProvider(kvStorageId)
}
protected lateinit var location: String
protected lateinit var storage: PluggableKeyValueStorage

@Test
Expand Down Expand Up @@ -145,7 +147,8 @@ abstract class PluggableKeyValueStorageTest {

@BeforeEach
fun setUp() {
storage = kvStorageSpi.newStorage(null).apply {
location = createTempDirectory(prefix = "pluggableKeyValueStorage").toString()
storage = kvStorageSpi.newStorage(location = location).apply {
isMapWithKeyDuplicates = { mapName -> mapName.endsWith("withDuplicates") }
}
}
Expand Down