From fbd14902bf025aafa4d5de37d739cf02c16662e1 Mon Sep 17 00:00:00 2001 From: Valentyn Sobol Date: Wed, 30 Apr 2025 14:06:10 +0300 Subject: [PATCH] [jacodb-storage] Add ability to fine-tune Xodus K/V storage using its own EnvironmentConfig As an example of the feature use, added a test which opens a read-only storage listening to changes made by another R/W storage. --- .../kotlin/org/jacodb/impl/JcErsSettings.kt | 5 ++++ .../storage/kv/xodus/XodusKeyValueStorage.kt | 11 ++++++-- .../kv/xodus/XodusKeyValueStorageSPI.kt | 10 +++++-- .../storage/kv/XodusKeyValueStorageTest.kt | 27 +++++++++++++++++++ .../kv/PluggableKeyValueStorageTest.kt | 7 +++-- 5 files changed, 54 insertions(+), 6 deletions(-) diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/JcErsSettings.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/JcErsSettings.kt index f3b4622cf..bbdf07102 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/JcErsSettings.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/JcErsSettings.kt @@ -16,8 +16,10 @@ package org.jacodb.impl +import jetbrains.exodus.env.EnvironmentConfig import org.jacodb.api.storage.ers.ErsSettings import org.jacodb.impl.storage.kv.lmdb.LMDB_KEY_VALUE_STORAGE_SPI +import org.jacodb.impl.storage.kv.xodus.XODUS_KEY_VALUE_STORAGE_SPI class RamErsSettings( val immutableDumpsPath: String? = null @@ -31,3 +33,6 @@ open class JcKvErsSettings(val kvId: String) : ErsSettings // by default, mapSize is 1Gb class JcLmdbErsSettings(val mapSize: Long = 0x40_00_00_00) : JcKvErsSettings(LMDB_KEY_VALUE_STORAGE_SPI) +class JcXodusErsSettings(val configurer: (EnvironmentConfig.() -> Unit)? = null) : + JcKvErsSettings(XODUS_KEY_VALUE_STORAGE_SPI) + diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt index c3ad84334..725913503 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorage.kt @@ -17,6 +17,7 @@ package org.jacodb.impl.storage.kv.xodus import jetbrains.exodus.env.Environment +import jetbrains.exodus.env.EnvironmentConfig import jetbrains.exodus.env.Environments import jetbrains.exodus.env.Store import jetbrains.exodus.env.StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING @@ -25,14 +26,20 @@ import jetbrains.exodus.env.TransactionBase import org.jacodb.api.storage.kv.PluggableKeyValueStorage import org.jacodb.api.storage.kv.Transaction -internal class XodusKeyValueStorage(location: String) : PluggableKeyValueStorage() { +internal class XodusKeyValueStorage(location: String, configurer: (EnvironmentConfig.() -> Unit)?) : + PluggableKeyValueStorage() { - private val env: Environment = Environments.newInstance(location, + private val env: Environment = Environments.newInstance( + location, environmentConfig { logFileSize = 32768 logCachePageSize = 65536 * 4 gcStartIn = 600_000 useVersion1Format = false // use v2 data format, as we use stores with prefixing, i.e., patricia trees + + // If a configurer is set, apply it after default settings ^^^ are defined. + // This allows overriding them as well. + configurer?.let { it() } } ) diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorageSPI.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorageSPI.kt index 8a9f82491..42418e718 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorageSPI.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/kv/xodus/XodusKeyValueStorageSPI.kt @@ -19,6 +19,7 @@ package org.jacodb.impl.storage.kv.xodus import org.jacodb.api.storage.ers.ErsSettings import org.jacodb.api.storage.kv.PluggableKeyValueStorage import org.jacodb.api.storage.kv.PluggableKeyValueStorageSPI +import org.jacodb.impl.JcXodusErsSettings import kotlin.io.path.createTempDirectory const val XODUS_KEY_VALUE_STORAGE_SPI = "org.jacodb.impl.storage.kv.xodus.XodusKeyValueStorageSPI" @@ -27,6 +28,11 @@ class XodusKeyValueStorageSPI : PluggableKeyValueStorageSPI { override val id = XODUS_KEY_VALUE_STORAGE_SPI - override fun newStorage(location: String?, settings: ErsSettings): PluggableKeyValueStorage = - XodusKeyValueStorage(location ?: createTempDirectory(prefix = "xodusKeyValueStorage").toString()) + override fun newStorage(location: String?, settings: ErsSettings): PluggableKeyValueStorage { + val configurer = (settings as? JcXodusErsSettings)?.configurer + return XodusKeyValueStorage( + location ?: createTempDirectory(prefix = "xodusKeyValueStorage").toString(), + configurer + ) + } } \ No newline at end of file diff --git a/jacodb-storage/src/test/kotlin/org/jacodb/testing/storage/kv/XodusKeyValueStorageTest.kt b/jacodb-storage/src/test/kotlin/org/jacodb/testing/storage/kv/XodusKeyValueStorageTest.kt index e79b78ec3..ab4fdc6cc 100644 --- a/jacodb-storage/src/test/kotlin/org/jacodb/testing/storage/kv/XodusKeyValueStorageTest.kt +++ b/jacodb-storage/src/test/kotlin/org/jacodb/testing/storage/kv/XodusKeyValueStorageTest.kt @@ -16,9 +16,36 @@ package org.jacodb.testing.storage.kv +import jetbrains.exodus.io.DataReaderWriterProvider +import org.jacodb.impl.JcXodusErsSettings import org.jacodb.impl.storage.kv.xodus.XODUS_KEY_VALUE_STORAGE_SPI +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import java.lang.Long.getLong class XodusKeyValueStorageTest : PluggableKeyValueStorageTest() { override val kvStorageId = XODUS_KEY_VALUE_STORAGE_SPI + + @Test + fun `test shared usage of the same db`() { + val settings = JcXodusErsSettings { + logDataReaderWriterProvider = DataReaderWriterProvider.WATCHING_READER_WRITER_PROVIDER + } + val roStorage = kvStorageSpi.newStorage(location = location, settings = settings) + roStorage.transactional { txn -> + assertTrue(txn.isReadonly) + assertNull(txn.get("a map", "key".asByteArray)) + } + putGet() + Thread.sleep(getLong("jetbrains.exodus.io.watching.forceCheckEach", 3000L) + 500) + roStorage.transactional { txn -> + val got = txn.get("a map", "key".asByteArray) + assertNotNull(got) + assertEquals("value", got?.asString) + } + } } \ No newline at end of file diff --git a/jacodb-storage/src/testFixtures/kotlin/org/jacodb/testing/storage/kv/PluggableKeyValueStorageTest.kt b/jacodb-storage/src/testFixtures/kotlin/org/jacodb/testing/storage/kv/PluggableKeyValueStorageTest.kt index cbca8c97b..13f87744d 100644 --- a/jacodb-storage/src/testFixtures/kotlin/org/jacodb/testing/storage/kv/PluggableKeyValueStorageTest.kt +++ b/jacodb-storage/src/testFixtures/kotlin/org/jacodb/testing/storage/kv/PluggableKeyValueStorageTest.kt @@ -28,14 +28,16 @@ import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import kotlin.io.path.createTempDirectory abstract class PluggableKeyValueStorageTest { abstract val kvStorageId: String - private val kvStorageSpi by lazy(LazyThreadSafetyMode.NONE) { + protected val kvStorageSpi by lazy(LazyThreadSafetyMode.NONE) { PluggableKeyValueStorageSPI.getProvider(kvStorageId) } + protected lateinit var location: String protected lateinit var storage: PluggableKeyValueStorage @Test @@ -145,7 +147,8 @@ abstract class PluggableKeyValueStorageTest { @BeforeEach fun setUp() { - storage = kvStorageSpi.newStorage(null).apply { + location = createTempDirectory(prefix = "pluggableKeyValueStorage").toString() + storage = kvStorageSpi.newStorage(location = location).apply { isMapWithKeyDuplicates = { mapName -> mapName.endsWith("withDuplicates") } } }