From 4c6ec36df7b79b6021e965bd6df1e3aa418b9487 Mon Sep 17 00:00:00 2001 From: spaceopen Date: Sat, 20 Jul 2019 06:38:53 +0200 Subject: [PATCH 1/2] Preliminary LMDB backend support, tested with modified todo example. --- build.gradle | 2 + src/main/kotlin/io/kweb/shoebox/Shoebox.kt | 19 +-- .../io/kweb/shoebox/stores/LmdbStore.kt | 150 ++++++++++++++++++ 3 files changed, 162 insertions(+), 9 deletions(-) create mode 100644 src/main/kotlin/io/kweb/shoebox/stores/LmdbStore.kt diff --git a/build.gradle b/build.gradle index 3dd4405..264e8b9 100644 --- a/build.gradle +++ b/build.gradle @@ -47,11 +47,13 @@ repositories { dependencies { compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version" + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version" compile 'com.github.salomonbrys.kotson:kotson:2.5.0' compile 'com.google.guava:guava:27.1-jre' compile 'net.incongru.watchservice:barbary-watchservice:1.0' compile 'com.fatboyindustrial.gson-javatime-serialisers:gson-javatime-serialisers:1.1.1' + implementation 'org.lmdbjava:lmdbjava:0.7.0' testCompile 'io.kotlintest:kotlintest:2.0.7' } diff --git a/src/main/kotlin/io/kweb/shoebox/Shoebox.kt b/src/main/kotlin/io/kweb/shoebox/Shoebox.kt index 0e77c4a..4fa7ded 100644 --- a/src/main/kotlin/io/kweb/shoebox/Shoebox.kt +++ b/src/main/kotlin/io/kweb/shoebox/Shoebox.kt @@ -9,12 +9,12 @@ import java.util.concurrent.ConcurrentHashMap import kotlin.reflect.KClass - /* - * TODO: 1) Add a lockfile mechanism to prevent multiple JVMs or threads from - * TODO: using the same directory - * TODO: 2) Handle changes that occur to the filesystem which aren't initiated here - * TODO: (then remove the previous lockfile mechanism) - */ +/* +* TODO: 1) Add a lockfile mechanism to prevent multiple JVMs or threads from +* TODO: using the same directory +* TODO: 2) Handle changes that occur to the filesystem which aren't initiated here +* TODO: (then remove the previous lockfile mechanism) +*/ /** * Create a [Shoebox], use this in preference to the Shoebox constructor to avoid having to provide a `KClass` @@ -28,7 +28,7 @@ import kotlin.reflect.KClass inline fun Shoebox(store : Store) = Shoebox(store, T::class) inline fun Shoebox(dir : Path) = Shoebox(DirectoryStore(dir), T::class) inline fun Shoebox() = Shoebox(MemoryStore(), T::class) - +inline fun Shoebox(name: String) = Shoebox(LmdbStore(name), T::class) /** * Can persistently store and retrieve objects, and notify listeners of changes to those objects @@ -139,13 +139,13 @@ class Shoebox(val store: Store, private val kc: KClass) { fun deleteRemoveListener(handle : Long) { removeListeners.remove(handle) } - + fun onChange(listener: (T, KeyValue, Source) -> Unit) : Long { val handle = listenerHandleSource.incrementAndGet() changeListeners.put(handle, listener) return handle } - + fun onChange(key: String, listener: (T, T, Source) -> Unit) : Long { val handle = listenerHandleSource.incrementAndGet() keySpecificChangeListeners.computeIfAbsent(key, { ConcurrentHashMap() }).put(handle, listener) @@ -170,6 +170,7 @@ class Shoebox(val store: Store, private val kc: KClass) { is MemoryStore -> MemoryStore() is DirectoryStore -> DirectoryStore(store.directory.parent.resolve("${store.directory.fileName}-$name-view")) + is LmdbStore -> LmdbStore("${store.name}-$name-view") else -> throw RuntimeException("Shoebox doesn't currently support creating a view for store type ${store::class.simpleName}") } return View(Shoebox(store), this, verify, by) diff --git a/src/main/kotlin/io/kweb/shoebox/stores/LmdbStore.kt b/src/main/kotlin/io/kweb/shoebox/stores/LmdbStore.kt new file mode 100644 index 0000000..dfffa48 --- /dev/null +++ b/src/main/kotlin/io/kweb/shoebox/stores/LmdbStore.kt @@ -0,0 +1,150 @@ +package io.kweb.shoebox.stores + +import com.fatboyindustrial.gsonjavatime.Converters +import com.google.gson.* +import com.google.gson.reflect.TypeToken +import io.kweb.shoebox.* +import java.nio.file.* +import java.time.* +import kotlin.reflect.KClass + +import org.lmdbjava.* +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteBuffer.allocateDirect +import java.nio.charset.StandardCharsets.UTF_8 +import kotlin.io.FileSystemException + + +/** + * TODO: remove dependence on gson + */ + +inline fun LmdbStore(name: String) = LmdbStore(name, T::class) +/* +val defaultGson: Gson = Converters.registerAll(GsonBuilder()).let { + it.registerTypeAdapter(object : TypeToken() {}.type, DurationConverter()) +}.create() +*/ +class LmdbStore(val name: String, private val kc: KClass, val gson: Gson = defaultGson) : Store { + + companion object { + private val home: String = System.getProperty("user.dir") + var env: Env = create("$home/data") + + fun create(path: String): Env { + println("LMDB database directory: $path") + val file = File(path) + if (!file.exists()) { + if (!file.mkdir()) { + throw FileSystemException(file, reason = "Failed to create LMDB database directory!") + } + } else { + if (!file.isDirectory) { + throw InvalidPathException("Not a directory", path) + } + } + return Env.create().setMapSize(10485760).setMaxDbs(100).open(file) + } + + protected fun finalize() { + env.close() + } + + } + + private val dbi: Dbi = env.openDbi(name, DbiFlags.MDB_CREATE) + + /** + * Retrieve the entries in this store, similar to [Map.entries] but lazy + * + * @return The keys and their corresponding values in this [Shoebox] + */ + override val entries: Iterable> get() { + val ret = mutableSetOf>() + env.txnRead().use { txn -> + dbi.iterate(txn).use { c -> + c.forEach { + val k = UTF_8.decode(it.key()).toString() + val v = gson.fromJson(UTF_8.decode(it.`val`()).toString(), kc.javaObjectType) + ret.add(KeyValue(k, v)) + } + } + txn.abort() + } + return ret + } + + /** + * Retrieve a value, similar to [Map.get] + * + * @param key The key associated with the desired value + * @return The value associated with the key, or null if no value is associated + */ + override operator fun get(key: String): T? { + require(key.isNotBlank()) {"key(\"$key\") must not be blank"} + val k = allocateDirect(env.maxKeySize) + k.put(key.toByteArray(UTF_8)).flip() + var ret: T? = null + env.txnRead().use { txn -> + val v: ByteBuffer? = dbi.get(txn, k) + if (v != null) { + ret = gson.fromJson(UTF_8.decode(v).toString(), kc.javaObjectType) + } + txn.abort() + } + return ret + } + + /** + * Remove a key-value pair + * + * @param key The key associated with the value to be removed, similar to [MutableMap.remove] + */ + override fun remove(key: String) : T? { + require(key.isNotBlank()) {"key(\"$key\") must not be blank"} + val k = allocateDirect(env.maxKeySize) + k.put(key.toByteArray(UTF_8)).flip() + var ret: T? = null + env.txnWrite().use { txn -> + // who needs the value? + val oldv: ByteBuffer? = dbi.get(txn, k) + if (oldv != null) { + ret = gson.fromJson(UTF_8.decode(oldv).toString(), kc.javaObjectType) + } + dbi.delete(txn, k) + txn.commit() + } + return ret + } + + /** + * Set or change a value, simliar to [MutableMap.set] + * + * @param key The key associated with the value to be set or changed + * @param value The new value + */ + override operator fun set(key: String, value: T) : T? { + require(key.isNotBlank()) {"key(\"$key\") must not be blank"} + val k = allocateDirect(env.maxKeySize) + k.put(key.toByteArray(UTF_8)).flip() + val bytes = gson.toJson(value, kc.javaObjectType).toByteArray(UTF_8) + val v = allocateDirect(bytes.size) + v.put(bytes).flip() + var ret: T? = null + env.txnWrite().use { txn -> + // is the old value necessary? + val oldv: ByteBuffer? = dbi.get(txn, k) + if (oldv != null) { + ret = gson.fromJson(UTF_8.decode(oldv).toString(), kc.javaObjectType) + } + dbi.put(txn, k, v) + txn.commit() + } + return ret + } + + protected fun finalize() { + dbi.close() + } +} From e6a5b84203fe311093b49d2779d84978178a04c6 Mon Sep 17 00:00:00 2001 From: spaceopen Date: Sun, 21 Jul 2019 20:04:21 +0200 Subject: [PATCH 2/2] Factory function name changed to LmdbStore --- src/main/kotlin/io/kweb/shoebox/Shoebox.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/io/kweb/shoebox/Shoebox.kt b/src/main/kotlin/io/kweb/shoebox/Shoebox.kt index 4fa7ded..509fac7 100644 --- a/src/main/kotlin/io/kweb/shoebox/Shoebox.kt +++ b/src/main/kotlin/io/kweb/shoebox/Shoebox.kt @@ -28,7 +28,7 @@ import kotlin.reflect.KClass inline fun Shoebox(store : Store) = Shoebox(store, T::class) inline fun Shoebox(dir : Path) = Shoebox(DirectoryStore(dir), T::class) inline fun Shoebox() = Shoebox(MemoryStore(), T::class) -inline fun Shoebox(name: String) = Shoebox(LmdbStore(name), T::class) +inline fun LmdbShoebox(name: String) = Shoebox(LmdbStore(name), T::class) /** * Can persistently store and retrieve objects, and notify listeners of changes to those objects