diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bdec818fb8..e7e3b043b5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,7 @@ jobs: node-version-file: '.nvmrc' - name: Combine SARIF files run: | - npx @microsoft/sarif-multitool merge --merge-runs --output-file merged.sarif $(find . -iname '*.sarif*') + npx @microsoft/sarif-multitool merge --output-file merged.sarif $(find . -iname '*.sarif*') env: # Disables globalization support. # This makes the @microsoft/sarif-multitool work without ICU package installed. diff --git a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt index 1468b726d9..779de04738 100644 --- a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt +++ b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt @@ -13,10 +13,10 @@ data class BTree(val root: BTreeNode) : IStreamExecutorProvider by r fun validate() { graph.getStreamExecutor().query { root.validate(true) - check(root.getEntries().toList().getBlocking(graph).map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getBlocking(graph)) { + check(root.getEntries().toList().getBlocking().map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getBlocking()) { "duplicate entries: $root" } - check(root.getEntries().map { it.key }.toList().getBlocking(graph).sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getBlocking(graph)) { + check(root.getEntries().map { it.key }.toList().getBlocking().sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getBlocking()) { "not sorted: $this" } IStream.of(Unit) diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt index 86c85a6eda..f8cba8b748 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt @@ -72,7 +72,7 @@ class BTreeTest { assertEquals( (100L..200L).map { it to it * 2 }, - tree.getAll((100L..200L)).toList().getBlocking(tree), + tree.getAll((100L..200L)).toList().getBlocking(), ) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt index 1e2734877f..0b8f727218 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt @@ -27,19 +27,19 @@ class HamtCollisionTest { ) var tree: HamtTree = HamtTree(HamtInternalNode.createEmpty(config)) - tree = tree.put(0b000000000, "a").getBlocking(tree) - tree = tree.put(0b000000001, "b").getBlocking(tree) - tree = tree.put(0b000000011, "c").getBlocking(tree) - tree = tree.put(0b100000000, "d").getBlocking(tree) - tree = tree.put(0b100000001, "e").getBlocking(tree) - tree = tree.put(0b100000011, "f").getBlocking(tree) - - assertEquals("a", tree.get(0b000000000).getBlocking(tree)) - assertEquals("b", tree.get(0b000000001).getBlocking(tree)) - assertEquals("c", tree.get(0b000000011).getBlocking(tree)) - assertEquals("d", tree.get(0b100000000).getBlocking(tree)) - assertEquals("e", tree.get(0b100000001).getBlocking(tree)) - assertEquals("f", tree.get(0b100000011).getBlocking(tree)) + tree = tree.put(0b000000000, "a").getBlocking() + tree = tree.put(0b000000001, "b").getBlocking() + tree = tree.put(0b000000011, "c").getBlocking() + tree = tree.put(0b100000000, "d").getBlocking() + tree = tree.put(0b100000001, "e").getBlocking() + tree = tree.put(0b100000011, "f").getBlocking() + + assertEquals("a", tree.get(0b000000000).getBlocking()) + assertEquals("b", tree.get(0b000000001).getBlocking()) + assertEquals("c", tree.get(0b000000011).getBlocking()) + assertEquals("d", tree.get(0b100000000).getBlocking()) + assertEquals("e", tree.get(0b100000001).getBlocking()) + assertEquals("f", tree.get(0b100000011).getBlocking()) } /** @@ -60,22 +60,22 @@ class HamtCollisionTest { ) var tree: HamtTree = HamtTree(HamtInternalNode.createEmpty(config)) - tree = tree.put(0b00, "a").getBlocking(tree) - tree = tree.put(0b01, "b").getBlocking(tree) - tree = tree.put(0b10, "c").getBlocking(tree) - tree = tree.put(0b11, "d").getBlocking(tree) + tree = tree.put(0b00, "a").getBlocking() + tree = tree.put(0b01, "b").getBlocking() + tree = tree.put(0b10, "c").getBlocking() + tree = tree.put(0b11, "d").getBlocking() - assertEquals("a", tree.get(0b00).getBlocking(tree)) - assertEquals("b", tree.get(0b01).getBlocking(tree)) - assertEquals("c", tree.get(0b10).getBlocking(tree)) - assertEquals("d", tree.get(0b11).getBlocking(tree)) + assertEquals("a", tree.get(0b00).getBlocking()) + assertEquals("b", tree.get(0b01).getBlocking()) + assertEquals("c", tree.get(0b10).getBlocking()) + assertEquals("d", tree.get(0b11).getBlocking()) - tree = tree.put(0b01, "changed").getBlocking(tree) + tree = tree.put(0b01, "changed").getBlocking() - assertEquals("a", tree.get(0b00).getBlocking(tree)) - assertEquals("changed", tree.get(0b01).getBlocking(tree)) - assertEquals("c", tree.get(0b10).getBlocking(tree)) - assertEquals("d", tree.get(0b11).getBlocking(tree)) + assertEquals("a", tree.get(0b00).getBlocking()) + assertEquals("changed", tree.get(0b01).getBlocking()) + assertEquals("c", tree.get(0b10).getBlocking()) + assertEquals("d", tree.get(0b11).getBlocking()) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt new file mode 100644 index 0000000000..5459d0964d --- /dev/null +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt @@ -0,0 +1,43 @@ +package org.modelix.datastructures + +import org.modelix.datastructures.hamt.HamtInternalNode +import org.modelix.datastructures.hamt.HamtNode +import org.modelix.datastructures.hamt.HamtTree +import org.modelix.datastructures.objects.IObjectGraph +import org.modelix.datastructures.objects.StringDataTypeConfiguration +import org.modelix.streams.getBlocking +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Direct coverage of [org.modelix.datastructures.IPersistentMap.getChanges] on the HAMT. Guards that a value change is + * reported as [EntryChangedEvent] (not add/remove): this relies on the streams engine evaluating a synchronous prefix + * eagerly, which `HamtLeafNode.getChanges` depends on via a `var` set by one stream and read by a later deferred one. + */ +class HamtGetChangesTest { + private fun newTree(): HamtTree { + val config = HamtNode.Config( + graph = IObjectGraph.FREE_FLOATING, + keyConfig = StringDataTypeConfiguration(), + valueConfig = StringDataTypeConfiguration(), + ) + return HamtTree(HamtInternalNode.createEmpty(config)) + } + + @Test + fun single_entry_value_change_is_a_change_not_add() { + val old = newTree().put("k1", "a").getBlocking() + val new = old.put("k1", "b").getBlocking() + val changes = new.getChanges(old, changesOnly = false).toList().getBlocking() + assertEquals(listOf(EntryChangedEvent("k1", "a", "b")), changes) + } + + @Test + fun one_changed_among_several() { + var old = newTree() + for (i in 0 until 20) old = old.put("k$i", "v$i").getBlocking() + val new = old.put("k7", "changed").getBlocking() + val changes = new.getChanges(old, changesOnly = false).toList().getBlocking() + assertEquals(listOf(EntryChangedEvent("k7", "v7", "changed")), changes) + } +} diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt index f3c058ba32..bae666724e 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt @@ -33,7 +33,7 @@ class HamtTest { if (expected.isNotEmpty()) { val key = expected.keys.random(rand) // println("remove $key") - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() expected.remove(key) } } @@ -41,13 +41,13 @@ class HamtTest { val key = "k" + rand.nextInt(keyRange).toString() val value = "v" + rand.nextInt(valueRange).toString() // println("insert $key -> $value") - tree = tree.put(key, value).getBlocking(tree) + tree = tree.put(key, value).getBlocking() expected[key] = value } } for (entry in expected.entries) { - assertEquals(entry.value, tree.get(entry.key).getBlocking(tree), "for key ${entry.key}") + assertEquals(entry.value, tree.get(entry.key).getBlocking(), "for key ${entry.key}") } } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt index 4d7156aed3..1a582f7722 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt @@ -25,10 +25,10 @@ class PatriciaTrieTest { fun assertTree() { val expectedEntries = addedEntries.associateWith { values[it]!! } - val actualEntries = tree.getAll().toList().getBlocking(tree).toMap() + val actualEntries = tree.getAll().toList().getBlocking().toMap() assertEquals(expectedEntries, actualEntries) - val expectedTree = addedEntries.fold(initialTree) { acc, it -> acc.put(it, values[it]!!).getBlocking(tree) } + val expectedTree = addedEntries.fold(initialTree) { acc, it -> acc.put(it, values[it]!!).getBlocking() } assertEquals(expectedTree, tree) assertEquals(expectedTree.asObject().getHash(), tree.asObject().getHash()) @@ -39,13 +39,13 @@ class PatriciaTrieTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } @@ -56,55 +56,55 @@ class PatriciaTrieTest { @Test fun `slice with shorter prefix and single entry`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("abcdef", "1").getBlocking(tree) + tree = tree.put("abcdef", "1").getBlocking() - assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) + assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) } @Test fun `slice with shorter prefix and two entries`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("abcdef", "1").getBlocking(tree) - tree = tree.put("abcdeg", "2").getBlocking(tree) + tree = tree.put("abcdef", "1").getBlocking() + tree = tree.put("abcdeg", "2").getBlocking() - assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("2", tree.slice("abc").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("2", tree.slice("abc").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test fun `slice with prefix between two entries`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() - assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) + assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) } @Test fun `slice with one before and two after`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) - tree = tree.put("abcdeg", "3").getBlocking(tree) + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() + tree = tree.put("abcdeg", "3").getBlocking() - assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("3", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("3", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test fun `slice with two before and two after at existing split`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("a", "0").getBlocking(tree) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) - tree = tree.put("abcdeg", "3").getBlocking(tree) - - assertEquals(null, tree.slice("ab").flatMapZeroOrOne { it.get("a") }.getBlocking(tree)) - assertEquals("1", tree.slice("ab").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("ab").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("3", tree.slice("ab").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + tree = tree.put("a", "0").getBlocking() + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() + tree = tree.put("abcdeg", "3").getBlocking() + + assertEquals(null, tree.slice("ab").flatMapZeroOrOne { it.get("a") }.getBlocking()) + assertEquals("1", tree.slice("ab").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("ab").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("3", tree.slice("ab").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test @@ -127,16 +127,16 @@ class PatriciaTrieTest { "abcdC", ) for (key in initialKeys) { - tree = tree.put(key, "value of $key").getBlocking(tree) + tree = tree.put(key, "value of $key").getBlocking() } assertEquals( initialKeys.associateWith { "value of $it" }, - tree.getAll().toList().getBlocking(tree).toMap(), + tree.getAll().toList().getBlocking().toMap(), ) for (key in initialKeys) { - assertEquals("value of $key", tree.get(key).getBlocking(tree)) + assertEquals("value of $key", tree.get(key).getBlocking()) } var replacementTree = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) @@ -151,16 +151,16 @@ class PatriciaTrieTest { "abcdB93", ) for (key in replacementKeys) { - replacementTree = replacementTree.put(key, "new value of $key").getBlocking(tree) + replacementTree = replacementTree.put(key, "new value of $key").getBlocking() } - tree = tree.replaceSlice("abcdB", replacementTree).getBlocking(tree) + tree = tree.replaceSlice("abcdB", replacementTree).getBlocking() assertEquals( initialKeys.filterNot { it.startsWith("abcdB") }.map { it to "value of $it" } .plus(replacementKeys.map { it to "new value of $it" }) .sortedBy { it.first } .joinToString("\n") { "${it.first} -> ${it.second}" }, - tree.getAll().toList().getBlocking(tree).sortedBy { it.first }.joinToString("\n") { "${it.first} -> ${it.second}" }, + tree.getAll().toList().getBlocking().sortedBy { it.first }.joinToString("\n") { "${it.first} -> ${it.second}" }, ) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt index b2e4a88ae3..b25d5f3f60 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt @@ -29,7 +29,7 @@ class TreeDiffTest { val restoringGraph = FullyLoadedObjectGraph() val initialDiff = tree.asObject().getDescendantsAndSelf() .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree).toMap() + .toList().getBlocking().toMap() val restoringConfig = PatriciaTrieConfig( restoringGraph, StringDataTypeConfiguration(), @@ -40,11 +40,11 @@ class TreeDiffTest { fun assertTree() { val diff = tree.asObject().objectDiff(restoredTree.asObject()) .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree).toMap() + .toList().getBlocking().toMap() restoredTree = PatriciaTrie(restoringGraph.loadObjects(tree.asObject().getHash(), restoringDeserializer, diff)) val expectedEntries = addedEntries.associateWith { values[it]!! } - val actualEntries = restoredTree.getAll().toList().getBlocking(tree).toMap() + val actualEntries = restoredTree.getAll().toList().getBlocking().toMap() assertEquals(expectedEntries, actualEntries) } @@ -53,13 +53,13 @@ class TreeDiffTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } @@ -85,7 +85,7 @@ class TreeDiffTest { for (previousTree in (1..5).map { previousTrees.random(rand) }) { val diff = tree.asObject().objectDiff(previousTree.asObject()) .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree) + .toList().getBlocking() val duplicateObjects = diff.groupBy { it.first }.filter { it.value.size > 1 }.map { it.value.first() } assertEquals(emptyList(), duplicateObjects) @@ -98,13 +98,13 @@ class TreeDiffTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt index e67daa5486..667dec57d6 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt @@ -1,10 +1,5 @@ package org.modelix.model.api.async -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.maybeOf -import com.badoo.reaktive.maybe.maybeOfEmpty -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.singleOf import org.modelix.model.api.ConceptReference import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.IConcept @@ -26,9 +21,6 @@ open class NodeAsAsyncNode(val node: INode) : IAsyncNode { return SimpleStreamExecutor } - private fun T?.asOptionalMono(): Maybe = if (this != null) maybeOf(this) else maybeOfEmpty() - private fun T.asMono(): Single = singleOf(this) - override fun asRegularNode(): INode = node override fun getConcept(): IStream.One { diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt b/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt index 44b7a2831a..3bcd23d9e8 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt @@ -100,7 +100,7 @@ class RandomModelChangeGenerator(val rootNode: INode, private val rand: Random) .map { it.asRegularNode() } .filter(condition) .firstOrNull() - .getBlocking(rootNode.asAsyncNode()) + .getBlocking() } } diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt index c285142e80..30f64ea956 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt @@ -153,5 +153,5 @@ interface IModelClientV2 { @DelicateModelixApi suspend fun IModelClientV2.diffAsMutationParameters(repositoryId: RepositoryId, newVersion: ObjectHash, oldVersion: ObjectHash): List> { val version = lazyLoadVersion(repositoryId, newVersion.toString()) as CLVersion - return version.historyAsMutationParameters(oldVersion).toList().getSuspending(version.graph) + return version.historyAsMutationParameters(oldVersion).toList().getSuspending() } diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt index 3776392a1e..c5685794ff 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt @@ -110,7 +110,7 @@ suspend fun IModelClientV2.migrateRoles( }, ), ).andThenUnit() - }.drainAll().executeBlocking(oldTree) + }.drainAll().executeBlocking() } push(branch, newVersion, oldVersion) diff --git a/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt b/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt index 6378dd0b42..759fd48201 100644 --- a/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt +++ b/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt @@ -511,7 +511,7 @@ internal class ClientJSImpl(private val modelClient: ModelClientV2) : ClientJS { version.historyAsMutationParameters(ObjectHash(oldVersion)) .map { it.toJS() } .toList() - .getSuspending(version.graph) + .getSuspending() .toTypedArray() } } diff --git a/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt b/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt index 8f6cfa4cb9..6927dfa1a8 100644 --- a/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt +++ b/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt @@ -66,7 +66,7 @@ internal class ChangeListener(private val tree: IMutableModelTree, private val c } override fun treeChanged(oldTree: IGenericModelTree, newTree: IGenericModelTree) { - newTree.getChanges(oldTree, false).iterateBlocking(newTree) { + newTree.getChanges(oldTree, false).iterateBlocking { when (it) { is ConceptChangedEvent -> changeCallback(ConceptChanged(nodeIdToInode(it.nodeId))) is ContainmentChangedEvent -> changeCallback(ContainmentChanged(nodeIdToInode(it.nodeId))) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt index 6b40d6807b..4a0e2551e6 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt @@ -126,7 +126,7 @@ sealed class HistoryIndexNode : IObjectData { } fun of(version1: Object, version2: Object): HistoryIndexNode { - return of(version1).asObject(version1.graph).merge(of(version2).asObject(version2.graph)).getBlocking(version1.graph).data + return of(version1).asObject(version1.graph).merge(of(version2).asObject(version2.graph)).getBlocking().data } /** diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt index 334ae2a54f..48cd2c18ca 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt @@ -28,7 +28,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : val interval = delay / 2 runUntilLimit { - index.data.splitAtInterval(EquidistantIntervalsSpec(interval).withTimeRangeFilter(timeRange)).iterateSuspending(index.graph) { node -> + index.data.splitAtInterval(EquidistantIntervalsSpec(interval).withTimeRangeFilter(timeRange)).iterateSuspending { node -> if (previousMinTime - node.maxTime >= delay) { if (sessions.lastIndex >= pagination.asRange().last) throw LimitReached() sessions += HistoryInterval( @@ -76,7 +76,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : var previousIntervalId: Long = Long.MAX_VALUE runUntilLimit { - index.data.splitAtInterval(intervalsSpec).iterateSuspending(index.graph) { node -> + index.data.splitAtInterval(intervalsSpec).iterateSuspending { node -> val intervalId = intervalsSpec.getIntervalIndex(node.maxTime) check(intervalId <= previousIntervalId) if (intervalId == previousIntervalId) { @@ -114,7 +114,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : pagination: PaginationParameters, ): List { val index: Object = historyIndex() - val inTimeRange = if (timeRange == null) index else index.getRange(timeRange).orNull().getSuspending(index.graph) + val inTimeRange = if (timeRange == null) index else index.getRange(timeRange).orNull().getSuspending() if (inTimeRange == null) return emptyList() return inTimeRange.data.getRange(pagination.asRange()) .flatMapOrdered { it.getAllVersionsReversed() } @@ -129,7 +129,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : ) } .toList() - .getSuspending(index.graph) + .getSuspending() } override suspend fun splitAt(splitPoints: List): List { diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt index 89b17d9e2f..8f93c83968 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt @@ -42,7 +42,7 @@ abstract class ModelTreeBuilder private constructor(protected val common return HamtInternalNode.createEmpty(config) .put(root.data.id, root.ref, common.graph) .orNull() - .getBlocking(common.graph)!! + .getBlocking()!! .let { HamtTree(it) } .autoResolveValues() .asModelTree(common.treeId, common.storeRoleIds) @@ -64,7 +64,7 @@ abstract class ModelTreeBuilder private constructor(protected val common ) return PatriciaTrie(config) .put(root.data.id, root.ref) - .getBlocking(common.graph) + .getBlocking() .autoResolveValues() .asModelTree(common.treeId, common.storeRoleIds) } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt index fb779ab997..89d4099259 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt @@ -107,7 +107,7 @@ constructor(private val idGenerator: IIdGenerator?) { } } } + transaction.tree.getChildren(transaction.tree.getRootNodeId(), ITree.DETACHED_NODES_LINK) - .toList().getBlocking(transaction.tree) + .toList().getBlocking() .map { DeleteNodeOp(it).apply(mutableTree) } mergedVersion = CLVersion.builder() diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt index 6ba9bb25dc..9f9df9a5a9 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt @@ -57,7 +57,7 @@ private fun createNewTreeData( HamtInternalNode.createEmpty(config) .put(root.id, graph.fromCreated(root), graph) .orNull() - .getBlocking(graph)!!, + .getBlocking()!!, ), trieWithNodeRefIds = null, usesRoleIds = useRoleIds, diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt index 92403c3cf6..d8fedbf381 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt @@ -320,7 +320,7 @@ class CLVersion(val obj: Object) : IVersion { } fun loadFromHash(hash: ObjectHash, graph: IObjectGraph): CLVersion { - return requestFromHash(hash, graph).getBlocking(graph) + return requestFromHash(hash, graph).getBlocking() } fun requestFromHash(hash: ObjectHash, graph: IObjectGraph): IStream.One { diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt index 9bb8e44ea1..d87629c22f 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt @@ -56,7 +56,7 @@ class OperationsCompressor(val resultTree: Object) { } for (id in createdNodes) { - if (!resultTree.data.getModelTree().containsNode(id).getBlocking(resultTree.graph)) { + if (!resultTree.data.getModelTree().containsNode(id).getBlocking()) { throw RuntimeException("Tree expected to contain node $id") } } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt index 8e99eaba66..63a2fe27b9 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt @@ -58,7 +58,7 @@ class SingleThreadMutableModelTree( override fun mutate(parameters: MutationParameters) { val oldTree = tree - val newTree = tree.mutate(parameters).getBlocking(tree) + val newTree = tree.mutate(parameters).getBlocking() tree = newTree for (listener in listeners) { listener.treeChanged(oldTree, newTree) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt index 19d6742631..f8e1456527 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt @@ -29,8 +29,8 @@ sealed class AbstractOperation : IOperation { } protected fun getNodePosition(tree: IModelTree, nodeId: INodeReference): PositionInRole { - val (parent, role) = requireNotNull(tree.getContainment(nodeId).getBlocking(tree)) { "Node has no parent: $nodeId" } - val index = tree.getChildren(parent, role).toList().getBlocking(tree).indexOf(nodeId) + val (parent, role) = requireNotNull(tree.getContainment(nodeId).getBlocking()) { "Node has no parent: $nodeId" } + val index = tree.getChildren(parent, role).toList().getBlocking().indexOf(nodeId) return PositionInRole(RoleInNode(parent, role), index) } @@ -41,7 +41,7 @@ sealed class AbstractOperation : IOperation { protected fun getDetachedNodesEndPosition(tree: IModelTree): PositionInRole { val detachedRole = RoleInNode(tree.getRootNodeId(), ITree.DETACHED_NODES_LINK) - val index = tree.getChildren(detachedRole.nodeId.toGlobal(tree.getId()), detachedRole.role).count().getBlocking(tree) + val index = tree.getChildren(detachedRole.nodeId.toGlobal(tree.getId()), detachedRole.role).count().getBlocking() return PositionInRole(detachedRole, index) } } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt index da63ac666b..0992c3da2b 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt @@ -63,20 +63,20 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept } override fun captureIntend(tree: IModelTree): IOperationIntend { - val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree) + val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking() return Intend(CapturedInsertPosition(position.index, children.toList())) } inner class Intend(val capturedPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - val parentExists = tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking(tree) - val childExists = IStream.many(childIdsAndConcepts).flatMap { tree.containsNode(it.first) }.toList().getBlocking(tree) + val parentExists = tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking() + val childExists = IStream.many(childIdsAndConcepts).flatMap { tree.containsNode(it.first) }.toList().getBlocking() val targetPosition = if (parentExists) { val newIndex = if (position.index < 0) { position.index } else { capturedPosition.findIndex( - tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree), + tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(), position.index, ) } @@ -91,7 +91,7 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept // handle already existing child IDs childIdsAndConcepts.zip(childExists).asReversed().flatMap { (idAndConcept, exists) -> val (childId, childConcept) = idAndConcept - val currentContainment = tree.getContainment(childId).getBlocking(tree) + val currentContainment = tree.getContainment(childId).getBlocking() // If the containment is correct, ignore the index to avoid unnecessary changes. As part of the // conflict resolution algorithm we are allowed to make any decision. There are no right or wrong @@ -105,7 +105,7 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept } if (exists) { - if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking(tree)) { + if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking()) { emptyList() } else { listOf(MoveNodeOp(childId, targetPosition)) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt index 12603c82d0..e0b3c30b62 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt @@ -28,15 +28,15 @@ class AddNewChildSubtreeOp( } override fun apply(tree: IMutableModelTree): IAppliedOperation { - decompress().iterateBlocking(resultTreeHash.graph) { it.apply(tree) } + decompress().iterateBlocking { it.apply(tree) } return Applied() } fun decompress(): IStream.Many { val resultTree = getResultTree() return resultTree.getDescendants(childId.toGlobal(resultTree.getId()), true).flatMapOrdered { node -> - val parent = resultTree.getParent(node).getBlocking(resultTree)!! - val roleInParent = resultTree.getRoleInParent(node).getBlocking(resultTree)!! + val parent = resultTree.getParent(node).getBlocking()!! + val roleInParent = resultTree.getRoleInParent(node).getBlocking()!! val pos = PositionInRole( parent, roleInParent, @@ -45,8 +45,8 @@ class AddNewChildSubtreeOp( decompressNode(resultTree, node, pos, false) } + resultTree.getDescendants(childId.toGlobal(resultTree.getId()), true).flatMapOrdered { node -> - val parent = resultTree.getParent(node).getBlocking(resultTree)!! - val roleInParent = resultTree.getRoleInParent(node).getBlocking(resultTree)!! + val parent = resultTree.getParent(node).getBlocking()!! + val roleInParent = resultTree.getRoleInParent(node).getBlocking()!! val pos = PositionInRole( parent, roleInParent, @@ -64,7 +64,7 @@ class AddNewChildSubtreeOp( SetReferenceOp(node, role, target) } } else { - IStream.of(AddNewChildOp(position!!, node, tree.getConceptReference(node).getBlocking(tree))) + + IStream.of(AddNewChildOp(position!!, node, tree.getConceptReference(node).getBlocking())) + tree.getProperties(node).map { (property, value) -> SetPropertyOp(node, property, value) } } } @@ -82,20 +82,20 @@ class AddNewChildSubtreeOp( .getDescendants(childId.toGlobal(resultTree.getId()), true) .map { DeleteNodeOp(it) } .toList() - .getBlocking(resultTree.asObject().graph) + .getBlocking() .asReversed() } } override fun captureIntend(tree: IModelTree): IOperationIntend { - val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree) + val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking() return Intend(CapturedInsertPosition(position.index, children)) } inner class Intend(val capturedPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - if (tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { - val newIndex = capturedPosition.findIndex(tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree)) + if (tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking()) { + val newIndex = capturedPosition.findIndex(tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking()) return listOf(withPosition(position.withIndex(newIndex))) } else { return listOf(withPosition(getDetachedNodesEndPosition(tree))) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt index 3e8048d673..7c037f2f50 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt @@ -56,7 +56,7 @@ class BulkUpdateOp( override fun restoreIntend(tree: IModelTree): List { // The intended change is to put the model into the given state. Any concurrent change can just be // overwritten as long as the subtree root as the starting point still exists. - return if (tree.containsNode(subtreeRootId.toGlobal(tree.getId())).getBlocking(tree)) { + return if (tree.containsNode(subtreeRootId.toGlobal(tree.getId())).getBlocking()) { listOf(getOriginalOp()) } else { listOf(NoOp()) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt index a6ba48a1bf..9170779a89 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt @@ -33,8 +33,8 @@ class DeleteNodeOp(val childId: INodeReference) : AbstractOperation(), IOperatio } override fun restoreIntend(tree: IModelTree): List { - if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking(tree)) return listOf(NoOp()) - val allChildren = tree.getChildren(childId.toGlobal(tree.getId())).toList().getBlocking(tree) + if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking()) return listOf(NoOp()) + val allChildren = tree.getChildren(childId.toGlobal(tree.getId())).toList().getBlocking() if (allChildren.isNotEmpty()) { val targetPos = getDetachedNodesEndPosition(tree) return allChildren diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt index 34049a6045..30b24d4886 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt @@ -44,24 +44,24 @@ class MoveNodeOp(val childId: INodeReference, val targetPosition: PositionInRole override fun captureIntend(tree: IModelTree): IOperationIntend { val capturedTargetPosition = CapturedInsertPosition( targetPosition.index, - tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(tree), + tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(), ) return Intend(capturedTargetPosition) } inner class Intend(val capturedTargetPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking(tree)) return emptyList() + if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking()) return emptyList() val newSourcePosition = getNodePosition(tree, childId.toGlobal(tree.getId())) - if (!tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { + if (!tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking()) { return listOf( withPos(getDetachedNodesEndPosition(tree)), ) } - if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking(tree)) return emptyList() - val newTargetPosition = if (tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { + if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking()) return emptyList() + val newTargetPosition = if (tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking()) { val newTargetIndex = capturedTargetPosition.findIndex( - tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(tree), + tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(), targetPosition.index, ) targetPosition.withIndex(newTargetIndex) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt index 5e146d1739..d93c9baa4f 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt @@ -31,7 +31,7 @@ class SetConceptOp(val nodeId: INodeReference, val concept: ConceptReference?) : override fun getOriginalOp(): IOperation = this override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt index f4961df74f..1aeddd62d7 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt @@ -21,7 +21,7 @@ class SetPropertyOp(val nodeId: INodeReference, val role: IPropertyReference, va } override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt index 23780ce220..1673410378 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt @@ -22,7 +22,7 @@ class SetReferenceOp(val sourceId: INodeReference, val role: IReferenceLinkRefer } override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(sourceId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(sourceId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt b/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt index 09d67a65ab..693ef60861 100644 --- a/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt +++ b/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt @@ -144,20 +144,20 @@ class DuplicateImportConflictTest : TreeTestBase() { } private fun assertSameTree(tree1: IGenericModelTree, tree2: IGenericModelTree) { - val changes = tree2.getChanges(tree1, false).toList().getBlocking(tree2) + val changes = tree2.getChanges(tree1, false).toList().getBlocking() for (changeEvent in changes) { when (changeEvent) { is ChildrenChangedEvent -> { - val children1 = tree1.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking(tree1) - val children2 = tree2.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking(tree2) + val children1 = tree1.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking() + val children2 = tree2.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking() println(changeEvent) println(" children1: $children1") println(" children2: $children2") } is ReferenceChangedEvent -> { - val target1 = tree1.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking(tree1) - val target2 = tree2.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking(tree2) + val target1 = tree1.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking() + val target2 = tree2.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking() println(changeEvent) println(" target1: $target1") println(" target2: $target2") diff --git a/model-datastructure/src/commonTest/kotlin/HamtTest.kt b/model-datastructure/src/commonTest/kotlin/HamtTest.kt index 0a746912d7..f3e8331f07 100644 --- a/model-datastructure/src/commonTest/kotlin/HamtTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HamtTest.kt @@ -35,25 +35,25 @@ class HamtTest { // add entry val key = rand.nextInt(1000).toLong() val value = rand.nextLong() - hamt = hamt.put(key, createEntry(value, graph)).getBlocking(hamt) + hamt = hamt.put(key, createEntry(value, graph)).getBlocking() expectedMap[key] = value } else { val keys: List = ArrayList(expectedMap.keys) val key = keys[rand.nextInt(keys.size)] if (rand.nextBoolean()) { // remove entry - hamt = hamt.remove(key).getBlocking(hamt) + hamt = hamt.remove(key).getBlocking() expectedMap.remove(key) } else { // replace entry val value = rand.nextLong() - hamt = hamt.put(key, createEntry(value, graph)).getBlocking(hamt) + hamt = hamt.put(key, createEntry(value, graph)).getBlocking() expectedMap[key] = value } } storeCache.clearCache() for ((key, value) in expectedMap) { - assertEquals(value, hamt.get(key).flatMapZeroOrOne { it.resolve() }.getBlocking(hamt)!!.data.id) + assertEquals(value, hamt.get(key).flatMapZeroOrOne { it.resolve() }.getBlocking()!!.data.id) } } } @@ -83,17 +83,17 @@ class HamtTest { valueConfig = ObjectReferenceDataTypeConfiguration(graph, CPNode), ) var hamt = HamtTree(HamtInternalNode.createEmpty(config)) - var getId = { e: IStream.ZeroOrOne> -> e.flatMapZeroOrOne { it.resolve() }.getBlocking(hamt)!!.data.id } + var getId = { e: IStream.ZeroOrOne> -> e.flatMapZeroOrOne { it.resolve() }.getBlocking()!!.data.id } - hamt = hamt.put(965L, createEntry(-6579471327666419615, graph)).getBlocking(hamt) - hamt = hamt.put(949L, createEntry(4912341421267007347, graph)).getBlocking(hamt) + hamt = hamt.put(965L, createEntry(-6579471327666419615, graph)).getBlocking() + hamt = hamt.put(949L, createEntry(4912341421267007347, graph)).getBlocking() assertEquals(4912341421267007347, getId(hamt.get(949L))) - hamt = hamt.put(260L, createEntry(4166750678024106842, graph)).getBlocking(hamt) + hamt = hamt.put(260L, createEntry(4166750678024106842, graph)).getBlocking() assertEquals(4166750678024106842, getId(hamt.get(260L))) - hamt = hamt.put(794L, createEntry(5492533034562136353, graph)).getBlocking(hamt) - hamt = hamt.put(104L, createEntry(-6505928823483070382, graph)).getBlocking(hamt) - hamt = hamt.put(47L, createEntry(3122507882718949737, graph)).getBlocking(hamt) - hamt = hamt.put(693L, createEntry(-2086105010854963537, graph)).getBlocking(hamt) + hamt = hamt.put(794L, createEntry(5492533034562136353, graph)).getBlocking() + hamt = hamt.put(104L, createEntry(-6505928823483070382, graph)).getBlocking() + hamt = hamt.put(47L, createEntry(3122507882718949737, graph)).getBlocking() + hamt = hamt.put(693L, createEntry(-2086105010854963537, graph)).getBlocking() storeCache.clearCache() // assertEquals(69239088, (hamt!!.getData() as CPHamtInternal).bitmap) // assertEquals(6, (hamt!!.getData() as CPHamtInternal).children.count()) @@ -130,8 +130,8 @@ class HamtTest { for (i in 1..10) { var map = emptyMap - entries.entries.shuffled(rand).forEach { map = map.put(it.key, it.value).getBlocking(map) } - keysToRemove.forEach { map = map.remove(it).getBlocking(map) } + entries.entries.shuffled(rand).forEach { map = map.put(it.key, it.value).getBlocking() } + keysToRemove.forEach { map = map.remove(it).getBlocking() } val hash = map.asObject().getHashString() if (i == 1) { expectedHash = hash diff --git a/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt b/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt index c8b3768788..3ce1c3c405 100644 --- a/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt @@ -67,7 +67,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val node = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(AttributeValuesAggregation.of("staging", "prod"), node.data.attributes.getValues("env")) assertEquals(AttributeValuesAggregation.of("1"), node.data.attributes.getValues("run")) } @@ -80,7 +80,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val node = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(AttributeValuesAggregation.of("a", "b"), node.data.attributes["env"]) assertEquals(AttributeValuesAggregation.of("1"), node.data.attributes["run"]) } @@ -104,7 +104,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val original = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() val serialized = original.data.serialize() // Range node serializes attributes in field index 8 (0-based, LEVEL1-delimited). // Extract and deserialize just that field rather than the full node, because the full diff --git a/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt b/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt index 3bdf3d3b10..ef65cd7e02 100644 --- a/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt @@ -67,7 +67,7 @@ class HistoryIndexTest { val graph = version1.graph val history1 = HistoryIndexNode.of(version1.obj, version2.obj).asObject(graph) val history2 = HistoryIndexNode.of(version3.obj).asObject(graph) - val history = history1.merge(history2).getBlocking(graph) + val history = history1.merge(history2).getBlocking() assertEquals(3, history.size) assertEquals(3, history.height) } @@ -82,7 +82,7 @@ class HistoryIndexTest { val history = HistoryIndexNode.of(version1.obj, version2.obj).asObject(graph) .merge(HistoryIndexNode.of(version3.obj).asObject(graph)) .merge(HistoryIndexNode.of(version4.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(4, history.size) assertEquals(4, history.height) } @@ -99,7 +99,7 @@ class HistoryIndexTest { .merge(HistoryIndexNode.of(version3.obj).asObject(graph)) .merge(HistoryIndexNode.of(version4.obj).asObject(graph)) .merge(HistoryIndexNode.of(version5.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(5, history.size) assertEquals(4, history.height) } @@ -135,16 +135,16 @@ class HistoryIndexTest { .merge(HistoryIndexNode.of(version4b.obj).asObject(graph)) .merge(HistoryIndexNode.of(version5b.obj).asObject(graph)) .merge(HistoryIndexNode.of(version6b.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() val history = historyA .merge(historyB) .merge(HistoryIndexNode.of(version7.obj).asObject(graph)) .merge(HistoryIndexNode.of(version8.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals( listOf(version1, version2, version3a, version3b, version4a, version4b, version5a, version5b, version6b, version7, version8).map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.size) assertEquals(5, history.height) @@ -158,12 +158,12 @@ class HistoryIndexTest { } val graph = versions.first().graph val history = versions.drop(1).fold(HistoryIndexNode.of(versions.first().asObject()).asObject(graph)) { acc, it -> - acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking(graph) + acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking() } assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.height) } @@ -176,12 +176,12 @@ class HistoryIndexTest { } val graph = versions.first().graph val history = versions.drop(1).shuffled(Random(78234554)).fold(HistoryIndexNode.of(versions.first().asObject()).asObject(graph)) { acc, it -> - acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking(graph) + acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking() } assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(13, history.height) } @@ -198,7 +198,7 @@ class HistoryIndexTest { assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.height) } @@ -215,7 +215,7 @@ class HistoryIndexTest { assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(14, history.height) } @@ -228,6 +228,6 @@ class HistoryIndexTest { val centerIndex = versions.size / 2 return buildHistory(versions.subList(0, centerIndex)) .merge(buildHistory(versions.subList(centerIndex, versions.size))) - .getBlocking(graph) + .getBlocking() } } diff --git a/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt b/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt index 6b8b969c9f..682d03cabf 100644 --- a/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt +++ b/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt @@ -87,9 +87,9 @@ class ObjectDiffTest { newTree = changeGenerator2.applyRandomChange(newTree, null) } - val diff = newTree.getTreeObject().objectDiff(initialTree.getTreeObject()).toList().getBlocking(store1) - val initialObjects = initialTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking(store1) - val newObjects = newTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking(store1) + val diff = newTree.getTreeObject().objectDiff(initialTree.getTreeObject()).toList().getBlocking() + val initialObjects = initialTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking() + val newObjects = newTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking() val unnecessaryObjects = (diff.associateBy { it.getHashString() } - newObjects.map { it.getHashString() }.toSet()).values.toSet() assertEquals(emptySet(), unnecessaryObjects) diff --git a/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt b/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt index 578a87ea63..084a623d74 100644 --- a/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt +++ b/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt @@ -295,7 +295,7 @@ private class TransactionAdapter(val transaction: IGenericMutableModelTree.Write override fun getChildren(parentId: Long, role: String?): Iterable { return transaction.tree.getChildren(parentId.translate(), IChildLinkReference.fromString(role)) - .map { it.translate() }.toList().getBlocking(transaction.tree) + .map { it.translate() }.toList().getBlocking() } override fun getAllChildren(parentId: Long): Iterable { diff --git a/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt b/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt index c4dffce19b..661cf18daa 100644 --- a/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt +++ b/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt @@ -29,7 +29,7 @@ class VersionAttributesTest { val restoredVersion = graph2.loadObjects( rootHash = originalVersion.getObjectHash(), rootDeserializer = CPVersion, - receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(graph1), + receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(), ).let { CLVersion(it) } assertEquals(mapOf(key to value), restoredVersion.getAttributes()) @@ -55,7 +55,7 @@ class VersionAttributesTest { val restoredVersion = graph2.loadObjects( rootHash = originalVersion.getObjectHash(), rootDeserializer = CPVersion, - receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(graph1), + receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(), ).let { CLVersion(it) } assertEquals(attrs, restoredVersion.getAttributes()) diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt index 675bba3ab9..38ecf48f86 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt @@ -91,10 +91,7 @@ class LionwebApiImpl(val repoManager: IRepositoriesManager) : LionwebApi() { // TODO use an index to find the nodes by their foreign ID (aka original ID) if (foreignIds.isNotEmpty()) { -// version.obj.graph.getStreamExecutor().iterateSuspending({ -// version.tree.nodesMap.getEntries().flatMap { it.second.resolve() } -// .filter { foreignIds.contains(it.data.getPropertyValue(NodeData.ID_PROPERTY_KEY)) } -// }) { +// version.obj.graph.getStreamExecutor().iterateSuspending { // modelixIds.add(it.data.id) // } } diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt index 3bc0af6084..24d4a83ebe 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt @@ -361,14 +361,14 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { // Deleting the root node isn't allowed val mainTree = newVersion.getModelTree() - check(mainTree.containsNode(mainTree.getRootNodeId()).getBlocking(mainTree)) + check(mainTree.containsNode(mainTree.getRootNodeId()).getBlocking()) // ensure there are no missing objects // newVersion.graph.getStreamExecutor().iterate({ newVersion.diff(oldVersion) }) { } if (oldVersion != null) { // If the object diff is buggy, client and server will skip over the same objects. // The model diff should also iterate over all new objects and is used for additional validation. - newVersion.getModelTree().getChanges(oldVersion.getModelTree(), false).iterateBlocking(newVersion.getModelTree()) { } + newVersion.getModelTree().getChanges(oldVersion.getModelTree(), false).iterateBlocking { } } // TODO check invariants of the model (consistent parent-child relations, single root, containment cycles) @@ -380,7 +380,7 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { check(newVersion.operations.count() == 0) } else { val mutableTree = baseVersion.getModelTree().asMutableSingleThreaded() - newVersion.operationsAsStream().iterateBlocking(mainTree) { op -> + newVersion.operationsAsStream().iterateBlocking { op -> op.apply(mutableTree) } check(mutableTree.getTransaction().tree.getHash() == newVersion.getModelTree().getHash()) { @@ -438,8 +438,8 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { val newElement = HistoryIndexNode.of(version.obj).asObject(graph) val newIndex = when (parentIndices.size) { 0 -> newElement - 1 -> parentIndices.single().merge(newElement).getBlocking(graph) - 2 -> parentIndices[0].merge(parentIndices[1]).merge(newElement).getBlocking(graph) + 1 -> parentIndices.single().merge(newElement).getBlocking() + 2 -> parentIndices[0].merge(parentIndices[1]).merge(newElement).getBlocking() else -> error("impossible") } newIndex.write() @@ -613,7 +613,7 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { val version = CLVersion.loadFromHash(versionHash, sourceStore) // Use diff with empty list to get all objects reachable from this version - version.diff(emptyList()).iterateBlocking(sourceStore) { obj -> + version.diff(emptyList()).iterateBlocking { obj -> reachableHashes.add(obj.getHashString()) } } diff --git a/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt index 4bd2434066..19a1827c92 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt @@ -454,7 +454,7 @@ private fun getChildren(modelTree: IMutableModelTree): SortedSet getChildren(it.tree) } private fun getChildren(modelTree: IGenericModelTree): SortedSet = - modelTree.getChildren(modelTree.getRootNodeId()).toList().getBlocking(modelTree) + modelTree.getChildren(modelTree.getRootNodeId()).toList().getBlocking() .toSortedSet(compareBy { it.serialize() }) private fun IMutableModelTree.treeHash(): String = runRead { t -> t.tree.asObject().getHashString() } diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt index 0b4c3d61dc..c2350d7d32 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt index 09205f8045..ef75f48dac 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt index 899e1c2723..ff758293ef 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt index 068f24843e..1a924c5fec 100644 --- a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt +++ b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt @@ -95,7 +95,7 @@ class GitExportTest : MPSTestBase() { val reimportedVersion = client.pull(modelixBranchReimport, null) // check that no changes got lost during the round-trip - val diff = reimportedVersion.getModelTree().getChanges(modifiedVersion.getModelTree(), false).toList().getSuspending(reimportedVersion.asObject().graph) + val diff = reimportedVersion.getModelTree().getChanges(modifiedVersion.getModelTree(), false).toList().getSuspending() val changes = ArrayList() for (event in diff) { when (event) { @@ -105,9 +105,9 @@ class GitExportTest : MPSTestBase() { is NodeRemovedEvent -> TODO() is ChildrenChangedEvent -> TODO() is PropertyChangedEvent -> { - changes += modifiedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending(modifiedVersion.asObject().graph) + + changes += modifiedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending() + " -> " + - reimportedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending(reimportedVersion.asObject().graph) + reimportedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending() } is ReferenceChangedEvent -> TODO() } diff --git a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt index 18e2c19370..cb517b8479 100644 --- a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt +++ b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt @@ -583,7 +583,7 @@ class GitImportTest : MPSTestBase() { for (version in latestVersion.historyAsSequence()) { for (parentVersion in version.getParentVersions()) { val baseObjects = parentVersion.getModelTree().asObject().getDescendantsAndSelf().map { it.getHash() }.toList() - .getBlocking(parentVersion.asObject().graph) + .getBlocking() val baseObjectsSet = baseObjects.toSet() // no object is returned twice @@ -602,13 +602,13 @@ class GitImportTest : MPSTestBase() { it } .toList() - .getBlocking(parentVersion.asObject().graph) + .getBlocking() // also the delta itself doesn't contain duplicate objects val duplicateObjects: List> = deltaObjects.groupBy { it.getHash() }.filter { it.value.size > 1 }.map { it.value.first() } if (duplicateObjects.isNotEmpty()) { // place breakpoint here for debugging - version.diff(parentVersion, filter).toList().getBlocking(parentVersion.asObject().graph) + version.diff(parentVersion, filter).toList().getBlocking() } assertEquals(emptyList>(), duplicateObjects) @@ -617,12 +617,12 @@ class GitImportTest : MPSTestBase() { if (unnecessaryObjects.isNotEmpty()) { // just for debugging parentVersion.getModelTree().asObject().getDescendantsAndSelf().toList() - .getBlocking(parentVersion.asObject().graph).joinToString("\n") { it.toString() } + .getBlocking().joinToString("\n") { it.toString() } .let { File("oldVersion.txt").writeText(it) } version.getModelTree().asObject().getDescendantsAndSelf().toList() - .getBlocking(parentVersion.asObject().graph).joinToString("\n") { it.toString() } + .getBlocking().joinToString("\n") { it.toString() } .let { File("newVersion.txt").writeText(it) } - version.diff(parentVersion, filter).toList().getBlocking(parentVersion.asObject().graph) + version.diff(parentVersion, filter).toList().getBlocking() } assertEquals(emptyList>(), unnecessaryObjects) } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index 2edaf21d16..7ae6bcfb86 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -446,7 +446,7 @@ class BindingWorker( val node = sourceModel.tryResolveNode(nodeId) ?: return invalidationTree.invalidate(node, false) } - newTree.getChanges(baseVersion.getModelTree(), changesOnly = true).iterateSuspending(newTree.asObject().graph) { event -> + newTree.getChanges(baseVersion.getModelTree(), changesOnly = true).iterateSuspending { event -> when (event) { is ContainmentChangedEvent, is NodeRemovedEvent, is NodeAddedEvent -> { // There will be a ChildrenChangedEvent that indirectly handles these cases. diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 8f8e6be9e9..c287e51f69 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -198,7 +198,7 @@ class ProjectSyncTest : ProjectSyncTestBase() { assertNotEquals(version2.getObjectHash(), version3.getObjectHash()) // version4 is the result of merging version3 into version2 - val diff = version4.getModelTree().getChanges(version2.getModelTree(), false).toList().getBlocking(version4.getModelTree()) + val diff = version4.getModelTree().getChanges(version2.getModelTree(), false).toList().getBlocking() assertEquals(emptyList>(), diff) assertEquals(version2.getTreeReference().getHash(), version4.getTreeReference().getHash()) @@ -291,8 +291,8 @@ class ProjectSyncTest : ProjectSyncTestBase() { assertEquals(1, changes.size) val change = changes.single() as PropertyChangedEvent assertEquals(MPSProperty(nameProperty).getUID(), change.role.getUID()) - assertEquals("MyClass", version1.getModelTree().getProperty(change.nodeId, change.role).getBlocking(version1.getModelTree())) - assertEquals("Changed", version2.getModelTree().getProperty(change.nodeId, change.role).getBlocking(version1.getModelTree())) + assertEquals("MyClass", version1.getModelTree().getProperty(change.nodeId, change.role).getBlocking()) + assertEquals("Changed", version2.getModelTree().getProperty(change.nodeId, change.role).getBlocking()) } fun `test descendants of new node are synchronized`() = runChangeInMpsTest { classNode -> diff --git a/streams-redesign.md b/streams-redesign.md new file mode 100644 index 0000000000..3bd96155f1 --- /dev/null +++ b/streams-redesign.md @@ -0,0 +1,245 @@ +# Streams redesign & migration + +This document records the redesign of the `streams` module: replacing its three interchangeable reactive backends +(Sequence / Flow / Reaktive) with a single round-based interpreter, while preserving the public API so that no +downstream module required API-driven changes. + +The engine was first prototyped in a separate `streams2` module and then merged into `streams` (the prototype's +parallel `IStream` API and engine were redundant once the design was proven). See [`streams/README.md`](streams/README.md) +for the resulting design overview; this document covers the redesign and the migration of the wider codebase. + +--- + +## 1. Motivation + +The `streams` module existed to provide one capability that plain coroutines/`Flow` could not give cheaply: +**automatic bulk-request batching** for lazy, partial loading of large content-addressed models. When a traversal +needs many objects from a remote store, the independent requests must be coalesced into a single round-trip. + +### Why there were three backends + +The old design carried three implementations because they had different execution semantics +(see the former doc-comment in `IStreamExecutor`): + +- **Sequence** — synchronous, eager, pull-based; minimal overhead when all data is local. +- **Flow** — suspendable, pull-based; best integration with `suspend` sources. +- **Reaktive** — push-based; the *only* one able to collect a set of pending requests before forcing any of them, + which is what bulk batching needs. Flows could parallelize requests only by launching a coroutine per object — + too much overhead. + +This meant ~1900 lines across `SequenceStreamBuilder`, `FlowStreamBuilder`, `ReaktiveStreamBuilder`, +`DeferredStreamBuilder`, the concrete stream classes, and the Reaktive third-party dependency. + +### The key insight + +Batching requires the runtime to **see a set of independent data requests before forcing any of them**. That is an +**applicative** property, not a monadic one: + +- `zip(a, b)` and the elements of a `Many` are **independent** → their fetches belong to the **same batch round**. +- `flatMap` is a **dependency** → the right-hand side's keys are unknown until the left resolves → a **later round**. + +Pull (Sequence/Flow) is demand-driven and sequential and cannot expose this frontier without per-item concurrency. +Push (Reaktive) can. But a purpose-built interpreter gets the same property **structurally**, from the shape of the +computation, with no external dependency — and collapses all three backends into one. + +Prior art for this pattern: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. + +--- + +## 2. The new design + +### `Step` — the intermediate representation + +A stream is described lazily and interpreted in **rounds**. The core type +([`streams/.../engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)). + +> **Note:** this section describes the *first* cut. The `Blocked`/`Pending`/`resume` representation below was later +> replaced by a lazily-evaluated node graph to restore a depth-first bounded request frontier — see §6a for the +> current model. The applicative/monadic split and round-based batching described here are unchanged. + +```kotlin +sealed interface Step +class Done(val values: List) // fully resolved +class Blocked(val pending: Pending, val resume: () -> Step) // needs a round of work first +class Failed(val cause: Throwable) +``` + +`Pending` is the deduplicated work for one round: bulk fetches grouped by data source, plus async leaves. The +combinators encode the applicative/monadic split: + +- `flatMapStep` (monadic) — defers the continuation into the next round. +- `combineConcat` / `zipN` / `zip2` (applicative) — **union** the pending work of independent branches into the same + round. That union *is* the batch. + +A subtree with no `Blocked` resolves straight to `Done`: the **synchronous fast path** for local data, with no +scheduler and nothing allocated (this subsumes the old Sequence backend's role). + +### The driver + +`Execution.drive` (and `driveSuspending`) is a loop where each iteration is one batch round: + +1. For each data source in `pending`, issue **one bulk call** (`execute` / `executeSuspending`), chunked to `batchSize`. +2. Run any async leaves. +3. Fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds). +4. `resume()` and repeat until `Done`. + +The loop is the **trampoline** that keeps fetch-dependent chains (the common case in tree traversal) stack-safe +regardless of depth. This subsumes the old Reaktive subscribe-collect-batch mechanism. + +### Batching is now structural + +Previously, batching was tied to an executor via a `ContextValue`: `enqueue(key)` registered into the +current query's queue. Now a fetch leaf carries its own data **source**, and the driver groups fetches **per source +per round**. Consequences: + +- `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to its `IBulkExecutor`. +- Any executor that drives a stream batches its fetches — including `SimpleStreamExecutor`, which therefore now issues + **strictly fewer** round-trips than before, never more. + +--- + +## 3. How the public API was preserved + +The entire public surface of `streams` is unchanged. The interface files (`IStream`, `IStreamBuilder`, +`IStreamExecutor`, `IExecutableStream`, `IStreamInternal`, `IBulkExecutor`, `Zip`, and all extension functions) were +kept as-is. Only the *implementation* was swapped: + +| Concern | Before | After | +|---|---|---| +| Stream instances | `SingleValueStream`, `CollectionAsStream`, `EmptyStream`, deferred wrappers, … | one `StreamImpl` backing every cardinality, plus `CompletableImpl` | +| Builder | `DeferredStreamBuilder` (+ 3 backend builders) | one `StreamBuilderImpl` over the engine | +| `SimpleStreamExecutor` / `BlockingStreamExecutor` | Sequence/Flow conversion | thin wrappers over `drive` / `driveSuspending` | +| `BulkRequestStreamExecutor` | Reaktive + `RequestQueue` + `CompletableObservable` | round driver; `enqueue` = fetch leaf | +| `convert(IStreamBuilder)` | converts to a backend representation | returns `this` (single representation) | +| `asFlow()` / `asSequence()` | backend conversion | materialize via the driver | +| `fromFlow` / `singleFromCoroutine` | Flow/Reaktive sources | **async leaves** resolved by the suspending driver (or `runBlockingIfJvm` under the blocking driver) | + +New files in `streams`: + +- [`engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt) — `Step`, `Pending`, + combinators, blocking + suspending drivers, fetch leaves, async leaves. +- [`StreamImpl.kt`](streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt) — `StreamImpl`, `CompletableImpl`, + `StreamBuilderImpl`, `StreamAssertionError`. + +Deleted: `SequenceStreamBuilder`, `FlowStreamBuilder`, `ReaktiveStreamBuilder`, `DeferredStreamBuilder`, +`SingleValueStream`, `CollectionAsStream`, `SequenceAsStream`, `EmptyStream`, `EmptyCompletableStream`, +`CompletableObservable`, `StreamExtensions` (Reaktive helpers). The Reaktive dependency was removed from +`streams/build.gradle.kts`. + +### Why a single module (the `streams2` prototype was merged in) + +The engine was prototyped in a separate `streams2` module, then merged into `streams` rather than kept as a separate +dependency, because: + +- the engine types are `internal` to `streams`; +- the real integration needs a **suspending driver** and **async leaves** that the minimal prototype deliberately + omitted; +- `streams`' `IBulkExecutor` already declares `executeSuspending`, which the engine uses directly; +- keeping both meant two competing `IBulkExecutor` / `IStream` hierarchies for no benefit — the prototype's API was a + strict subset of the one `streams` already exposes. + +The prototype's batching/dedup/stack-safety tests were kept, rewritten against the real `BulkRequestStreamExecutor` +API (`BulkRequestBatchingTest`). + +--- + +## 4. Migration of the wider codebase + +**No downstream module required API-driven changes.** The only edits outside `streams` were removals of **dead code** +that compiled solely because `streams` used to re-export Reaktive via an `api` dependency: + +- `model-api/.../async/NodeAsAsyncNode.kt` — two unused private helpers (`asOptionalMono`, `asMono`) and their imports. +- `modelql-untyped/.../AllChildrenTraversalStep.kt`, `AllReferencesTraversalStep.kt`, `DescendantsTraversalStep.kt` — + unused `com.badoo.reaktive.observable.{map, flatMap}` imports (the `.map`/`.flatMap` calls resolve to `IStream` + members). + +These were latent couplings to a transitive dependency, not uses of the `streams` API. + +### Verification + +- `streams`: compiles JVM + JS; unit tests pass. +- `datastructures`, `model-datastructure`: compile **untouched**; full JVM test suites pass. +- `modelql-core`, `modelql-untyped`: compile + tests pass. +- `model-client`, `model-server`, `model-server-api`, `modelql-typed/-html/-client/-server`, `bulk-model-sync-lib`: + compile (JVM + JS where applicable). +- **`model-server` `LazyLoadingTest`** passes — the access-pattern / batching-correctness test, which confirms the + structural batching reproduces the old Reaktive collect-and-batch behavior. + +--- + +## 5. Follow-up: the executor is no longer required to run a stream + +Because batching is now structural — a fetch leaf carries its own `IBulkExecutor` source — a `Step` is fully +self-contained. The `IStreamExecutor` passed to the terminal operations no longer carries any batching context; it +only supplied a batch size and (for `BulkRequestStreamExecutor`) set `IStreamExecutor.CONTEXT`. Two changes removed +that residual coupling: + +- **Batch size moved onto the source.** `IBulkExecutor` now declares `val batchSize` (default + `DEFAULT_BULK_REQUEST_BATCH_SIZE = 5000`); the driver chunks each source's keys to *its own* batch size. The driver + no longer takes a batch-size parameter. `BulkRequestStreamExecutor(source, batchSize)` keeps working — it exposes + the constructor batch size on the source the leaves bind to. +- **Executor-less terminals.** New `getBlocking()` / `getSuspending()` / `iterateBlocking { }` / + `iterateSuspending { }` / `executeBlocking()` / `executeSuspending()` drive a fresh execution directly. The + `*(executor: IStreamExecutor | IStreamExecutorProvider)` overloads are `@Deprecated(ReplaceWith(...))` and keep their + original behavior for compatibility. + +All in-repo call sites (~180, mostly `getBlocking`) were migrated to the executor-less form. `IStreamExecutor` / +`IStreamExecutorProvider` and `BulkRequestStreamExecutor.enqueue` remain — `enqueue` is still how fetch leaves are +created, and `IStreamExecutor.CONTEXT` is still set during `BulkRequestStreamExecutor` runs because ModelQL resolves +the "current executor" via `IStreamExecutor.getInstance()`. + +--- + +## 6. Behavioral tradeoffs + +These follow from the "no incremental emission" decision and are intentional. They change performance characteristics, +not results. + +1. **`iterate` / `iterateSuspending` fully materialize before visiting.** Reaktive previously pushed elements and + drained between batches to bound memory; the new driver builds the whole result list first. For very large + server-side iterations (e.g. walking all objects) this raises peak memory. *If this matters for a hot path, the + clean fix is to add genuine per-round streaming to just the `iterate*` drivers without disturbing the rest of the + engine.* +2. **`cached()` multicasts (evaluates once per run).** A stream consumed by multiple branches is resolved once via a + shared memo cell (`MemoCell`) that the round driver advances at most once per round, so work *and side effects* are + not duplicated and the result is shared. ModelQL depends on this for shared/side-effecting query steps (see + `ModelQLClientTest.testCaching`). +3. **`take` / `skip` operate on materialized results** — they do not prune upstream fetches. +4. **`SimpleStreamExecutor` now batches** per source/round — strictly fewer round-trips than before. + +## 6a. Depth-first, bounded request frontier + +The first cut of the round driver expanded the *entire* breadth of a traversal level per round: it materialized a +`Step` per child (the applicative `combineConcat` over a node's children) and fetched the whole level at once. That +lost a property of the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` — a depth-first traversal that kept +the live request frontier bounded — so peak memory grew with the *width* of a tree level rather than its depth. + +The engine was reworked to restore it. `Step` is now a lazily-evaluated node graph (`Done` / `FetchStep` / `MapStep` / +`FlatMapStep` / `ZipStep` / `FanStep` / `AsyncStep` / `Recover`/`OnError`/`MemoStep`) instead of `Blocked` + +`resume`. The driver walks the graph each round collecting the first-unmet request of every pending branch into a +shared pool, but: + +- it **stops adding requests once the pool reaches the source's `batchSize`** (the cap), and +- **`FanStep` builds its children lazily** — left-to-right, dropped once resolved — so live *unresolved* children also + stay within the cap. + +Combinators still evaluate eagerly when their inputs are already `Done`, so a synchronous prefix (and its side effects) +runs at build time in build order — several call sites depend on this (e.g. `HamtLeafNode.getChanges` sets a `var` in +one synchronous stream and reads it in a later `deferZeroOrOne`). Only blocking children are deferred and bounded. + +The result: **peak request frontier `≤ batchSize`** regardless of level width, while sibling requests that fit under +the cap still share one round (batching preserved); an unbounded cap collapses to one round per level. Verified by +`FrontierSizeTest` (a wide+deep tree whose widest level far exceeds the cap) and `HamtGetChangesTest` / +`BulkRequestBatchingTest` (batching + eager-evaluation semantics). + +The result list itself is still fully materialized — tradeoff #1 (streaming `iterate*`) is unchanged and orthogonal. + +## 7. Known limitations / future work + +- **Within-round stack safety.** The driver trampolines across rounds (the common fetch-dependent case), and the + per-round walk recurses on traversal depth (bounded in practice) — matching the previous engine. A pathological deep + *pure* `flatMap` chain that never blocks would still recurse natively; the fix is an explicit-stack walk if needed. +- **Optional streaming `iterate*`** — see tradeoff #1. +- **Retire the executor entirely.** With the executor no longer required to run a stream (§5), `IStreamExecutor` / + `IStreamExecutorProvider` could be removed over time — the remaining users are `enqueue` (fetch-leaf creation) and + ModelQL's `getInstance()`-based "current executor" lookup, both of which can be reworked. diff --git a/streams/README.md b/streams/README.md new file mode 100644 index 0000000000..dbb0d60399 --- /dev/null +++ b/streams/README.md @@ -0,0 +1,121 @@ +# streams + +A streaming abstraction for **batched, lazy data loading**, built around a single round-based interpreter. + +It provides the `IStream` API used throughout modelix for traversing large, content-addressed models where the data +is loaded on demand from a (possibly remote) store. Its defining capability is **automatic bulk-request batching**: +independent data requests reachable in a traversal are coalesced into a single round-trip instead of being fetched +one at a time. + +> History: this module previously had three interchangeable backends (Sequence / Flow / Reaktive). They were replaced +> by the round-based `Step` engine described below. See [`streams-redesign.md`](../streams-redesign.md) for the +> redesign and migration record. + +## Why a custom implementation + +Batching requires the runtime to *see a set of independent data requests before forcing any of them*. This is an +**applicative** property, not a monadic one: + +- `zip(a, b)` and the elements of a `Many` are **independent** → all their fetches belong to the **same batch round**. +- `flatMap*` is a **dependency** → the right-hand side's keys can't be known until the left resolves → **next round**. + +Pull-based streams (`Sequence`, `Flow`) are demand-driven and sequential: they force one element, block, then ask for +the next. They cannot expose the request frontier without spawning a coroutine per pending item (too much overhead). +A push-based library (Reaktive) can, which is why this module used to carry it. The `Step` engine gets the same +property **structurally**, from the shape of the computation, with no external dependency. + +Prior art for this idea: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. + +## The core idea: `Step` + +The engine ([`engine/StepEngine.kt`](src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)) describes a +stream lazily and interprets it in **rounds**: + +```kotlin +sealed class Step // a node in a lazily-evaluated computation +class Done(val values: List) // fully resolved +class FetchStep(source, key) // a bulk-fetch leaf +class MapStep / FlatMapStep / ZipStep / FanStep // transform / dependency / applicative join / fan-out +// + AsyncStep, RecoverStep, OnErrorStep, MemoStep +``` + +A `Step` is a node the driver walks. The combinators encode the applicative/monadic split: + +- `flatMapStep` (monadic) — defers the continuation into the next round. +- `combineConcat` / `zipN` / `fanOut` (applicative) — let the requests of independent branches be collected into the + same round. That *is* the batch. + +Combinators evaluate **eagerly when their inputs are already `Done`** (the synchronous fast path for local data: no +node allocated, side effects in `map`/`flatMap` lambdas run at build time in build order). A lazy node is built only +when an input would block on a request. + +## Execution + +The driver (`Execution.drive` / `driveSuspending`) is a loop where each iteration is one batch round: + +1. **walk** the live node graph, collecting the first-unmet request of every pending branch into a shared pool; +2. issue **one bulk call per data source** (chunked to that source's `IBulkExecutor.batchSize`), run any async leaves, + and fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds); +3. walk again, until the root resolves. + +The loop is the **trampoline** across rounds, so fetch-dependent depth costs rounds, not stack. + +### Depth-first, bounded frontier + +The walk does **not** expand the whole breadth of a traversal at once. It stops adding requests to the round once the +pool reaches the source's `batchSize` (the cap), and `FanStep` children are built lazily — instantiated left-to-right +and dropped once resolved — so the number of live *unresolved* children never exceeds the cap either. Once a request +resolves, the next walk descends deeper before expanding shallower siblings that didn't fit. The result is a +depth-first traversal whose **peak request frontier is `≤ batchSize`**, independent of how wide a tree level is — the +property the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` provided. Sibling requests still share a round +whenever they fit under the cap (the common case), so batching is preserved; with a cap above any level the behaviour +collapses to one round per level. (See `FrontierSizeTest`.) + +Batching is **structural**: a fetch leaf carries its own data source, so the driver groups fetches per source per +round regardless of which executor runs it. `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to +its `IBulkExecutor`. Because the source is carried by the stream, the **batch size lives on `IBulkExecutor`** (not on +the executor), and **terminal operations don't need an executor**: `getBlocking()` / `getSuspending()` / +`iterateBlocking { }` / `iterateSuspending { }` / `executeBlocking()` drive the self-contained stream directly. The +executor-taking overloads are `@Deprecated` and kept for compatibility. + +## Public API + +Cardinality is encoded in the type: `IStream.Many` (0+), `IStream.ZeroOrOne` (0..1), `IStream.OneOrMany` (1+), +`IStream.One` (exactly 1), and `IStream.Completable` (completion, no value). + +- Builders: `IStream.of`, `empty`, `many`, `zip`, `fromFlow`, `singleFromCoroutine`, … +- Terminals (no executor needed): `getBlocking()`, `getSuspending()`, `iterateBlocking { }`, + `iterateSuspending { }`, `executeBlocking()`. The `*(executor)` overloads remain, deprecated. +- Executors (still available; `BulkRequestStreamExecutor` also provides `enqueue`): `IStreamExecutor`, + `SimpleStreamExecutor`, `BulkRequestStreamExecutor`, `IExecutableStream`. +- Batchable source: `IBulkExecutor` (`execute` + `executeSuspending` + `batchSize`). + +## Layout + +``` +src/commonMain/kotlin/org/modelix/streams/ +├── engine/StepEngine.kt // Step IR, combinators, drivers, fetch + async leaves +├── StreamImpl.kt // backing impl for every cardinality + Completable + the builder +├── IStream.kt // public cardinality types + operators/extensions +├── IStreamBuilder.kt // builder interface (of/many/zip/fromFlow/…) +├── IStreamExecutor.kt // executor interface + extensions +├── IExecutableStream.kt // deferred/composable execution +├── IStreamInternal.kt // @DelicateModelixApi blocking/suspending accessors +├── BulkRequestStreamExecutor.kt // batching executor + IBulkExecutor +├── SimpleStreamExecutor.kt +└── Zip.kt // arity-2..9 zip overloads +``` + +## Known limitations / tradeoffs + +These follow from the engine resolving each query fully (no incremental emission): + +1. `iterate` / `iterateSuspending` fully materialize the *result* before visiting — higher peak memory for very large + iterations. Note this is independent of the request frontier, which **is** bounded (see "Depth-first, bounded + frontier" above): the fetch working set stays `≤ batchSize` even though the result list does not. The clean fix for + the result side, if a hot path needs it, is per-round streaming in just the `iterate*` drivers. +2. `cached()` multicasts: a stream consumed by multiple branches is evaluated once per run (via a shared memo cell + resolved once and reused), so side effects and work aren't duplicated. ModelQL relies on this for shared query steps. +3. `take` / `skip` operate on materialized results (don't prune upstream fetches). +4. Within-round stack safety covers fetch-dependent chains (the common case). A pathological deep *pure* `flatMap` + chain that never blocks would still recurse; the fix is to encode `Step` as a stack-safe free monad if needed. diff --git a/streams/build.gradle.kts b/streams/build.gradle.kts index fc6383d8df..c93634dbcb 100644 --- a/streams/build.gradle.kts +++ b/streams/build.gradle.kts @@ -9,9 +9,6 @@ kotlin { dependencies { implementation(project(":kotlin-utils")) implementation(libs.kotlin.coroutines.core) - - api(libs.reaktive) - api(libs.reaktive.coroutines.interop) } } commonTest { diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt index 1b0c677eff..e6b6e33d91 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt @@ -1,165 +1,71 @@ package org.modelix.streams -import com.badoo.reaktive.observable.subscribe -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.notNull -import com.badoo.reaktive.single.subscribe -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.onSuccess -import kotlinx.coroutines.yield -import org.modelix.kotlin.utils.ContextValue -import org.modelix.kotlin.utils.runSynchronized +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.Step +import org.modelix.streams.engine.fetchStep + +/** Default maximum number of keys requested in a single bulk call. */ +const val DEFAULT_BULK_REQUEST_BATCH_SIZE: Int = 5000 interface IBulkExecutor { fun execute(keys: List): Map suspend fun executeSuspending(keys: List): Map -} - -class BulkRequestStreamExecutor(private val bulkExecutor: IBulkExecutor, val batchSize: Int = 5000) : IStreamExecutor, IStreamExecutorProvider { - private val requestQueue = ContextValue() - private val streamBuilder = ReaktiveStreamBuilder() - private inner class RequestQueue { - val queue: MutableMap = LinkedHashMap() - - fun process() { - while (queue.isNotEmpty()) { - sendNextBatch { bulkExecutor.execute(it) } - } - } - - suspend fun processSuspending(afterBatch: suspend () -> Unit) { - while (queue.isNotEmpty()) { - sendNextBatch { bulkExecutor.executeSuspending(it) } - // Give stream consumers time to process the new elements. - yield() - afterBatch() - } - } + /** + * Maximum number of keys to request in a single [execute]/[executeSuspending] call. Each batch round chunks this + * source's pending keys to this size. The batch size is a property of the data source itself, so it applies + * regardless of which executor drives the stream. + */ + val batchSize: Int get() = DEFAULT_BULK_REQUEST_BATCH_SIZE +} - private inline fun sendNextBatch(executor: (keys: List) -> Map) { - val requests = runSynchronized(queue) { - // The callback of a request usually enqueues new requests until it reaches the leafs of the - // data structure. By executing the latest (instead of the oldest) request we basically do a depth - // first traversal which keeps the maximum size of the queue smaller. - val requests = queue.entries.tailSequence(batchSize).map { it.value.hash to it.value }.toList() - requests.forEach { queue.remove(it.first) } - requests - } - if (requests.isEmpty()) return - try { - val map = executor(requests.map { it.first }) - for ((_, queueElement) in requests) { - queueElement - val value = map[queueElement.hash] - queueElement.requestResult.complete(value) - } - } catch (ex: Throwable) { - for ((_, queueElement) in requests) { - queueElement.requestResult.failed(ex) - } +/** + * Executor that batches the individual fetches enqueued via [enqueue] into bulk calls against the wrapped source. + * + * Since the [Step] engine batches structurally (per source per round) and the batch size now lives on the + * [IBulkExecutor] itself, this type is no longer required for batching to work — any executor (e.g. + * [SimpleStreamExecutor]) drives the same fetch leaves with the same batching. It is retained for API compatibility + * and as the holder of [enqueue]. + */ +class BulkRequestStreamExecutor( + bulkExecutor: IBulkExecutor, + val batchSize: Int = DEFAULT_BULK_REQUEST_BATCH_SIZE, +) : IStreamExecutor, IStreamExecutorProvider { + + // Honor the batch size passed to this constructor by exposing it on the source the fetch leaves are bound to. + private val source: IBulkExecutor = + if (bulkExecutor.batchSize == batchSize) { + bulkExecutor + } else { + object : IBulkExecutor by bulkExecutor { + override val batchSize: Int get() = this@BulkRequestStreamExecutor.batchSize } } - fun query(hash: K): IStream.ZeroOrOne { - return (queue.getOrPut(hash) { QueueElement(hash, queue.size) }).value.notNull() - .let { streamBuilder.WrapperMaybe(it) } - } - } - - private inner class QueueElement(val hash: K, val position: Int) { - val requestResult = CompletableObservable() - val value: Single = requestResult.single - } - override fun getStreamExecutor(): IStreamExecutor = this - fun enqueue(key: K): IStream.ZeroOrOne { - val currentQueue = requestQueue.getValueOrNull() - return if (currentQueue == null) { - IStream.deferZeroOrOne { requestQueue.getValue().query(key) } - } else { - currentQueue.query(key) - } - } + @Suppress("UNCHECKED_CAST") + fun enqueue(key: K): IStream.ZeroOrOne = + StreamImpl { execution -> fetchStep(execution, source as IBulkExecutor, key) as Step } override fun query(body: () -> IStream.One): T { - fun doProcess(queue: RequestQueue): T { - var result: Result? = null - val reaktiveStream = streamBuilder.convert(body()) - val subscription = reaktiveStream.subscribe( - onSuccess = { result = Result.success(it) }, - onError = { result = Result.failure(it) }, - ) - try { - queue.process() - } finally { - subscription.dispose() - } - return checkNotNull(result) { "Empty stream" }.getOrThrow() - } - - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.computeWith(newQueue) { - IStreamExecutor.CONTEXT.computeWith(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + return IStreamExecutor.CONTEXT.computeWith(this) { + val execution = Execution() + execution.drive(body().asStep(execution)).single() } } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - suspend fun doProcess(queue: RequestQueue): T { - var result: Result? = null - val reaktiveStream = streamBuilder.convert(body()) - val subscription = reaktiveStream.subscribe( - onSuccess = { result = Result.success(it) }, - onError = { result = Result.failure(it) }, - ) - try { - queue.processSuspending({}) - } finally { - subscription.dispose() - } - return checkNotNull(result) { "Empty stream" }.getOrThrow() - } - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.runInCoroutine(newQueue) { - IStreamExecutor.CONTEXT.runInCoroutine(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + return IStreamExecutor.CONTEXT.runInCoroutine(this) { + val execution = Execution() + execution.driveSuspending(body().asStep(execution)).single() } } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - fun doProcess(queue: RequestQueue) { - val reaktiveStream = streamBuilder.convert(streamProvider()) - val subscription = reaktiveStream.subscribe(onNext = visitor) - try { - queue.process() - } finally { - subscription.dispose() - } - } - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.computeWith(newQueue) { - IStreamExecutor.CONTEXT.computeWith(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + IStreamExecutor.CONTEXT.computeWith(this) { + val execution = Execution() + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } } @@ -167,49 +73,9 @@ class BulkRequestStreamExecutor(private val bulkExecutor: IBulkExecutor IStream.Many, visitor: suspend (T) -> Unit, ) { - suspend fun doProcess(queue: RequestQueue) { - val reaktiveStream = streamBuilder.convert(streamProvider()) - val channel = Channel(capacity = Channel.Factory.UNLIMITED) - val subscription = reaktiveStream.subscribe( - onNext = { channel.trySend(it).getOrThrow() }, - ) - try { - suspend fun drainChannel() { - while (channel.tryReceive().onSuccess { visitor(it) }.isSuccess) { - // element already processed in onSuccess - } - } - - queue.processSuspending({ - // force processing of elements after each batch to avoid the channel from growing to big - drainChannel() - }) - drainChannel() - } finally { - subscription.dispose() - } - } - - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.runInCoroutine(newQueue) { - IStreamExecutor.CONTEXT.runInCoroutine(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + IStreamExecutor.CONTEXT.runInCoroutine(this) { + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } } } - -private fun Sequence.tailSequence(size: Int, tailSize: Int): Sequence { - if (size <= tailSize) return this - if (tailSize <= 0) return emptySequence() - return drop(size - tailSize) -} - -private fun Collection.tailSequence(tailSize: Int): Sequence { - return asSequence().tailSequence(size, tailSize) -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt deleted file mode 100644 index 96d58cd2a4..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt +++ /dev/null @@ -1,203 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -open class CollectionAsStream(val collection: Collection) : IStream.Many, IStreamInternal { - protected open fun convertLater() = DeferredStreamBuilder.ConvertibleMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.Many { - return converter.many(collection) - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return CollectionAsStream(collection.filter(predicate)) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return CollectionAsStream(collection.map(mapper)) - } - - override fun mapNotNull(mapper: (E) -> R?): IStream.Many { - return CollectionAsStream(collection.mapNotNull(mapper)) - } - - override fun firstOrNull(): IStream.One { - return SingleValueStream(collection.firstOrNull()) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapOrdered(mapper) - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapUnordered(mapper) - } - - override fun flatMapIterable(mapper: (E) -> Iterable): IStream.Many { - return SequenceAsStream(collection.asSequence().flatMap { mapper(it) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return when (other) { - is SingleValueStream -> CollectionAsStream(collection + other.value) - is SequenceAsStream -> SequenceAsStream(collection.asSequence() + other.wrapped) - is EmptyStream -> this - is CollectionAsStream -> CollectionAsStream(collection + other.collection) - else -> convertLater().concat(other) - } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return when (other) { - is SingleValueStream -> CollectionAsStreamOneOrMany(collection + other.value) - is SequenceAsStreamOneOrMany -> SequenceAsStreamOneOrMany(collection.asSequence() + other.wrapped) - is CollectionAsStreamOneOrMany -> CollectionAsStreamOneOrMany(collection + other.collection) - else -> convertLater().concat(other) - } - } - - override fun distinct(): IStream.Many { - return CollectionAsStream(collection.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - if (collection.isNotEmpty()) throw StreamAssertionError("Not empty: $collection") - return EmptyCompletableStream() - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - if (collection.isEmpty()) throw NoSuchElementException("Empty stream") - return CollectionAsStreamOneOrMany(collection) - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(collection.fold(initial, operation)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(collection.associate { keySelector(it) to valueSelector(it) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - val (a, b) = collection.partition(predicate) - return merger(CollectionAsStream(a), CollectionAsStream(b)) - } - - override fun skip(count: Long): IStream.Many { - return CollectionAsStream(collection.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return SingleValueStream(collection.single()) - } - - override fun count(): IStream.One { - return SingleValueStream(collection.size) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(if (collection.isEmpty()) defaultValue() else collection.first()) - } - - override fun firstOrDefault(defaultValue: E): IStream.One { - return SingleValueStream(if (collection.isEmpty()) defaultValue else collection.first()) - } - - override fun take(n: Int): IStream.Many { - return CollectionAsStream(collection.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return if (collection.isEmpty()) EmptyStream() else SingleValueStream(collection.first()) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return if (collection.isEmpty()) alternative() else this - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(collection.isEmpty()) - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return if (collection.isEmpty()) SingleValueStream(alternative()) else CollectionAsStreamOneOrMany(collection) - } - - override fun withIndex(): IStream.Many> { - return SequenceAsStream(collection.withIndex().asSequence()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return this - } - - override fun asFlow(): Flow { - return collection.asFlow() - } - - override fun asSequence(): Sequence { - return collection.asSequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream((collection as? List) ?: collection.toList()) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(collection.indexOf(element)) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - collection.forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - collection.forEach { visitor(it) } - } -} - -class CollectionAsStreamOneOrMany(list: Collection) : CollectionAsStream(list), IStream.OneOrMany { - protected override fun convertLater() = DeferredStreamBuilder.ConvertibleOneOrMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return converter.many(collection).assertNotEmpty { "Empty stream" } - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return CollectionAsStreamOneOrMany(collection.map(mapper)) - } - - override fun distinct(): IStream.OneOrMany { - return CollectionAsStreamOneOrMany(collection.distinct()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return this - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return convertLater().flatMapOne(mapper) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt b/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt deleted file mode 100644 index ea03ef37aa..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt +++ /dev/null @@ -1,68 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.single.SingleEmitter -import com.badoo.reaktive.single.single -import org.modelix.kotlin.utils.runSynchronized - -/** - * Similar to a CompletableFuture, observers can wait for a value that is provided in the future by some - * asynchronous process. - */ -class CompletableObservable(val afterSubscribe: () -> Unit = {}) { - val single = single { - runSynchronized(this) { - if (done) { - emitResult(it) - } else { - observers += it - } - } - afterSubscribe() - } - private var value: E? = null - private var throwable: Throwable? = null - private var done: Boolean = false - private var observers: List> = emptyList() - - private fun emitResult(observers: List>) { - for (observer in observers) { - emitResult(observer) - } - } - - private fun emitResult(emitter: SingleEmitter) { - if (throwable == null) { - emitter.onSuccess(value as E) - } else { - emitter.onError(throwable!!) - } - } - - private fun clearObservers(): List> { - return observers.also { observers = emptyList() } - } - - fun isDone() = runSynchronized(this) { done } - - fun complete(newValue: E) { - emitResult( - runSynchronized(this) { - check(!done) { "Already done" } - value = newValue - done = true - clearObservers() - }, - ) - } - - fun failed(ex: Throwable) { - emitResult( - runSynchronized(this) { - check(!done) { "Already done" } - throwable = ex - done = true - clearObservers() - }, - ) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt deleted file mode 100644 index fe6f1efbde..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt +++ /dev/null @@ -1,376 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.single -import org.modelix.kotlin.utils.DelicateModelixApi - -class DeferredStreamBuilder : IStreamBuilder { - override fun zero(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun empty(): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun of(element: T): IStream.One { - return SingleValueStream(element) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { it.deferZeroOrOne(supplier) } - } - - override fun many(elements: Collection): IStream.Many { - return CollectionAsStream(elements) - } - - override fun many(elements: Sequence): IStream.Many { - return SequenceAsStream(elements) - } - - override fun many(elements: Iterable): IStream.Many { - return SequenceAsStream(elements.asSequence()) - } - - override fun many(elements: Array): IStream.Many { - return CollectionAsStream(elements.asList()) - } - - override fun many(elements: LongArray): IStream.Many { - return CollectionAsStream(elements.asList()) - } - - override fun fromFlow(flow: Flow): IStream.Many { - return ConvertibleMany { it.fromFlow(flow) } - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - return ConvertibleMany { it.zip(input, mapper) } - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - if (input.all { it is SingleValueStream }) { - return SingleValueStream(mapper(input.map { (it as SingleValueStream).value })) - } - return ConvertibleOne { c -> - c.zip(input.map { it.convert(c) }, mapper) - } - } - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return ConvertibleOne { it.singleFromCoroutine(block) } - } - - class ConvertibleOne(conversion: (IStreamBuilder) -> IStream.One) : ConvertibleZeroOrOne(conversion), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - return super.convert(converter) as IStream.One - } - - @DelicateModelixApi - override fun getBlocking(): E = SequenceStreamBuilder.INSTANCE.convert(this).single() - - @DelicateModelixApi - override suspend fun getSuspending(): E = FlowStreamBuilder.INSTANCE.convert(this).single() - - override fun map(mapper: (E) -> R): IStream.One { - return ConvertibleOne { convert(it).map(mapper) } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return ConvertibleOne { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return ConvertibleOne { convert(it).doOnBeforeError(consumer) } - } - - override fun distinct(): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).distinct() } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return ConvertibleOne { c -> convert(c).flatMapOne { mapper(it).convert(c) } } - } - - override fun cached(): IStream.One { - return ConvertibleOne { convert(it).cached() } - } - } - - open class ConvertibleZeroOrOne(conversion: (IStreamBuilder) -> IStream.ZeroOrOne) : ConvertibleMany(conversion), IStream.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - return super.convert(converter) as IStream.ZeroOrOne - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).filter(predicate) } - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).map(mapper) } - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return ConvertibleOne { convert(it).assertNotEmpty(message) } - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return ConvertibleOne { convert(it).ifEmpty_(defaultValue) } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).doOnBeforeError(consumer) } - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return ConvertibleOne { convert(it).exceptionIfEmpty(exception) } - } - - override fun orNull(): IStream.One { - return ConvertibleOne { convert(it).orNull() } - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { converter -> - convert(converter).flatMapZeroOrOne { mapper(it).convert(converter) } - } - } - } - - open class ConvertibleMany(conversion: (IStreamBuilder) -> IStream.Many) : ConvertibleBase(conversion), IStream.Many { - override fun convert(converter: IStreamBuilder): IStream.Many { - return super.convert(converter) as IStream.Many - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return ConvertibleMany { convert(it).filter(predicate) } - } - - override fun map(mapper: (E) -> R): IStream.Many { - return ConvertibleMany { convert(it).map(mapper) } - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return ConvertibleMany { converter -> - convert(converter).flatMapUnordered { mapper(it).convert(converter) } - } - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return ConvertibleMany { converter -> - convert(converter).flatMapOrdered { mapper(it).convert(converter) } - } - } - - override fun concat(other: IStream.Many): IStream.Many { - return ConvertibleMany { convert(it).concat(other.convert(it)) } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).concat(other.convert(it)) } - } - - override fun distinct(): IStream.Many { - return ConvertibleMany { convert(it).distinct() } - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return ConvertibleCompletable { convert(it).assertEmpty(message) } - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).assertNotEmpty(message) } - } - - override fun drainAll(): IStream.Completable { - return ConvertibleCompletable { convert(it).drainAll() } - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return ConvertibleOne { convert(it).fold(initial, operation) } - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return ConvertibleOne { convert(it).toMap(keySelector, valueSelector) } - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return ConvertibleMany { c -> - convert(c).splitMerge(predicate) { a, b -> merger(a.convert(c), b.convert(c)) } - } - } - - override fun skip(count: Long): IStream.Many { - return ConvertibleMany { convert(it).skip(count) } - } - - override fun exactlyOne(): IStream.One { - return ConvertibleOne { convert(it).exactlyOne() } - } - - override fun count(): IStream.One { - return ConvertibleOne { convert(it).count() } - } - - override fun indexOf(element: E): IStream.One { - return ConvertibleOne { convert(it).indexOf(element) } - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return ConvertibleOne { convert(it).firstOrDefault(defaultValue) } - } - - override fun take(n: Int): IStream.Many { - return ConvertibleMany { convert(it).take(n) } - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).firstOrEmpty() } - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return ConvertibleMany { c -> convert(c).switchIfEmpty_ { alternative().convert(c) } } - } - - override fun isEmpty(): IStream.One { - return ConvertibleOne { convert(it).isEmpty() } - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).ifEmpty_(alternative) } - } - - override fun withIndex(): IStream.Many> { - return ConvertibleMany { convert(it).withIndex() } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return ConvertibleMany { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return ConvertibleMany { convert(it).doOnBeforeError(consumer) } - } - } - - class ConvertibleOneOrMany(conversion: (IStreamBuilder) -> IStream.OneOrMany) : ConvertibleMany(conversion), IStream.OneOrMany { - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return super.convert(converter) as IStream.OneOrMany - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).map(mapper) } - } - - override fun distinct(): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).distinct() } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).doOnBeforeError(consumer) } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return ConvertibleOneOrMany { c -> convert(c).flatMapOne { mapper(it).convert(c) } } - } - } - - class ConvertibleCompletable(conversion: (IStreamBuilder) -> IStream.Completable) : ConvertibleBase(conversion), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - return super.convert(converter) as IStream.Completable - } - - @DelicateModelixApi - override fun executeBlocking() = SequenceStreamBuilder.INSTANCE.convert(this).forEach { } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (Any?) -> Unit) { - executeBlocking() - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { - executeSuspending() - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - FlowStreamBuilder.INSTANCE.convert(this).collect {} - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return ConvertibleCompletable { convert(it).andThen(other.convert(it)) } - } - override fun plus(other: IStream.Many): IStream.Many { - return ConvertibleMany { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.One): IStream.One { - return ConvertibleOne { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).plus(other.convert(it)) } - } - } - - abstract class ConvertibleBase(private val conversion: (IStreamBuilder) -> IStream) : IStreamInternal { - private var converter: IStreamBuilder? = null - - /** - * Without caching of the converted stream [IStream.One.cached] doesn't work. - */ - private val converted: Pair, IStreamBuilder> by lazy { converter.let { conversion(it!!) to it } } - - override fun convert(converter: IStreamBuilder): IStream { - this.converter = converter - check(converted.second == converter) { "Converter changed ${converted.second} -> $converter" } - return converted.first - } - - override fun asFlow(): Flow { - val converted = convert(FlowStreamBuilder.INSTANCE) - return (converted as FlowStreamBuilder.WrapperBase).wrapped - } - - override fun asSequence(): Sequence { - val converted = convert(SequenceStreamBuilder.INSTANCE) - return (converted as SequenceStreamBuilder.WrapperBase).wrapped - } - - override fun toList(): IStream.One> = ConvertibleOne { convert(it).toList() } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - SequenceStreamBuilder.INSTANCE.convert(this).forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - FlowStreamBuilder.INSTANCE.convert(this).collect(visitor) - } - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt deleted file mode 100644 index 2149a2c46c..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt +++ /dev/null @@ -1,56 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.emptyFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class EmptyCompletableStream() : IStreamInternal.Completable { - - override fun convert(converter: IStreamBuilder): IStream.Completable { - return converter.zero() - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return other - } - - override fun plus(other: IStream.Many): IStream.Many { - return other - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return other - } - - override fun plus(other: IStream.One): IStream.One { - return other - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return other - } - - override fun asFlow(): Flow { - return emptyFlow() - } - - override fun asSequence(): Sequence { - return emptySequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream(emptyList()) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (Any?) -> Unit) {} - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) {} - - @DelicateModelixApi - override fun executeBlocking() {} - - @DelicateModelixApi - override suspend fun executeSuspending() {} -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt deleted file mode 100644 index 39bd09e4d8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt +++ /dev/null @@ -1,157 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.emptyFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class EmptyStream : IStreamInternal.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - return converter.empty() - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return this - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return SingleValueStream(defaultValue()) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - throw exception() - } - - override fun orNull(): IStream.One { - return SingleValueStream(null) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return this - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - throw StreamAssertionError(message()) - } - - override fun asFlow(): Flow { - return emptyFlow() - } - - override fun asSequence(): Sequence { - return asSequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream(emptyList()) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) {} - - @DelicateModelixApi - override suspend fun getSuspending(): E? = null - - @DelicateModelixApi - override fun getBlocking(): E? = null - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) {} - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return EmptyStream() - } - - override fun concat(other: IStream.Many): IStream.Many { - return other - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return other - } - - override fun distinct(): IStream.Many { - return this - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return EmptyCompletableStream() - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(initial) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(emptyMap()) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return merger(this, this) - } - - override fun skip(count: Long): IStream.Many { - return this - } - - override fun exactlyOne(): IStream.One { - throw NoSuchElementException("Empty stream") - } - - override fun count(): IStream.One { - return SingleValueStream(0) - } - - override fun filterBySingle(condition: (E) -> IStream.One): IStream.Many { - return this - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(defaultValue()) - } - - override fun take(n: Int): IStream.Many { - return this - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return alternative() - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(true) - } - - override fun withIndex(): IStream.Many> { - return EmptyStream() - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(-1) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt deleted file mode 100644 index b63ed1bb77..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt +++ /dev/null @@ -1,320 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.catch -import kotlinx.coroutines.flow.count -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.drop -import kotlinx.coroutines.flow.emitAll -import kotlinx.coroutines.flow.emptyFlow -import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.firstOrNull -import kotlinx.coroutines.flow.flatMapConcat -import kotlinx.coroutines.flow.flatMapMerge -import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.flow.flowOf -import kotlinx.coroutines.flow.fold -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.onEmpty -import kotlinx.coroutines.flow.single -import kotlinx.coroutines.flow.take -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.flow.withIndex -import kotlinx.coroutines.flow.zip -import org.modelix.kotlin.utils.DelicateModelixApi - -class FlowStreamBuilder() : IStreamBuilder { - - companion object { - val INSTANCE = FlowStreamBuilder() - } - - fun convert(stream: IStream) = (stream.convert(this) as WrapperBase).wrapped - - override fun of(element: T): IStream.One = Wrapper(flowOf(element)) - override fun many(elements: Sequence): IStream.Many = Wrapper(elements.asFlow()) - override fun of(vararg elements: T): IStream.Many = Wrapper(elements.asFlow()) - override fun empty(): IStream.ZeroOrOne = Wrapper(emptyFlow()) - override fun fromFlow(flow: Flow): IStream.Many = Wrapper(flow) - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return Wrapper( - flow { - coroutineScope { - emit(block()) - } - }, - ) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(flow> { emit(supplier()) }).flatten() - } - - override fun zero(): IStream.Completable { - return Completable(emptyFlow()) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return Wrapper( - flow { - emit(mapper(input.map { convert(it).single() })) - }, - ) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return Wrapper( - convert(source1).zip(convert(source2)) { a, b -> - mapper(a, b) - }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - TODO("Not yet implemented") - } - - abstract inner class WrapperBase(val wrapped: Flow) : IStreamInternal { - override fun asFlow(): Flow = wrapped - override fun toList(): IStream.One> = Wrapper(flow { emit(wrapped.toList()) }) - override fun asSequence(): Sequence = throw UnsupportedOperationException() - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use suspendable queries") - } - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.collect(visitor) - } - } - - inner class Completable(wrapped: Flow) : WrapperBase(wrapped), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@FlowStreamBuilder) - return this - } - - override fun executeBlocking() { - throw UnsupportedOperationException("Use suspendable queries") - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - wrapped.collect { } - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return Completable( - flow { - wrapped.collect { } - other.asFlow().collect { } - }, - ) - } - - override fun plus(other: IStream.Many): IStream.Many { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.One): IStream.One { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return plusFlow(convert(other)) - } - - fun plusFlow(other: Flow): Wrapper { - return Wrapper( - flow { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeSuspending() - emitAll(other) - }, - ) - } - } - - inner class Wrapper(wrapped: Flow) : WrapperBase(wrapped), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@FlowStreamBuilder) - return this - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return Wrapper(wrapped.map { mapper(it) }) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return Wrapper(wrapped.filter { predicate(it) }) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.onEmpty { emit(defaultValue()) }) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return Wrapper(wrapped.onEmpty { throw exception() }) - } - - override fun orNull(): IStream.One { - return Wrapper(wrapped.onEmpty { emit(null) }) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMapMerge { convert(mapper(it)) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return Wrapper(wrapped + convert(other)) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return Wrapper(wrapped + convert(other)) - } - - override fun getBlocking(): E { - throw UnsupportedOperationException("Use suspendable queries") - } - - override suspend fun getSuspending(): E { - return wrapped.single() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return Wrapper( - flow { - emit(wrapped.fold(initial) { acc, value -> operation(acc, value) }) - }, - ) - } - - override fun distinct(): IStream.OneOrMany { - return Wrapper(wrapped.distinctUntilChanged()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return Completable(wrapped.onEach { throw StreamAssertionError(message(it)) }) - } - - override fun drainAll(): IStream.Completable { - return Completable(wrapped.filter { false }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return Wrapper( - flow { - val map = LinkedHashMap() - wrapped.collect { - map[keySelector(it)] = valueSelector(it) - } - emit(map) - }, - ) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - throw UnsupportedOperationException() - } - - override fun cached(): IStream.One { - throw UnsupportedOperationException() - } - - override fun skip(count: Long): IStream.Many { - return Wrapper(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return Wrapper(flow { emit(wrapped.single()) }) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return Wrapper(wrapped.onEmpty { throw NoSuchElementException(message()) }) - } - - override fun count(): IStream.One { - return Wrapper(flow { emit(wrapped.count()) }) - } - - override fun indexOf(element: E): IStream.One { - return Wrapper(flow { emit(wrapped.withIndex().firstOrNull { it == element }?.index ?: -1) }) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.onEmpty { emit(defaultValue()) }.take(1)) - } - - override fun take(n: Int): IStream.Many { - return Wrapper(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return Wrapper(wrapped.take(1)) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return Wrapper(wrapped.onEmpty { emitAll(convert(alternative())) }) - } - - override fun isEmpty(): IStream.One { - return Wrapper(wrapped.map { false }.onEmpty { emit(true) }.take(1)) - } - - override fun withIndex(): IStream.Many> { - return Wrapper(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return Wrapper(wrapped.catch { emit(valueSupplier(it)) }) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return Wrapper( - wrapped.catch { - consumer(it) - throw it - }, - ) - } - } -} - -private operator fun Flow.plus(other: Flow) = onCompletion { if (it == null) emitAll(other) } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt index 315f46cfe1..a8b11ded51 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt @@ -4,6 +4,8 @@ import kotlinx.coroutines.flow.Flow import org.modelix.streams.IStream.Many import org.modelix.streams.IStream.One import org.modelix.streams.IStream.OneOrMany +import org.modelix.streams.IStream.ZeroOrOne +import kotlin.jvm.JvmName interface IStream { fun convert(converter: IStreamBuilder): IStream @@ -108,7 +110,7 @@ interface IStream { override fun doOnBeforeError(consumer: (Throwable) -> Unit): One } - companion object : IStreamBuilder by DeferredStreamBuilder() { + companion object : IStreamBuilder by StreamBuilderImpl { } } @@ -156,3 +158,146 @@ fun IStream.Many.onEach(action: (T) -> Unit): IStream.Many = map { action(it) it } + +// --- Boolean reductions --- + +/** Emits `true` if at least one element satisfies [predicate], `false` otherwise (and for an empty stream). */ +fun IStream.Many.any(predicate: (T) -> Boolean): IStream.One = filter(predicate).isNotEmpty() + +/** Emits `true` if every element satisfies [predicate]. An empty stream emits `true`. */ +fun IStream.Many.all(predicate: (T) -> Boolean): IStream.One = filter { !predicate(it) }.isEmpty() + +/** Emits `true` if no element satisfies [predicate]. An empty stream emits `true`. */ +fun IStream.Many.none(predicate: (T) -> Boolean): IStream.One = filter(predicate).isEmpty() + +/** Emits `true` if the stream contains no elements. */ +fun IStream.Many<*>.none(): IStream.One = isEmpty() + +// --- Filtering --- + +/** Keeps the elements that do *not* satisfy [predicate]. */ +fun IStream.Many.filterNot(predicate: (T) -> Boolean): IStream.Many = filter { !predicate(it) } + +/** Keeps only elements that are instances of [R]. */ +inline fun IStream.Many<*>.filterIsInstance(): IStream.Many = mapNotNull { it as? R } + +/** Keeps the elements for which [predicate] (receiving the element's index) returns `true`. */ +fun IStream.Many.filterIndexed(predicate: (Int, T) -> Boolean): IStream.Many = + withIndex().filter { predicate(it.index, it.value) }.map { it.value } + +/** Emits at most [count] elements counted from the start of the stream. Alias for [IStream.Many.skip]. */ +fun IStream.Many.drop(count: Int): IStream.Many = skip(count.toLong()) + +// --- Element access --- + +/** Emits the first element, or fails with [NoSuchElementException] if the stream is empty. */ +fun IStream.Many.first(): IStream.One = + firstOrEmpty().exceptionIfEmpty { NoSuchElementException("Stream is empty") } + +/** Emits the first element satisfying [predicate], or fails with [NoSuchElementException] if there is none. */ +fun IStream.Many.first(predicate: (T) -> Boolean): IStream.One = filter(predicate).first() + +/** Emits the first element satisfying [predicate], or `null` if there is none. */ +fun IStream.Many.firstOrNull(predicate: (T) -> Boolean): IStream.One = filter(predicate).firstOrNull() + +/** Emits the last element, or fails with [NoSuchElementException] if the stream is empty. */ +fun IStream.Many.last(): IStream.One = toList().map { it.last() } + +/** Emits the last element, or `null` if the stream is empty. */ +fun IStream.Many.lastOrNull(): IStream.One = toList().map { it.lastOrNull() } + +// --- Indexed / element-wise mapping --- + +/** Maps each element together with its index. */ +fun IStream.Many.mapIndexed(mapper: (Int, T) -> R): IStream.Many = + withIndex().map { mapper(it.index, it.value) } + +// --- Collection conversions --- + +/** Collects all elements into a [Set]. */ +fun IStream.Many.toSet(): IStream.One> = toList().map { it.toSet() } + +/** Groups the elements by the key returned by [keySelector]. */ +fun IStream.Many.groupBy(keySelector: (T) -> K): IStream.One>> = + toList().map { it.groupBy(keySelector) } + +/** Groups the elements by [keySelector], mapping each grouped element through [valueSelector]. */ +fun IStream.Many.groupBy(keySelector: (T) -> K, valueSelector: (T) -> V): IStream.One>> = + toList().map { it.groupBy(keySelector, valueSelector) } + +/** Associates each element with the key produced by [keySelector]. Later duplicates win. */ +fun IStream.Many.associateBy(keySelector: (T) -> K): IStream.One> = toMap(keySelector) { it } + +/** Associates each element (used as the key) with the value produced by [valueSelector]. */ +fun IStream.Many.associateWith(valueSelector: (T) -> V): IStream.One> = toMap({ it }, valueSelector) + +/** Collects a stream of pairs into a [Map]. Later duplicate keys win. */ +fun IStream.Many>.toMap(): IStream.One> = toMap({ it.first }, { it.second }) + +// --- Sorting --- + +fun > IStream.Many.sorted(): IStream.Many = toList().flatMapIterable { it.sorted() } +fun > IStream.Many.sortedDescending(): IStream.Many = toList().flatMapIterable { it.sortedDescending() } +fun > IStream.Many.sortedBy(selector: (T) -> R): IStream.Many = + toList().flatMapIterable { it.sortedBy(selector) } +fun > IStream.Many.sortedByDescending(selector: (T) -> R): IStream.Many = + toList().flatMapIterable { it.sortedByDescending(selector) } +fun IStream.Many.sortedWith(comparator: Comparator): IStream.Many = + toList().flatMapIterable { it.sortedWith(comparator) } + +// --- distinct / dedup --- + +/** Removes elements that share a key (as returned by [selector]) with an earlier element. */ +fun IStream.Many.distinctBy(selector: (T) -> K): IStream.Many = + toList().flatMapIterable { it.distinctBy(selector) } + +// --- Numeric and general reductions --- + +/** Reduces the stream with [operation], starting from the first element. Fails on an empty stream. */ +fun IStream.Many.reduce(operation: (acc: T, T) -> T): IStream.One = toList().map { it.reduce(operation) } + +/** Sums the [Int] values produced by [selector]. */ +fun IStream.Many.sumOf(selector: (T) -> Int): IStream.One = fold(0) { acc, e -> acc + selector(e) } + +@JvmName("sumInt") +fun IStream.Many.sum(): IStream.One = fold(0) { acc, e -> acc + e } + +@JvmName("sumLong") +fun IStream.Many.sum(): IStream.One = fold(0L) { acc, e -> acc + e } + +@JvmName("sumDouble") +fun IStream.Many.sum(): IStream.One = fold(0.0) { acc, e -> acc + e } + +/** Emits the element yielding the largest value by [selector], or `null` if the stream is empty. */ +fun > IStream.Many.maxByOrNull(selector: (T) -> R): IStream.One = + toList().map { it.maxByOrNull(selector) } + +/** Emits the element yielding the smallest value by [selector], or `null` if the stream is empty. */ +fun > IStream.Many.minByOrNull(selector: (T) -> R): IStream.One = + toList().map { it.minByOrNull(selector) } + +/** Emits the largest element according to [comparator], or `null` if the stream is empty. */ +fun IStream.Many.maxWithOrNull(comparator: Comparator): IStream.One = + toList().map { it.maxWithOrNull(comparator) } + +/** Emits the smallest element according to [comparator], or `null` if the stream is empty. */ +fun IStream.Many.minWithOrNull(comparator: Comparator): IStream.One = + toList().map { it.minWithOrNull(comparator) } + +// --- Prepending / appending single elements --- + +/** Emits [value] before all elements of this stream. */ +fun IStream.Many.startWith(value: T): IStream.Many = IStream.of(value).concat(this) + +/** Emits [value] after all elements of this stream. */ +fun IStream.Many.endWith(value: T): IStream.OneOrMany = concat(IStream.of(value)) + +// --- String joining --- + +/** Joins all elements into a single [String]. */ +fun IStream.Many.joinToString( + separator: CharSequence = ", ", + prefix: CharSequence = "", + postfix: CharSequence = "", + transform: ((T) -> CharSequence)? = null, +): IStream.One = toList().map { it.joinToString(separator, prefix, postfix, transform = transform) } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt index 26aaf5887f..e5a5da3314 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt @@ -1,6 +1,7 @@ package org.modelix.streams import org.modelix.kotlin.utils.ContextValue +import org.modelix.streams.engine.Execution /** * There reason that there are three different types of implementations if they all have different execution semantics. @@ -88,26 +89,90 @@ fun IStreamExecutor.asProvider(): IStreamExecutorProvider = SimpleStreamExecutor // fun IStream.One.getSynchronous(executor: IStreamExecutorProvider): T = getSynchronous(executor.getStreamExecutor()) // fun IStream.One.getSynchronous(executor: IStreamExecutor): T = executor.query { this } +// Terminal operations. The stream carries its own data source(s), so no executor is required to run it: each call +// drives a fresh execution, batching fetches per source per round (chunked to IBulkExecutor.batchSize). The +// executor-taking overloads are retained for compatibility and delegate to these. + +fun IStream.One.getBlocking(): T { + val execution = Execution() + return execution.drive(asStep(execution)).single() +} + +fun IStream.ZeroOrOne.getBlocking(): T? { + val execution = Execution() + return execution.drive(asStep(execution)).firstOrNull() +} + +suspend fun IStream.One.getSuspending(): T { + val execution = Execution() + return execution.driveSuspending(asStep(execution)).single() +} + +suspend fun IStream.ZeroOrOne.getSuspending(): T? { + val execution = Execution() + return execution.driveSuspending(asStep(execution)).firstOrNull() +} + +fun IStream.Many.iterateBlocking(visitor: (T) -> Unit) { + val execution = Execution() + execution.drive(asStep(execution)).forEach(visitor) +} + +suspend fun IStream.Many.iterateSuspending(visitor: suspend (T) -> Unit) { + val execution = Execution() + execution.driveSuspending(asStep(execution)).forEach { visitor(it) } +} + +fun IStream.Completable.executeBlocking() { + val execution = Execution() + execution.drive(asStep(execution)) +} + +suspend fun IStream.Completable.executeSuspending() { + val execution = Execution() + execution.driveSuspending(asStep(execution)) +} + +private const val EXECUTOR_DEPRECATION = "The executor is no longer required; the stream carries its own data source(s)." + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.One.getBlocking(executor: IStreamExecutorProvider): T = getBlocking(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.One.getBlocking(executor: IStreamExecutor): T = executor.query { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.ZeroOrOne.getBlocking(executor: IStreamExecutorProvider): T? = getBlocking(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.ZeroOrOne.getBlocking(executor: IStreamExecutor): T? = executor.query { this.orNull() } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.One.getSuspending(executor: IStreamExecutorProvider): T = getSuspending(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.One.getSuspending(executor: IStreamExecutor): T = executor.querySuspending { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutorProvider): T? = getSuspending(executor.getStreamExecutor()) -suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutor): T? = executor.querySuspending { this.orNull() } -// fun IStream.Many.iterateSynchronous(executor: IStreamExecutorProvider, visitor: (T) -> Unit): Unit = iterateSynchronous(executor.getStreamExecutor(), visitor) -// fun IStream.Many.iterateSynchronous(executor: IStreamExecutor, visitor: (T) -> Unit): Unit = executor.iterate({ this }, visitor) +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) +suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutor): T? = executor.querySuspending { this.orNull() } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateBlocking(visitor)")) fun IStream.Many.iterateBlocking(executor: IStreamExecutorProvider, visitor: (T) -> Unit): Unit = iterateBlocking(executor.getStreamExecutor(), visitor) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateBlocking(visitor)")) fun IStream.Many.iterateBlocking(executor: IStreamExecutor, visitor: (T) -> Unit): Unit = executor.iterate({ this }, visitor) +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("executeBlocking()")) fun IStream.Completable.executeBlocking(executor: IStreamExecutorProvider): Unit = executor.execute { this } + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("executeSuspending()")) suspend fun IStream.Completable.executeSuspending(executor: IStreamExecutorProvider): Unit = executor.executeSuspending { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateSuspending(visitor)")) suspend fun IStream.Many.iterateSuspending(executor: IStreamExecutorProvider, visitor: suspend (T) -> Unit): Unit = iterateSuspending(executor.getStreamExecutor(), visitor) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateSuspending(visitor)")) suspend fun IStream.Many.iterateSuspending(executor: IStreamExecutor, visitor: suspend (T) -> Unit): Unit = executor.iterateSuspending({ this }, visitor) diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt deleted file mode 100644 index e577f63fa8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt +++ /dev/null @@ -1,650 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.base.Source -import com.badoo.reaktive.completable.Completable -import com.badoo.reaktive.completable.andThen -import com.badoo.reaktive.completable.asMaybe -import com.badoo.reaktive.completable.asObservable -import com.badoo.reaktive.completable.completableOfEmpty -import com.badoo.reaktive.coroutinesinterop.asObservable -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.asCompletable -import com.badoo.reaktive.maybe.asObservable -import com.badoo.reaktive.maybe.asSingle -import com.badoo.reaktive.maybe.asSingleOrError -import com.badoo.reaktive.maybe.defaultIfEmpty -import com.badoo.reaktive.maybe.doOnBeforeError -import com.badoo.reaktive.maybe.filter -import com.badoo.reaktive.maybe.flatMap -import com.badoo.reaktive.maybe.flatMapObservable -import com.badoo.reaktive.maybe.map -import com.badoo.reaktive.maybe.maybeDefer -import com.badoo.reaktive.maybe.maybeOfEmpty -import com.badoo.reaktive.maybe.onErrorReturn -import com.badoo.reaktive.observable.Observable -import com.badoo.reaktive.observable.asCompletable -import com.badoo.reaktive.observable.autoConnect -import com.badoo.reaktive.observable.concatWith -import com.badoo.reaktive.observable.doOnBeforeError -import com.badoo.reaktive.observable.filter -import com.badoo.reaktive.observable.firstOrComplete -import com.badoo.reaktive.observable.firstOrDefault -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.flatMapSingle -import com.badoo.reaktive.observable.map -import com.badoo.reaktive.observable.observableOf -import com.badoo.reaktive.observable.onErrorReturn -import com.badoo.reaktive.observable.publish -import com.badoo.reaktive.observable.skip -import com.badoo.reaktive.observable.switchIfEmpty -import com.badoo.reaktive.observable.take -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.observable.toMap -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.asCompletable -import com.badoo.reaktive.single.asMaybe -import com.badoo.reaktive.single.asObservable -import com.badoo.reaktive.single.doOnBeforeError -import com.badoo.reaktive.single.filter -import com.badoo.reaktive.single.flatMap -import com.badoo.reaktive.single.flatMapMaybe -import com.badoo.reaktive.single.flatMapObservable -import com.badoo.reaktive.single.map -import com.badoo.reaktive.single.onErrorReturn -import com.badoo.reaktive.single.singleOf -import com.badoo.reaktive.single.subscribe -import com.badoo.reaktive.single.zipWith -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import org.modelix.kotlin.utils.DelicateModelixApi -import org.modelix.streams.IStream.OneOrMany - -class ReaktiveStreamBuilder() : IStreamBuilder { - - fun convert(stream: IStream.Completable) = (stream.convert(this) as Wrapper<*>).wrappedAsCompletable() - fun convert(stream: IStream.One) = (stream.convert(this) as Wrapper).wrappedAsSingle() - fun convert(stream: IStream.ZeroOrOne) = (stream.convert(this) as Wrapper).wrappedAsMaybe() - fun convert(stream: IStream.Many) = (stream.convert(this) as Wrapper).wrappedAsObservable() - - override fun of(element: T): IStream.One = WrapperSingle(singleOf(element)) - override fun many(elements: Sequence): IStream.Many = WrapperMany(elements.asObservable()) - override fun of(vararg elements: T): IStream.Many = WrapperMany(elements.asObservable()) - override fun empty(): IStream.ZeroOrOne = WrapperMaybe(maybeOfEmpty()) - override fun fromFlow(flow: Flow): IStream.Many = WrapperMany(flow.asObservable()) - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return WrapperSingle(com.badoo.reaktive.coroutinesinterop.singleFromCoroutine(block)) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe( - maybeDefer { - supplier().toReaktive() - }, - ) - } - - override fun zero(): IStream.Completable { - return WrapperCompletable(completableOfEmpty()) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return WrapperSingle( - com.badoo.reaktive.single.zip(*input.map { it.toReaktive() }.toTypedArray()) { - mapper(it) - }, - ) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return WrapperSingle(source1.toReaktive().zipWith(source2.toReaktive(), mapper)) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - return WrapperMany( - com.badoo.reaktive.observable.zip(*input.map { it.toReaktive() }.toTypedArray()) { - mapper(it) - }, - ) - } - - abstract inner class Wrapper { - abstract fun wrappedAsSingle(): Single - abstract fun wrappedAsMaybe(): Maybe - abstract fun wrappedAsObservable(): Observable - abstract fun wrappedAsCompletable(): Completable - } - - abstract inner class ReaktiveWrapper : Wrapper(), IStream.Many, IStreamInternal { - abstract val wrapped: Source<*> - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun skip(count: Long): IStream.Many { - require(count >= 0L) - return WrapperMany(wrappedAsObservable().skip(count)) - } - - override fun count(): IStream.One { - return WrapperSingle(wrappedAsObservable().count()) - } - - override fun take(n: Int): IStream.Many { - return WrapperMany(wrappedAsObservable().take(n)) - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrappedAsObservable().toList()) - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrappedAsObservable().isEmpty()) - } - - override fun exactlyOne(): IStream.One { - return WrapperSingle(wrappedAsSingle()) - } - - override fun concat(other: OneOrMany): OneOrMany { - return WrapperOneOrMany(wrappedAsObservable().concatWith(other.toReaktive())) - } - } - - inner class WrapperCompletable(val wrapped: Completable) : - Wrapper(), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsSingle(): Single = throw NoSuchElementException() - override fun wrappedAsMaybe(): Maybe = wrapped.asMaybe() - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsCompletable(): Completable = wrapped - - override fun executeBlocking() { - throw UnsupportedOperationException("Use IStreamExecutor.query") - } - - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return WrapperCompletable(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.One): IStream.One { - return WrapperSingle(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: OneOrMany): OneOrMany { - return WrapperOneOrMany(wrapped.andThen(other.toReaktive())) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrapped.asObservable().toList()) - } - - override fun iterateBlocking(visitor: (Any?) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - } - - open inner class WrapperMany(override val wrapped: Observable) : - ReaktiveWrapper(), IStream.Many { - override fun convert(converter: IStreamBuilder): IStream.Many { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped - override fun wrappedAsSingle(): Single = wrapped.exactlyOne() - override fun wrappedAsMaybe(): Maybe = wrapped.firstOrComplete() - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return WrapperMany(wrapped.filter(predicate)) - } - - override fun ifEmpty_(alternative: () -> E): OneOrMany { - return WrapperOneOrMany(wrapped.switchIfEmpty { observableOf(alternative()) }) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return WrapperMany(wrapped.map(mapper)) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMap(maxConcurrency = Int.MAX_VALUE) { mapper(it).toReaktive() }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMap(maxConcurrency = 1) { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.concatWith(other.toReaktive())) - } - - override fun distinct(): IStream.Many { - return WrapperMany(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return WrapperCompletable(wrapped.assertEmpty(message)) - } - - override fun assertNotEmpty(message: () -> String): OneOrMany { - return WrapperOneOrMany(wrapped.assertNotEmpty(message)) - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.fold(initial, operation)) - } - - override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> { - return WrapperSingle(wrapped.toMap(keySelector, valueSelector)) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - val sharedInput = wrappedAsObservable().publish().autoConnect(2) - val a = sharedInput.filter { predicate(it) == true } - val b = sharedInput.filter { predicate(it) == false } - return merger(WrapperMany(a), WrapperMany(b)) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.firstOrDefault(defaultValue)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return WrapperMaybe(wrappedAsMaybe()) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.switchIfEmpty { alternative().toReaktive() }) - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrapped.isEmpty()) - } - - override fun withIndex(): IStream.Many> { - return WrapperMany(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return WrapperMany(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return WrapperMany(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.withIndex().firstOrNull().map { it?.index ?: -1 }) - } - } - - inner class WrapperOneOrMany(wrapped: Observable) : - WrapperMany(wrapped), OneOrMany { - override fun convert(converter: IStreamBuilder): OneOrMany { - require(converter == this@ReaktiveStreamBuilder) - return this - } - - override fun wrappedAsObservable(): Observable = wrapped - override fun wrappedAsSingle(): Single = wrapped.exactlyOne() - override fun wrappedAsMaybe(): Maybe = wrapped.firstOrComplete() - - override fun map(mapper: (E) -> R): OneOrMany { - return WrapperOneOrMany(wrapped.map(mapper)) - } - - override fun distinct(): OneOrMany { - return WrapperOneOrMany(wrapped.distinct()) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): OneOrMany { - return WrapperOneOrMany(wrapped.flatMapSingle(maxConcurrency = 1) { mapper(it).toReaktive() }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): OneOrMany { - return WrapperOneOrMany(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): OneOrMany { - return WrapperOneOrMany(wrapped.doOnBeforeError(consumer)) - } - } - - inner class WrapperSingle(override val wrapped: Single) : ReaktiveWrapper(), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsSingle(): Single = wrapped - override fun wrappedAsMaybe(): Maybe = wrapped.asMaybe() - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - fun getAsync(onError: ((Throwable) -> Unit)?, onSuccess: ((E) -> Unit)?) { - wrappedAsSingle().subscribe(onError = onError, onSuccess = onSuccess) - } - - override fun cached(): IStream.One { - return WrapperSingle(wrappedAsSingle().cached()) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return WrapperSingle(wrapped.flatMap { mapper(it).toReaktive() }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return WrapperSingle(wrapped.map(mapper)) - } - - override fun getBlocking(): E { - throw UnsupportedOperationException("Use IStreamExecutor.query") - } - - override suspend fun getSuspending(): E { - throw UnsupportedOperationException("Use IStreamExecutor.querySuspending") - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asObservable().asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrapped.map { listOf(it) }) - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.filter(predicate)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return this // cannot be empty - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return this // cannot be empty - } - - override fun orNull(): IStream.One { - return this // cannot be empty - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.flatMapMaybe { mapper(it).toReaktive() }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMapObservable { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().concatWith(other.toReaktive())) - } - - override fun distinct(): OneOrMany { - return this // single element is always distinct - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - throw StreamAssertionError("Single will never be empty: $wrapped") - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return this // cannot be empty - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.map { operation(initial, it) }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return WrapperSingle(wrapped.map { mapOf(keySelector(it) to valueSelector(it)) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - TODO("Not yet implemented") - } - - override fun exactlyOne(): IStream.One { - return this - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return this // there is always a first element - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this // there is always a first element - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return this // never empty - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrapped.asObservable().isEmpty()) - } - - override fun withIndex(): IStream.Many> { - return WrapperSingle(wrapped.map { IndexedValue(0, it) }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return WrapperSingle(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return WrapperSingle(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.map { if (it == element) 0 else -1 }) - } - } - - inner class WrapperMaybe(override val wrapped: Maybe) : ReaktiveWrapper(), IStream.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsSingle(): Single = wrapped.asSingleOrError() - override fun wrappedAsMaybe(): Maybe = wrapped - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return WrapperSingle(wrapped.asSingleOrError(exception)) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.flatMap { mapper(it).toReaktive() }) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.filter(predicate)) - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.map(mapper)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.asSingle(defaultValue)) - } - - override fun orNull(): IStream.One { - return WrapperSingle(wrapped.orNull()) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asObservable().asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMapObservable { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().concatWith(other.toReaktive())) - } - - override fun distinct(): IStream.Many { - return this // there is never more than one element - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return WrapperCompletable(wrapped.assertEmpty(message)) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return WrapperSingle( - wrapped.asSingleOrError { - throw StreamAssertionError("At least one element was expected. xxx " + message()) - }, - ) - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.asObservable().fold(initial, operation)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return WrapperSingle(wrapped.asObservable().toMap(keySelector, valueSelector)) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - TODO("Not yet implemented") - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.asSingle(defaultValue)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().switchIfEmpty { alternative().toReaktive() }) - } - - override fun withIndex(): IStream.Many> { - return WrapperMaybe(wrapped.map { IndexedValue(0, it) }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.map { if (it == element) 0 else -1 }.defaultIfEmpty(-1)) - } - } - fun IStream.One.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.ZeroOrOne.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.Many.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.Completable.toReaktive() = this@ReaktiveStreamBuilder.convert(this) -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt deleted file mode 100644 index 32359db3a3..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt +++ /dev/null @@ -1,185 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -/** - * Runs operations eagerly, if they return a single value. - * If an operation returns many values, it remains a lazy sequence to avoid unnecessary memory consumption. - */ -open class SequenceAsStream(val wrapped: Sequence) : IStream.Many, IStreamInternal { - protected open fun convertLater() = DeferredStreamBuilder.ConvertibleMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.Many { - return converter.many(wrapped) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return SequenceAsStream(wrapped.map(mapper)) - } - - override fun asFlow(): Flow { - return wrapped.asFlow() - } - - override fun asSequence(): Sequence { - return wrapped - } - - override fun toList(): IStream.One> { - return SingleValueStream(wrapped.toList()) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - wrapped.forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.forEach { visitor(it) } - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return SequenceAsStream(wrapped.filter(predicate)) - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.ifEmpty { sequenceOf(alternative()) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapOrdered(mapper) - } - - override fun concat(other: IStream.Many): IStream.Many { - return convertLater().concat(other) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return convertLater().concat(other) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(wrapped.fold(initial, operation)) - } - - override fun distinct(): IStream.Many { - return SequenceAsStream(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - wrapped.forEach { throw StreamAssertionError(message(it)) } - return EmptyCompletableStream() - } - - override fun drainAll(): IStream.Completable { - wrapped.forEach {} - return EmptyCompletableStream() - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(wrapped.associate { keySelector(it) to valueSelector(it) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - // XXX Sequence.partition reads all entries into two lists, which consumes more memory. - // An alternative would be to create two sequences using Sequence.filter, but then the input is iterated - // twice. It depends on the use case which one is preferred. - // Currently, there is only a single usage of this method in Modelix and that is for bulk queries, which - // means the size of the input is limited. - // Also, in cases where bulk queries are used the stream will be a Reaktive one and this sequence based - // implementation is never called. - val (a, b) = wrapped.partition(predicate) - return merger(SequenceAsStream(a.asSequence()), SequenceAsStream(b.asSequence())) - } - - override fun skip(count: Long): IStream.Many { - return SequenceAsStream(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return SingleValueStream(wrapped.single()) - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.ifEmpty { throw StreamAssertionError(message()) }) - } - - override fun count(): IStream.One { - return SingleValueStream(wrapped.count()) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(wrapped.indexOf(element)) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(wrapped.ifEmpty { sequenceOf(defaultValue()) }.first()) - } - - override fun take(n: Int): IStream.Many { - return SequenceAsStream(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return wrapped.map { SingleValueStream(it) }.take(1).ifEmpty { sequenceOf(EmptyStream()) }.single() - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return convertLater().switchIfEmpty_(alternative) - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(wrapped.map { false }.take(1).ifEmpty { sequenceOf(true) }.single()) - } - - override fun withIndex(): IStream.Many> { - return SequenceAsStream(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return convertLater().onErrorReturn(valueSupplier) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return convertLater().doOnBeforeError(consumer) - } -} - -class SequenceAsStreamOneOrMany(wrapped: Sequence) : SequenceAsStream(wrapped), IStream.OneOrMany { - override fun convertLater(): DeferredStreamBuilder.ConvertibleOneOrMany { - return DeferredStreamBuilder.ConvertibleOneOrMany { convert(it) } - } - - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return super.convert(converter).assertNotEmpty { "Empty stream" } - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.map(mapper)) - } - - override fun distinct(): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.distinct()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return convertLater().onErrorReturn(valueSupplier) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return convertLater().doOnBeforeError(consumer) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return convertLater().flatMapOne(mapper) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt deleted file mode 100644 index a1118911e8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt +++ /dev/null @@ -1,323 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class SequenceStreamBuilder() : IStreamBuilder { - - companion object { - val INSTANCE = SequenceStreamBuilder() - } - - fun convert(stream: IStream) = (stream.convert(this) as WrapperBase).wrapped - - override fun of(element: T): IStream.One = Wrapper(sequenceOf(element)) - override fun many(elements: Sequence): IStream.Many = Wrapper(elements) - override fun of(vararg elements: T): IStream.Many = Wrapper(elements.asSequence()) - override fun empty(): IStream.ZeroOrOne = Wrapper(emptySequence()) - - override fun fromFlow(flow: Flow): IStream.Many { - throw UnsupportedOperationException("Use FlowStreamBuilder") - } - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return throw UnsupportedOperationException("Use Reaktive based streams") - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(sequence> { yield(supplier()) }).flatten() - } - - override fun zero(): IStream.Completable { - return Completable(emptySequence()) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return Wrapper( - convert(source1) - .zip(convert(source2)) - .map { mapper(it.first, it.second) }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): Wrapper { - val input = input.toList() - val sequences = input.map { convert(it) } - return Wrapper( - when (sequences.size) { - 0 -> emptySequence() - 1 -> sequences.single().map { mapper(listOf(it)) } - else -> sequences.map { it.map { listOf(it) } }.reduce { a, b -> a.zip(b) { a, b -> a + b } }.map(mapper) - }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return zip(input.map { it.assertNotEmpty { "Empty" } } as List>, mapper) - } - - abstract inner class WrapperBase(val wrapped: Sequence) : IStreamInternal { - override fun asFlow(): Flow = wrapped.asFlow() - override fun toList(): IStream.One> = Wrapper(sequence { yield(wrapped.toList()) }) - override fun asSequence(): Sequence = wrapped - - override fun iterateBlocking(visitor: (E) -> Unit) { - wrapped.forEach(visitor) - } - - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.forEach { visitor(it) } - } - } - - inner class Completable(wrapped: Sequence) : WrapperBase(wrapped), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@SequenceStreamBuilder) - return this - } - - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - override fun executeBlocking() { - wrapped.forEach { } - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - wrapped.forEach { } - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return Completable( - sequence { - executeBlocking() - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - (other as IStreamInternal.Completable).executeBlocking() - }, - ) - } - - override fun plus(other: IStream.Many): IStream.Many { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.One): IStream.One { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return plusSequence(convert(other)) - } - - fun plusSequence(other: WrapperBase): Wrapper { - return Wrapper( - sequence { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeBlocking() - yieldAll(other.asSequence()) - }, - ) - } - - fun plusSequence(other: Sequence): Wrapper { - return Wrapper( - sequence { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeBlocking() - yieldAll(other) - }, - ) - } - } - - inner class Wrapper(wrapped: Sequence) : WrapperBase(wrapped), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@SequenceStreamBuilder) - return this - } - fun getAsync(onError: ((Throwable) -> Unit)?, onSuccess: ((E) -> Unit)?) { - try { - for (element in wrapped) { - onSuccess?.invoke(element) - } - } catch (ex: Throwable) { - onError?.invoke(ex) - } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return Wrapper(wrapped.map(mapper)) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return Wrapper(wrapped.filter(predicate)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(defaultValue()) }) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return Wrapper(wrapped.ifEmpty { throw exception() }) - } - - override fun orNull(): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(null) }) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return Wrapper(wrapped + convert(other)) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return Wrapper(wrapped + convert(other)) - } - - override fun getBlocking(): E { - return wrapped.single() - } - - override suspend fun getSuspending(): E { - return wrapped.single() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return Wrapper(sequenceOf(wrapped.fold(initial, operation))) - } - - override fun distinct(): IStream.OneOrMany { - return Wrapper(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return Completable(wrapped.onEach { throw StreamAssertionError(message(it)) }) - } - - override fun drainAll(): IStream.Completable { - return Completable(wrapped.filter { false }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return Wrapper(sequenceOf(wrapped.associate { keySelector(it) to valueSelector(it) })) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - // XXX Sequence.partition reads all entries into two lists, which consumes more memory. - // An alternative would be to create two sequences using Sequence.filter, but then the input is iterated - // twice. It depends on the use case which one is preferred. - // Currently, there is only a single usage of this method in Modelix and that is for bulk queries, which - // means the size of the input is limited. - // Also, in cases where bulk queries are used the stream will be a Reaktive one and this sequence based - // implementation is never called. - val (a, b) = wrapped.partition(predicate) - return merger(Wrapper(a.asSequence()), Wrapper(b.asSequence())) - } - - override fun cached(): IStream.One { - val cached by lazy { wrapped.toList() } - return Wrapper(sequenceOf({ cached }).flatMap { it() }) - } - - override fun skip(count: Long): IStream.Many { - return Wrapper(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return Wrapper(sequence { yield(wrapped.single()) }) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return Wrapper(wrapped.ifEmpty { throw StreamAssertionError(message()) }) - } - - override fun count(): IStream.One { - return Wrapper(sequence { yield(wrapped.count()) }) - } - - override fun indexOf(element: E): IStream.One { - return Wrapper(sequence { yield(wrapped.indexOf(element)) }) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(defaultValue()) }.take(1)) - } - - override fun take(n: Int): IStream.Many { - return Wrapper(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return Wrapper(wrapped.take(1)) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return Wrapper(wrapped.ifEmpty { convert(alternative()) }) - } - - override fun isEmpty(): IStream.One { - return Wrapper(wrapped.map { false }.ifEmpty { sequenceOf(true) }.take(1)) - } - - override fun withIndex(): IStream.Many> { - return Wrapper(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return Wrapper( - sequence { - try { - yieldAll(wrapped) - } catch (ex: Throwable) { - yield(valueSupplier(ex)) - } - }, - ) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return Wrapper( - sequence { - try { - yieldAll(wrapped) - } catch (ex: Throwable) { - consumer(ex) - throw ex - } - }, - ) - } - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt index 34bd8a1916..178224805a 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt @@ -1,24 +1,33 @@ package org.modelix.streams -import kotlinx.coroutines.flow.single +import org.modelix.streams.engine.Execution +/** + * Default executor. Fetches embedded in a stream (via [BulkRequestStreamExecutor.enqueue]) are batched per source per + * round, chunked to each source's [IBulkExecutor.batchSize], so this is safe to use even when a stream contains data + * requests. + */ object SimpleStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { - return SequenceStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.drive(body().asStep(execution)).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - return FlowStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.driveSuspending(body().asStep(execution)).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - SequenceStreamBuilder.INSTANCE.convert(streamProvider()).forEach(visitor) + val execution = Execution() + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } override suspend fun iterateSuspending( streamProvider: suspend () -> IStream.Many, visitor: suspend (T) -> Unit, ) { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect(visitor) + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt deleted file mode 100644 index 7503d75b2f..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt +++ /dev/null @@ -1,200 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flowOf -import org.modelix.kotlin.utils.DelicateModelixApi - -class SingleValueStream(val value: E) : IStreamInternal.One { - protected fun convertLater() = DeferredStreamBuilder.ConvertibleOne { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.One { - return converter.of(value) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return mapper(value) - } - - override fun map(mapper: (E) -> R): IStream.One { - return SingleValueStream(mapper(value)) - } - - @DelicateModelixApi - override fun getBlocking(): E { - return value - } - - @DelicateModelixApi - override suspend fun getSuspending(): E { - return value - } - - override fun cached(): IStream.One { - return this - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return this - } - - override fun asFlow(): Flow { - return flowOf(value) - } - - override fun asSequence(): Sequence { - return sequenceOf(value) - } - - override fun toList(): IStream.One> { - return SingleValueStream(listOf(value)) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - visitor(value) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - visitor(value) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return if (predicate(value)) this else EmptyStream() - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return this - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return this - } - - override fun orNull(): IStream.One { - return this - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return mapper(value) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return this - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return mapper(value) - } - - override fun flatMapIterable(mapper: (E) -> Iterable): IStream.Many { - return SequenceAsStream(mapper(value).asSequence()) - } - - override fun concat(other: IStream.Many): IStream.Many { - return when (other) { - is SingleValueStream -> CollectionAsStream(listOf(value, other.value)) - is SequenceAsStream -> SequenceAsStream(sequenceOf(value) + other.wrapped) - is EmptyStream -> this - is CollectionAsStream -> CollectionAsStream(listOf(value) + other.collection) - else -> convertLater().concat(other) - } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return when (other) { - is SingleValueStream -> CollectionAsStreamOneOrMany(listOf(value, other.value)) - is SequenceAsStreamOneOrMany -> SequenceAsStreamOneOrMany(sequenceOf(value) + other.wrapped) - is CollectionAsStreamOneOrMany -> CollectionAsStreamOneOrMany(listOf(value) + other.collection) - else -> convertLater().concat(other) - } - } - - override fun distinct(): IStream.OneOrMany { - return this - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - throw StreamAssertionError(message(value)) - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(operation(initial, value)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(mapOf(keySelector(value) to valueSelector(value))) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return if (predicate(value)) merger(this, EmptyStream()) else merger(EmptyStream(), this) - } - - override fun skip(count: Long): IStream.Many { - return if (count >= 1) EmptyStream() else this - } - - override fun exactlyOne(): IStream.One { - return this - } - - override fun count(): IStream.One { - return SingleValueStream(1) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return this - } - - override fun firstOrDefault(defaultValue: E): IStream.One { - return this - } - - override fun zipWith( - other: IStream.One, - mapper: (E, T) -> R, - ): IStream.One { - return when (other) { - is SingleValueStream -> SingleValueStream(mapper(value, other.value)) - else -> convertLater().zipWith(other, mapper) - } - } - - override fun take(n: Int): IStream.Many { - return if (n >= 1) this else EmptyStream() - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return this - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(false) - } - - override fun withIndex(): IStream.Many> { - return SingleValueStream(IndexedValue(0, value)) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(if (element == value) 0 else -1) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt deleted file mode 100644 index 9c07e4a788..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt +++ /dev/null @@ -1,93 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.base.tryCatch -import com.badoo.reaktive.completable.Completable -import com.badoo.reaktive.completable.CompletableCallbacks -import com.badoo.reaktive.disposable.Disposable -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.asCompletable -import com.badoo.reaktive.maybe.defaultIfEmpty -import com.badoo.reaktive.maybe.map -import com.badoo.reaktive.observable.Observable -import com.badoo.reaktive.observable.ObservableObserver -import com.badoo.reaktive.observable.asCompletable -import com.badoo.reaktive.observable.asObservable -import com.badoo.reaktive.observable.autoConnect -import com.badoo.reaktive.observable.collect -import com.badoo.reaktive.observable.filter -import com.badoo.reaktive.observable.firstOrDefault -import com.badoo.reaktive.observable.firstOrError -import com.badoo.reaktive.observable.flatMapSingle -import com.badoo.reaktive.observable.map -import com.badoo.reaktive.observable.observable -import com.badoo.reaktive.observable.replay -import com.badoo.reaktive.observable.switchIfEmpty -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.asObservable -import com.badoo.reaktive.single.flatMap -import com.badoo.reaktive.single.map - -class StreamAssertionError(message: String) : IllegalArgumentException(message) - -fun Observable<*>.count(): Single = collect({ arrayOf(0) }) { acc, it -> acc[0]++ }.map { it[0] } -fun Observable.fold(initial: R, operation: (R, T) -> R): Single { - return collect({ mutableListOf(initial) }) { acc, it -> acc[0] = operation(acc[0], it) }.map { it[0] } -} - -fun Sequence.asObservable(): Observable = asIterable().asObservable() -fun Observable.exactlyOne(): Single = toList().map { it.single() } -fun Observable.firstOrNull(): Single = firstOrDefault(null) -fun Observable<*>.isEmpty(): Single = map { false }.firstOrDefault(true) -fun Observable.filterBySingle(condition: (T) -> Single): Observable { - return this.flatMapSingle { element -> - condition(element).map { included -> - element to included - } - }.filter { it.second }.map { it.first } -} -fun Observable.withIndex(): Observable> { - var index = 0 - return map { IndexedValue(index++, it) } -} -fun Observable.assertNotEmpty(message: () -> String): Observable { - return this.switchIfEmpty { throw StreamAssertionError("At least one element was expected. xxx " + message()) } -} -fun Maybe.assertEmpty(message: (T) -> String): Completable { - return map { throw StreamAssertionError(message(it)) }.asCompletable() -} -fun Observable.assertEmpty(message: (T) -> String): Completable { - return map { throw StreamAssertionError(message(it)) }.asCompletable() -} -fun Single.cached(): Single { - return this.asObservable().replay(1).autoConnect() - .firstOrError { IllegalStateException("Single was empty. Should not happen.") } -} - -fun Maybe.orNull(): Single = defaultIfEmpty(null) -fun Array.asObservable(): Observable = asIterable().asObservable() -fun LongArray.asObservable(): Observable = asIterable().asObservable() - -fun Observable.distinct(): Observable { - return observable { emitter -> - val emittedValues = HashSet() - subscribe( - object : ObservableObserver, CompletableCallbacks by emitter { - override fun onSubscribe(disposable: Disposable) { - emitter.setDisposable(disposable) - } - - override fun onNext(value: T) { - emitter.tryCatch(block = { emittedValues.add(value) }) { - if (it) { - emitter.onNext(value) - } - } - } - }, - ) - } -} - -fun Maybe<*>.exists(): Single = map { true }.defaultIfEmpty(false) -fun Single>.flatten(): Single = flatMap { it } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt new file mode 100644 index 0000000000..dad9d00e96 --- /dev/null +++ b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt @@ -0,0 +1,311 @@ +package org.modelix.streams + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.toList +import org.modelix.kotlin.utils.DelicateModelixApi +import org.modelix.kotlin.utils.runBlockingIfJvm +import org.modelix.streams.engine.Done +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.Step +import org.modelix.streams.engine.asyncStep +import org.modelix.streams.engine.combineConcat +import org.modelix.streams.engine.doOnError +import org.modelix.streams.engine.fanOut +import org.modelix.streams.engine.flatMapStep +import org.modelix.streams.engine.mapValues +import org.modelix.streams.engine.memoStep +import org.modelix.streams.engine.recover +import org.modelix.streams.engine.zipN + +class StreamAssertionError(message: String) : IllegalArgumentException(message) + +/** Common bridge from a public [IStream] to its internal [Step] representation for the current run. */ +internal interface HasStep { + fun buildStep(execution: Execution): Step +} + +@Suppress("UNCHECKED_CAST") +internal fun IStream.asStep(execution: Execution): Step = (this as HasStep).buildStep(execution) + +/** + * The single backing implementation of every non-[Completable] [IStream] cardinality. The runtime instance always + * satisfies the strongest interface ([IStreamInternal.One]); static typing at the API boundary restricts which + * operators are reachable. Operators return new [StreamImpl] instances typed at their most-derived cardinality. + */ +internal class StreamImpl(val build: (Execution) -> Step) : IStreamInternal.One, HasStep { + + override fun buildStep(execution: Execution): Step = build(execution) + + override fun convert(converter: IStreamBuilder): IStream.One = this + + override fun asFlow(): Flow = flow { + val execution = Execution() + for (value in execution.driveSuspending(build(execution))) emit(value) + } + + override fun asSequence(): Sequence { + val execution = Execution() + return execution.drive(build(execution)).asSequence() + } + + override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } + + override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).mapValues { it.filter(predicate) } } + + override fun map(mapper: (E) -> R): IStream.One = + StreamImpl { execution -> build(execution).mapValues { it.map(mapper) } } + + override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { values -> fanOut(values) { mapper(it).asStep(execution) } } } + + override fun concat(other: IStream.Many): IStream.Many = + StreamImpl { execution -> combineConcat(listOf(build(execution), other.asStep(execution))) } + + override fun concat(other: IStream.OneOrMany): IStream.OneOrMany = + StreamImpl { execution -> combineConcat(listOf(build(execution), other.asStep(execution))) } + + override fun distinct(): IStream.OneOrMany = + StreamImpl { execution -> build(execution).mapValues { it.distinct() } } + + override fun assertEmpty(message: (E) -> String): IStream.Completable = + CompletableImpl { execution -> + build(execution).mapValues { values -> + if (values.isNotEmpty()) throw StreamAssertionError(message(values.first())) + emptyList() + } + } + + override fun assertNotEmpty(message: () -> String): IStream.One = + StreamImpl { execution -> + build(execution).mapValues { values -> + if (values.isEmpty()) throw StreamAssertionError("At least one element was expected. " + message()) + values + } + } + + override fun drainAll(): IStream.Completable = + CompletableImpl { execution -> build(execution).mapValues { emptyList() } } + + override fun fold(initial: R, operation: (R, E) -> R): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.fold(initial, operation)) } } + + override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> = + StreamImpl { execution -> build(execution).mapValues { values -> listOf(values.associate { keySelector(it) to valueSelector(it) }) } } + + override fun splitMerge(predicate: (E) -> Boolean, merger: (IStream.Many, IStream.Many) -> IStream.Many): IStream.Many = + StreamImpl { execution -> + build(execution).flatMapStep { values -> + val (matching, notMatching) = values.partition(predicate) + merger(IStream.many(matching), IStream.many(notMatching)).asStep(execution) + } + } + + override fun skip(count: Long): IStream.Many = + StreamImpl { execution -> build(execution).mapValues { it.drop(count.toInt()) } } + + override fun exactlyOne(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.single()) } } + + override fun count(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.size) } } + + override fun firstOrDefault(defaultValue: () -> E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> listOf(if (values.isEmpty()) defaultValue() else values.first()) } } + + override fun take(n: Int): IStream.Many = + StreamImpl { execution -> build(execution).mapValues { it.take(n) } } + + override fun firstOrEmpty(): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).mapValues { it.take(1) } } + + override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { values -> if (values.isEmpty()) alternative().asStep(execution) else Done(values) } } + + override fun isEmpty(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.isEmpty()) } } + + override fun ifEmpty_(defaultValue: () -> E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) listOf(defaultValue()) else values } } + + override fun withIndex(): IStream.Many> = + StreamImpl { execution -> build(execution).mapValues { it.withIndex().toList() } } + + override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One = + StreamImpl { execution -> + try { + build(execution).recover { listOf(valueSupplier(it)) } + } catch (ex: Throwable) { + Done(listOf(valueSupplier(ex))) + } + } + + override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One = + StreamImpl { execution -> + try { + build(execution).doOnError(consumer) + } catch (ex: Throwable) { + consumer(ex) + throw ex + } + } + + override fun indexOf(element: E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.indexOf(element)) } } + + override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One = + StreamImpl { execution -> build(execution).flatMapStep { values -> mapper(values.single()).asStep(execution) } } + + override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).flatMapStep { values -> fanOut(values) { mapper(it).asStep(execution) } } } + + override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) throw exception() else values } } + + override fun orNull(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) listOf(null) else listOf(values.single()) } } + + @Suppress("UNCHECKED_CAST") + override fun cached(): IStream.One { + val self = this + val token = Any() + return StreamImpl { execution -> memoStep(execution, token) { self.build(execution) as Step } as Step } + } + + @DelicateModelixApi + override fun iterateBlocking(visitor: (E) -> Unit) { + val execution = Execution() + execution.drive(build(execution)).forEach(visitor) + } + + @DelicateModelixApi + override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { + val execution = Execution() + execution.driveSuspending(build(execution)).forEach { visitor(it) } + } + + @Suppress("UNCHECKED_CAST") + @DelicateModelixApi + override fun getBlocking(): E { + val execution = Execution() + val values = execution.drive(build(execution)) + return (if (values.isEmpty()) null else values.first()) as E + } + + @Suppress("UNCHECKED_CAST") + @DelicateModelixApi + override suspend fun getSuspending(): E { + val execution = Execution() + val values = execution.driveSuspending(build(execution)) + return (if (values.isEmpty()) null else values.first()) as E + } +} + +/** Backing implementation of [IStream.Completable]. A completion carries no values; its step resolves to an empty list. */ +internal class CompletableImpl(val build: (Execution) -> Step) : IStreamInternal.Completable, HasStep { + + override fun buildStep(execution: Execution): Step = build(execution) + + override fun convert(converter: IStreamBuilder): IStream.Completable = this + + override fun asFlow(): Flow = flow { + val execution = Execution() + for (value in execution.driveSuspending(build(execution))) emit(value) + } + + override fun asSequence(): Sequence { + val execution = Execution() + return execution.drive(build(execution)).asSequence() + } + + override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } + + override fun andThen(other: IStream.Completable): IStream.Completable = + CompletableImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.One): IStream.One = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.OneOrMany): IStream.OneOrMany = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + @DelicateModelixApi + override fun iterateBlocking(visitor: (Any?) -> Unit) { + val execution = Execution() + execution.drive(build(execution)).forEach(visitor) + } + + @DelicateModelixApi + override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { + val execution = Execution() + execution.driveSuspending(build(execution)).forEach { visitor(it) } + } + + @DelicateModelixApi + override fun executeBlocking() { + val execution = Execution() + execution.drive(build(execution)) + } + + @DelicateModelixApi + override suspend fun executeSuspending() { + val execution = Execution() + execution.driveSuspending(build(execution)) + } +} + +/** The single [IStreamBuilder] backed by the [Step] engine. Replaces the Sequence/Flow/Reaktive/Deferred builders. */ +internal object StreamBuilderImpl : IStreamBuilder { + override fun zero(): IStream.Completable = CompletableImpl { Done(emptyList()) } + + override fun empty(): IStream.ZeroOrOne = StreamImpl { Done(emptyList()) } + + override fun of(element: T): IStream.One = StreamImpl { Done(listOf(element)) } + + override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> supplier().asStep(execution) } + + override fun many(elements: Sequence): IStream.Many = StreamImpl { Done(elements.toList()) } + + @Suppress("UNCHECKED_CAST") + override fun fromFlow(flow: Flow): IStream.Many = StreamImpl { execution -> + asyncStep( + execution, + Any(), + produceBlocking = { runBlockingIfJvm { flow.toList() } }, + produceSuspending = { flow.toList() }, + ) as Step + } + + override fun zip(input: List>, mapper: (List) -> R): IStream.Many = + StreamImpl { execution -> + zipN(input.map { it.asStep(execution) }) { valueLists -> + val count = if (valueLists.isEmpty()) 0 else valueLists.minOf { it.size } + (0 until count).map { i -> mapper(valueLists.map { it[i] }) } + } + } + + override fun zip(input: List>, mapper: (List) -> R): IStream.One = + StreamImpl { execution -> + zipN(input.map { it.asStep(execution) }) { valueLists -> listOf(mapper(valueLists.map { it.single() })) } + } + + @Suppress("UNCHECKED_CAST") + override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One = StreamImpl { execution -> + asyncStep( + execution, + Any(), + produceBlocking = { runBlockingIfJvm { listOf(coroutineScope { block() }) } }, + produceSuspending = { listOf(coroutineScope { block() }) }, + ) as Step + } +} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt new file mode 100644 index 0000000000..d65a71de5c --- /dev/null +++ b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt @@ -0,0 +1,373 @@ +package org.modelix.streams.engine + +import kotlinx.coroutines.yield +import org.modelix.streams.IBulkExecutor + +/** + * The round-based interpreter that backs every [org.modelix.streams.IStream] instance. + * + * A [Step] is a node in a lazily-evaluated computation that produces zero or more values. Evaluation proceeds in + * *rounds*: the driver walks the live node graph, collecting the data requests it depends on, resolves them with one + * bulk call per source, and walks again — until the root resolves. The combinators encode the applicative/monadic split + * that makes bulk-request batching automatic: + * - applicative composition ([combineConcat], [zipN], [fanOut]) lets the independent requests of sibling branches be + * collected into the *same* round — that is the batch; + * - monadic composition ([flatMapStep]) introduces a dependency, pushing the right-hand side into a *later* round. + * + * ### Depth-first, bounded frontier + * + * The driver does not expand the whole breadth of a traversal at once. A single walk collects the first-unmet request + * of every pending branch into a shared pool, but **stops adding requests once the pool reaches the source's + * [IBulkExecutor.batchSize]** (the cap). Fan-out children ([FanStep]) are built lazily, left-to-right, and dropped once + * resolved, so the number of live *unresolved* child nodes never exceeds the cap either. Once a branch's request + * resolves, the next walk descends one level deeper into it before expanding the breadth of shallower siblings that did + * not fit under the cap. The result is a depth-first traversal whose peak request frontier is `<= batchSize`, + * regardless of how wide a tree level is — the property the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` + * provided. Independent sibling requests still share a round whenever they fit under the cap (the common case), so + * applicative batching is preserved; with a cap above any level the behaviour collapses to a single round per level. + * + * Errors are not represented as nodes: an exception thrown while resolving a node propagates natively and is caught by + * an enclosing [recover]/[doOnError]. A fetch that throws inside the driver propagates out of [drive] uncaught, exactly + * as before. + */ +internal sealed class Step { + /** Memoized resolved value list; `null` until this node is fully resolved within the current execution. */ + var cached: List? = null +} + +/** An already-resolved node. */ +internal class Done(val values: List) : Step() + +/** A fetch leaf with zero-or-one semantics: a `null`/absent value resolves to an empty list. */ +internal class FetchStep(val source: IBulkExecutor, val key: Any?) : Step() + +/** A leaf that produces its values via a (potentially suspending) computation rather than a bulk fetch. */ +internal class AsyncStep( + val token: Any, + val produceBlocking: () -> List, + val produceSuspending: suspend () -> List, +) : Step() + +/** Linear transform of the resolved value list of [src]. */ +internal class MapStep(val src: Step, val f: (List) -> List) : Step() + +/** Monadic dependency: once [src] resolves, the values pick the next step. */ +internal class FlatMapStep(val src: Step, val f: (List) -> Step) : Step() { + var inner: Step? = null +} + +/** Applicative join: all parts resolve, then their value lists are handed to [f] together. */ +internal class ZipStep(val parts: List>, val f: (List>) -> List) : Step() + +/** + * Lazily-built ordered concatenation of [size] children. The only place a breadth frontier is born. Children are + * instantiated on demand via [makeChild] (left-to-right) and dropped once resolved, so the live unresolved children are + * bounded by the round cap. + */ +internal class FanStep(val size: Int, val makeChild: (Int) -> Step) : Step() { + val childResults: Array?> = arrayOfNulls(size) + val childNodes: Array?> = arrayOfNulls(size) +} + +/** Recovers from an exception thrown while resolving [src] by producing replacement values. */ +internal class RecoverStep(val src: Step, val handler: (Throwable) -> List) : Step() + +/** Runs [consumer] before an exception thrown while resolving [src] propagates. */ +internal class OnErrorStep(val src: Step, val consumer: (Throwable) -> Unit) : Step() + +/** A view onto a shared, single-resolution node (the [org.modelix.streams.IStream.One.cached] mechanism). */ +internal class MemoStep(val token: Any, val init: () -> Step) : Step() + +// --------------------------------------------------------------------------------------------------------------------- +// Combinators (the API surface used by StreamImpl; signatures are kept stable) +// --------------------------------------------------------------------------------------------------------------------- + +// Each combinator evaluates *eagerly* when its inputs are already resolved (`Done`), exactly as the previous engine +// did, so the synchronous prefix of a computation — including any side effects in `map`/`flatMap` lambdas — runs at +// build time and in build order. Some call sites rely on this (e.g. a `var` set by one synchronous stream and read by a +// later `deferZeroOrOne`). Only when an input would block on a request is a lazy node built and deferred to the driver. + +internal fun Step.mapValues(f: (List) -> List): Step = + if (this is Done) Done(f(values)) else MapStep(this, f) + +internal fun Step.flatMapStep(f: (List) -> Step): Step = + if (this is Done) f(values) else FlatMapStep(this, f) + +internal fun Step.recover(handler: (Throwable) -> List): Step = + if (this is Done) this else RecoverStep(this, handler) + +internal fun Step.doOnError(consumer: (Throwable) -> Unit): Step = + if (this is Done) this else OnErrorStep(this, consumer) + +/** Applicative concatenation of the values of independent, already-built steps in order. */ +internal fun combineConcat(steps: List>): Step = + if (steps.all { it is Done }) Done(steps.flatMap { (it as Done).values }) else FanStep(steps.size) { steps[it] } + +/** Applicative combination handing all resolved value lists to [f] together. */ +internal fun zipN(steps: List>, f: (List>) -> List): Step = + if (steps.all { it is Done }) Done(f(steps.map { (it as Done).values })) else ZipStep(steps, f) + +/** + * Applicative concatenation of children built lazily from [items] — the breadth-bounded fan-out. + * + * The leading run of children that resolve synchronously (to a [Done]) is built and concatenated eagerly, preserving + * the build-time evaluation of a synchronous prefix. At the first child that would block on a request, the rest of the + * children become a lazy [FanStep]: instantiated on demand and dropped once resolved, so the live unresolved children + * stay bounded by the round cap. A wide traversal whose children are all fetches has an empty eager prefix and is + * therefore fully lazy. + */ +internal fun fanOut(items: List, makeChild: (A) -> Step): Step { + if (items.isEmpty()) return Done(emptyList()) + val prefix = ArrayList() + var i = 0 + var firstBlocking: Step? = null + while (i < items.size) { + val child = makeChild(items[i]) + if (child is Done) { + prefix.addAll(child.values) + i++ + } else { + firstBlocking = child + break + } + } + if (firstBlocking == null) return Done(prefix) // every child was synchronous + val startIndex = i + val alreadyBuilt = firstBlocking + val lazyTail = FanStep(items.size - startIndex) { idx -> + if (idx == 0) alreadyBuilt else makeChild(items[startIndex + idx]) + } + return if (prefix.isEmpty()) lazyTail else FanStep(2) { idx -> if (idx == 0) Done(prefix) else lazyTail } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Leaves +// --------------------------------------------------------------------------------------------------------------------- + +/** A fetch leaf. The [execution] is unused at build time; the cache is consulted by the driver while it walks. */ +@Suppress("UNUSED_PARAMETER") +internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: Any?): Step = + FetchStep(source, key) + +@Suppress("UNUSED_PARAMETER") +internal fun asyncStep( + execution: Execution, + token: Any, + produceBlocking: () -> List, + produceSuspending: suspend () -> List, +): Step = AsyncStep(token, produceBlocking, produceSuspending) + +@Suppress("UNUSED_PARAMETER") +internal fun memoStep(execution: Execution, token: Any, init: () -> Step): Step = MemoStep(token, init) + +// --------------------------------------------------------------------------------------------------------------------- +// Per-run state and the depth-first, bounded-frontier driver +// --------------------------------------------------------------------------------------------------------------------- + +private val MISSING = Any() + +/** + * Per-query state shared by every [Step] built for a single execution. Holds the fetch cache (each key fetched at most + * once across the whole traversal), async results, and shared memo cells, plus the mutable state of the current round. + */ +internal class Execution { + private val fetchCaches = HashMap, HashMap>() + private val asyncResults = HashMap>() + private val memoCells = HashMap>() + + // Mutable state of the round currently being collected; reset by [newRound]. + private var pendingFetches = LinkedHashMap, LinkedHashSet>() + private var pendingAsync = ArrayList>() + private val seenAsyncTokens = HashSet() + private var pendingCount = 0 + private var cap = Int.MAX_VALUE + + private fun cacheFor(source: IBulkExecutor) = fetchCaches.getOrPut(source) { HashMap() } + + private fun memoCell(token: Any, init: () -> Step): Step = memoCells.getOrPut(token, init) + + private fun newRound() { + pendingFetches = LinkedHashMap() + pendingAsync = ArrayList() + seenAsyncTokens.clear() + pendingCount = 0 + cap = Int.MAX_VALUE + } + + private fun roundIsFull(): Boolean = pendingCount >= cap + + /** Registers a fetch demand for the current round, honoring the per-source batch-size cap. */ + private fun addFetch(source: IBulkExecutor, key: Any?) { + val set = pendingFetches.getOrPut(source) { LinkedHashSet() } + if (key in set) return + cap = if (cap == Int.MAX_VALUE) source.batchSize else minOf(cap, source.batchSize) + if (pendingCount >= cap) return + set.add(key) + pendingCount++ + } + + private fun addAsync(step: AsyncStep<*>) { + if (seenAsyncTokens.add(step.token)) pendingAsync.add(step) + } + + /** + * Returns the node's resolved value list, or `null` if it is still blocked on a request that was registered for the + * current round. Recurses on the node graph; the recursion depth is the traversal depth (bounded in practice), + * matching the recursion characteristics of the previous engine. + */ + private fun expand(step: Step<*>): List? { + step.cached?.let { return it } + val result: List? = when (step) { + is Done -> step.values + is FetchStep -> { + val cache = cacheFor(step.source) + if (cache.containsKey(step.key)) { + val v = cache[step.key] + if (v === MISSING) emptyList() else listOf(v) + } else { + addFetch(step.source, step.key) + null + } + } + is AsyncStep -> { + if (asyncResults.containsKey(step.token)) { + asyncResults.getValue(step.token) + } else { + addAsync(step) + null + } + } + is MapStep<*, *> -> { + @Suppress("UNCHECKED_CAST") + expand(step.src)?.let { (step.f as (List) -> List)(it) } + } + is FlatMapStep<*, *> -> { + val sv = expand(step.src) + if (sv == null) { + null + } else { + @Suppress("UNCHECKED_CAST") + val typed = step as FlatMapStep + val inner = typed.inner ?: typed.f(sv).also { typed.inner = it } + expand(inner) + } + } + is ZipStep<*, *> -> { + var allResolved = true + val partValues = ArrayList>(step.parts.size) + for (part in step.parts) { + val pv = expand(part) + if (pv == null) allResolved = false else if (allResolved) partValues.add(pv) + } + if (allResolved) { + @Suppress("UNCHECKED_CAST") + (step.f as (List>) -> List)(partValues) + } else { + null + } + } + is FanStep<*> -> { + @Suppress("UNCHECKED_CAST") + val fan = step as FanStep + val out = ArrayList() + var allResolved = true + var i = 0 + while (i < fan.size) { + val done = fan.childResults[i] + if (done != null) { + if (allResolved) out.addAll(done) + i++ + continue + } + val child = fan.childNodes[i] ?: fan.makeChild(i).also { fan.childNodes[i] = it } + val cv = expand(child) + if (cv != null) { + fan.childResults[i] = cv + fan.childNodes[i] = null // free the resolved child node + if (allResolved) out.addAll(cv) + } else { + allResolved = false + // Keep scanning siblings to batch their requests into this same round, until the cap is hit. + if (roundIsFull()) break + } + i++ + } + if (allResolved) out else null + } + is RecoverStep<*> -> { + try { + expand(step.src) + } catch (ex: Throwable) { + @Suppress("UNCHECKED_CAST") + (step.handler as (Throwable) -> List)(ex) + } + } + is OnErrorStep<*> -> { + try { + expand(step.src) + } catch (ex: Throwable) { + step.consumer(ex) + throw ex + } + } + is MemoStep<*> -> expand(memoCell(step.token, step.init)) + } + if (result != null) step.cached = result + return result + } + + /** + * Drives a step to completion, blocking. Each loop iteration is one round: collect the bounded request frontier, + * resolve it (one bulk call per source, chunked to that source's [IBulkExecutor.batchSize], plus any async leaves), + * then walk again. The loop is the trampoline across rounds; fetch-dependent depth costs rounds, not stack. + */ + fun drive(initial: Step): List { + while (true) { + newRound() + val resolved = expand(initial) + if (resolved != null) { + @Suppress("UNCHECKED_CAST") + return resolved as List + } + check(pendingFetches.isNotEmpty() || pendingAsync.isNotEmpty()) { + "stream made no progress: root unresolved but no request was demanded" + } + for ((source, keys) in pendingFetches) { + for (chunk in keys.chunked(source.batchSize)) { + fillFetch(source, chunk, source.execute(chunk)) + } + } + for (action in pendingAsync) { + asyncResults[action.token] = action.produceBlocking() + } + } + } + + suspend fun driveSuspending(initial: Step): List { + while (true) { + newRound() + val resolved = expand(initial) + if (resolved != null) { + @Suppress("UNCHECKED_CAST") + return resolved as List + } + check(pendingFetches.isNotEmpty() || pendingAsync.isNotEmpty()) { + "stream made no progress: root unresolved but no request was demanded" + } + for ((source, keys) in pendingFetches) { + for (chunk in keys.chunked(source.batchSize)) { + fillFetch(source, chunk, source.executeSuspending(chunk)) + } + } + for (action in pendingAsync) { + asyncResults[action.token] = action.produceSuspending() + } + yield() + } + } + + private fun fillFetch(source: IBulkExecutor, keys: List, results: Map) { + val cache = cacheFor(source) + for (key in keys) cache[key] = if (results.containsKey(key)) results[key] else MISSING + } +} diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt new file mode 100644 index 0000000000..ae8842e61d --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt @@ -0,0 +1,114 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** Records every bulk call so tests can assert how many rounds happened and which keys were batched together. */ +private class RecordingSource(private val backing: Map) : IBulkExecutor { + val calls = mutableListOf>() + + override fun execute(keys: List): Map { + calls.add(keys.sorted()) + return keys.filter { backing.containsKey(it) }.associateWith { backing.getValue(it) } + } + + override suspend fun executeSuspending(keys: List): Map = execute(keys) + + val roundCount: Int get() = calls.size + val totalKeysFetched: Int get() = calls.sumOf { it.size } +} + +/** + * Verifies the round-based engine's batching behaviour through the real [BulkRequestStreamExecutor] / [enqueue] path: + * independent fetches share a round, dependent fetches use separate rounds, duplicate keys are fetched once, and deep + * dependent chains stay stack-safe. + */ +class BulkRequestBatchingTest { + private val data = (0..1000).associateWith { "v$it" } + + @Test + fun single_fetch() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + assertEquals("v3", executor.query { executor.enqueue(3).exceptionIfEmpty() }) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(3)), source.calls) + } + + @Test + fun independent_fetches_are_batched_into_one_round() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val result = executor.query { + IStream.many(listOf(1, 2, 3, 4)).flatMapOrdered { executor.enqueue(it).orNull() }.toList() + } + assertEquals(listOf("v1", "v2", "v3", "v4"), result) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(1, 2, 3, 4)), source.calls) + } + + @Test + fun zip_batches_both_sides_together() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val combined = executor.query { + executor.enqueue(5).exceptionIfEmpty().zipWith(executor.enqueue(6).exceptionIfEmpty()) { a, b -> "$a+$b" } + } + assertEquals("v5+v6", combined) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(5, 6)), source.calls) + } + + @Test + fun dependent_fetches_use_separate_rounds() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + // The second fetch's key is derived from the first result -> it cannot be known until round 1 resolves. + val result = executor.query { + executor.enqueue(1).exceptionIfEmpty().flatMapOne { first -> + executor.enqueue(first.removePrefix("v").toInt() + 4).exceptionIfEmpty() // "v1" -> 5 + } + } + assertEquals("v5", result) + assertEquals(2, source.roundCount) + assertEquals(listOf(listOf(1), listOf(5)), source.calls) + } + + @Test + fun duplicate_keys_are_fetched_once() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val result = executor.query { + IStream.many(listOf(2, 2, 2, 7, 7)).flatMapOrdered { executor.enqueue(it).orNull() }.toList() + } + assertEquals(listOf("v2", "v2", "v2", "v7", "v7"), result) + assertEquals(1, source.roundCount) + assertEquals(2, source.totalKeysFetched) // only keys 2 and 7 actually fetched + } + + @Test + fun missing_key_resolves_to_empty() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + assertEquals(null, executor.query { executor.enqueue(9999).orNull() }) + assertFailsWith { + executor.query { executor.enqueue(9999).exceptionIfEmpty() } + } + } + + @Test + fun deep_dependent_chain_is_stack_safe() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + // 1000 sequential dependent fetches -> 1000 extra rounds, must not overflow the stack. + var stream: IStream.One = executor.enqueue(0).exceptionIfEmpty() + repeat(1000) { + stream = stream.flatMapOne { v -> executor.enqueue(v.removePrefix("v").toInt() + 1).exceptionIfEmpty() } + } + assertEquals("v1000", executor.query { stream }) + assertEquals(1001, source.roundCount) + assertTrue(source.calls.all { it.size == 1 }) + } +} diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt new file mode 100644 index 0000000000..d2863cbb54 --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt @@ -0,0 +1,59 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Verifies the multicast semantics of [IStream.One.cached]: a stream consumed by multiple downstream branches is + * evaluated exactly once (including any side effects), and the single result is shared. This is what ModelQL relies on + * for shared/memoized query steps (see ModelQLClientTest.testCaching). + */ +class CachedStreamTest { + private val executor = SimpleStreamExecutor + + @Test + fun `cached one is evaluated once across multiple consumers`() { + var evaluations = 0 + val cached = IStream.of(Unit).map { + evaluations++ + 42 + }.cached() + + val result = executor.query { + cached.map { it + 1000 }.plus(cached.map { it + 2000 }).toList() + } + + assertEquals(listOf(1042, 2042), result) + assertEquals(1, evaluations) // the side effect ran exactly once, not once per consumer + } + + @Test + fun `without cached the side effect runs per consumer`() { + var evaluations = 0 + val notCached = IStream.of(Unit).map { + evaluations++ + 42 + } + + val result = executor.query { + notCached.map { it + 1000 }.plus(notCached.map { it + 2000 }).toList() + } + + assertEquals(listOf(1042, 2042), result) + assertEquals(2, evaluations) // confirms the distinction: each consumer re-evaluates + } + + @Test + fun `cached one shared via zip evaluates once`() { + var evaluations = 0 + val cached = IStream.of(7).map { + evaluations++ + it + }.cached() + + val result = executor.query { cached.zipWith(cached) { a, b -> a + b } } + + assertEquals(14, result) + assertEquals(1, evaluations) + } +} diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt new file mode 100644 index 0000000000..0f8e8817b1 --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt @@ -0,0 +1,91 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * A synthetic content-addressed tree exposed as an [IBulkExecutor]. The node id is the key; the value is the + * comma-separated list of child ids. Nodes are laid out as a complete [branching]-ary tree in BFS order + * (root = 0, children of i are i*branching+1 .. i*branching+branching), so children can be computed on the fly + * without materializing the whole tree. + * + * Crucially it records, per bulk round, how many keys were requested together. With a [batchSize] larger than any + * level, each [execute] call corresponds to exactly one engine round, so [maxRoundSize] is the peak fetch frontier — + * the quantity the old `BulkRequestStreamExecutor.sendNextBatch` depth-first traversal kept bounded. + */ +private class TreeSource(val branching: Int, val depth: Int) : IBulkExecutor { + val nodeCount: Int = ((pow(branching, depth + 1) - 1) / (branching - 1)) + val widestLevel: Int = pow(branching, depth) + + var maxRoundSize: Int = 0 + private set + var roundCount: Int = 0 + private set + var totalFetched: Int = 0 + private set + + override fun execute(keys: List): Map { + roundCount++ + totalFetched += keys.size + if (keys.size > maxRoundSize) maxRoundSize = keys.size + return keys.associateWith { id -> + (1..branching).map { id * branching + it }.filter { it < nodeCount }.joinToString(",") + } + } + + override suspend fun executeSuspending(keys: List): Map = execute(keys) + + companion object { + private fun pow(base: Int, exp: Int): Int { + var r = 1 + repeat(exp) { r *= base } + return r + } + } +} + +class FrontierSizeTest { + + /** Builds `getDescendants(id)` the same shape the real tree uses: of(id) + children.flatMap { descendants }. */ + private fun descendants(executor: BulkRequestStreamExecutor, id: Int): IStream.Many { + val childIds: IStream.Many = executor.enqueue(id).orNull().flatMapOrdered { csv -> + if (csv.isNullOrEmpty()) IStream.many(emptyList()) else IStream.many(csv.split(",").map { it.toInt() }) + } + return IStream.of(id).concat(childIds.flatMapOrdered { descendants(executor, it) }) + } + + @Test + fun frontier_is_bounded_by_batch_size_on_wide_deep_tree() { + val source = TreeSource(branching = 4, depth = 7) // 21845 nodes, widest level 16384 + val batchSize = 1000 + val executor = BulkRequestStreamExecutor(source, batchSize = batchSize) + + val count = executor.query { descendants(executor, 0).count() } + + // The traversal is correct and does no redundant work... + assertEquals(source.nodeCount, count) + assertEquals(source.nodeCount, source.totalFetched) + // ...and the peak request frontier stays within the batch size, even though the widest level is far larger. + // This is the depth-first bound the old sendNextBatch provided; the new round-based engine now preserves it. + assertTrue(source.widestLevel > batchSize, "test should exercise a level wider than the cap") + assertTrue( + source.maxRoundSize <= batchSize, + "peak frontier ${source.maxRoundSize} must stay within batchSize $batchSize (widest level ${source.widestLevel})", + ) + } + + @Test + fun unbounded_batch_size_batches_each_level_into_one_round() { + // With a batch size above any level, no frontier is trimmed: each tree level is fetched in a single round, so + // applicative batching is fully preserved when memory is not a constraint. (depth + 1 = 8 rounds.) + val source = TreeSource(branching = 4, depth = 7) + val executor = BulkRequestStreamExecutor(source, batchSize = Int.MAX_VALUE) + + val count = executor.query { descendants(executor, 0).count() } + + assertEquals(source.nodeCount, count) + assertEquals(8, source.roundCount) + assertEquals(source.widestLevel, source.maxRoundSize) + } +} diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt index b8ba19acfa..33d31a0070 100644 --- a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt +++ b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt @@ -1,18 +1,141 @@ package org.modelix.streams -import com.badoo.reaktive.observable.observableOf -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.single.blockingGet import kotlin.test.Test import kotlin.test.assertEquals class StreamExtensionsTests { + private val executor = SimpleStreamExecutor + @Test fun `distinct removes duplicates`() { assertEquals( listOf("g", "a", "d", "h", "z"), - observableOf("g", "g", "a", "d", "h", "z", "g", "h").distinct().toList().blockingGet(), + executor.query { IStream.many(listOf("g", "g", "a", "d", "h", "z", "g", "h")).distinct().toList() }, + ) + } + + @Test + fun `map and filter`() { + assertEquals( + listOf(20, 40), + executor.query { IStream.many(1..4).map { it * 10 }.filter { it % 20 == 0 }.toList() }, + ) + } + + @Test + fun `flatMap concatenates in order`() { + assertEquals( + listOf(1, -1, 2, -2, 3, -3), + executor.query { IStream.many(1..3).flatMapOrdered { IStream.many(listOf(it, -it)) }.toList() }, + ) + } + + @Test + fun `zip combines single values`() { + assertEquals( + "a1", + executor.query { IStream.of("a").zipWith(IStream.of(1)) { a, b -> "$a$b" } }, ) } + + @Test + fun `fold and count`() { + assertEquals(10, executor.query { IStream.many(1..4).fold(0) { acc, v -> acc + v } }) + assertEquals(4, executor.query { IStream.many(1..4).count() }) + } + + @Test + fun `any all none`() { + assertEquals(true, executor.query { IStream.many(1..4).any { it > 3 } }) + assertEquals(false, executor.query { IStream.many(1..4).any { it > 4 } }) + assertEquals(true, executor.query { IStream.many(1..4).all { it > 0 } }) + assertEquals(false, executor.query { IStream.many(1..4).all { it > 1 } }) + assertEquals(true, executor.query { IStream.many(1..4).none { it > 4 } }) + assertEquals(true, executor.query { IStream.many(emptyList()).all { it > 0 } }) + assertEquals(true, executor.query { IStream.many(emptyList()).none() }) + } + + @Test + fun `filterNot and filterIsInstance`() { + assertEquals(listOf(1, 3), executor.query { IStream.many(1..4).filterNot { it % 2 == 0 }.toList() }) + assertEquals( + listOf("a", "b"), + executor.query { IStream.many(listOf("a", 1, "b", 2)).filterIsInstance().toList() }, + ) + } + + @Test + fun `first and last`() { + assertEquals(1, executor.query { IStream.many(1..4).first() }) + assertEquals(2, executor.query { IStream.many(1..4).first { it % 2 == 0 } }) + assertEquals(null, executor.query { IStream.many(1..4).firstOrNull { it > 4 } }) + assertEquals(4, executor.query { IStream.many(1..4).last() }) + assertEquals(null, executor.query { IStream.many(emptyList()).lastOrNull() }) + } + + @Test + fun `mapIndexed and filterIndexed`() { + assertEquals( + listOf("0:a", "1:b"), + executor.query { IStream.many(listOf("a", "b")).mapIndexed { i, v -> "$i:$v" }.toList() }, + ) + assertEquals( + listOf("a", "c"), + executor.query { IStream.many(listOf("a", "b", "c", "d")).filterIndexed { i, _ -> i % 2 == 0 }.toList() }, + ) + } + + @Test + fun `collection conversions`() { + assertEquals(setOf(1, 2, 3), executor.query { IStream.many(listOf(1, 2, 2, 3)).toSet() }) + assertEquals( + mapOf(0 to listOf(2, 4), 1 to listOf(1, 3)), + executor.query { IStream.many(1..4).groupBy { it % 2 } }, + ) + assertEquals( + mapOf(1 to "a", 2 to "bb"), + executor.query { IStream.many(listOf("a", "bb")).associateBy { it.length } }, + ) + assertEquals( + mapOf("a" to 1, "bb" to 2), + executor.query { IStream.many(listOf("a", "bb")).associateWith { it.length } }, + ) + assertEquals( + mapOf(1 to "a", 2 to "b"), + executor.query { IStream.many(listOf(1 to "a", 2 to "b")).toMap() }, + ) + } + + @Test + fun `sorting and distinctBy`() { + assertEquals(listOf(1, 2, 3, 4), executor.query { IStream.many(listOf(3, 1, 4, 2)).sorted().toList() }) + assertEquals(listOf(4, 3, 2, 1), executor.query { IStream.many(listOf(3, 1, 4, 2)).sortedDescending().toList() }) + assertEquals( + listOf("a", "bb", "ccc"), + executor.query { IStream.many(listOf("ccc", "a", "bb")).sortedBy { it.length }.toList() }, + ) + assertEquals( + listOf("a", "bb"), + executor.query { IStream.many(listOf("a", "bb", "cc", "d")).distinctBy { it.length }.toList() }, + ) + } + + @Test + fun `reductions`() { + assertEquals(24, executor.query { IStream.many(1..4).reduce { acc, v -> acc * v } }) + assertEquals(10, executor.query { IStream.many(1..4).sumOf { it } }) + assertEquals(10, executor.query { IStream.many(1..4).sum() }) + assertEquals(10L, executor.query { IStream.many(listOf(1L, 2L, 3L, 4L)).sum() }) + assertEquals("ccc", executor.query { IStream.many(listOf("a", "bb", "ccc")).maxByOrNull { it.length } }) + assertEquals("a", executor.query { IStream.many(listOf("a", "bb", "ccc")).minByOrNull { it.length } }) + assertEquals(null, executor.query { IStream.many(emptyList()).maxByOrNull { it.length } }) + } + + @Test + fun `startWith endWith and joinToString`() { + assertEquals(listOf(0, 1, 2, 3), executor.query { IStream.many(1..3).startWith(0).toList() }) + assertEquals(listOf(1, 2, 3, 4), executor.query { IStream.many(1..3).endWith(4).toList() }) + assertEquals("[1, 2, 3]", executor.query { IStream.many(1..3).joinToString(prefix = "[", postfix = "]") }) + } } diff --git a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt index 1fb843bd43..cafd2fb4e2 100644 --- a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt +++ b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt @@ -1,29 +1,33 @@ package org.modelix.streams -import kotlinx.coroutines.flow.single -import org.modelix.kotlin.utils.runBlockingIfJvm +import org.modelix.streams.engine.Execution +/** + * JVM executor that always drives streams to completion blocking, resolving async leaves (e.g. flows) via + * [org.modelix.kotlin.utils.runBlockingIfJvm] inside the engine. With the unified engine this behaves like + * [SimpleStreamExecutor]; it is retained for API compatibility. + */ object BlockingStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { - return runBlockingIfJvm { - FlowStreamBuilder.INSTANCE.convert(body()).single() - } + val execution = Execution() + return execution.drive(body().asStep(execution)).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - return FlowStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.driveSuspending(body().asStep(execution)).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - runBlockingIfJvm { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect { visitor(it) } - } + val execution = Execution() + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } override suspend fun iterateSuspending( streamProvider: suspend () -> IStream.Many, visitor: suspend (T) -> Unit, ) { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect(visitor) + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } }