From e7b72ae67350275abfc63b030f8f1ad926b79921 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 20:23:51 +0200 Subject: [PATCH 1/7] feat(streams): replace Sequence/Flow/Reaktive backends with a round-based Step engine The streams module previously carried three interchangeable backends (Sequence, Flow, Reaktive) plus a deferred builder. Reaktive existed solely to provide the push semantics needed for bulk-request batching: collecting a set of independent data requests before forcing any of them. That batching requirement is an *applicative* property and can be satisfied structurally by a single round-based interpreter, removing the need for three backends and the third-party Reaktive dependency. This change: - Adds streams2: a standalone, dependency-free reference implementation of the Step-based engine (prototype + tests + README). - Ports the engine into the streams module behind its existing public API: - engine/StepEngine.kt: Step IR, applicative/monadic combinators, blocking + suspending round drivers, fetch leaves, async leaves (fromFlow/ singleFromCoroutine). - StreamImpl.kt: one backing impl for every IStream cardinality, plus CompletableImpl and the single StreamBuilderImpl. - SimpleStreamExecutor / BlockingStreamExecutor / BulkRequestStreamExecutor are now thin wrappers over the driver; enqueue() is a fetch leaf and batching is structural (per source, per round). - Deletes the four backend builders, the concrete stream classes, the Reaktive helpers, and the Reaktive dependency. The public API (IStream.*, IStreamBuilder, IStreamExecutor, IExecutableStream, IStreamInternal, IBulkExecutor, BulkRequestStreamExecutor, all operators) is unchanged. No downstream module required API-driven changes; the only other edits remove dead code (unused Reaktive imports/helpers in model-api and modelql-untyped) that compiled only via the former transitive Reaktive export. Verified: streams compiles JVM+JS and tests pass; datastructures and model-datastructure compile untouched and their suites pass; modelql-core/-untyped tests pass; model-client/model-server/modelql-* compile; model-server LazyLoadingTest (batching/access-pattern correctness) passes. See streams-redesign.md and streams2/README.md for details and tradeoffs. Co-Authored-By: Claude Opus 4.8 --- .../model/api/async/NodeAsAsyncNode.kt | 8 - .../untyped/AllChildrenTraversalStep.kt | 2 - .../untyped/AllReferencesTraversalStep.kt | 2 - .../untyped/DescendantsTraversalStep.kt | 2 - settings.gradle.kts | 1 + streams-redesign.md | 183 +++++ streams/build.gradle.kts | 3 - .../streams/BulkRequestStreamExecutor.kt | 216 +----- .../org/modelix/streams/CollectionAsStream.kt | 203 ------ .../modelix/streams/CompletableObservable.kt | 68 -- .../modelix/streams/DeferredStreamBuilder.kt | 376 ---------- .../modelix/streams/EmptyCompletableStream.kt | 56 -- .../kotlin/org/modelix/streams/EmptyStream.kt | 157 ----- .../org/modelix/streams/FlowStreamBuilder.kt | 320 --------- .../kotlin/org/modelix/streams/IStream.kt | 2 +- .../modelix/streams/ReaktiveStreamBuilder.kt | 650 ------------------ .../org/modelix/streams/SequenceAsStream.kt | 185 ----- .../modelix/streams/SequenceStreamBuilder.kt | 323 --------- .../modelix/streams/SimpleStreamExecutor.kt | 21 +- .../org/modelix/streams/SingleValueStream.kt | 200 ------ .../org/modelix/streams/StreamExtensions.kt | 93 --- .../kotlin/org/modelix/streams/StreamImpl.kt | 306 +++++++++ .../org/modelix/streams/engine/StepEngine.kt | 249 +++++++ .../modelix/streams/StreamExtensionsTests.kt | 37 +- .../modelix/streams/BlockingStreamExecutor.kt | 26 +- streams2/README.md | 127 ++++ streams2/build.gradle.kts | 18 + .../kotlin/org/modelix/streams2/Execution.kt | 87 +++ .../org/modelix/streams2/IBulkExecutor.kt | 12 + .../kotlin/org/modelix/streams2/IStream.kt | 181 +++++ .../org/modelix/streams2/IStreamExecutor.kt | 24 + .../kotlin/org/modelix/streams2/Step.kt | 92 +++ .../org/modelix/streams2/Streams2Test.kt | 122 ++++ 33 files changed, 1500 insertions(+), 2852 deletions(-) create mode 100644 streams-redesign.md delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt delete mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt create mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt create mode 100644 streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt create mode 100644 streams2/README.md create mode 100644 streams2/build.gradle.kts create mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt create mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt create mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt create mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt create mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt create mode 100644 streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt b/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt index e67daa5486..667dec57d6 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/api/async/NodeAsAsyncNode.kt @@ -1,10 +1,5 @@ package org.modelix.model.api.async -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.maybeOf -import com.badoo.reaktive.maybe.maybeOfEmpty -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.singleOf import org.modelix.model.api.ConceptReference import org.modelix.model.api.IChildLinkReference import org.modelix.model.api.IConcept @@ -26,9 +21,6 @@ open class NodeAsAsyncNode(val node: INode) : IAsyncNode { return SimpleStreamExecutor } - private fun T?.asOptionalMono(): Maybe = if (this != null) maybeOf(this) else maybeOfEmpty() - private fun T.asMono(): Single = singleOf(this) - override fun asRegularNode(): INode = node override fun getConcept(): IStream.One { diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt index 0b4c3d61dc..c2350d7d32 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllChildrenTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt index 09205f8045..ef75f48dac 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/AllReferencesTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt index 899e1c2723..ff758293ef 100644 --- a/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt +++ b/modelql-untyped/src/commonMain/kotlin/org/modelix/modelql/untyped/DescendantsTraversalStep.kt @@ -1,7 +1,5 @@ package org.modelix.modelql.untyped -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.map import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/settings.gradle.kts b/settings.gradle.kts index 4507b79b74..bf4322aaab 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -41,5 +41,6 @@ include("mps-multiplatform-lib") include("mps-repository-concepts") include("mps-sync-plugin3") include("streams") +include("streams2") include("ts-model-api") include("vue-model-api") diff --git a/streams-redesign.md b/streams-redesign.md new file mode 100644 index 0000000000..1b1d994a98 --- /dev/null +++ b/streams-redesign.md @@ -0,0 +1,183 @@ +# Streams redesign & migration + +This document records the redesign of the `streams` module: replacing its three interchangeable reactive backends +(Sequence / Flow / Reaktive) with a single round-based interpreter, while preserving the public API so that no +downstream module required API-driven changes. + +The new engine was first prototyped in the [`streams2`](streams2/README.md) module; this document covers porting that +design into `streams` and the migration of the wider codebase. + +--- + +## 1. Motivation + +The `streams` module existed to provide one capability that plain coroutines/`Flow` could not give cheaply: +**automatic bulk-request batching** for lazy, partial loading of large content-addressed models. When a traversal +needs many objects from a remote store, the independent requests must be coalesced into a single round-trip. + +### Why there were three backends + +The old design carried three implementations because they had different execution semantics +(see the former doc-comment in `IStreamExecutor`): + +- **Sequence** — synchronous, eager, pull-based; minimal overhead when all data is local. +- **Flow** — suspendable, pull-based; best integration with `suspend` sources. +- **Reaktive** — push-based; the *only* one able to collect a set of pending requests before forcing any of them, + which is what bulk batching needs. Flows could parallelize requests only by launching a coroutine per object — + too much overhead. + +This meant ~1900 lines across `SequenceStreamBuilder`, `FlowStreamBuilder`, `ReaktiveStreamBuilder`, +`DeferredStreamBuilder`, the concrete stream classes, and the Reaktive third-party dependency. + +### The key insight + +Batching requires the runtime to **see a set of independent data requests before forcing any of them**. That is an +**applicative** property, not a monadic one: + +- `zip(a, b)` and the elements of a `Many` are **independent** → their fetches belong to the **same batch round**. +- `flatMap` is a **dependency** → the right-hand side's keys are unknown until the left resolves → a **later round**. + +Pull (Sequence/Flow) is demand-driven and sequential and cannot expose this frontier without per-item concurrency. +Push (Reaktive) can. But a purpose-built interpreter gets the same property **structurally**, from the shape of the +computation, with no external dependency — and collapses all three backends into one. + +Prior art for this pattern: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. + +--- + +## 2. The new design + +### `Step` — the intermediate representation + +A stream is described lazily and interpreted in **rounds**. The core type +([`streams/.../engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)): + +```kotlin +sealed interface Step +class Done(val values: List) // fully resolved +class Blocked(val pending: Pending, val resume: () -> Step) // needs a round of work first +class Failed(val cause: Throwable) +``` + +`Pending` is the deduplicated work for one round: bulk fetches grouped by data source, plus async leaves. The +combinators encode the applicative/monadic split: + +- `flatMapStep` (monadic) — defers the continuation into the next round. +- `combineConcat` / `zipN` / `zip2` (applicative) — **union** the pending work of independent branches into the same + round. That union *is* the batch. + +A subtree with no `Blocked` resolves straight to `Done`: the **synchronous fast path** for local data, with no +scheduler and nothing allocated (this subsumes the old Sequence backend's role). + +### The driver + +`Execution.drive` (and `driveSuspending`) is a loop where each iteration is one batch round: + +1. For each data source in `pending`, issue **one bulk call** (`execute` / `executeSuspending`), chunked to `batchSize`. +2. Run any async leaves. +3. Fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds). +4. `resume()` and repeat until `Done`. + +The loop is the **trampoline** that keeps fetch-dependent chains (the common case in tree traversal) stack-safe +regardless of depth. This subsumes the old Reaktive subscribe-collect-batch mechanism. + +### Batching is now structural + +Previously, batching was tied to an executor via a `ContextValue`: `enqueue(key)` registered into the +current query's queue. Now a fetch leaf carries its own data **source**, and the driver groups fetches **per source +per round**. Consequences: + +- `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to its `IBulkExecutor`. +- Any executor that drives a stream batches its fetches — including `SimpleStreamExecutor`, which therefore now issues + **strictly fewer** round-trips than before, never more. + +--- + +## 3. How the public API was preserved + +The entire public surface of `streams` is unchanged. The interface files (`IStream`, `IStreamBuilder`, +`IStreamExecutor`, `IExecutableStream`, `IStreamInternal`, `IBulkExecutor`, `Zip`, and all extension functions) were +kept as-is. Only the *implementation* was swapped: + +| Concern | Before | After | +|---|---|---| +| Stream instances | `SingleValueStream`, `CollectionAsStream`, `EmptyStream`, deferred wrappers, … | one `StreamImpl` backing every cardinality, plus `CompletableImpl` | +| Builder | `DeferredStreamBuilder` (+ 3 backend builders) | one `StreamBuilderImpl` over the engine | +| `SimpleStreamExecutor` / `BlockingStreamExecutor` | Sequence/Flow conversion | thin wrappers over `drive` / `driveSuspending` | +| `BulkRequestStreamExecutor` | Reaktive + `RequestQueue` + `CompletableObservable` | round driver; `enqueue` = fetch leaf | +| `convert(IStreamBuilder)` | converts to a backend representation | returns `this` (single representation) | +| `asFlow()` / `asSequence()` | backend conversion | materialize via the driver | +| `fromFlow` / `singleFromCoroutine` | Flow/Reaktive sources | **async leaves** resolved by the suspending driver (or `runBlockingIfJvm` under the blocking driver) | + +New files in `streams`: + +- [`engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt) — `Step`, `Pending`, + combinators, blocking + suspending drivers, fetch leaves, async leaves. +- [`StreamImpl.kt`](streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt) — `StreamImpl`, `CompletableImpl`, + `StreamBuilderImpl`, `StreamAssertionError`. + +Deleted: `SequenceStreamBuilder`, `FlowStreamBuilder`, `ReaktiveStreamBuilder`, `DeferredStreamBuilder`, +`SingleValueStream`, `CollectionAsStream`, `SequenceAsStream`, `EmptyStream`, `EmptyCompletableStream`, +`CompletableObservable`, `StreamExtensions` (Reaktive helpers). The Reaktive dependency was removed from +`streams/build.gradle.kts`. + +### Why the engine lives in `streams`, not as a dependency on `streams2` + +The engine was ported into `streams` rather than wiring `streams` → `streams2`, because: + +- the engine types must be `internal` to `streams`; +- the real integration needs a **suspending driver** and **async leaves** that the minimal `streams2` prototype + deliberately omits; +- `streams`' `IBulkExecutor` already declares `executeSuspending`, which the engine uses directly; +- it avoids two competing `IBulkExecutor` / `IStream` types across modules. + +`streams2` remains as the standalone, dependency-free reference implementation. + +--- + +## 4. Migration of the wider codebase + +**No downstream module required API-driven changes.** The only edits outside `streams` were removals of **dead code** +that compiled solely because `streams` used to re-export Reaktive via an `api` dependency: + +- `model-api/.../async/NodeAsAsyncNode.kt` — two unused private helpers (`asOptionalMono`, `asMono`) and their imports. +- `modelql-untyped/.../AllChildrenTraversalStep.kt`, `AllReferencesTraversalStep.kt`, `DescendantsTraversalStep.kt` — + unused `com.badoo.reaktive.observable.{map, flatMap}` imports (the `.map`/`.flatMap` calls resolve to `IStream` + members). + +These were latent couplings to a transitive dependency, not uses of the `streams` API. + +### Verification + +- `streams`: compiles JVM + JS; unit tests pass. +- `datastructures`, `model-datastructure`: compile **untouched**; full JVM test suites pass. +- `modelql-core`, `modelql-untyped`: compile + tests pass. +- `model-client`, `model-server`, `model-server-api`, `modelql-typed/-html/-client/-server`, `bulk-model-sync-lib`: + compile (JVM + JS where applicable). +- **`model-server` `LazyLoadingTest`** passes — the access-pattern / batching-correctness test, which confirms the + structural batching reproduces the old Reaktive collect-and-batch behavior. + +--- + +## 5. Behavioral tradeoffs + +These follow from the "no incremental emission" decision and are intentional. They change performance characteristics, +not results. + +1. **`iterate` / `iterateSuspending` fully materialize before visiting.** Reaktive previously pushed elements and + drained between batches to bound memory; the new driver builds the whole result list first. For very large + server-side iterations (e.g. walking all objects) this raises peak memory. *If this matters for a hot path, the + clean fix is to add genuine per-round streaming to just the `iterate*` drivers without disturbing the rest of the + engine.* +2. **`cached()` is currently a no-op.** Fetch-level dedup (the expensive part) is handled by the per-run cache; only + pure-recompute memoization is lost. +3. **`take` / `skip` operate on materialized results** — they do not prune upstream fetches. +4. **`SimpleStreamExecutor` now batches** per source/round — strictly fewer round-trips than before. + +## 6. Known limitations / future work + +- **Within-round stack safety.** The round driver trampolines across `Blocked` (the common fetch-dependent case). A + pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode `Step` + as a stack-safe free monad (explicit interpreter loop) if needed. +- **Optional streaming `iterate*`** — see tradeoff #1. +- **Restore `cached()` memoization** if ModelQL recompute cost proves material. diff --git a/streams/build.gradle.kts b/streams/build.gradle.kts index fc6383d8df..c93634dbcb 100644 --- a/streams/build.gradle.kts +++ b/streams/build.gradle.kts @@ -9,9 +9,6 @@ kotlin { dependencies { implementation(project(":kotlin-utils")) implementation(libs.kotlin.coroutines.core) - - api(libs.reaktive) - api(libs.reaktive.coroutines.interop) } } commonTest { diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt index 1b0c677eff..583614a95d 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt @@ -1,165 +1,53 @@ package org.modelix.streams -import com.badoo.reaktive.observable.subscribe -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.notNull -import com.badoo.reaktive.single.subscribe -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.channels.onSuccess -import kotlinx.coroutines.yield -import org.modelix.kotlin.utils.ContextValue -import org.modelix.kotlin.utils.runSynchronized +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.Step +import org.modelix.streams.engine.drive +import org.modelix.streams.engine.driveSuspending +import org.modelix.streams.engine.fetchStep interface IBulkExecutor { fun execute(keys: List): Map suspend fun executeSuspending(keys: List): Map } -class BulkRequestStreamExecutor(private val bulkExecutor: IBulkExecutor, val batchSize: Int = 5000) : IStreamExecutor, IStreamExecutorProvider { - private val requestQueue = ContextValue() - private val streamBuilder = ReaktiveStreamBuilder() - - private inner class RequestQueue { - val queue: MutableMap = LinkedHashMap() - - fun process() { - while (queue.isNotEmpty()) { - sendNextBatch { bulkExecutor.execute(it) } - } - } - - suspend fun processSuspending(afterBatch: suspend () -> Unit) { - while (queue.isNotEmpty()) { - sendNextBatch { bulkExecutor.executeSuspending(it) } - // Give stream consumers time to process the new elements. - yield() - afterBatch() - } - } - - private inline fun sendNextBatch(executor: (keys: List) -> Map) { - val requests = runSynchronized(queue) { - // The callback of a request usually enqueues new requests until it reaches the leafs of the - // data structure. By executing the latest (instead of the oldest) request we basically do a depth - // first traversal which keeps the maximum size of the queue smaller. - val requests = queue.entries.tailSequence(batchSize).map { it.value.hash to it.value }.toList() - requests.forEach { queue.remove(it.first) } - requests - } - if (requests.isEmpty()) return - try { - val map = executor(requests.map { it.first }) - for ((_, queueElement) in requests) { - queueElement - val value = map[queueElement.hash] - queueElement.requestResult.complete(value) - } - } catch (ex: Throwable) { - for ((_, queueElement) in requests) { - queueElement.requestResult.failed(ex) - } - } - } - - fun query(hash: K): IStream.ZeroOrOne { - return (queue.getOrPut(hash) { QueueElement(hash, queue.size) }).value.notNull() - .let { streamBuilder.WrapperMaybe(it) } - } - } - - private inner class QueueElement(val hash: K, val position: Int) { - val requestResult = CompletableObservable() - val value: Single = requestResult.single - } +/** + * Executor that batches the individual fetches enqueued via [enqueue] into bulk calls against [bulkExecutor]. + * + * Batching is structural: [enqueue] returns a stream backed by a fetch leaf bound to [bulkExecutor], and the round + * driver groups all fetches reachable in a round (across independent stream branches) into a single + * [IBulkExecutor.execute] call, chunked to [batchSize]. Dependent fetches (reached through `flatMap`) fall into later + * rounds. This replaces the previous Reaktive subscribe-collect-batch mechanism. + */ +class BulkRequestStreamExecutor( + private val bulkExecutor: IBulkExecutor, + val batchSize: Int = 5000, +) : IStreamExecutor, IStreamExecutorProvider { override fun getStreamExecutor(): IStreamExecutor = this - fun enqueue(key: K): IStream.ZeroOrOne { - val currentQueue = requestQueue.getValueOrNull() - return if (currentQueue == null) { - IStream.deferZeroOrOne { requestQueue.getValue().query(key) } - } else { - currentQueue.query(key) - } - } + @Suppress("UNCHECKED_CAST") + fun enqueue(key: K): IStream.ZeroOrOne = + StreamImpl { execution -> fetchStep(execution, bulkExecutor as IBulkExecutor, key) as Step } override fun query(body: () -> IStream.One): T { - fun doProcess(queue: RequestQueue): T { - var result: Result? = null - val reaktiveStream = streamBuilder.convert(body()) - val subscription = reaktiveStream.subscribe( - onSuccess = { result = Result.success(it) }, - onError = { result = Result.failure(it) }, - ) - try { - queue.process() - } finally { - subscription.dispose() - } - return checkNotNull(result) { "Empty stream" }.getOrThrow() - } - - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.computeWith(newQueue) { - IStreamExecutor.CONTEXT.computeWith(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + return IStreamExecutor.CONTEXT.computeWith(this) { + val execution = Execution() + execution.drive(body().asStep(execution), batchSize).single() } } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - suspend fun doProcess(queue: RequestQueue): T { - var result: Result? = null - val reaktiveStream = streamBuilder.convert(body()) - val subscription = reaktiveStream.subscribe( - onSuccess = { result = Result.success(it) }, - onError = { result = Result.failure(it) }, - ) - try { - queue.processSuspending({}) - } finally { - subscription.dispose() - } - return checkNotNull(result) { "Empty stream" }.getOrThrow() - } - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.runInCoroutine(newQueue) { - IStreamExecutor.CONTEXT.runInCoroutine(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + return IStreamExecutor.CONTEXT.runInCoroutine(this) { + val execution = Execution() + execution.driveSuspending(body().asStep(execution), batchSize).single() } } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - fun doProcess(queue: RequestQueue) { - val reaktiveStream = streamBuilder.convert(streamProvider()) - val subscription = reaktiveStream.subscribe(onNext = visitor) - try { - queue.process() - } finally { - subscription.dispose() - } - } - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.computeWith(newQueue) { - IStreamExecutor.CONTEXT.computeWith(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + IStreamExecutor.CONTEXT.computeWith(this) { + val execution = Execution() + execution.drive(streamProvider().asStep(execution), batchSize).forEach(visitor) } } @@ -167,49 +55,9 @@ class BulkRequestStreamExecutor(private val bulkExecutor: IBulkExecutor IStream.Many, visitor: suspend (T) -> Unit, ) { - suspend fun doProcess(queue: RequestQueue) { - val reaktiveStream = streamBuilder.convert(streamProvider()) - val channel = Channel(capacity = Channel.Factory.UNLIMITED) - val subscription = reaktiveStream.subscribe( - onNext = { channel.trySend(it).getOrThrow() }, - ) - try { - suspend fun drainChannel() { - while (channel.tryReceive().onSuccess { visitor(it) }.isSuccess) { - // element already processed in onSuccess - } - } - - queue.processSuspending({ - // force processing of elements after each batch to avoid the channel from growing to big - drainChannel() - }) - drainChannel() - } finally { - subscription.dispose() - } - } - - val existingQueue = requestQueue.getValueOrNull() - return if (existingQueue == null) { - val newQueue = RequestQueue() - requestQueue.runInCoroutine(newQueue) { - IStreamExecutor.CONTEXT.runInCoroutine(this) { - doProcess(newQueue) - } - } - } else { - doProcess(existingQueue) + IStreamExecutor.CONTEXT.runInCoroutine(this) { + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution), batchSize).forEach { visitor(it) } } } } - -private fun Sequence.tailSequence(size: Int, tailSize: Int): Sequence { - if (size <= tailSize) return this - if (tailSize <= 0) return emptySequence() - return drop(size - tailSize) -} - -private fun Collection.tailSequence(tailSize: Int): Sequence { - return asSequence().tailSequence(size, tailSize) -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt deleted file mode 100644 index 96d58cd2a4..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/CollectionAsStream.kt +++ /dev/null @@ -1,203 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -open class CollectionAsStream(val collection: Collection) : IStream.Many, IStreamInternal { - protected open fun convertLater() = DeferredStreamBuilder.ConvertibleMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.Many { - return converter.many(collection) - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return CollectionAsStream(collection.filter(predicate)) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return CollectionAsStream(collection.map(mapper)) - } - - override fun mapNotNull(mapper: (E) -> R?): IStream.Many { - return CollectionAsStream(collection.mapNotNull(mapper)) - } - - override fun firstOrNull(): IStream.One { - return SingleValueStream(collection.firstOrNull()) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapOrdered(mapper) - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapUnordered(mapper) - } - - override fun flatMapIterable(mapper: (E) -> Iterable): IStream.Many { - return SequenceAsStream(collection.asSequence().flatMap { mapper(it) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return when (other) { - is SingleValueStream -> CollectionAsStream(collection + other.value) - is SequenceAsStream -> SequenceAsStream(collection.asSequence() + other.wrapped) - is EmptyStream -> this - is CollectionAsStream -> CollectionAsStream(collection + other.collection) - else -> convertLater().concat(other) - } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return when (other) { - is SingleValueStream -> CollectionAsStreamOneOrMany(collection + other.value) - is SequenceAsStreamOneOrMany -> SequenceAsStreamOneOrMany(collection.asSequence() + other.wrapped) - is CollectionAsStreamOneOrMany -> CollectionAsStreamOneOrMany(collection + other.collection) - else -> convertLater().concat(other) - } - } - - override fun distinct(): IStream.Many { - return CollectionAsStream(collection.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - if (collection.isNotEmpty()) throw StreamAssertionError("Not empty: $collection") - return EmptyCompletableStream() - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - if (collection.isEmpty()) throw NoSuchElementException("Empty stream") - return CollectionAsStreamOneOrMany(collection) - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(collection.fold(initial, operation)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(collection.associate { keySelector(it) to valueSelector(it) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - val (a, b) = collection.partition(predicate) - return merger(CollectionAsStream(a), CollectionAsStream(b)) - } - - override fun skip(count: Long): IStream.Many { - return CollectionAsStream(collection.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return SingleValueStream(collection.single()) - } - - override fun count(): IStream.One { - return SingleValueStream(collection.size) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(if (collection.isEmpty()) defaultValue() else collection.first()) - } - - override fun firstOrDefault(defaultValue: E): IStream.One { - return SingleValueStream(if (collection.isEmpty()) defaultValue else collection.first()) - } - - override fun take(n: Int): IStream.Many { - return CollectionAsStream(collection.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return if (collection.isEmpty()) EmptyStream() else SingleValueStream(collection.first()) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return if (collection.isEmpty()) alternative() else this - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(collection.isEmpty()) - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return if (collection.isEmpty()) SingleValueStream(alternative()) else CollectionAsStreamOneOrMany(collection) - } - - override fun withIndex(): IStream.Many> { - return SequenceAsStream(collection.withIndex().asSequence()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return this - } - - override fun asFlow(): Flow { - return collection.asFlow() - } - - override fun asSequence(): Sequence { - return collection.asSequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream((collection as? List) ?: collection.toList()) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(collection.indexOf(element)) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - collection.forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - collection.forEach { visitor(it) } - } -} - -class CollectionAsStreamOneOrMany(list: Collection) : CollectionAsStream(list), IStream.OneOrMany { - protected override fun convertLater() = DeferredStreamBuilder.ConvertibleOneOrMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return converter.many(collection).assertNotEmpty { "Empty stream" } - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return CollectionAsStreamOneOrMany(collection.map(mapper)) - } - - override fun distinct(): IStream.OneOrMany { - return CollectionAsStreamOneOrMany(collection.distinct()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return this - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return convertLater().flatMapOne(mapper) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt b/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt deleted file mode 100644 index ea03ef37aa..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/CompletableObservable.kt +++ /dev/null @@ -1,68 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.single.SingleEmitter -import com.badoo.reaktive.single.single -import org.modelix.kotlin.utils.runSynchronized - -/** - * Similar to a CompletableFuture, observers can wait for a value that is provided in the future by some - * asynchronous process. - */ -class CompletableObservable(val afterSubscribe: () -> Unit = {}) { - val single = single { - runSynchronized(this) { - if (done) { - emitResult(it) - } else { - observers += it - } - } - afterSubscribe() - } - private var value: E? = null - private var throwable: Throwable? = null - private var done: Boolean = false - private var observers: List> = emptyList() - - private fun emitResult(observers: List>) { - for (observer in observers) { - emitResult(observer) - } - } - - private fun emitResult(emitter: SingleEmitter) { - if (throwable == null) { - emitter.onSuccess(value as E) - } else { - emitter.onError(throwable!!) - } - } - - private fun clearObservers(): List> { - return observers.also { observers = emptyList() } - } - - fun isDone() = runSynchronized(this) { done } - - fun complete(newValue: E) { - emitResult( - runSynchronized(this) { - check(!done) { "Already done" } - value = newValue - done = true - clearObservers() - }, - ) - } - - fun failed(ex: Throwable) { - emitResult( - runSynchronized(this) { - check(!done) { "Already done" } - throwable = ex - done = true - clearObservers() - }, - ) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt deleted file mode 100644 index fe6f1efbde..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/DeferredStreamBuilder.kt +++ /dev/null @@ -1,376 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.single -import org.modelix.kotlin.utils.DelicateModelixApi - -class DeferredStreamBuilder : IStreamBuilder { - override fun zero(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun empty(): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun of(element: T): IStream.One { - return SingleValueStream(element) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { it.deferZeroOrOne(supplier) } - } - - override fun many(elements: Collection): IStream.Many { - return CollectionAsStream(elements) - } - - override fun many(elements: Sequence): IStream.Many { - return SequenceAsStream(elements) - } - - override fun many(elements: Iterable): IStream.Many { - return SequenceAsStream(elements.asSequence()) - } - - override fun many(elements: Array): IStream.Many { - return CollectionAsStream(elements.asList()) - } - - override fun many(elements: LongArray): IStream.Many { - return CollectionAsStream(elements.asList()) - } - - override fun fromFlow(flow: Flow): IStream.Many { - return ConvertibleMany { it.fromFlow(flow) } - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - return ConvertibleMany { it.zip(input, mapper) } - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - if (input.all { it is SingleValueStream }) { - return SingleValueStream(mapper(input.map { (it as SingleValueStream).value })) - } - return ConvertibleOne { c -> - c.zip(input.map { it.convert(c) }, mapper) - } - } - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return ConvertibleOne { it.singleFromCoroutine(block) } - } - - class ConvertibleOne(conversion: (IStreamBuilder) -> IStream.One) : ConvertibleZeroOrOne(conversion), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - return super.convert(converter) as IStream.One - } - - @DelicateModelixApi - override fun getBlocking(): E = SequenceStreamBuilder.INSTANCE.convert(this).single() - - @DelicateModelixApi - override suspend fun getSuspending(): E = FlowStreamBuilder.INSTANCE.convert(this).single() - - override fun map(mapper: (E) -> R): IStream.One { - return ConvertibleOne { convert(it).map(mapper) } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return ConvertibleOne { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return ConvertibleOne { convert(it).doOnBeforeError(consumer) } - } - - override fun distinct(): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).distinct() } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return ConvertibleOne { c -> convert(c).flatMapOne { mapper(it).convert(c) } } - } - - override fun cached(): IStream.One { - return ConvertibleOne { convert(it).cached() } - } - } - - open class ConvertibleZeroOrOne(conversion: (IStreamBuilder) -> IStream.ZeroOrOne) : ConvertibleMany(conversion), IStream.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - return super.convert(converter) as IStream.ZeroOrOne - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).filter(predicate) } - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).map(mapper) } - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return ConvertibleOne { convert(it).assertNotEmpty(message) } - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return ConvertibleOne { convert(it).ifEmpty_(defaultValue) } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).doOnBeforeError(consumer) } - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return ConvertibleOne { convert(it).exceptionIfEmpty(exception) } - } - - override fun orNull(): IStream.One { - return ConvertibleOne { convert(it).orNull() } - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { converter -> - convert(converter).flatMapZeroOrOne { mapper(it).convert(converter) } - } - } - } - - open class ConvertibleMany(conversion: (IStreamBuilder) -> IStream.Many) : ConvertibleBase(conversion), IStream.Many { - override fun convert(converter: IStreamBuilder): IStream.Many { - return super.convert(converter) as IStream.Many - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return ConvertibleMany { convert(it).filter(predicate) } - } - - override fun map(mapper: (E) -> R): IStream.Many { - return ConvertibleMany { convert(it).map(mapper) } - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return ConvertibleMany { converter -> - convert(converter).flatMapUnordered { mapper(it).convert(converter) } - } - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return ConvertibleMany { converter -> - convert(converter).flatMapOrdered { mapper(it).convert(converter) } - } - } - - override fun concat(other: IStream.Many): IStream.Many { - return ConvertibleMany { convert(it).concat(other.convert(it)) } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).concat(other.convert(it)) } - } - - override fun distinct(): IStream.Many { - return ConvertibleMany { convert(it).distinct() } - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return ConvertibleCompletable { convert(it).assertEmpty(message) } - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).assertNotEmpty(message) } - } - - override fun drainAll(): IStream.Completable { - return ConvertibleCompletable { convert(it).drainAll() } - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return ConvertibleOne { convert(it).fold(initial, operation) } - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return ConvertibleOne { convert(it).toMap(keySelector, valueSelector) } - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return ConvertibleMany { c -> - convert(c).splitMerge(predicate) { a, b -> merger(a.convert(c), b.convert(c)) } - } - } - - override fun skip(count: Long): IStream.Many { - return ConvertibleMany { convert(it).skip(count) } - } - - override fun exactlyOne(): IStream.One { - return ConvertibleOne { convert(it).exactlyOne() } - } - - override fun count(): IStream.One { - return ConvertibleOne { convert(it).count() } - } - - override fun indexOf(element: E): IStream.One { - return ConvertibleOne { convert(it).indexOf(element) } - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return ConvertibleOne { convert(it).firstOrDefault(defaultValue) } - } - - override fun take(n: Int): IStream.Many { - return ConvertibleMany { convert(it).take(n) } - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).firstOrEmpty() } - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return ConvertibleMany { c -> convert(c).switchIfEmpty_ { alternative().convert(c) } } - } - - override fun isEmpty(): IStream.One { - return ConvertibleOne { convert(it).isEmpty() } - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).ifEmpty_(alternative) } - } - - override fun withIndex(): IStream.Many> { - return ConvertibleMany { convert(it).withIndex() } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return ConvertibleMany { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return ConvertibleMany { convert(it).doOnBeforeError(consumer) } - } - } - - class ConvertibleOneOrMany(conversion: (IStreamBuilder) -> IStream.OneOrMany) : ConvertibleMany(conversion), IStream.OneOrMany { - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return super.convert(converter) as IStream.OneOrMany - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).map(mapper) } - } - - override fun distinct(): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).distinct() } - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).onErrorReturn(valueSupplier) } - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).doOnBeforeError(consumer) } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return ConvertibleOneOrMany { c -> convert(c).flatMapOne { mapper(it).convert(c) } } - } - } - - class ConvertibleCompletable(conversion: (IStreamBuilder) -> IStream.Completable) : ConvertibleBase(conversion), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - return super.convert(converter) as IStream.Completable - } - - @DelicateModelixApi - override fun executeBlocking() = SequenceStreamBuilder.INSTANCE.convert(this).forEach { } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (Any?) -> Unit) { - executeBlocking() - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { - executeSuspending() - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - FlowStreamBuilder.INSTANCE.convert(this).collect {} - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return ConvertibleCompletable { convert(it).andThen(other.convert(it)) } - } - override fun plus(other: IStream.Many): IStream.Many { - return ConvertibleMany { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return ConvertibleZeroOrOne { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.One): IStream.One { - return ConvertibleOne { convert(it).plus(other.convert(it)) } - } - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return ConvertibleOneOrMany { convert(it).plus(other.convert(it)) } - } - } - - abstract class ConvertibleBase(private val conversion: (IStreamBuilder) -> IStream) : IStreamInternal { - private var converter: IStreamBuilder? = null - - /** - * Without caching of the converted stream [IStream.One.cached] doesn't work. - */ - private val converted: Pair, IStreamBuilder> by lazy { converter.let { conversion(it!!) to it } } - - override fun convert(converter: IStreamBuilder): IStream { - this.converter = converter - check(converted.second == converter) { "Converter changed ${converted.second} -> $converter" } - return converted.first - } - - override fun asFlow(): Flow { - val converted = convert(FlowStreamBuilder.INSTANCE) - return (converted as FlowStreamBuilder.WrapperBase).wrapped - } - - override fun asSequence(): Sequence { - val converted = convert(SequenceStreamBuilder.INSTANCE) - return (converted as SequenceStreamBuilder.WrapperBase).wrapped - } - - override fun toList(): IStream.One> = ConvertibleOne { convert(it).toList() } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - SequenceStreamBuilder.INSTANCE.convert(this).forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - FlowStreamBuilder.INSTANCE.convert(this).collect(visitor) - } - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt deleted file mode 100644 index 2149a2c46c..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyCompletableStream.kt +++ /dev/null @@ -1,56 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.emptyFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class EmptyCompletableStream() : IStreamInternal.Completable { - - override fun convert(converter: IStreamBuilder): IStream.Completable { - return converter.zero() - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return other - } - - override fun plus(other: IStream.Many): IStream.Many { - return other - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return other - } - - override fun plus(other: IStream.One): IStream.One { - return other - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return other - } - - override fun asFlow(): Flow { - return emptyFlow() - } - - override fun asSequence(): Sequence { - return emptySequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream(emptyList()) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (Any?) -> Unit) {} - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) {} - - @DelicateModelixApi - override fun executeBlocking() {} - - @DelicateModelixApi - override suspend fun executeSuspending() {} -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt deleted file mode 100644 index 39bd09e4d8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/EmptyStream.kt +++ /dev/null @@ -1,157 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.emptyFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class EmptyStream : IStreamInternal.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - return converter.empty() - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return this - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return SingleValueStream(defaultValue()) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - throw exception() - } - - override fun orNull(): IStream.One { - return SingleValueStream(null) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return EmptyStream() - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return this - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - throw StreamAssertionError(message()) - } - - override fun asFlow(): Flow { - return emptyFlow() - } - - override fun asSequence(): Sequence { - return asSequence() - } - - override fun toList(): IStream.One> { - return SingleValueStream(emptyList()) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) {} - - @DelicateModelixApi - override suspend fun getSuspending(): E? = null - - @DelicateModelixApi - override fun getBlocking(): E? = null - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) {} - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return EmptyStream() - } - - override fun concat(other: IStream.Many): IStream.Many { - return other - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return other - } - - override fun distinct(): IStream.Many { - return this - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return EmptyCompletableStream() - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(initial) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(emptyMap()) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return merger(this, this) - } - - override fun skip(count: Long): IStream.Many { - return this - } - - override fun exactlyOne(): IStream.One { - throw NoSuchElementException("Empty stream") - } - - override fun count(): IStream.One { - return SingleValueStream(0) - } - - override fun filterBySingle(condition: (E) -> IStream.One): IStream.Many { - return this - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(defaultValue()) - } - - override fun take(n: Int): IStream.Many { - return this - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return alternative() - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(true) - } - - override fun withIndex(): IStream.Many> { - return EmptyStream() - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(-1) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt deleted file mode 100644 index b63ed1bb77..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/FlowStreamBuilder.kt +++ /dev/null @@ -1,320 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.catch -import kotlinx.coroutines.flow.count -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.drop -import kotlinx.coroutines.flow.emitAll -import kotlinx.coroutines.flow.emptyFlow -import kotlinx.coroutines.flow.filter -import kotlinx.coroutines.flow.firstOrNull -import kotlinx.coroutines.flow.flatMapConcat -import kotlinx.coroutines.flow.flatMapMerge -import kotlinx.coroutines.flow.flow -import kotlinx.coroutines.flow.flowOf -import kotlinx.coroutines.flow.fold -import kotlinx.coroutines.flow.map -import kotlinx.coroutines.flow.onCompletion -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.onEmpty -import kotlinx.coroutines.flow.single -import kotlinx.coroutines.flow.take -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.flow.withIndex -import kotlinx.coroutines.flow.zip -import org.modelix.kotlin.utils.DelicateModelixApi - -class FlowStreamBuilder() : IStreamBuilder { - - companion object { - val INSTANCE = FlowStreamBuilder() - } - - fun convert(stream: IStream) = (stream.convert(this) as WrapperBase).wrapped - - override fun of(element: T): IStream.One = Wrapper(flowOf(element)) - override fun many(elements: Sequence): IStream.Many = Wrapper(elements.asFlow()) - override fun of(vararg elements: T): IStream.Many = Wrapper(elements.asFlow()) - override fun empty(): IStream.ZeroOrOne = Wrapper(emptyFlow()) - override fun fromFlow(flow: Flow): IStream.Many = Wrapper(flow) - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return Wrapper( - flow { - coroutineScope { - emit(block()) - } - }, - ) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(flow> { emit(supplier()) }).flatten() - } - - override fun zero(): IStream.Completable { - return Completable(emptyFlow()) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return Wrapper( - flow { - emit(mapper(input.map { convert(it).single() })) - }, - ) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return Wrapper( - convert(source1).zip(convert(source2)) { a, b -> - mapper(a, b) - }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - TODO("Not yet implemented") - } - - abstract inner class WrapperBase(val wrapped: Flow) : IStreamInternal { - override fun asFlow(): Flow = wrapped - override fun toList(): IStream.One> = Wrapper(flow { emit(wrapped.toList()) }) - override fun asSequence(): Sequence = throw UnsupportedOperationException() - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use suspendable queries") - } - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.collect(visitor) - } - } - - inner class Completable(wrapped: Flow) : WrapperBase(wrapped), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@FlowStreamBuilder) - return this - } - - override fun executeBlocking() { - throw UnsupportedOperationException("Use suspendable queries") - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - wrapped.collect { } - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return Completable( - flow { - wrapped.collect { } - other.asFlow().collect { } - }, - ) - } - - override fun plus(other: IStream.Many): IStream.Many { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.One): IStream.One { - return plusFlow(convert(other)) - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return plusFlow(convert(other)) - } - - fun plusFlow(other: Flow): Wrapper { - return Wrapper( - flow { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeSuspending() - emitAll(other) - }, - ) - } - } - - inner class Wrapper(wrapped: Flow) : WrapperBase(wrapped), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@FlowStreamBuilder) - return this - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return Wrapper(wrapped.map { mapper(it) }) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return Wrapper(wrapped.filter { predicate(it) }) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.onEmpty { emit(defaultValue()) }) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return Wrapper(wrapped.onEmpty { throw exception() }) - } - - override fun orNull(): IStream.One { - return Wrapper(wrapped.onEmpty { emit(null) }) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMapMerge { convert(mapper(it)) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMapConcat { convert(mapper(it)) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return Wrapper(wrapped + convert(other)) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return Wrapper(wrapped + convert(other)) - } - - override fun getBlocking(): E { - throw UnsupportedOperationException("Use suspendable queries") - } - - override suspend fun getSuspending(): E { - return wrapped.single() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return Wrapper( - flow { - emit(wrapped.fold(initial) { acc, value -> operation(acc, value) }) - }, - ) - } - - override fun distinct(): IStream.OneOrMany { - return Wrapper(wrapped.distinctUntilChanged()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return Completable(wrapped.onEach { throw StreamAssertionError(message(it)) }) - } - - override fun drainAll(): IStream.Completable { - return Completable(wrapped.filter { false }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return Wrapper( - flow { - val map = LinkedHashMap() - wrapped.collect { - map[keySelector(it)] = valueSelector(it) - } - emit(map) - }, - ) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - throw UnsupportedOperationException() - } - - override fun cached(): IStream.One { - throw UnsupportedOperationException() - } - - override fun skip(count: Long): IStream.Many { - return Wrapper(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return Wrapper(flow { emit(wrapped.single()) }) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return Wrapper(wrapped.onEmpty { throw NoSuchElementException(message()) }) - } - - override fun count(): IStream.One { - return Wrapper(flow { emit(wrapped.count()) }) - } - - override fun indexOf(element: E): IStream.One { - return Wrapper(flow { emit(wrapped.withIndex().firstOrNull { it == element }?.index ?: -1) }) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.onEmpty { emit(defaultValue()) }.take(1)) - } - - override fun take(n: Int): IStream.Many { - return Wrapper(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return Wrapper(wrapped.take(1)) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return Wrapper(wrapped.onEmpty { emitAll(convert(alternative())) }) - } - - override fun isEmpty(): IStream.One { - return Wrapper(wrapped.map { false }.onEmpty { emit(true) }.take(1)) - } - - override fun withIndex(): IStream.Many> { - return Wrapper(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return Wrapper(wrapped.catch { emit(valueSupplier(it)) }) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return Wrapper( - wrapped.catch { - consumer(it) - throw it - }, - ) - } - } -} - -private operator fun Flow.plus(other: Flow) = onCompletion { if (it == null) emitAll(other) } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt index 315f46cfe1..339b9eab93 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt @@ -108,7 +108,7 @@ interface IStream { override fun doOnBeforeError(consumer: (Throwable) -> Unit): One } - companion object : IStreamBuilder by DeferredStreamBuilder() { + companion object : IStreamBuilder by StreamBuilderImpl { } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt deleted file mode 100644 index e577f63fa8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/ReaktiveStreamBuilder.kt +++ /dev/null @@ -1,650 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.base.Source -import com.badoo.reaktive.completable.Completable -import com.badoo.reaktive.completable.andThen -import com.badoo.reaktive.completable.asMaybe -import com.badoo.reaktive.completable.asObservable -import com.badoo.reaktive.completable.completableOfEmpty -import com.badoo.reaktive.coroutinesinterop.asObservable -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.asCompletable -import com.badoo.reaktive.maybe.asObservable -import com.badoo.reaktive.maybe.asSingle -import com.badoo.reaktive.maybe.asSingleOrError -import com.badoo.reaktive.maybe.defaultIfEmpty -import com.badoo.reaktive.maybe.doOnBeforeError -import com.badoo.reaktive.maybe.filter -import com.badoo.reaktive.maybe.flatMap -import com.badoo.reaktive.maybe.flatMapObservable -import com.badoo.reaktive.maybe.map -import com.badoo.reaktive.maybe.maybeDefer -import com.badoo.reaktive.maybe.maybeOfEmpty -import com.badoo.reaktive.maybe.onErrorReturn -import com.badoo.reaktive.observable.Observable -import com.badoo.reaktive.observable.asCompletable -import com.badoo.reaktive.observable.autoConnect -import com.badoo.reaktive.observable.concatWith -import com.badoo.reaktive.observable.doOnBeforeError -import com.badoo.reaktive.observable.filter -import com.badoo.reaktive.observable.firstOrComplete -import com.badoo.reaktive.observable.firstOrDefault -import com.badoo.reaktive.observable.flatMap -import com.badoo.reaktive.observable.flatMapSingle -import com.badoo.reaktive.observable.map -import com.badoo.reaktive.observable.observableOf -import com.badoo.reaktive.observable.onErrorReturn -import com.badoo.reaktive.observable.publish -import com.badoo.reaktive.observable.skip -import com.badoo.reaktive.observable.switchIfEmpty -import com.badoo.reaktive.observable.take -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.observable.toMap -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.asCompletable -import com.badoo.reaktive.single.asMaybe -import com.badoo.reaktive.single.asObservable -import com.badoo.reaktive.single.doOnBeforeError -import com.badoo.reaktive.single.filter -import com.badoo.reaktive.single.flatMap -import com.badoo.reaktive.single.flatMapMaybe -import com.badoo.reaktive.single.flatMapObservable -import com.badoo.reaktive.single.map -import com.badoo.reaktive.single.onErrorReturn -import com.badoo.reaktive.single.singleOf -import com.badoo.reaktive.single.subscribe -import com.badoo.reaktive.single.zipWith -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import org.modelix.kotlin.utils.DelicateModelixApi -import org.modelix.streams.IStream.OneOrMany - -class ReaktiveStreamBuilder() : IStreamBuilder { - - fun convert(stream: IStream.Completable) = (stream.convert(this) as Wrapper<*>).wrappedAsCompletable() - fun convert(stream: IStream.One) = (stream.convert(this) as Wrapper).wrappedAsSingle() - fun convert(stream: IStream.ZeroOrOne) = (stream.convert(this) as Wrapper).wrappedAsMaybe() - fun convert(stream: IStream.Many) = (stream.convert(this) as Wrapper).wrappedAsObservable() - - override fun of(element: T): IStream.One = WrapperSingle(singleOf(element)) - override fun many(elements: Sequence): IStream.Many = WrapperMany(elements.asObservable()) - override fun of(vararg elements: T): IStream.Many = WrapperMany(elements.asObservable()) - override fun empty(): IStream.ZeroOrOne = WrapperMaybe(maybeOfEmpty()) - override fun fromFlow(flow: Flow): IStream.Many = WrapperMany(flow.asObservable()) - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return WrapperSingle(com.badoo.reaktive.coroutinesinterop.singleFromCoroutine(block)) - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe( - maybeDefer { - supplier().toReaktive() - }, - ) - } - - override fun zero(): IStream.Completable { - return WrapperCompletable(completableOfEmpty()) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return WrapperSingle( - com.badoo.reaktive.single.zip(*input.map { it.toReaktive() }.toTypedArray()) { - mapper(it) - }, - ) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return WrapperSingle(source1.toReaktive().zipWith(source2.toReaktive(), mapper)) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.Many { - return WrapperMany( - com.badoo.reaktive.observable.zip(*input.map { it.toReaktive() }.toTypedArray()) { - mapper(it) - }, - ) - } - - abstract inner class Wrapper { - abstract fun wrappedAsSingle(): Single - abstract fun wrappedAsMaybe(): Maybe - abstract fun wrappedAsObservable(): Observable - abstract fun wrappedAsCompletable(): Completable - } - - abstract inner class ReaktiveWrapper : Wrapper(), IStream.Many, IStreamInternal { - abstract val wrapped: Source<*> - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun skip(count: Long): IStream.Many { - require(count >= 0L) - return WrapperMany(wrappedAsObservable().skip(count)) - } - - override fun count(): IStream.One { - return WrapperSingle(wrappedAsObservable().count()) - } - - override fun take(n: Int): IStream.Many { - return WrapperMany(wrappedAsObservable().take(n)) - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrappedAsObservable().toList()) - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrappedAsObservable().isEmpty()) - } - - override fun exactlyOne(): IStream.One { - return WrapperSingle(wrappedAsSingle()) - } - - override fun concat(other: OneOrMany): OneOrMany { - return WrapperOneOrMany(wrappedAsObservable().concatWith(other.toReaktive())) - } - } - - inner class WrapperCompletable(val wrapped: Completable) : - Wrapper(), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsSingle(): Single = throw NoSuchElementException() - override fun wrappedAsMaybe(): Maybe = wrapped.asMaybe() - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsCompletable(): Completable = wrapped - - override fun executeBlocking() { - throw UnsupportedOperationException("Use IStreamExecutor.query") - } - - override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return WrapperCompletable(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: IStream.One): IStream.One { - return WrapperSingle(wrapped.andThen(other.toReaktive())) - } - - override fun plus(other: OneOrMany): OneOrMany { - return WrapperOneOrMany(wrapped.andThen(other.toReaktive())) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrapped.asObservable().toList()) - } - - override fun iterateBlocking(visitor: (Any?) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - throw UnsupportedOperationException("Use IStreamExecutor.iterateSuspending") - } - } - - open inner class WrapperMany(override val wrapped: Observable) : - ReaktiveWrapper(), IStream.Many { - override fun convert(converter: IStreamBuilder): IStream.Many { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped - override fun wrappedAsSingle(): Single = wrapped.exactlyOne() - override fun wrappedAsMaybe(): Maybe = wrapped.firstOrComplete() - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return WrapperMany(wrapped.filter(predicate)) - } - - override fun ifEmpty_(alternative: () -> E): OneOrMany { - return WrapperOneOrMany(wrapped.switchIfEmpty { observableOf(alternative()) }) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return WrapperMany(wrapped.map(mapper)) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun flatMapUnordered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMap(maxConcurrency = Int.MAX_VALUE) { mapper(it).toReaktive() }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMap(maxConcurrency = 1) { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.concatWith(other.toReaktive())) - } - - override fun distinct(): IStream.Many { - return WrapperMany(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return WrapperCompletable(wrapped.assertEmpty(message)) - } - - override fun assertNotEmpty(message: () -> String): OneOrMany { - return WrapperOneOrMany(wrapped.assertNotEmpty(message)) - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.fold(initial, operation)) - } - - override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> { - return WrapperSingle(wrapped.toMap(keySelector, valueSelector)) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - val sharedInput = wrappedAsObservable().publish().autoConnect(2) - val a = sharedInput.filter { predicate(it) == true } - val b = sharedInput.filter { predicate(it) == false } - return merger(WrapperMany(a), WrapperMany(b)) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.firstOrDefault(defaultValue)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return WrapperMaybe(wrappedAsMaybe()) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.switchIfEmpty { alternative().toReaktive() }) - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrapped.isEmpty()) - } - - override fun withIndex(): IStream.Many> { - return WrapperMany(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return WrapperMany(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return WrapperMany(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.withIndex().firstOrNull().map { it?.index ?: -1 }) - } - } - - inner class WrapperOneOrMany(wrapped: Observable) : - WrapperMany(wrapped), OneOrMany { - override fun convert(converter: IStreamBuilder): OneOrMany { - require(converter == this@ReaktiveStreamBuilder) - return this - } - - override fun wrappedAsObservable(): Observable = wrapped - override fun wrappedAsSingle(): Single = wrapped.exactlyOne() - override fun wrappedAsMaybe(): Maybe = wrapped.firstOrComplete() - - override fun map(mapper: (E) -> R): OneOrMany { - return WrapperOneOrMany(wrapped.map(mapper)) - } - - override fun distinct(): OneOrMany { - return WrapperOneOrMany(wrapped.distinct()) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): OneOrMany { - return WrapperOneOrMany(wrapped.flatMapSingle(maxConcurrency = 1) { mapper(it).toReaktive() }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): OneOrMany { - return WrapperOneOrMany(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): OneOrMany { - return WrapperOneOrMany(wrapped.doOnBeforeError(consumer)) - } - } - - inner class WrapperSingle(override val wrapped: Single) : ReaktiveWrapper(), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsSingle(): Single = wrapped - override fun wrappedAsMaybe(): Maybe = wrapped.asMaybe() - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - fun getAsync(onError: ((Throwable) -> Unit)?, onSuccess: ((E) -> Unit)?) { - wrappedAsSingle().subscribe(onError = onError, onSuccess = onSuccess) - } - - override fun cached(): IStream.One { - return WrapperSingle(wrappedAsSingle().cached()) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return WrapperSingle(wrapped.flatMap { mapper(it).toReaktive() }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return WrapperSingle(wrapped.map(mapper)) - } - - override fun getBlocking(): E { - throw UnsupportedOperationException("Use IStreamExecutor.query") - } - - override suspend fun getSuspending(): E { - throw UnsupportedOperationException("Use IStreamExecutor.querySuspending") - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asObservable().asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun toList(): IStream.One> { - return WrapperSingle(wrapped.map { listOf(it) }) - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.filter(predicate)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return this // cannot be empty - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return this // cannot be empty - } - - override fun orNull(): IStream.One { - return this // cannot be empty - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.flatMapMaybe { mapper(it).toReaktive() }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMapObservable { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().concatWith(other.toReaktive())) - } - - override fun distinct(): OneOrMany { - return this // single element is always distinct - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - throw StreamAssertionError("Single will never be empty: $wrapped") - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return this // cannot be empty - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.map { operation(initial, it) }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return WrapperSingle(wrapped.map { mapOf(keySelector(it) to valueSelector(it)) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - TODO("Not yet implemented") - } - - override fun exactlyOne(): IStream.One { - return this - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return this // there is always a first element - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this // there is always a first element - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return this // never empty - } - - override fun isEmpty(): IStream.One { - return WrapperSingle(wrapped.asObservable().isEmpty()) - } - - override fun withIndex(): IStream.Many> { - return WrapperSingle(wrapped.map { IndexedValue(0, it) }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return WrapperSingle(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return WrapperSingle(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.map { if (it == element) 0 else -1 }) - } - } - - inner class WrapperMaybe(override val wrapped: Maybe) : ReaktiveWrapper(), IStream.ZeroOrOne { - override fun convert(converter: IStreamBuilder): IStream.ZeroOrOne { - require(converter == this@ReaktiveStreamBuilder) - return this - } - override fun wrappedAsObservable(): Observable = wrapped.asObservable() - override fun wrappedAsSingle(): Single = wrapped.asSingleOrError() - override fun wrappedAsMaybe(): Maybe = wrapped - override fun wrappedAsCompletable(): Completable = wrapped.asCompletable() - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return WrapperSingle(wrapped.asSingleOrError(exception)) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.flatMap { mapper(it).toReaktive() }) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.filter(predicate)) - } - - override fun map(mapper: (E) -> R): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.map(mapper)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.asSingle(defaultValue)) - } - - override fun orNull(): IStream.One { - return WrapperSingle(wrapped.orNull()) - } - - override fun asFlow(): Flow { - throw UnsupportedOperationException() - // return wrapped.asObservable().asFlow() - } - - override fun asSequence(): Sequence { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun iterateBlocking(visitor: (E) -> Unit) { - throw UnsupportedOperationException("Use IStreamExecutor.iterate") - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.flatMapObservable { mapper(it).toReaktive() }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().concatWith(other.toReaktive())) - } - - override fun distinct(): IStream.Many { - return this // there is never more than one element - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return WrapperCompletable(wrapped.assertEmpty(message)) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return WrapperSingle( - wrapped.asSingleOrError { - throw StreamAssertionError("At least one element was expected. xxx " + message()) - }, - ) - } - - override fun drainAll(): IStream.Completable { - return WrapperCompletable(wrapped.asCompletable()) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return WrapperSingle(wrapped.asObservable().fold(initial, operation)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return WrapperSingle(wrapped.asObservable().toMap(keySelector, valueSelector)) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - TODO("Not yet implemented") - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return WrapperSingle(wrapped.asSingle(defaultValue)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return WrapperMany(wrapped.asObservable().switchIfEmpty { alternative().toReaktive() }) - } - - override fun withIndex(): IStream.Many> { - return WrapperMaybe(wrapped.map { IndexedValue(0, it) }) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.onErrorReturn(valueSupplier)) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.ZeroOrOne { - return WrapperMaybe(wrapped.doOnBeforeError(consumer)) - } - - override fun indexOf(element: E): IStream.One { - return WrapperSingle(wrapped.map { if (it == element) 0 else -1 }.defaultIfEmpty(-1)) - } - } - fun IStream.One.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.ZeroOrOne.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.Many.toReaktive() = this@ReaktiveStreamBuilder.convert(this) - fun IStream.Completable.toReaktive() = this@ReaktiveStreamBuilder.convert(this) -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt deleted file mode 100644 index 32359db3a3..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceAsStream.kt +++ /dev/null @@ -1,185 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -/** - * Runs operations eagerly, if they return a single value. - * If an operation returns many values, it remains a lazy sequence to avoid unnecessary memory consumption. - */ -open class SequenceAsStream(val wrapped: Sequence) : IStream.Many, IStreamInternal { - protected open fun convertLater() = DeferredStreamBuilder.ConvertibleMany { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.Many { - return converter.many(wrapped) - } - - override fun map(mapper: (E) -> R): IStream.Many { - return SequenceAsStream(wrapped.map(mapper)) - } - - override fun asFlow(): Flow { - return wrapped.asFlow() - } - - override fun asSequence(): Sequence { - return wrapped - } - - override fun toList(): IStream.One> { - return SingleValueStream(wrapped.toList()) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - wrapped.forEach(visitor) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.forEach { visitor(it) } - } - - override fun filter(predicate: (E) -> Boolean): IStream.Many { - return SequenceAsStream(wrapped.filter(predicate)) - } - - override fun ifEmpty_(alternative: () -> E): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.ifEmpty { sequenceOf(alternative()) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return convertLater().flatMapOrdered(mapper) - } - - override fun concat(other: IStream.Many): IStream.Many { - return convertLater().concat(other) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return convertLater().concat(other) - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(wrapped.fold(initial, operation)) - } - - override fun distinct(): IStream.Many { - return SequenceAsStream(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - wrapped.forEach { throw StreamAssertionError(message(it)) } - return EmptyCompletableStream() - } - - override fun drainAll(): IStream.Completable { - wrapped.forEach {} - return EmptyCompletableStream() - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(wrapped.associate { keySelector(it) to valueSelector(it) }) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - // XXX Sequence.partition reads all entries into two lists, which consumes more memory. - // An alternative would be to create two sequences using Sequence.filter, but then the input is iterated - // twice. It depends on the use case which one is preferred. - // Currently, there is only a single usage of this method in Modelix and that is for bulk queries, which - // means the size of the input is limited. - // Also, in cases where bulk queries are used the stream will be a Reaktive one and this sequence based - // implementation is never called. - val (a, b) = wrapped.partition(predicate) - return merger(SequenceAsStream(a.asSequence()), SequenceAsStream(b.asSequence())) - } - - override fun skip(count: Long): IStream.Many { - return SequenceAsStream(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return SingleValueStream(wrapped.single()) - } - - override fun assertNotEmpty(message: () -> String): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.ifEmpty { throw StreamAssertionError(message()) }) - } - - override fun count(): IStream.One { - return SingleValueStream(wrapped.count()) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(wrapped.indexOf(element)) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return SingleValueStream(wrapped.ifEmpty { sequenceOf(defaultValue()) }.first()) - } - - override fun take(n: Int): IStream.Many { - return SequenceAsStream(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return wrapped.map { SingleValueStream(it) }.take(1).ifEmpty { sequenceOf(EmptyStream()) }.single() - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return convertLater().switchIfEmpty_(alternative) - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(wrapped.map { false }.take(1).ifEmpty { sequenceOf(true) }.single()) - } - - override fun withIndex(): IStream.Many> { - return SequenceAsStream(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.Many { - return convertLater().onErrorReturn(valueSupplier) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.Many { - return convertLater().doOnBeforeError(consumer) - } -} - -class SequenceAsStreamOneOrMany(wrapped: Sequence) : SequenceAsStream(wrapped), IStream.OneOrMany { - override fun convertLater(): DeferredStreamBuilder.ConvertibleOneOrMany { - return DeferredStreamBuilder.ConvertibleOneOrMany { convert(it) } - } - - override fun convert(converter: IStreamBuilder): IStream.OneOrMany { - return super.convert(converter).assertNotEmpty { "Empty stream" } - } - - override fun map(mapper: (E) -> R): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.map(mapper)) - } - - override fun distinct(): IStream.OneOrMany { - return SequenceAsStreamOneOrMany(wrapped.distinct()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.OneOrMany { - return convertLater().onErrorReturn(valueSupplier) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.OneOrMany { - return convertLater().doOnBeforeError(consumer) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.OneOrMany { - return convertLater().flatMapOne(mapper) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt deleted file mode 100644 index a1118911e8..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SequenceStreamBuilder.kt +++ /dev/null @@ -1,323 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.asFlow -import org.modelix.kotlin.utils.DelicateModelixApi - -class SequenceStreamBuilder() : IStreamBuilder { - - companion object { - val INSTANCE = SequenceStreamBuilder() - } - - fun convert(stream: IStream) = (stream.convert(this) as WrapperBase).wrapped - - override fun of(element: T): IStream.One = Wrapper(sequenceOf(element)) - override fun many(elements: Sequence): IStream.Many = Wrapper(elements) - override fun of(vararg elements: T): IStream.Many = Wrapper(elements.asSequence()) - override fun empty(): IStream.ZeroOrOne = Wrapper(emptySequence()) - - override fun fromFlow(flow: Flow): IStream.Many { - throw UnsupportedOperationException("Use FlowStreamBuilder") - } - - override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One { - return throw UnsupportedOperationException("Use Reaktive based streams") - } - - override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(sequence> { yield(supplier()) }).flatten() - } - - override fun zero(): IStream.Completable { - return Completable(emptySequence()) - } - - override fun zip( - source1: IStream.One, - source2: IStream.One, - mapper: (T1, T2) -> R, - ): IStream.One { - return Wrapper( - convert(source1) - .zip(convert(source2)) - .map { mapper(it.first, it.second) }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): Wrapper { - val input = input.toList() - val sequences = input.map { convert(it) } - return Wrapper( - when (sequences.size) { - 0 -> emptySequence() - 1 -> sequences.single().map { mapper(listOf(it)) } - else -> sequences.map { it.map { listOf(it) } }.reduce { a, b -> a.zip(b) { a, b -> a + b } }.map(mapper) - }, - ) - } - - override fun zip( - input: List>, - mapper: (List) -> R, - ): IStream.One { - return zip(input.map { it.assertNotEmpty { "Empty" } } as List>, mapper) - } - - abstract inner class WrapperBase(val wrapped: Sequence) : IStreamInternal { - override fun asFlow(): Flow = wrapped.asFlow() - override fun toList(): IStream.One> = Wrapper(sequence { yield(wrapped.toList()) }) - override fun asSequence(): Sequence = wrapped - - override fun iterateBlocking(visitor: (E) -> Unit) { - wrapped.forEach(visitor) - } - - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - wrapped.forEach { visitor(it) } - } - } - - inner class Completable(wrapped: Sequence) : WrapperBase(wrapped), IStreamInternal.Completable { - override fun convert(converter: IStreamBuilder): IStream.Completable { - require(converter == this@SequenceStreamBuilder) - return this - } - - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - override fun executeBlocking() { - wrapped.forEach { } - } - - @DelicateModelixApi - override suspend fun executeSuspending() { - wrapped.forEach { } - } - - override fun andThen(other: IStream.Completable): IStream.Completable { - return Completable( - sequence { - executeBlocking() - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - (other as IStreamInternal.Completable).executeBlocking() - }, - ) - } - - override fun plus(other: IStream.Many): IStream.Many { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.One): IStream.One { - return plusSequence(convert(other)) - } - - override fun plus(other: IStream.OneOrMany): IStream.OneOrMany { - return plusSequence(convert(other)) - } - - fun plusSequence(other: WrapperBase): Wrapper { - return Wrapper( - sequence { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeBlocking() - yieldAll(other.asSequence()) - }, - ) - } - - fun plusSequence(other: Sequence): Wrapper { - return Wrapper( - sequence { - @OptIn(DelicateModelixApi::class) // usage inside IStreamExecutor is allowed - executeBlocking() - yieldAll(other) - }, - ) - } - } - - inner class Wrapper(wrapped: Sequence) : WrapperBase(wrapped), IStreamInternal.One { - override fun convert(converter: IStreamBuilder): IStream.One { - require(converter == this@SequenceStreamBuilder) - return this - } - fun getAsync(onError: ((Throwable) -> Unit)?, onSuccess: ((E) -> Unit)?) { - try { - for (element in wrapped) { - onSuccess?.invoke(element) - } - } catch (ex: Throwable) { - onError?.invoke(ex) - } - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun map(mapper: (E) -> R): IStream.One { - return Wrapper(wrapped.map(mapper)) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return Wrapper(wrapped.filter(predicate)) - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(defaultValue()) }) - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return Wrapper(wrapped.ifEmpty { throw exception() }) - } - - override fun orNull(): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(null) }) - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return Wrapper(wrapped.flatMap { convert(mapper(it)) }) - } - - override fun concat(other: IStream.Many): IStream.Many { - return Wrapper(wrapped + convert(other)) - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return Wrapper(wrapped + convert(other)) - } - - override fun getBlocking(): E { - return wrapped.single() - } - - override suspend fun getSuspending(): E { - return wrapped.single() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return Wrapper(sequenceOf(wrapped.fold(initial, operation))) - } - - override fun distinct(): IStream.OneOrMany { - return Wrapper(wrapped.distinct()) - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - return Completable(wrapped.onEach { throw StreamAssertionError(message(it)) }) - } - - override fun drainAll(): IStream.Completable { - return Completable(wrapped.filter { false }) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return Wrapper(sequenceOf(wrapped.associate { keySelector(it) to valueSelector(it) })) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - // XXX Sequence.partition reads all entries into two lists, which consumes more memory. - // An alternative would be to create two sequences using Sequence.filter, but then the input is iterated - // twice. It depends on the use case which one is preferred. - // Currently, there is only a single usage of this method in Modelix and that is for bulk queries, which - // means the size of the input is limited. - // Also, in cases where bulk queries are used the stream will be a Reaktive one and this sequence based - // implementation is never called. - val (a, b) = wrapped.partition(predicate) - return merger(Wrapper(a.asSequence()), Wrapper(b.asSequence())) - } - - override fun cached(): IStream.One { - val cached by lazy { wrapped.toList() } - return Wrapper(sequenceOf({ cached }).flatMap { it() }) - } - - override fun skip(count: Long): IStream.Many { - return Wrapper(wrapped.drop(count.toInt())) - } - - override fun exactlyOne(): IStream.One { - return Wrapper(sequence { yield(wrapped.single()) }) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return Wrapper(wrapped.ifEmpty { throw StreamAssertionError(message()) }) - } - - override fun count(): IStream.One { - return Wrapper(sequence { yield(wrapped.count()) }) - } - - override fun indexOf(element: E): IStream.One { - return Wrapper(sequence { yield(wrapped.indexOf(element)) }) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return Wrapper(wrapped.ifEmpty { sequenceOf(defaultValue()) }.take(1)) - } - - override fun take(n: Int): IStream.Many { - return Wrapper(wrapped.take(n)) - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return Wrapper(wrapped.take(1)) - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return Wrapper(wrapped.ifEmpty { convert(alternative()) }) - } - - override fun isEmpty(): IStream.One { - return Wrapper(wrapped.map { false }.ifEmpty { sequenceOf(true) }.take(1)) - } - - override fun withIndex(): IStream.Many> { - return Wrapper(wrapped.withIndex()) - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return Wrapper( - sequence { - try { - yieldAll(wrapped) - } catch (ex: Throwable) { - yield(valueSupplier(ex)) - } - }, - ) - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return Wrapper( - sequence { - try { - yieldAll(wrapped) - } catch (ex: Throwable) { - consumer(ex) - throw ex - } - }, - ) - } - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt index 34bd8a1916..38f9018a81 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt @@ -1,24 +1,35 @@ package org.modelix.streams -import kotlinx.coroutines.flow.single +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.drive +import org.modelix.streams.engine.driveSuspending +/** + * Default executor. Drives streams with no batch-size limit. Fetches embedded in a stream (via + * [BulkRequestStreamExecutor.enqueue]) are still batched per source per round, so this is safe to use even when a + * stream contains data requests; it simply doesn't impose a maximum batch size. + */ object SimpleStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { - return SequenceStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.drive(body().asStep(execution), Int.MAX_VALUE).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - return FlowStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.driveSuspending(body().asStep(execution), Int.MAX_VALUE).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - SequenceStreamBuilder.INSTANCE.convert(streamProvider()).forEach(visitor) + val execution = Execution() + execution.drive(streamProvider().asStep(execution), Int.MAX_VALUE).forEach(visitor) } override suspend fun iterateSuspending( streamProvider: suspend () -> IStream.Many, visitor: suspend (T) -> Unit, ) { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect(visitor) + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution), Int.MAX_VALUE).forEach { visitor(it) } } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt deleted file mode 100644 index 7503d75b2f..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SingleValueStream.kt +++ /dev/null @@ -1,200 +0,0 @@ -package org.modelix.streams - -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.flowOf -import org.modelix.kotlin.utils.DelicateModelixApi - -class SingleValueStream(val value: E) : IStreamInternal.One { - protected fun convertLater() = DeferredStreamBuilder.ConvertibleOne { convert(it) } - - override fun convert(converter: IStreamBuilder): IStream.One { - return converter.of(value) - } - - override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One { - return mapper(value) - } - - override fun map(mapper: (E) -> R): IStream.One { - return SingleValueStream(mapper(value)) - } - - @DelicateModelixApi - override fun getBlocking(): E { - return value - } - - @DelicateModelixApi - override suspend fun getSuspending(): E { - return value - } - - override fun cached(): IStream.One { - return this - } - - override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One { - return this - } - - override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One { - return this - } - - override fun asFlow(): Flow { - return flowOf(value) - } - - override fun asSequence(): Sequence { - return sequenceOf(value) - } - - override fun toList(): IStream.One> { - return SingleValueStream(listOf(value)) - } - - @DelicateModelixApi - override fun iterateBlocking(visitor: (E) -> Unit) { - visitor(value) - } - - @DelicateModelixApi - override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { - visitor(value) - } - - override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne { - return if (predicate(value)) this else EmptyStream() - } - - override fun ifEmpty_(defaultValue: () -> E): IStream.One { - return this - } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One { - return this - } - - override fun orNull(): IStream.One { - return this - } - - override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne { - return mapper(value) - } - - override fun assertNotEmpty(message: () -> String): IStream.One { - return this - } - - override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many { - return mapper(value) - } - - override fun flatMapIterable(mapper: (E) -> Iterable): IStream.Many { - return SequenceAsStream(mapper(value).asSequence()) - } - - override fun concat(other: IStream.Many): IStream.Many { - return when (other) { - is SingleValueStream -> CollectionAsStream(listOf(value, other.value)) - is SequenceAsStream -> SequenceAsStream(sequenceOf(value) + other.wrapped) - is EmptyStream -> this - is CollectionAsStream -> CollectionAsStream(listOf(value) + other.collection) - else -> convertLater().concat(other) - } - } - - override fun concat(other: IStream.OneOrMany): IStream.OneOrMany { - return when (other) { - is SingleValueStream -> CollectionAsStreamOneOrMany(listOf(value, other.value)) - is SequenceAsStreamOneOrMany -> SequenceAsStreamOneOrMany(sequenceOf(value) + other.wrapped) - is CollectionAsStreamOneOrMany -> CollectionAsStreamOneOrMany(listOf(value) + other.collection) - else -> convertLater().concat(other) - } - } - - override fun distinct(): IStream.OneOrMany { - return this - } - - override fun assertEmpty(message: (E) -> String): IStream.Completable { - throw StreamAssertionError(message(value)) - } - - override fun drainAll(): IStream.Completable { - return EmptyCompletableStream() - } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One { - return SingleValueStream(operation(initial, value)) - } - - override fun toMap( - keySelector: (E) -> K, - valueSelector: (E) -> V, - ): IStream.One> { - return SingleValueStream(mapOf(keySelector(value) to valueSelector(value))) - } - - override fun splitMerge( - predicate: (E) -> Boolean, - merger: (IStream.Many, IStream.Many) -> IStream.Many, - ): IStream.Many { - return if (predicate(value)) merger(this, EmptyStream()) else merger(EmptyStream(), this) - } - - override fun skip(count: Long): IStream.Many { - return if (count >= 1) EmptyStream() else this - } - - override fun exactlyOne(): IStream.One { - return this - } - - override fun count(): IStream.One { - return SingleValueStream(1) - } - - override fun firstOrDefault(defaultValue: () -> E): IStream.One { - return this - } - - override fun firstOrDefault(defaultValue: E): IStream.One { - return this - } - - override fun zipWith( - other: IStream.One, - mapper: (E, T) -> R, - ): IStream.One { - return when (other) { - is SingleValueStream -> SingleValueStream(mapper(value, other.value)) - else -> convertLater().zipWith(other, mapper) - } - } - - override fun take(n: Int): IStream.Many { - return if (n >= 1) this else EmptyStream() - } - - override fun firstOrEmpty(): IStream.ZeroOrOne { - return this - } - - override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many { - return this - } - - override fun isEmpty(): IStream.One { - return SingleValueStream(false) - } - - override fun withIndex(): IStream.Many> { - return SingleValueStream(IndexedValue(0, value)) - } - - override fun indexOf(element: E): IStream.One { - return SingleValueStream(if (element == value) 0 else -1) - } -} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt deleted file mode 100644 index 9c07e4a788..0000000000 --- a/streams/src/commonMain/kotlin/org/modelix/streams/StreamExtensions.kt +++ /dev/null @@ -1,93 +0,0 @@ -package org.modelix.streams - -import com.badoo.reaktive.base.tryCatch -import com.badoo.reaktive.completable.Completable -import com.badoo.reaktive.completable.CompletableCallbacks -import com.badoo.reaktive.disposable.Disposable -import com.badoo.reaktive.maybe.Maybe -import com.badoo.reaktive.maybe.asCompletable -import com.badoo.reaktive.maybe.defaultIfEmpty -import com.badoo.reaktive.maybe.map -import com.badoo.reaktive.observable.Observable -import com.badoo.reaktive.observable.ObservableObserver -import com.badoo.reaktive.observable.asCompletable -import com.badoo.reaktive.observable.asObservable -import com.badoo.reaktive.observable.autoConnect -import com.badoo.reaktive.observable.collect -import com.badoo.reaktive.observable.filter -import com.badoo.reaktive.observable.firstOrDefault -import com.badoo.reaktive.observable.firstOrError -import com.badoo.reaktive.observable.flatMapSingle -import com.badoo.reaktive.observable.map -import com.badoo.reaktive.observable.observable -import com.badoo.reaktive.observable.replay -import com.badoo.reaktive.observable.switchIfEmpty -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.single.Single -import com.badoo.reaktive.single.asObservable -import com.badoo.reaktive.single.flatMap -import com.badoo.reaktive.single.map - -class StreamAssertionError(message: String) : IllegalArgumentException(message) - -fun Observable<*>.count(): Single = collect({ arrayOf(0) }) { acc, it -> acc[0]++ }.map { it[0] } -fun Observable.fold(initial: R, operation: (R, T) -> R): Single { - return collect({ mutableListOf(initial) }) { acc, it -> acc[0] = operation(acc[0], it) }.map { it[0] } -} - -fun Sequence.asObservable(): Observable = asIterable().asObservable() -fun Observable.exactlyOne(): Single = toList().map { it.single() } -fun Observable.firstOrNull(): Single = firstOrDefault(null) -fun Observable<*>.isEmpty(): Single = map { false }.firstOrDefault(true) -fun Observable.filterBySingle(condition: (T) -> Single): Observable { - return this.flatMapSingle { element -> - condition(element).map { included -> - element to included - } - }.filter { it.second }.map { it.first } -} -fun Observable.withIndex(): Observable> { - var index = 0 - return map { IndexedValue(index++, it) } -} -fun Observable.assertNotEmpty(message: () -> String): Observable { - return this.switchIfEmpty { throw StreamAssertionError("At least one element was expected. xxx " + message()) } -} -fun Maybe.assertEmpty(message: (T) -> String): Completable { - return map { throw StreamAssertionError(message(it)) }.asCompletable() -} -fun Observable.assertEmpty(message: (T) -> String): Completable { - return map { throw StreamAssertionError(message(it)) }.asCompletable() -} -fun Single.cached(): Single { - return this.asObservable().replay(1).autoConnect() - .firstOrError { IllegalStateException("Single was empty. Should not happen.") } -} - -fun Maybe.orNull(): Single = defaultIfEmpty(null) -fun Array.asObservable(): Observable = asIterable().asObservable() -fun LongArray.asObservable(): Observable = asIterable().asObservable() - -fun Observable.distinct(): Observable { - return observable { emitter -> - val emittedValues = HashSet() - subscribe( - object : ObservableObserver, CompletableCallbacks by emitter { - override fun onSubscribe(disposable: Disposable) { - emitter.setDisposable(disposable) - } - - override fun onNext(value: T) { - emitter.tryCatch(block = { emittedValues.add(value) }) { - if (it) { - emitter.onNext(value) - } - } - } - }, - ) - } -} - -fun Maybe<*>.exists(): Single = map { true }.defaultIfEmpty(false) -fun Single>.flatten(): Single = flatMap { it } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt new file mode 100644 index 0000000000..ebb6f0fc81 --- /dev/null +++ b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt @@ -0,0 +1,306 @@ +package org.modelix.streams + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.toList +import org.modelix.kotlin.utils.DelicateModelixApi +import org.modelix.kotlin.utils.runBlockingIfJvm +import org.modelix.streams.engine.Done +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.Step +import org.modelix.streams.engine.asyncStep +import org.modelix.streams.engine.combineConcat +import org.modelix.streams.engine.doOnError +import org.modelix.streams.engine.drive +import org.modelix.streams.engine.driveSuspending +import org.modelix.streams.engine.flatMapStep +import org.modelix.streams.engine.mapValues +import org.modelix.streams.engine.recover +import org.modelix.streams.engine.zipN + +class StreamAssertionError(message: String) : IllegalArgumentException(message) + +/** Common bridge from a public [IStream] to its internal [Step] representation for the current run. */ +internal interface HasStep { + fun buildStep(execution: Execution): Step +} + +@Suppress("UNCHECKED_CAST") +internal fun IStream.asStep(execution: Execution): Step = (this as HasStep).buildStep(execution) + +/** + * The single backing implementation of every non-[Completable] [IStream] cardinality. The runtime instance always + * satisfies the strongest interface ([IStreamInternal.One]); static typing at the API boundary restricts which + * operators are reachable. Operators return new [StreamImpl] instances typed at their most-derived cardinality. + */ +internal class StreamImpl(val build: (Execution) -> Step) : IStreamInternal.One, HasStep { + + override fun buildStep(execution: Execution): Step = build(execution) + + override fun convert(converter: IStreamBuilder): IStream.One = this + + override fun asFlow(): Flow = flow { + val execution = Execution() + for (value in execution.driveSuspending(build(execution), Int.MAX_VALUE)) emit(value) + } + + override fun asSequence(): Sequence { + val execution = Execution() + return execution.drive(build(execution), Int.MAX_VALUE).asSequence() + } + + override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } + + override fun filter(predicate: (E) -> Boolean): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).mapValues { it.filter(predicate) } } + + override fun map(mapper: (E) -> R): IStream.One = + StreamImpl { execution -> build(execution).mapValues { it.map(mapper) } } + + override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { values -> combineConcat(values.map { mapper(it).asStep(execution) }) } } + + override fun concat(other: IStream.Many): IStream.Many = + StreamImpl { execution -> combineConcat(listOf(build(execution), other.asStep(execution))) } + + override fun concat(other: IStream.OneOrMany): IStream.OneOrMany = + StreamImpl { execution -> combineConcat(listOf(build(execution), other.asStep(execution))) } + + override fun distinct(): IStream.OneOrMany = + StreamImpl { execution -> build(execution).mapValues { it.distinct() } } + + override fun assertEmpty(message: (E) -> String): IStream.Completable = + CompletableImpl { execution -> + build(execution).mapValues { values -> + if (values.isNotEmpty()) throw StreamAssertionError(message(values.first())) + emptyList() + } + } + + override fun assertNotEmpty(message: () -> String): IStream.One = + StreamImpl { execution -> + build(execution).mapValues { values -> + if (values.isEmpty()) throw StreamAssertionError("At least one element was expected. " + message()) + values + } + } + + override fun drainAll(): IStream.Completable = + CompletableImpl { execution -> build(execution).mapValues { emptyList() } } + + override fun fold(initial: R, operation: (R, E) -> R): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.fold(initial, operation)) } } + + override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> = + StreamImpl { execution -> build(execution).mapValues { values -> listOf(values.associate { keySelector(it) to valueSelector(it) }) } } + + override fun splitMerge(predicate: (E) -> Boolean, merger: (IStream.Many, IStream.Many) -> IStream.Many): IStream.Many = + StreamImpl { execution -> + build(execution).flatMapStep { values -> + val (matching, notMatching) = values.partition(predicate) + merger(IStream.many(matching), IStream.many(notMatching)).asStep(execution) + } + } + + override fun skip(count: Long): IStream.Many = + StreamImpl { execution -> build(execution).mapValues { it.drop(count.toInt()) } } + + override fun exactlyOne(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.single()) } } + + override fun count(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.size) } } + + override fun firstOrDefault(defaultValue: () -> E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> listOf(if (values.isEmpty()) defaultValue() else values.first()) } } + + override fun take(n: Int): IStream.Many = + StreamImpl { execution -> build(execution).mapValues { it.take(n) } } + + override fun firstOrEmpty(): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).mapValues { it.take(1) } } + + override fun switchIfEmpty_(alternative: () -> IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { values -> if (values.isEmpty()) alternative().asStep(execution) else Done(values) } } + + override fun isEmpty(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.isEmpty()) } } + + override fun ifEmpty_(defaultValue: () -> E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) listOf(defaultValue()) else values } } + + override fun withIndex(): IStream.Many> = + StreamImpl { execution -> build(execution).mapValues { it.withIndex().toList() } } + + override fun onErrorReturn(valueSupplier: (Throwable) -> E): IStream.One = + StreamImpl { execution -> + try { + build(execution).recover { listOf(valueSupplier(it)) } + } catch (ex: Throwable) { + Done(listOf(valueSupplier(ex))) + } + } + + override fun doOnBeforeError(consumer: (Throwable) -> Unit): IStream.One = + StreamImpl { execution -> + try { + build(execution).doOnError(consumer) + } catch (ex: Throwable) { + consumer(ex) + throw ex + } + } + + override fun indexOf(element: E): IStream.One = + StreamImpl { execution -> build(execution).mapValues { listOf(it.indexOf(element)) } } + + override fun flatMapOne(mapper: (E) -> IStream.One): IStream.One = + StreamImpl { execution -> build(execution).flatMapStep { values -> mapper(values.single()).asStep(execution) } } + + override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).flatMapStep { values -> combineConcat(values.map { mapper(it).asStep(execution) }) } } + + override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) throw exception() else values } } + + override fun orNull(): IStream.One = + StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) listOf(null) else listOf(values.single()) } } + + override fun cached(): IStream.One = this + + @DelicateModelixApi + override fun iterateBlocking(visitor: (E) -> Unit) { + val execution = Execution() + execution.drive(build(execution), Int.MAX_VALUE).forEach(visitor) + } + + @DelicateModelixApi + override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { + val execution = Execution() + execution.driveSuspending(build(execution), Int.MAX_VALUE).forEach { visitor(it) } + } + + @Suppress("UNCHECKED_CAST") + @DelicateModelixApi + override fun getBlocking(): E { + val execution = Execution() + val values = execution.drive(build(execution), Int.MAX_VALUE) + return (if (values.isEmpty()) null else values.first()) as E + } + + @Suppress("UNCHECKED_CAST") + @DelicateModelixApi + override suspend fun getSuspending(): E { + val execution = Execution() + val values = execution.driveSuspending(build(execution), Int.MAX_VALUE) + return (if (values.isEmpty()) null else values.first()) as E + } +} + +/** Backing implementation of [IStream.Completable]. A completion carries no values; its step resolves to an empty list. */ +internal class CompletableImpl(val build: (Execution) -> Step) : IStreamInternal.Completable, HasStep { + + override fun buildStep(execution: Execution): Step = build(execution) + + override fun convert(converter: IStreamBuilder): IStream.Completable = this + + override fun asFlow(): Flow = flow { + val execution = Execution() + for (value in execution.driveSuspending(build(execution), Int.MAX_VALUE)) emit(value) + } + + override fun asSequence(): Sequence { + val execution = Execution() + return execution.drive(build(execution), Int.MAX_VALUE).asSequence() + } + + override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } + + override fun andThen(other: IStream.Completable): IStream.Completable = + CompletableImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.Many): IStream.Many = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.One): IStream.One = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + override fun plus(other: IStream.OneOrMany): IStream.OneOrMany = + StreamImpl { execution -> build(execution).flatMapStep { other.asStep(execution) } } + + @DelicateModelixApi + override fun iterateBlocking(visitor: (Any?) -> Unit) { + val execution = Execution() + execution.drive(build(execution), Int.MAX_VALUE).forEach(visitor) + } + + @DelicateModelixApi + override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { + val execution = Execution() + execution.driveSuspending(build(execution), Int.MAX_VALUE).forEach { visitor(it) } + } + + @DelicateModelixApi + override fun executeBlocking() { + val execution = Execution() + execution.drive(build(execution), Int.MAX_VALUE) + } + + @DelicateModelixApi + override suspend fun executeSuspending() { + val execution = Execution() + execution.driveSuspending(build(execution), Int.MAX_VALUE) + } +} + +/** The single [IStreamBuilder] backed by the [Step] engine. Replaces the Sequence/Flow/Reaktive/Deferred builders. */ +internal object StreamBuilderImpl : IStreamBuilder { + override fun zero(): IStream.Completable = CompletableImpl { Done(emptyList()) } + + override fun empty(): IStream.ZeroOrOne = StreamImpl { Done(emptyList()) } + + override fun of(element: T): IStream.One = StreamImpl { Done(listOf(element)) } + + override fun deferZeroOrOne(supplier: () -> IStream.ZeroOrOne): IStream.ZeroOrOne = + StreamImpl { execution -> supplier().asStep(execution) } + + override fun many(elements: Sequence): IStream.Many = StreamImpl { Done(elements.toList()) } + + @Suppress("UNCHECKED_CAST") + override fun fromFlow(flow: Flow): IStream.Many = StreamImpl { execution -> + asyncStep( + execution, + Any(), + produceBlocking = { runBlockingIfJvm { flow.toList() } }, + produceSuspending = { flow.toList() }, + ) as Step + } + + override fun zip(input: List>, mapper: (List) -> R): IStream.Many = + StreamImpl { execution -> + zipN(input.map { it.asStep(execution) }) { valueLists -> + val count = if (valueLists.isEmpty()) 0 else valueLists.minOf { it.size } + (0 until count).map { i -> mapper(valueLists.map { it[i] }) } + } + } + + override fun zip(input: List>, mapper: (List) -> R): IStream.One = + StreamImpl { execution -> + zipN(input.map { it.asStep(execution) }) { valueLists -> listOf(mapper(valueLists.map { it.single() })) } + } + + @Suppress("UNCHECKED_CAST") + override fun singleFromCoroutine(block: suspend CoroutineScope.() -> T): IStream.One = StreamImpl { execution -> + asyncStep( + execution, + Any(), + produceBlocking = { runBlockingIfJvm { listOf(coroutineScope { block() }) } }, + produceSuspending = { listOf(coroutineScope { block() }) }, + ) as Step + } +} diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt new file mode 100644 index 0000000000..27f2c75c91 --- /dev/null +++ b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt @@ -0,0 +1,249 @@ +package org.modelix.streams.engine + +import kotlinx.coroutines.yield +import org.modelix.streams.IBulkExecutor + +/** + * The round-based interpreter that backs every [org.modelix.streams.IStream] instance. + * + * A [Step] produces zero or more values. Evaluation proceeds in *rounds*: a step is [Done], [Failed], or [Blocked] + * on a set of pending work ([Pending]) that must be resolved before it can continue. The combinators encode the + * applicative/monadic split that makes bulk-request batching automatic: + * - applicative composition ([combineConcat], [zipN], [zip2]) unions the pending work of independent branches into + * the *same* round — that is the batch; + * - monadic composition ([flatMapStep]) introduces a dependency, pushing the right-hand side into a *later* round. + * + * A subtree with no pending work resolves straight to [Done] without allocating anything — the synchronous fast path. + */ +internal sealed interface Step + +internal class Done(val values: List) : Step + +internal class Blocked(val pending: Pending, val resume: () -> Step) : Step + +internal class Failed(val cause: Throwable) : Step + +/** A leaf that produces its values via a (potentially suspending) computation rather than a bulk fetch. */ +internal class AsyncAction( + val token: Any, + val produceBlocking: () -> List, + val produceSuspending: suspend () -> List, +) + +/** The deduplicated set of work pending for a single round: bulk fetches grouped by source, plus async leaves. */ +internal class Pending private constructor( + val fetches: Map, Set>, + val asyncActions: List, +) { + fun union(other: Pending): Pending { + if (this === EMPTY) return other + if (other === EMPTY) return this + val fetches = HashMap, MutableSet>() + for ((source, keys) in this.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) + for ((source, keys) in other.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) + return Pending(fetches, this.asyncActions + other.asyncActions) + } + + companion object { + val EMPTY = Pending(emptyMap(), emptyList()) + + @Suppress("UNCHECKED_CAST") + fun fetch(source: IBulkExecutor<*, *>, key: Any?): Pending = + Pending(mapOf((source as IBulkExecutor) to setOf(key)), emptyList()) + + fun async(action: AsyncAction): Pending = Pending(emptyMap(), listOf(action)) + } +} + +internal fun Step.mapValues(f: (List) -> List): Step = when (this) { + is Done -> Done(f(values)) + is Blocked -> Blocked(pending) { resume().mapValues(f) } + is Failed -> this +} + +internal fun Step.flatMapStep(f: (List) -> Step): Step = when (this) { + is Done -> f(values) + is Blocked -> Blocked(pending) { resume().flatMapStep(f) } + is Failed -> this +} + +/** Recover from a failure (or an exception thrown while resuming) by producing replacement values. */ +internal fun Step.recover(handler: (Throwable) -> List): Step = when (this) { + is Done -> this + is Failed -> Done(handler(cause)) + is Blocked -> Blocked(pending) { + try { + resume().recover(handler) + } catch (ex: Throwable) { + Done(handler(ex)) + } + } +} + +/** Run [consumer] before a failure (or thrown exception) propagates. */ +internal fun Step.doOnError(consumer: (Throwable) -> Unit): Step = when (this) { + is Done -> this + is Failed -> { + consumer(cause) + this + } + is Blocked -> Blocked(pending) { + try { + resume().doOnError(consumer) + } catch (ex: Throwable) { + consumer(ex) + throw ex + } + } +} + +/** Applicative combination concatenating the values of independent steps in order, batching their pending work. */ +internal fun combineConcat(steps: List>): Step { + var pending = Pending.EMPTY + var allDone = true + for (step in steps) { + when (step) { + is Failed -> return step + is Blocked -> { + allDone = false + pending = pending.union(step.pending) + } + is Done -> {} + } + } + if (allDone) return Done(steps.flatMap { (it as Done).values }) + return Blocked(pending) { combineConcat(steps.map { if (it is Blocked) it.resume() else it }) } +} + +/** Applicative combination handing all resolved value lists to [f] together. */ +internal fun zipN(steps: List>, f: (List>) -> List): Step { + var pending = Pending.EMPTY + var allDone = true + for (step in steps) { + when (step) { + is Failed -> return step + is Blocked -> { + allDone = false + pending = pending.union(step.pending) + } + is Done -> {} + } + } + if (allDone) return Done(f(steps.map { (it as Done).values })) + return Blocked(pending) { zipN(steps.map { if (it is Blocked) it.resume() else it }, f) } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Per-run state and leaves +// --------------------------------------------------------------------------------------------------------------------- + +private val MISSING = Any() + +/** + * Per-query state shared by every [Step] built for a single execution. Holds the fetch cache so that each key is + * fetched at most once across the whole traversal (dedup within and across rounds), and the results of async leaves. + */ +internal class Execution { + private val fetchCaches = HashMap, HashMap>() + private val asyncResults = HashMap>() + + @Suppress("UNCHECKED_CAST") + private fun cacheFor(source: IBulkExecutor<*, *>) = + fetchCaches.getOrPut(source as IBulkExecutor) { HashMap() } + + fun isFetched(source: IBulkExecutor<*, *>, key: Any?): Boolean = cacheFor(source).containsKey(key) + + fun fetchedValue(source: IBulkExecutor<*, *>, key: Any?): Any? { + val value = cacheFor(source)[key] + return if (value === MISSING) null else value + } + + fun fillFetch(source: IBulkExecutor, keys: Set, results: Map) { + val cache = cacheFor(source) + for (key in keys) cache[key] = if (results.containsKey(key)) results[key] else MISSING + } + + fun hasAsync(token: Any): Boolean = asyncResults.containsKey(token) + fun asyncResult(token: Any): List = asyncResults.getValue(token) + fun fillAsync(token: Any, values: List) { asyncResults[token] = values } +} + +/** A fetch leaf with zero-or-one semantics: a `null`/absent value resolves to an empty step. */ +internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: Any?): Step { + if (execution.isFetched(source, key)) { + val value = execution.fetchedValue(source, key) + return if (value == null) Done(emptyList()) else Done(listOf(value)) + } + return Blocked(Pending.fetch(source, key)) { fetchStep(execution, source, key) } +} + +internal fun asyncStep( + execution: Execution, + token: Any, + produceBlocking: () -> List, + produceSuspending: suspend () -> List, +): Step { + if (execution.hasAsync(token)) return Done(execution.asyncResult(token)) + return Blocked(Pending.async(AsyncAction(token, produceBlocking, produceSuspending))) { + asyncStep(execution, token, produceBlocking, produceSuspending) + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Drivers +// --------------------------------------------------------------------------------------------------------------------- + +/** + * Drives a step to completion blocking. Each loop iteration is one round: one bulk call per source (chunked to + * [batchSize]) plus any async leaves, then resume. The loop is the trampoline that keeps fetch-dependent chains + * stack-safe regardless of depth. + */ +internal fun Execution.drive(initial: Step, batchSize: Int): List { + var step = initial + while (true) { + when (val current = step) { + is Done -> return current.values + is Failed -> throw current.cause + is Blocked -> { + for ((source, keys) in current.pending.fetches) { + for (chunk in keys.chunked(batchSize)) { + @Suppress("UNCHECKED_CAST") + val results = source.execute(chunk) as Map + fillFetch(source, chunk.toSet(), results) + } + } + for (action in current.pending.asyncActions) { + fillAsync(action.token, action.produceBlocking()) + } + step = current.resume() + } + } + } +} + +internal suspend fun Execution.driveSuspending(initial: Step, batchSize: Int): List { + var step = initial + while (true) { + when (val current = step) { + is Done -> return current.values + is Failed -> throw current.cause + is Blocked -> { + for ((source, keys) in current.pending.fetches) { + for (chunk in keys.chunked(batchSize)) { + @Suppress("UNCHECKED_CAST") + val results = source.executeSuspending(chunk) as Map + fillFetch(source, chunk.toSet(), results) + } + } + for (action in current.pending.asyncActions) { + fillAsync(action.token, action.produceSuspending()) + } + yield() + step = current.resume() + } + } + } +} + +private fun Set.chunked(size: Int): List> = + if (this.size <= size) listOf(this.toList()) else this.toList().chunked(size) diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt index b8ba19acfa..6538c7483a 100644 --- a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt +++ b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt @@ -1,18 +1,47 @@ package org.modelix.streams -import com.badoo.reaktive.observable.observableOf -import com.badoo.reaktive.observable.toList -import com.badoo.reaktive.single.blockingGet import kotlin.test.Test import kotlin.test.assertEquals class StreamExtensionsTests { + private val executor = SimpleStreamExecutor + @Test fun `distinct removes duplicates`() { assertEquals( listOf("g", "a", "d", "h", "z"), - observableOf("g", "g", "a", "d", "h", "z", "g", "h").distinct().toList().blockingGet(), + executor.query { IStream.many(listOf("g", "g", "a", "d", "h", "z", "g", "h")).distinct().toList() }, + ) + } + + @Test + fun `map and filter`() { + assertEquals( + listOf(20, 40), + executor.query { IStream.many(1..4).map { it * 10 }.filter { it % 20 == 0 }.toList() }, ) } + + @Test + fun `flatMap concatenates in order`() { + assertEquals( + listOf(1, -1, 2, -2, 3, -3), + executor.query { IStream.many(1..3).flatMapOrdered { IStream.many(listOf(it, -it)) }.toList() }, + ) + } + + @Test + fun `zip combines single values`() { + assertEquals( + "a1", + executor.query { IStream.of("a").zipWith(IStream.of(1)) { a, b -> "$a$b" } }, + ) + } + + @Test + fun `fold and count`() { + assertEquals(10, executor.query { IStream.many(1..4).fold(0) { acc, v -> acc + v } }) + assertEquals(4, executor.query { IStream.many(1..4).count() }) + } } diff --git a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt index 1fb843bd43..ca52ce54a0 100644 --- a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt +++ b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt @@ -1,29 +1,35 @@ package org.modelix.streams -import kotlinx.coroutines.flow.single -import org.modelix.kotlin.utils.runBlockingIfJvm +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.drive +import org.modelix.streams.engine.driveSuspending +/** + * JVM executor that always drives streams to completion blocking, resolving async leaves (e.g. flows) via + * [org.modelix.kotlin.utils.runBlockingIfJvm] inside the engine. With the unified engine this behaves like + * [SimpleStreamExecutor]; it is retained for API compatibility. + */ object BlockingStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { - return runBlockingIfJvm { - FlowStreamBuilder.INSTANCE.convert(body()).single() - } + val execution = Execution() + return execution.drive(body().asStep(execution), Int.MAX_VALUE).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { - return FlowStreamBuilder.INSTANCE.convert(body()).single() + val execution = Execution() + return execution.driveSuspending(body().asStep(execution), Int.MAX_VALUE).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { - runBlockingIfJvm { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect { visitor(it) } - } + val execution = Execution() + execution.drive(streamProvider().asStep(execution), Int.MAX_VALUE).forEach(visitor) } override suspend fun iterateSuspending( streamProvider: suspend () -> IStream.Many, visitor: suspend (T) -> Unit, ) { - FlowStreamBuilder.INSTANCE.convert(streamProvider()).collect(visitor) + val execution = Execution() + execution.driveSuspending(streamProvider().asStep(execution), Int.MAX_VALUE).forEach { visitor(it) } } } diff --git a/streams2/README.md b/streams2/README.md new file mode 100644 index 0000000000..539deea690 --- /dev/null +++ b/streams2/README.md @@ -0,0 +1,127 @@ +# streams2 + +A small, dependency-free streaming abstraction for **batched, lazy data loading**. It is a clean-room +reimplementation of the `streams` module's execution model, built around a single round-based interpreter +instead of three interchangeable reactive backends. + +This module is the **reference implementation / prototype** of the design. The same design has been ported into +the `streams` module to back its (richer, legacy) public API — see +[`streams-redesign.md`](../streams-redesign.md) at the repo root. + +## Why a custom implementation + +The one capability that justifies not using `Flow`/`RxJava`/coroutines directly is **automatic bulk-request +batching**: when a traversal needs many objects from a remote store, the independent requests should be coalesced +into a single round-trip rather than fetched one at a time. + +That requires the runtime to *see a whole set of independent data requests before forcing any of them*. This is an +**applicative** property, not a monadic one: + +- `zip(a, b)` and the elements of a `Many` are **independent** → all their fetches belong to the **same batch round**. +- `flatMap` is a **dependency** → the right-hand side's keys can't be known until the left resolves → **next round**. + +Pull-based streams (`Sequence`, `Flow`) are demand-driven and sequential: they force one element, block, then ask for +the next. They cannot expose the request frontier without spawning a coroutine per pending item (too much overhead). +A push-based library (Reaktive) can, which is why the old `streams` module carried it. streams2 gets the same +property **structurally**, from the shape of the computation, with no external dependency. + +Prior art for this idea: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. + +## Scope (intentional limitations) + +- **No incremental emission.** A stream is fully materialized when queried; `take`/`skip` truncate the materialized + result rather than stopping upstream work. +- **No coroutine/Flow integration.** Execution is synchronous and blocking. + +(The port into `streams` re-adds a suspending driver and async leaves for `fromFlow`/`singleFromCoroutine`, because +its legacy API requires them. The streams2 engine itself stays minimal.) + +## The core idea: `Step` + +Everything hinges on one internal type that encodes the applicative/monadic split mechanically (the ZIO Query +`Result` trick). See [`Step.kt`](src/commonMain/kotlin/org/modelix/streams2/Step.kt). + +```kotlin +sealed interface Step +class Done(val values: List) : Step // fully resolved +class Blocked(val requests: RequestMap, val resume: () -> Step) // needs a batch first +class Failed(val cause: Throwable) : Step +``` + +A `Step` produces 0+ values of `T`. Evaluation proceeds in **rounds**. The combinators are where batching falls out: + +```kotlin +// MONADIC: dependency. Right side's requests aren't known until left is Done → next round. +fun Step.flatMapStep(f: (List) -> Step): Step + +// APPLICATIVE: independence. Request sets UNION into the SAME round. This is the batch. +fun combineConcat(steps: List>): Step +fun zipN(steps: List>, f: (List>) -> List): Step +``` + +A subtree containing no `Blocked` resolves straight to `Done` — the **synchronous fast path** for locally-available +data, with no scheduler and no request set allocated. + +## Execution + +A `DataSource` is an [`IBulkExecutor`](src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt): + +```kotlin +interface IBulkExecutor { + fun execute(keys: List): Map +} +``` + +The driver ([`Execution.drive`](src/commonMain/kotlin/org/modelix/streams2/Execution.kt)) is a loop where each +iteration is one batch round: issue **one bulk call per source**, fill the per-run cache (dedup within *and* across +rounds), then resume. The loop is the **trampoline** that keeps fetch-dependent chains stack-safe regardless of depth. + +```kotlin +val executor = StreamExecutor() +val source: IBulkExecutor = ... + +// independent fetches -> ONE round +val pair = executor.query(IStream.fetch(source, 1).zipWith(IStream.fetch(source, 2)) { a, b -> "$a+$b" }) + +// dependent fetches -> SEPARATE rounds +val chained = executor.query( + IStream.fetch(source, 1).flatMapOne { v -> IStream.fetch(source, keyFrom(v)) }, +) +``` + +## Public API + +Cardinality is encoded in the type — see [`IStream.kt`](src/commonMain/kotlin/org/modelix/streams2/IStream.kt): + +| Type | Meaning | +|-----------------------|--------------| +| `IStream.Many` | 0 or more | +| `IStream.ZeroOrOne`| 0 or 1 | +| `IStream.One` | exactly 1 | + +Builders (`IStream.Companion`): `of`, `empty`, `many`, `fetch`, `zip`. +Executor ([`IStreamExecutor`](src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt)): `query`, `queryAll`, +`iterate`. + +Operators: `map`, `mapNotNull`, `filter`, `flatMap`, `concat`, `distinct`, `take`, `skip`, `withIndex`, `fold`, +`toList`, `toMap`, `count`, `isEmpty`, `drainAll`, `assertEmpty`, `firstOrEmpty`, `firstOrDefault`, `exactlyOne`, +`orNull`, `ifEmpty`, `exceptionIfEmpty`, `flatMapZeroOrOne`, `flatMapOne`, `zipWith`. + +## Layout + +``` +src/commonMain/kotlin/org/modelix/streams2/ +├── Step.kt // Step IR + applicative/monadic combinators +├── Execution.kt // per-run fetch cache, RequestMap, the round driver +├── IBulkExecutor.kt // batchable data source +├── IStream.kt // public cardinality types + StreamBase impl + builders +└── IStreamExecutor.kt // materializes streams +``` + +## Known limitations / future work + +1. **Within-round stack safety.** The round driver trampolines across `Blocked` (the common fetch-dependent case). + A pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode + `Step` itself as a stack-safe free monad (an explicit interpreter loop) if that ever bites. +2. **Error-recovery operators** (`onErrorReturn`-style) aren't exposed yet; the `Failed` channel exists in the IR. +3. **`take`/`skip` don't prune upstream fetches** (a consequence of no incremental emission). diff --git a/streams2/build.gradle.kts b/streams2/build.gradle.kts new file mode 100644 index 0000000000..e0cbe3c06f --- /dev/null +++ b/streams2/build.gradle.kts @@ -0,0 +1,18 @@ +plugins { + `maven-publish` + `modelix-kotlin-multiplatform` +} + +kotlin { + sourceSets { + commonMain { + dependencies { + } + } + commonTest { + dependencies { + implementation(kotlin("test")) + } + } + } +} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt new file mode 100644 index 0000000000..f4a98ce64b --- /dev/null +++ b/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt @@ -0,0 +1,87 @@ +package org.modelix.streams2 + +/** The deduped set of pending fetches for a single round, grouped by data source. */ +internal class RequestMap private constructor( + val byExecutor: Map, Set>, +) { + val isEmpty: Boolean get() = byExecutor.isEmpty() + + fun union(other: RequestMap): RequestMap { + if (byExecutor.isEmpty()) return other + if (other.byExecutor.isEmpty()) return this + val result = HashMap, MutableSet>() + for ((executor, keys) in byExecutor) result.getOrPut(executor) { HashSet() }.addAll(keys) + for ((executor, keys) in other.byExecutor) result.getOrPut(executor) { HashSet() }.addAll(keys) + return RequestMap(result) + } + + companion object { + val EMPTY = RequestMap(emptyMap()) + + @Suppress("UNCHECKED_CAST") + fun single(executor: IBulkExecutor<*, *>, key: Any?): RequestMap = + RequestMap(mapOf((executor as IBulkExecutor) to setOf(key))) + } +} + +/** Sentinel stored in the cache for keys that were requested but absent from the bulk response. */ +private val MISSING = Any() + +/** + * Per-run state shared by every [Step] built for a single query. Holds the fetched-value cache so that the same key + * is fetched at most once across the whole traversal (dedup within and across rounds), regardless of how many stream + * branches request it. + */ +internal class Execution { + private val caches = HashMap, HashMap>() + + @Suppress("UNCHECKED_CAST") + private fun cacheFor(executor: IBulkExecutor<*, *>): HashMap = + caches.getOrPut(executor as IBulkExecutor) { HashMap() } + + fun isCached(executor: IBulkExecutor<*, *>, key: Any?): Boolean = cacheFor(executor).containsKey(key) + + fun cachedValue(executor: IBulkExecutor<*, *>, key: Any?): Any? { + val value = cacheFor(executor)[key] + if (value === MISSING) throw NoSuchElementException("No value returned for key: $key") + return value + } + + fun fill(executor: IBulkExecutor, requestedKeys: Set, results: Map) { + val cache = cacheFor(executor) + for (key in requestedKeys) { + cache[key] = if (results.containsKey(key)) results[key] else MISSING + } + } +} + +@Suppress("UNCHECKED_CAST") +internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: K): Step { + return if (execution.isCached(source, key)) { + Done(listOf(execution.cachedValue(source, key) as V)) + } else { + Blocked(RequestMap.single(source, key)) { fetchStep(execution, source, key) } + } +} + +/** + * Drives a [Step] to completion. Each iteration of the loop is one batch round: it issues a single bulk call per + * pending data source, fills the cache, then resumes. The loop is the trampoline that keeps fetch-dependent chains + * (the common case in tree traversal) stack-safe regardless of depth. + */ +internal fun Execution.drive(initial: Step): List { + var step = initial + while (true) { + when (val current = step) { + is Done -> return current.values + is Failed -> throw current.cause + is Blocked -> { + for ((source, keys) in current.requests.byExecutor) { + val results = source.execute(keys.toList()) as Map + fill(source, keys, results) + } + step = current.resume() + } + } + } +} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt new file mode 100644 index 0000000000..87520ad860 --- /dev/null +++ b/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt @@ -0,0 +1,12 @@ +package org.modelix.streams2 + +/** + * A batchable data source. The whole point of this stream implementation is to coalesce many individual + * [fetch][IStream.Companion.fetch] requests against the same source into a single bulk call. + * + * Implementations must return a value for every requested key. A key that is absent from the returned map is + * treated as a missing value and surfaces as a [NoSuchElementException] in the stream that requested it. + */ +interface IBulkExecutor { + fun execute(keys: List): Map +} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt new file mode 100644 index 0000000000..2daea00661 --- /dev/null +++ b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt @@ -0,0 +1,181 @@ +package org.modelix.streams2 + +/** + * A lazily-described stream of values, evaluated in batch rounds by an [IStreamExecutor]. + * + * This is a clean-room reimplementation of the streaming abstraction with a single execution model: + * - No incremental emission. A stream is fully materialized when queried; operators like [Many.take] truncate the + * materialized result rather than stopping upstream work. + * - No coroutine/Flow integration. Execution is synchronous and blocking. + * + * The one capability that justifies a custom implementation is **automatic bulk-request batching**: independent + * [fetch][Companion.fetch] requests reachable through applicative composition (the elements of a [Many], the sides of + * a [zip][Companion.zip] or [One.zipWith]) are coalesced into a single [IBulkExecutor.execute] call, while [flatMap] + * dependencies fall into later rounds. See [Step] for the mechanism. + * + * Cardinality is encoded in the type: [Many] (0+), [ZeroOrOne] (0..1) and [One] (exactly 1). + */ +sealed interface IStream { + + interface Many : IStream { + fun map(transform: (E) -> R): Many + fun mapNotNull(transform: (E) -> R?): Many + fun filter(predicate: (E) -> Boolean): Many + fun flatMap(transform: (E) -> Many): Many + fun concat(other: Many<@UnsafeVariance E>): Many + fun distinct(): Many + fun take(n: Int): Many + fun skip(n: Int): Many + fun withIndex(): Many> + + fun fold(initial: R, operation: (R, E) -> R): One + fun toList(): One> + fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): One> + fun count(): One + fun isEmpty(): One + fun drainAll(): One + fun assertEmpty(message: (E) -> String): One + + fun firstOrEmpty(): ZeroOrOne + fun firstOrDefault(defaultValue: () -> @UnsafeVariance E): One + fun exactlyOne(): One + } + + interface ZeroOrOne : Many { + override fun map(transform: (E) -> R): ZeroOrOne + fun flatMapZeroOrOne(transform: (E) -> ZeroOrOne): ZeroOrOne + fun orNull(): One + fun ifEmpty(defaultValue: () -> @UnsafeVariance E): One + fun exceptionIfEmpty(exception: () -> Throwable): One + } + + interface One : ZeroOrOne { + override fun map(transform: (E) -> R): One + fun flatMapOne(transform: (E) -> One): One + fun zipWith(other: One, combine: (E, T) -> R): One + } + + companion object { + fun of(value: T): One = stream { Done(listOf(value)) } + + fun empty(): ZeroOrOne = stream { Done(emptyList()) } + + fun many(values: Iterable): Many = stream { Done(values.toList()) } + + fun many(values: Sequence): Many = stream { Done(values.toList()) } + + /** A single value fetched from a batchable [source]. The unit of batching. */ + fun fetch(source: IBulkExecutor, key: K): One = + stream { execution -> fetchStep(execution, source, key) } + + /** Applicatively combine independent single-value streams; their fetches share a batch round. */ + fun zip(streams: List>, combine: (List) -> R): One = + stream { execution -> + zipN(streams.map { it.toStep(execution) }) { valueLists -> listOf(combine(valueLists.map { it.single() })) } + } + } +} + +/** Bridges a public [IStream] back to its internal [Step] representation for the current run. */ +internal fun IStream.toStep(execution: Execution): Step = (this as StreamBase).buildStep(execution) + +/** + * The single backing implementation of every [IStream] cardinality. The runtime instance always satisfies the + * strongest interface ([IStream.One]); static typing at the API boundary restricts which operators are reachable. + */ +internal abstract class StreamBase : IStream.One { + + internal abstract fun buildStep(execution: Execution): Step + + override fun map(transform: (E) -> R): IStream.One = + stream { execution -> buildStep(execution).mapValues { values -> values.map(transform) } } + + override fun mapNotNull(transform: (E) -> R?): IStream.Many = + stream { execution -> buildStep(execution).mapValues { values -> values.mapNotNull(transform) } } + + override fun filter(predicate: (E) -> Boolean): IStream.Many = + stream { execution -> buildStep(execution).mapValues { values -> values.filter(predicate) } } + + override fun flatMap(transform: (E) -> IStream.Many): IStream.Many = + stream { execution -> + buildStep(execution).flatMapStep { values -> combineConcat(values.map { transform(it).toStep(execution) }) } + } + + override fun concat(other: IStream.Many<@UnsafeVariance E>): IStream.Many = + stream { execution -> combineConcat(listOf(buildStep(execution), other.toStep(execution))) } + + override fun distinct(): IStream.Many = + stream { execution -> buildStep(execution).mapValues { it.distinct() } } + + override fun take(n: Int): IStream.Many = + stream { execution -> buildStep(execution).mapValues { it.take(n) } } + + override fun skip(n: Int): IStream.Many = + stream { execution -> buildStep(execution).mapValues { it.drop(n) } } + + override fun withIndex(): IStream.Many> = + stream { execution -> buildStep(execution).mapValues { it.withIndex().toList() } } + + override fun fold(initial: R, operation: (R, E) -> R): IStream.One = + stream { execution -> buildStep(execution).mapValues { listOf(it.fold(initial, operation)) } } + + override fun toList(): IStream.One> = + stream { execution -> buildStep(execution).mapValues { listOf(it) } } + + override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> = + stream { execution -> buildStep(execution).mapValues { values -> listOf(values.associate { keySelector(it) to valueSelector(it) }) } } + + override fun count(): IStream.One = + stream { execution -> buildStep(execution).mapValues { listOf(it.size) } } + + override fun isEmpty(): IStream.One = + stream { execution -> buildStep(execution).mapValues { listOf(it.isEmpty()) } } + + override fun drainAll(): IStream.One = + stream { execution -> buildStep(execution).mapValues { listOf(Unit) } } + + override fun assertEmpty(message: (E) -> String): IStream.One = + stream { execution -> + buildStep(execution).mapValues { values -> + if (values.isNotEmpty()) throw IllegalStateException(message(values.first())) + listOf(Unit) + } + } + + override fun firstOrEmpty(): IStream.ZeroOrOne = + stream { execution -> buildStep(execution).mapValues { it.take(1) } } + + override fun firstOrDefault(defaultValue: () -> @UnsafeVariance E): IStream.One = + stream { execution -> buildStep(execution).mapValues { values -> listOf(values.firstOrNull() ?: defaultValue()) } } + + override fun exactlyOne(): IStream.One = + stream { execution -> buildStep(execution).mapValues { listOf(it.single()) } } + + override fun flatMapZeroOrOne(transform: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne = + stream { execution -> + buildStep(execution).flatMapStep { values -> combineConcat(values.map { transform(it).toStep(execution) }) } + } + + override fun orNull(): IStream.One = + stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) listOf(null) else listOf(values.single()) } } + + override fun ifEmpty(defaultValue: () -> @UnsafeVariance E): IStream.One = + stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) listOf(defaultValue()) else values } } + + override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One = + stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) throw exception() else values } } + + override fun flatMapOne(transform: (E) -> IStream.One): IStream.One = + stream { execution -> buildStep(execution).flatMapStep { values -> transform(values.single()).toStep(execution) } } + + override fun zipWith(other: IStream.One, combine: (E, T) -> R): IStream.One = + stream { execution -> + zip2(buildStep(execution), other.toStep(execution)) { a, b -> listOf(combine(a.single(), b.single())) } + } +} + +private class StreamNode(private val builder: (Execution) -> Step) : StreamBase() { + override fun buildStep(execution: Execution): Step = builder(execution) +} + +internal fun stream(builder: (Execution) -> Step): StreamBase = StreamNode(builder) diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt new file mode 100644 index 0000000000..2754a84210 --- /dev/null +++ b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt @@ -0,0 +1,24 @@ +package org.modelix.streams2 + +/** + * Materializes streams. Each query runs an independent round loop with its own fetch cache, issuing one bulk call + * per data source per round (see [Execution.drive]). + */ +interface IStreamExecutor { + fun query(stream: IStream.One): T + fun queryAll(stream: IStream.Many): List + fun iterate(stream: IStream.Many, visitor: (T) -> Unit) +} + +class StreamExecutor : IStreamExecutor { + override fun queryAll(stream: IStream.Many): List { + val execution = Execution() + return execution.drive(stream.toStep(execution)) + } + + override fun query(stream: IStream.One): T = queryAll(stream).single() + + override fun iterate(stream: IStream.Many, visitor: (T) -> Unit) { + for (value in queryAll(stream)) visitor(value) + } +} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt new file mode 100644 index 0000000000..a36ea82f61 --- /dev/null +++ b/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt @@ -0,0 +1,92 @@ +package org.modelix.streams2 + +/** + * The internal intermediate representation of a partially-evaluated stream. + * + * A [Step] produces zero or more values of [T]. Evaluation proceeds in *rounds*: a [Step] is either fully resolved + * ([Done]), needs a batch of data fetches before it can continue ([Blocked]), or has failed ([Failed]). + * + * The combinators on [Step] encode the applicative/monadic split that makes batching automatic: + * - Applicative composition ([zipN], [combineConcat]) unions the pending requests of independent branches into the + * *same* round. That is the batch. + * - Monadic composition ([flatMapStep]) introduces a dependency, so the right-hand side's requests can only be + * discovered after the left side resolves, i.e. in a *later* round. + * + * Nothing is ever forced eagerly: a subtree that contains no [Blocked] resolves straight to [Done] without ever + * allocating a request set, which is the synchronous fast path for locally-available data. + */ +internal sealed interface Step + +internal class Done(val values: List) : Step + +internal class Blocked(val requests: RequestMap, val resume: () -> Step) : Step + +internal class Failed(val cause: Throwable) : Step + +/** Transform the resolved value list, threading through the round boundaries. */ +internal fun Step.mapValues(f: (List) -> List): Step = when (this) { + is Done -> Done(f(values)) + is Blocked -> Blocked(requests) { resume().mapValues(f) } + is Failed -> this +} + +/** Monadic bind: the dependency that introduces a round boundary. */ +internal fun Step.flatMapStep(f: (List) -> Step): Step = when (this) { + is Done -> f(values) + is Blocked -> Blocked(requests) { resume().flatMapStep(f) } + is Failed -> this +} + +/** + * Applicative combination of independent steps that concatenates their values in order. + * + * Iterates over [steps] within a single round (no per-element native recursion); only recurses once per round via + * the [Blocked] thunk, so the recursion depth is bounded by the number of fetch rounds rather than the number of + * elements. + */ +internal fun combineConcat(steps: List>): Step { + var allDone = true + var requests = RequestMap.EMPTY + for (step in steps) { + when (step) { + is Failed -> return step + is Blocked -> { + allDone = false + requests = requests.union(step.requests) + } + is Done -> {} + } + } + if (allDone) return Done(steps.flatMap { (it as Done).values }) + return Blocked(requests) { combineConcat(steps.map { if (it is Blocked) it.resume() else it }) } +} + +/** Applicative combination that hands all resolved value lists to [f] together. */ +internal fun zipN(steps: List>, f: (List>) -> List): Step { + var allDone = true + var requests = RequestMap.EMPTY + for (step in steps) { + when (step) { + is Failed -> return step + is Blocked -> { + allDone = false + requests = requests.union(step.requests) + } + is Done -> {} + } + } + if (allDone) return Done(f(steps.map { (it as Done).values })) + return Blocked(requests) { zipN(steps.map { if (it is Blocked) it.resume() else it }, f) } +} + +internal fun zip2(a: Step, b: Step, f: (List, List) -> List): Step { + if (a is Failed) return a + if (b is Failed) return b + if (a is Done && b is Done) return Done(f(a.values, b.values)) + var requests = RequestMap.EMPTY + if (a is Blocked) requests = requests.union(a.requests) + if (b is Blocked) requests = requests.union(b.requests) + return Blocked(requests) { + zip2(if (a is Blocked) a.resume() else a, if (b is Blocked) b.resume() else b, f) + } +} diff --git a/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt b/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt new file mode 100644 index 0000000000..495f38d5b9 --- /dev/null +++ b/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt @@ -0,0 +1,122 @@ +package org.modelix.streams2 + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** Records every bulk call so tests can assert how many rounds happened and which keys were batched together. */ +private class RecordingSource(private val backing: Map) : IBulkExecutor { + val calls = mutableListOf>() + + override fun execute(keys: List): Map { + calls.add(keys.sorted()) + return keys.associateWith { backing.getValue(it) } + } + + val roundCount: Int get() = calls.size + val totalKeysFetched: Int get() = calls.sumOf { it.size } +} + +class Streams2Test { + private val executor = StreamExecutor() + private val data = (0..9).associateWith { "v$it" } + + @Test + fun constant_stream_does_not_touch_a_source() { + // The eager fast path: no Fetch nodes means the round loop returns immediately. + val result = executor.query(IStream.of(21).map { it * 2 }) + assertEquals(42, result) + } + + @Test + fun many_operators() { + val stream = IStream.many(1..5).map { it * 10 }.filter { it > 20 } + assertEquals(listOf(30, 40, 50), executor.queryAll(stream)) + assertEquals(120, executor.query(IStream.many(1..5).fold(0) { acc, v -> acc + v }.map { it * 8 })) + } + + @Test + fun single_fetch() { + val source = RecordingSource(data) + assertEquals("v3", executor.query(IStream.fetch(source, 3))) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(3)), source.calls) + } + + @Test + fun independent_fetches_are_batched_into_one_round() { + val source = RecordingSource(data) + val stream = IStream.many(listOf(1, 2, 3, 4)) + .flatMap { key -> IStream.fetch(source, key) } + assertEquals(listOf("v1", "v2", "v3", "v4"), executor.queryAll(stream)) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(1, 2, 3, 4)), source.calls) + } + + @Test + fun zip_batches_both_sides_together() { + val source = RecordingSource(data) + val combined = IStream.fetch(source, 5).zipWith(IStream.fetch(source, 6)) { a, b -> "$a+$b" } + assertEquals("v5+v6", executor.query(combined)) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(5, 6)), source.calls) + } + + @Test + fun dependent_fetches_use_separate_rounds() { + val source = RecordingSource(data) + // The second fetch's key is derived from the first result -> it cannot be known until round 1 resolves. + val stream = IStream.fetch(source, 1).flatMapOne { first -> + val nextKey = first.removePrefix("v").toInt() + 4 // "v1" -> 5 + IStream.fetch(source, nextKey) + } + assertEquals("v5", executor.query(stream)) + assertEquals(2, source.roundCount) + assertEquals(listOf(listOf(1), listOf(5)), source.calls) + } + + @Test + fun duplicate_keys_are_fetched_once() { + val source = RecordingSource(data) + val stream = IStream.many(listOf(2, 2, 2, 7, 7)) + .flatMap { key -> IStream.fetch(source, key) } + assertEquals(listOf("v2", "v2", "v2", "v7", "v7"), executor.queryAll(stream)) + assertEquals(1, source.roundCount) + assertEquals(2, source.totalKeysFetched) // only keys 2 and 7 actually fetched + } + + @Test + fun zero_or_one_handling() { + assertEquals("default", executor.query(IStream.empty().ifEmpty { "default" })) + assertEquals(null, executor.query(IStream.empty().orNull())) + assertEquals("x", executor.query(IStream.of("x").orNull())) + assertFailsWith { + executor.query(IStream.empty().exceptionIfEmpty { IllegalStateException("empty") }) + } + } + + @Test + fun missing_key_surfaces_as_error() { + val source = RecordingSource(data) + assertFailsWith { + executor.query(IStream.fetch(source, 999)) + } + } + + @Test + fun deep_dependent_chain_is_stack_safe() { + val source = RecordingSource((0..1000).associateWith { "v$it" }) + // 1000 sequential dependent fetches -> 1000 rounds, must not overflow the stack. + var stream: IStream.One = IStream.fetch(source, 0) + repeat(1000) { + stream = stream.flatMapOne { v -> + val next = v.removePrefix("v").toInt() + 1 + IStream.fetch(source, next) + } + } + assertEquals("v1000", executor.query(stream)) + assertEquals(1001, source.roundCount) + assertTrue(source.calls.all { it.size == 1 }) + } +} From 7fd34073717df951dec097ca6e42a454ce91e8eb Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 20:41:31 +0200 Subject: [PATCH 2/7] refactor(streams): merge streams2 prototype into streams The streams2 module duplicated both the IStream interface hierarchy and the Step engine. Since the engine was already ported into streams in a more complete form (Pending with async leaves, recover/doOnError, the suspending driver, batch chunking) and nothing depended on streams2, the separate module was pure redundancy. - Remove the streams2 module and its settings.gradle.kts registration. - Keep its batching/dedup/stack-safety tests, rewritten against the real BulkRequestStreamExecutor / enqueue API (BulkRequestBatchingTest). - Add streams/README.md with the design overview (previously in streams2/README). - Update streams-redesign.md to describe the prototype-then-merge history. Co-Authored-By: Claude Opus 4.8 --- settings.gradle.kts | 1 - streams-redesign.md | 22 ++- streams/README.md | 100 ++++++++++ .../streams/BulkRequestBatchingTest.kt | 114 +++++++++++ streams2/README.md | 127 ------------ streams2/build.gradle.kts | 18 -- .../kotlin/org/modelix/streams2/Execution.kt | 87 --------- .../org/modelix/streams2/IBulkExecutor.kt | 12 -- .../kotlin/org/modelix/streams2/IStream.kt | 181 ------------------ .../org/modelix/streams2/IStreamExecutor.kt | 24 --- .../kotlin/org/modelix/streams2/Step.kt | 92 --------- .../org/modelix/streams2/Streams2Test.kt | 122 ------------ 12 files changed, 227 insertions(+), 673 deletions(-) create mode 100644 streams/README.md create mode 100644 streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt delete mode 100644 streams2/README.md delete mode 100644 streams2/build.gradle.kts delete mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt delete mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt delete mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt delete mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt delete mode 100644 streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt delete mode 100644 streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt diff --git a/settings.gradle.kts b/settings.gradle.kts index bf4322aaab..4507b79b74 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -41,6 +41,5 @@ include("mps-multiplatform-lib") include("mps-repository-concepts") include("mps-sync-plugin3") include("streams") -include("streams2") include("ts-model-api") include("vue-model-api") diff --git a/streams-redesign.md b/streams-redesign.md index 1b1d994a98..74eab38abe 100644 --- a/streams-redesign.md +++ b/streams-redesign.md @@ -4,8 +4,9 @@ This document records the redesign of the `streams` module: replacing its three (Sequence / Flow / Reaktive) with a single round-based interpreter, while preserving the public API so that no downstream module required API-driven changes. -The new engine was first prototyped in the [`streams2`](streams2/README.md) module; this document covers porting that -design into `streams` and the migration of the wider codebase. +The engine was first prototyped in a separate `streams2` module and then merged into `streams` (the prototype's +parallel `IStream` API and engine were redundant once the design was proven). See [`streams/README.md`](streams/README.md) +for the resulting design overview; this document covers the redesign and the migration of the wider codebase. --- @@ -121,17 +122,20 @@ Deleted: `SequenceStreamBuilder`, `FlowStreamBuilder`, `ReaktiveStreamBuilder`, `CompletableObservable`, `StreamExtensions` (Reaktive helpers). The Reaktive dependency was removed from `streams/build.gradle.kts`. -### Why the engine lives in `streams`, not as a dependency on `streams2` +### Why a single module (the `streams2` prototype was merged in) -The engine was ported into `streams` rather than wiring `streams` → `streams2`, because: +The engine was prototyped in a separate `streams2` module, then merged into `streams` rather than kept as a separate +dependency, because: -- the engine types must be `internal` to `streams`; -- the real integration needs a **suspending driver** and **async leaves** that the minimal `streams2` prototype - deliberately omits; +- the engine types are `internal` to `streams`; +- the real integration needs a **suspending driver** and **async leaves** that the minimal prototype deliberately + omitted; - `streams`' `IBulkExecutor` already declares `executeSuspending`, which the engine uses directly; -- it avoids two competing `IBulkExecutor` / `IStream` types across modules. +- keeping both meant two competing `IBulkExecutor` / `IStream` hierarchies for no benefit — the prototype's API was a + strict subset of the one `streams` already exposes. -`streams2` remains as the standalone, dependency-free reference implementation. +The prototype's batching/dedup/stack-safety tests were kept, rewritten against the real `BulkRequestStreamExecutor` +API (`BulkRequestBatchingTest`). --- diff --git a/streams/README.md b/streams/README.md new file mode 100644 index 0000000000..801983bf3a --- /dev/null +++ b/streams/README.md @@ -0,0 +1,100 @@ +# streams + +A streaming abstraction for **batched, lazy data loading**, built around a single round-based interpreter. + +It provides the `IStream` API used throughout modelix for traversing large, content-addressed models where the data +is loaded on demand from a (possibly remote) store. Its defining capability is **automatic bulk-request batching**: +independent data requests reachable in a traversal are coalesced into a single round-trip instead of being fetched +one at a time. + +> History: this module previously had three interchangeable backends (Sequence / Flow / Reaktive). They were replaced +> by the round-based `Step` engine described below. See [`streams-redesign.md`](../streams-redesign.md) for the +> redesign and migration record. + +## Why a custom implementation + +Batching requires the runtime to *see a set of independent data requests before forcing any of them*. This is an +**applicative** property, not a monadic one: + +- `zip(a, b)` and the elements of a `Many` are **independent** → all their fetches belong to the **same batch round**. +- `flatMap*` is a **dependency** → the right-hand side's keys can't be known until the left resolves → **next round**. + +Pull-based streams (`Sequence`, `Flow`) are demand-driven and sequential: they force one element, block, then ask for +the next. They cannot expose the request frontier without spawning a coroutine per pending item (too much overhead). +A push-based library (Reaktive) can, which is why this module used to carry it. The `Step` engine gets the same +property **structurally**, from the shape of the computation, with no external dependency. + +Prior art for this idea: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. + +## The core idea: `Step` + +The engine ([`engine/StepEngine.kt`](src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)) describes a +stream lazily and interprets it in **rounds**: + +```kotlin +sealed interface Step +class Done(val values: List) // fully resolved +class Blocked(val pending: Pending, val resume: () -> Step) // needs a round of work first +class Failed(val cause: Throwable) +``` + +`Pending` is the deduplicated work for one round: bulk fetches grouped by data source, plus async leaves. The +combinators encode the applicative/monadic split: + +- `flatMapStep` (monadic) — defers the continuation into the next round. +- `combineConcat` / `zipN` / `zip2` (applicative) — **union** the pending work of independent branches into the same + round. That union *is* the batch. + +A subtree with no `Blocked` resolves straight to `Done`: the **synchronous fast path** for local data, with no +scheduler and nothing allocated. + +## Execution + +The driver (`Execution.drive` / `driveSuspending`) is a loop where each iteration is one batch round: + +1. issue **one bulk call per data source** (chunked to `batchSize`), and run any async leaves; +2. fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds); +3. `resume()` and repeat until `Done`. + +The loop is the **trampoline** that keeps fetch-dependent chains stack-safe regardless of depth. + +Batching is **structural**: a fetch leaf carries its own data source, so the driver groups fetches per source per +round regardless of which executor runs it. `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to +its `IBulkExecutor`. + +## Public API + +Cardinality is encoded in the type: `IStream.Many` (0+), `IStream.ZeroOrOne` (0..1), `IStream.OneOrMany` (1+), +`IStream.One` (exactly 1), and `IStream.Completable` (completion, no value). + +- Builders: `IStream.of`, `empty`, `many`, `zip`, `fromFlow`, `singleFromCoroutine`, … +- Execution: `IStreamExecutor` (`query`, `iterate`, suspending variants), `SimpleStreamExecutor`, + `BulkRequestStreamExecutor`, `IExecutableStream`. +- Batchable source: `IBulkExecutor` (`execute` + `executeSuspending`). + +## Layout + +``` +src/commonMain/kotlin/org/modelix/streams/ +├── engine/StepEngine.kt // Step IR, combinators, drivers, fetch + async leaves +├── StreamImpl.kt // backing impl for every cardinality + Completable + the builder +├── IStream.kt // public cardinality types + operators/extensions +├── IStreamBuilder.kt // builder interface (of/many/zip/fromFlow/…) +├── IStreamExecutor.kt // executor interface + extensions +├── IExecutableStream.kt // deferred/composable execution +├── IStreamInternal.kt // @DelicateModelixApi blocking/suspending accessors +├── BulkRequestStreamExecutor.kt // batching executor + IBulkExecutor +├── SimpleStreamExecutor.kt +└── Zip.kt // arity-2..9 zip overloads +``` + +## Known limitations / tradeoffs + +These follow from the engine resolving each query fully (no incremental emission): + +1. `iterate` / `iterateSuspending` fully materialize before visiting — higher peak memory for very large iterations. + The clean fix, if a hot path needs it, is per-round streaming in just the `iterate*` drivers. +2. `cached()` is currently a no-op; fetch-level dedup (the expensive part) is handled by the per-run cache. +3. `take` / `skip` operate on materialized results (don't prune upstream fetches). +4. Within-round stack safety covers fetch-dependent chains (the common case). A pathological deep *pure* `flatMap` + chain that never blocks would still recurse; the fix is to encode `Step` as a stack-safe free monad if needed. diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt new file mode 100644 index 0000000000..ae8842e61d --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/BulkRequestBatchingTest.kt @@ -0,0 +1,114 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** Records every bulk call so tests can assert how many rounds happened and which keys were batched together. */ +private class RecordingSource(private val backing: Map) : IBulkExecutor { + val calls = mutableListOf>() + + override fun execute(keys: List): Map { + calls.add(keys.sorted()) + return keys.filter { backing.containsKey(it) }.associateWith { backing.getValue(it) } + } + + override suspend fun executeSuspending(keys: List): Map = execute(keys) + + val roundCount: Int get() = calls.size + val totalKeysFetched: Int get() = calls.sumOf { it.size } +} + +/** + * Verifies the round-based engine's batching behaviour through the real [BulkRequestStreamExecutor] / [enqueue] path: + * independent fetches share a round, dependent fetches use separate rounds, duplicate keys are fetched once, and deep + * dependent chains stay stack-safe. + */ +class BulkRequestBatchingTest { + private val data = (0..1000).associateWith { "v$it" } + + @Test + fun single_fetch() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + assertEquals("v3", executor.query { executor.enqueue(3).exceptionIfEmpty() }) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(3)), source.calls) + } + + @Test + fun independent_fetches_are_batched_into_one_round() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val result = executor.query { + IStream.many(listOf(1, 2, 3, 4)).flatMapOrdered { executor.enqueue(it).orNull() }.toList() + } + assertEquals(listOf("v1", "v2", "v3", "v4"), result) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(1, 2, 3, 4)), source.calls) + } + + @Test + fun zip_batches_both_sides_together() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val combined = executor.query { + executor.enqueue(5).exceptionIfEmpty().zipWith(executor.enqueue(6).exceptionIfEmpty()) { a, b -> "$a+$b" } + } + assertEquals("v5+v6", combined) + assertEquals(1, source.roundCount) + assertEquals(listOf(listOf(5, 6)), source.calls) + } + + @Test + fun dependent_fetches_use_separate_rounds() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + // The second fetch's key is derived from the first result -> it cannot be known until round 1 resolves. + val result = executor.query { + executor.enqueue(1).exceptionIfEmpty().flatMapOne { first -> + executor.enqueue(first.removePrefix("v").toInt() + 4).exceptionIfEmpty() // "v1" -> 5 + } + } + assertEquals("v5", result) + assertEquals(2, source.roundCount) + assertEquals(listOf(listOf(1), listOf(5)), source.calls) + } + + @Test + fun duplicate_keys_are_fetched_once() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + val result = executor.query { + IStream.many(listOf(2, 2, 2, 7, 7)).flatMapOrdered { executor.enqueue(it).orNull() }.toList() + } + assertEquals(listOf("v2", "v2", "v2", "v7", "v7"), result) + assertEquals(1, source.roundCount) + assertEquals(2, source.totalKeysFetched) // only keys 2 and 7 actually fetched + } + + @Test + fun missing_key_resolves_to_empty() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + assertEquals(null, executor.query { executor.enqueue(9999).orNull() }) + assertFailsWith { + executor.query { executor.enqueue(9999).exceptionIfEmpty() } + } + } + + @Test + fun deep_dependent_chain_is_stack_safe() { + val source = RecordingSource(data) + val executor = BulkRequestStreamExecutor(source) + // 1000 sequential dependent fetches -> 1000 extra rounds, must not overflow the stack. + var stream: IStream.One = executor.enqueue(0).exceptionIfEmpty() + repeat(1000) { + stream = stream.flatMapOne { v -> executor.enqueue(v.removePrefix("v").toInt() + 1).exceptionIfEmpty() } + } + assertEquals("v1000", executor.query { stream }) + assertEquals(1001, source.roundCount) + assertTrue(source.calls.all { it.size == 1 }) + } +} diff --git a/streams2/README.md b/streams2/README.md deleted file mode 100644 index 539deea690..0000000000 --- a/streams2/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# streams2 - -A small, dependency-free streaming abstraction for **batched, lazy data loading**. It is a clean-room -reimplementation of the `streams` module's execution model, built around a single round-based interpreter -instead of three interchangeable reactive backends. - -This module is the **reference implementation / prototype** of the design. The same design has been ported into -the `streams` module to back its (richer, legacy) public API — see -[`streams-redesign.md`](../streams-redesign.md) at the repo root. - -## Why a custom implementation - -The one capability that justifies not using `Flow`/`RxJava`/coroutines directly is **automatic bulk-request -batching**: when a traversal needs many objects from a remote store, the independent requests should be coalesced -into a single round-trip rather than fetched one at a time. - -That requires the runtime to *see a whole set of independent data requests before forcing any of them*. This is an -**applicative** property, not a monadic one: - -- `zip(a, b)` and the elements of a `Many` are **independent** → all their fetches belong to the **same batch round**. -- `flatMap` is a **dependency** → the right-hand side's keys can't be known until the left resolves → **next round**. - -Pull-based streams (`Sequence`, `Flow`) are demand-driven and sequential: they force one element, block, then ask for -the next. They cannot expose the request frontier without spawning a coroutine per pending item (too much overhead). -A push-based library (Reaktive) can, which is why the old `streams` module carried it. streams2 gets the same -property **structurally**, from the shape of the computation, with no external dependency. - -Prior art for this idea: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch, Clump. - -## Scope (intentional limitations) - -- **No incremental emission.** A stream is fully materialized when queried; `take`/`skip` truncate the materialized - result rather than stopping upstream work. -- **No coroutine/Flow integration.** Execution is synchronous and blocking. - -(The port into `streams` re-adds a suspending driver and async leaves for `fromFlow`/`singleFromCoroutine`, because -its legacy API requires them. The streams2 engine itself stays minimal.) - -## The core idea: `Step` - -Everything hinges on one internal type that encodes the applicative/monadic split mechanically (the ZIO Query -`Result` trick). See [`Step.kt`](src/commonMain/kotlin/org/modelix/streams2/Step.kt). - -```kotlin -sealed interface Step -class Done(val values: List) : Step // fully resolved -class Blocked(val requests: RequestMap, val resume: () -> Step) // needs a batch first -class Failed(val cause: Throwable) : Step -``` - -A `Step` produces 0+ values of `T`. Evaluation proceeds in **rounds**. The combinators are where batching falls out: - -```kotlin -// MONADIC: dependency. Right side's requests aren't known until left is Done → next round. -fun Step.flatMapStep(f: (List) -> Step): Step - -// APPLICATIVE: independence. Request sets UNION into the SAME round. This is the batch. -fun combineConcat(steps: List>): Step -fun zipN(steps: List>, f: (List>) -> List): Step -``` - -A subtree containing no `Blocked` resolves straight to `Done` — the **synchronous fast path** for locally-available -data, with no scheduler and no request set allocated. - -## Execution - -A `DataSource` is an [`IBulkExecutor`](src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt): - -```kotlin -interface IBulkExecutor { - fun execute(keys: List): Map -} -``` - -The driver ([`Execution.drive`](src/commonMain/kotlin/org/modelix/streams2/Execution.kt)) is a loop where each -iteration is one batch round: issue **one bulk call per source**, fill the per-run cache (dedup within *and* across -rounds), then resume. The loop is the **trampoline** that keeps fetch-dependent chains stack-safe regardless of depth. - -```kotlin -val executor = StreamExecutor() -val source: IBulkExecutor = ... - -// independent fetches -> ONE round -val pair = executor.query(IStream.fetch(source, 1).zipWith(IStream.fetch(source, 2)) { a, b -> "$a+$b" }) - -// dependent fetches -> SEPARATE rounds -val chained = executor.query( - IStream.fetch(source, 1).flatMapOne { v -> IStream.fetch(source, keyFrom(v)) }, -) -``` - -## Public API - -Cardinality is encoded in the type — see [`IStream.kt`](src/commonMain/kotlin/org/modelix/streams2/IStream.kt): - -| Type | Meaning | -|-----------------------|--------------| -| `IStream.Many` | 0 or more | -| `IStream.ZeroOrOne`| 0 or 1 | -| `IStream.One` | exactly 1 | - -Builders (`IStream.Companion`): `of`, `empty`, `many`, `fetch`, `zip`. -Executor ([`IStreamExecutor`](src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt)): `query`, `queryAll`, -`iterate`. - -Operators: `map`, `mapNotNull`, `filter`, `flatMap`, `concat`, `distinct`, `take`, `skip`, `withIndex`, `fold`, -`toList`, `toMap`, `count`, `isEmpty`, `drainAll`, `assertEmpty`, `firstOrEmpty`, `firstOrDefault`, `exactlyOne`, -`orNull`, `ifEmpty`, `exceptionIfEmpty`, `flatMapZeroOrOne`, `flatMapOne`, `zipWith`. - -## Layout - -``` -src/commonMain/kotlin/org/modelix/streams2/ -├── Step.kt // Step IR + applicative/monadic combinators -├── Execution.kt // per-run fetch cache, RequestMap, the round driver -├── IBulkExecutor.kt // batchable data source -├── IStream.kt // public cardinality types + StreamBase impl + builders -└── IStreamExecutor.kt // materializes streams -``` - -## Known limitations / future work - -1. **Within-round stack safety.** The round driver trampolines across `Blocked` (the common fetch-dependent case). - A pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode - `Step` itself as a stack-safe free monad (an explicit interpreter loop) if that ever bites. -2. **Error-recovery operators** (`onErrorReturn`-style) aren't exposed yet; the `Failed` channel exists in the IR. -3. **`take`/`skip` don't prune upstream fetches** (a consequence of no incremental emission). diff --git a/streams2/build.gradle.kts b/streams2/build.gradle.kts deleted file mode 100644 index e0cbe3c06f..0000000000 --- a/streams2/build.gradle.kts +++ /dev/null @@ -1,18 +0,0 @@ -plugins { - `maven-publish` - `modelix-kotlin-multiplatform` -} - -kotlin { - sourceSets { - commonMain { - dependencies { - } - } - commonTest { - dependencies { - implementation(kotlin("test")) - } - } - } -} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt deleted file mode 100644 index f4a98ce64b..0000000000 --- a/streams2/src/commonMain/kotlin/org/modelix/streams2/Execution.kt +++ /dev/null @@ -1,87 +0,0 @@ -package org.modelix.streams2 - -/** The deduped set of pending fetches for a single round, grouped by data source. */ -internal class RequestMap private constructor( - val byExecutor: Map, Set>, -) { - val isEmpty: Boolean get() = byExecutor.isEmpty() - - fun union(other: RequestMap): RequestMap { - if (byExecutor.isEmpty()) return other - if (other.byExecutor.isEmpty()) return this - val result = HashMap, MutableSet>() - for ((executor, keys) in byExecutor) result.getOrPut(executor) { HashSet() }.addAll(keys) - for ((executor, keys) in other.byExecutor) result.getOrPut(executor) { HashSet() }.addAll(keys) - return RequestMap(result) - } - - companion object { - val EMPTY = RequestMap(emptyMap()) - - @Suppress("UNCHECKED_CAST") - fun single(executor: IBulkExecutor<*, *>, key: Any?): RequestMap = - RequestMap(mapOf((executor as IBulkExecutor) to setOf(key))) - } -} - -/** Sentinel stored in the cache for keys that were requested but absent from the bulk response. */ -private val MISSING = Any() - -/** - * Per-run state shared by every [Step] built for a single query. Holds the fetched-value cache so that the same key - * is fetched at most once across the whole traversal (dedup within and across rounds), regardless of how many stream - * branches request it. - */ -internal class Execution { - private val caches = HashMap, HashMap>() - - @Suppress("UNCHECKED_CAST") - private fun cacheFor(executor: IBulkExecutor<*, *>): HashMap = - caches.getOrPut(executor as IBulkExecutor) { HashMap() } - - fun isCached(executor: IBulkExecutor<*, *>, key: Any?): Boolean = cacheFor(executor).containsKey(key) - - fun cachedValue(executor: IBulkExecutor<*, *>, key: Any?): Any? { - val value = cacheFor(executor)[key] - if (value === MISSING) throw NoSuchElementException("No value returned for key: $key") - return value - } - - fun fill(executor: IBulkExecutor, requestedKeys: Set, results: Map) { - val cache = cacheFor(executor) - for (key in requestedKeys) { - cache[key] = if (results.containsKey(key)) results[key] else MISSING - } - } -} - -@Suppress("UNCHECKED_CAST") -internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: K): Step { - return if (execution.isCached(source, key)) { - Done(listOf(execution.cachedValue(source, key) as V)) - } else { - Blocked(RequestMap.single(source, key)) { fetchStep(execution, source, key) } - } -} - -/** - * Drives a [Step] to completion. Each iteration of the loop is one batch round: it issues a single bulk call per - * pending data source, fills the cache, then resumes. The loop is the trampoline that keeps fetch-dependent chains - * (the common case in tree traversal) stack-safe regardless of depth. - */ -internal fun Execution.drive(initial: Step): List { - var step = initial - while (true) { - when (val current = step) { - is Done -> return current.values - is Failed -> throw current.cause - is Blocked -> { - for ((source, keys) in current.requests.byExecutor) { - val results = source.execute(keys.toList()) as Map - fill(source, keys, results) - } - step = current.resume() - } - } - } -} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt deleted file mode 100644 index 87520ad860..0000000000 --- a/streams2/src/commonMain/kotlin/org/modelix/streams2/IBulkExecutor.kt +++ /dev/null @@ -1,12 +0,0 @@ -package org.modelix.streams2 - -/** - * A batchable data source. The whole point of this stream implementation is to coalesce many individual - * [fetch][IStream.Companion.fetch] requests against the same source into a single bulk call. - * - * Implementations must return a value for every requested key. A key that is absent from the returned map is - * treated as a missing value and surfaces as a [NoSuchElementException] in the stream that requested it. - */ -interface IBulkExecutor { - fun execute(keys: List): Map -} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt deleted file mode 100644 index 2daea00661..0000000000 --- a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStream.kt +++ /dev/null @@ -1,181 +0,0 @@ -package org.modelix.streams2 - -/** - * A lazily-described stream of values, evaluated in batch rounds by an [IStreamExecutor]. - * - * This is a clean-room reimplementation of the streaming abstraction with a single execution model: - * - No incremental emission. A stream is fully materialized when queried; operators like [Many.take] truncate the - * materialized result rather than stopping upstream work. - * - No coroutine/Flow integration. Execution is synchronous and blocking. - * - * The one capability that justifies a custom implementation is **automatic bulk-request batching**: independent - * [fetch][Companion.fetch] requests reachable through applicative composition (the elements of a [Many], the sides of - * a [zip][Companion.zip] or [One.zipWith]) are coalesced into a single [IBulkExecutor.execute] call, while [flatMap] - * dependencies fall into later rounds. See [Step] for the mechanism. - * - * Cardinality is encoded in the type: [Many] (0+), [ZeroOrOne] (0..1) and [One] (exactly 1). - */ -sealed interface IStream { - - interface Many : IStream { - fun map(transform: (E) -> R): Many - fun mapNotNull(transform: (E) -> R?): Many - fun filter(predicate: (E) -> Boolean): Many - fun flatMap(transform: (E) -> Many): Many - fun concat(other: Many<@UnsafeVariance E>): Many - fun distinct(): Many - fun take(n: Int): Many - fun skip(n: Int): Many - fun withIndex(): Many> - - fun fold(initial: R, operation: (R, E) -> R): One - fun toList(): One> - fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): One> - fun count(): One - fun isEmpty(): One - fun drainAll(): One - fun assertEmpty(message: (E) -> String): One - - fun firstOrEmpty(): ZeroOrOne - fun firstOrDefault(defaultValue: () -> @UnsafeVariance E): One - fun exactlyOne(): One - } - - interface ZeroOrOne : Many { - override fun map(transform: (E) -> R): ZeroOrOne - fun flatMapZeroOrOne(transform: (E) -> ZeroOrOne): ZeroOrOne - fun orNull(): One - fun ifEmpty(defaultValue: () -> @UnsafeVariance E): One - fun exceptionIfEmpty(exception: () -> Throwable): One - } - - interface One : ZeroOrOne { - override fun map(transform: (E) -> R): One - fun flatMapOne(transform: (E) -> One): One - fun zipWith(other: One, combine: (E, T) -> R): One - } - - companion object { - fun of(value: T): One = stream { Done(listOf(value)) } - - fun empty(): ZeroOrOne = stream { Done(emptyList()) } - - fun many(values: Iterable): Many = stream { Done(values.toList()) } - - fun many(values: Sequence): Many = stream { Done(values.toList()) } - - /** A single value fetched from a batchable [source]. The unit of batching. */ - fun fetch(source: IBulkExecutor, key: K): One = - stream { execution -> fetchStep(execution, source, key) } - - /** Applicatively combine independent single-value streams; their fetches share a batch round. */ - fun zip(streams: List>, combine: (List) -> R): One = - stream { execution -> - zipN(streams.map { it.toStep(execution) }) { valueLists -> listOf(combine(valueLists.map { it.single() })) } - } - } -} - -/** Bridges a public [IStream] back to its internal [Step] representation for the current run. */ -internal fun IStream.toStep(execution: Execution): Step = (this as StreamBase).buildStep(execution) - -/** - * The single backing implementation of every [IStream] cardinality. The runtime instance always satisfies the - * strongest interface ([IStream.One]); static typing at the API boundary restricts which operators are reachable. - */ -internal abstract class StreamBase : IStream.One { - - internal abstract fun buildStep(execution: Execution): Step - - override fun map(transform: (E) -> R): IStream.One = - stream { execution -> buildStep(execution).mapValues { values -> values.map(transform) } } - - override fun mapNotNull(transform: (E) -> R?): IStream.Many = - stream { execution -> buildStep(execution).mapValues { values -> values.mapNotNull(transform) } } - - override fun filter(predicate: (E) -> Boolean): IStream.Many = - stream { execution -> buildStep(execution).mapValues { values -> values.filter(predicate) } } - - override fun flatMap(transform: (E) -> IStream.Many): IStream.Many = - stream { execution -> - buildStep(execution).flatMapStep { values -> combineConcat(values.map { transform(it).toStep(execution) }) } - } - - override fun concat(other: IStream.Many<@UnsafeVariance E>): IStream.Many = - stream { execution -> combineConcat(listOf(buildStep(execution), other.toStep(execution))) } - - override fun distinct(): IStream.Many = - stream { execution -> buildStep(execution).mapValues { it.distinct() } } - - override fun take(n: Int): IStream.Many = - stream { execution -> buildStep(execution).mapValues { it.take(n) } } - - override fun skip(n: Int): IStream.Many = - stream { execution -> buildStep(execution).mapValues { it.drop(n) } } - - override fun withIndex(): IStream.Many> = - stream { execution -> buildStep(execution).mapValues { it.withIndex().toList() } } - - override fun fold(initial: R, operation: (R, E) -> R): IStream.One = - stream { execution -> buildStep(execution).mapValues { listOf(it.fold(initial, operation)) } } - - override fun toList(): IStream.One> = - stream { execution -> buildStep(execution).mapValues { listOf(it) } } - - override fun toMap(keySelector: (E) -> K, valueSelector: (E) -> V): IStream.One> = - stream { execution -> buildStep(execution).mapValues { values -> listOf(values.associate { keySelector(it) to valueSelector(it) }) } } - - override fun count(): IStream.One = - stream { execution -> buildStep(execution).mapValues { listOf(it.size) } } - - override fun isEmpty(): IStream.One = - stream { execution -> buildStep(execution).mapValues { listOf(it.isEmpty()) } } - - override fun drainAll(): IStream.One = - stream { execution -> buildStep(execution).mapValues { listOf(Unit) } } - - override fun assertEmpty(message: (E) -> String): IStream.One = - stream { execution -> - buildStep(execution).mapValues { values -> - if (values.isNotEmpty()) throw IllegalStateException(message(values.first())) - listOf(Unit) - } - } - - override fun firstOrEmpty(): IStream.ZeroOrOne = - stream { execution -> buildStep(execution).mapValues { it.take(1) } } - - override fun firstOrDefault(defaultValue: () -> @UnsafeVariance E): IStream.One = - stream { execution -> buildStep(execution).mapValues { values -> listOf(values.firstOrNull() ?: defaultValue()) } } - - override fun exactlyOne(): IStream.One = - stream { execution -> buildStep(execution).mapValues { listOf(it.single()) } } - - override fun flatMapZeroOrOne(transform: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne = - stream { execution -> - buildStep(execution).flatMapStep { values -> combineConcat(values.map { transform(it).toStep(execution) }) } - } - - override fun orNull(): IStream.One = - stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) listOf(null) else listOf(values.single()) } } - - override fun ifEmpty(defaultValue: () -> @UnsafeVariance E): IStream.One = - stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) listOf(defaultValue()) else values } } - - override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One = - stream { execution -> buildStep(execution).mapValues { values -> if (values.isEmpty()) throw exception() else values } } - - override fun flatMapOne(transform: (E) -> IStream.One): IStream.One = - stream { execution -> buildStep(execution).flatMapStep { values -> transform(values.single()).toStep(execution) } } - - override fun zipWith(other: IStream.One, combine: (E, T) -> R): IStream.One = - stream { execution -> - zip2(buildStep(execution), other.toStep(execution)) { a, b -> listOf(combine(a.single(), b.single())) } - } -} - -private class StreamNode(private val builder: (Execution) -> Step) : StreamBase() { - override fun buildStep(execution: Execution): Step = builder(execution) -} - -internal fun stream(builder: (Execution) -> Step): StreamBase = StreamNode(builder) diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt deleted file mode 100644 index 2754a84210..0000000000 --- a/streams2/src/commonMain/kotlin/org/modelix/streams2/IStreamExecutor.kt +++ /dev/null @@ -1,24 +0,0 @@ -package org.modelix.streams2 - -/** - * Materializes streams. Each query runs an independent round loop with its own fetch cache, issuing one bulk call - * per data source per round (see [Execution.drive]). - */ -interface IStreamExecutor { - fun query(stream: IStream.One): T - fun queryAll(stream: IStream.Many): List - fun iterate(stream: IStream.Many, visitor: (T) -> Unit) -} - -class StreamExecutor : IStreamExecutor { - override fun queryAll(stream: IStream.Many): List { - val execution = Execution() - return execution.drive(stream.toStep(execution)) - } - - override fun query(stream: IStream.One): T = queryAll(stream).single() - - override fun iterate(stream: IStream.Many, visitor: (T) -> Unit) { - for (value in queryAll(stream)) visitor(value) - } -} diff --git a/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt b/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt deleted file mode 100644 index a36ea82f61..0000000000 --- a/streams2/src/commonMain/kotlin/org/modelix/streams2/Step.kt +++ /dev/null @@ -1,92 +0,0 @@ -package org.modelix.streams2 - -/** - * The internal intermediate representation of a partially-evaluated stream. - * - * A [Step] produces zero or more values of [T]. Evaluation proceeds in *rounds*: a [Step] is either fully resolved - * ([Done]), needs a batch of data fetches before it can continue ([Blocked]), or has failed ([Failed]). - * - * The combinators on [Step] encode the applicative/monadic split that makes batching automatic: - * - Applicative composition ([zipN], [combineConcat]) unions the pending requests of independent branches into the - * *same* round. That is the batch. - * - Monadic composition ([flatMapStep]) introduces a dependency, so the right-hand side's requests can only be - * discovered after the left side resolves, i.e. in a *later* round. - * - * Nothing is ever forced eagerly: a subtree that contains no [Blocked] resolves straight to [Done] without ever - * allocating a request set, which is the synchronous fast path for locally-available data. - */ -internal sealed interface Step - -internal class Done(val values: List) : Step - -internal class Blocked(val requests: RequestMap, val resume: () -> Step) : Step - -internal class Failed(val cause: Throwable) : Step - -/** Transform the resolved value list, threading through the round boundaries. */ -internal fun Step.mapValues(f: (List) -> List): Step = when (this) { - is Done -> Done(f(values)) - is Blocked -> Blocked(requests) { resume().mapValues(f) } - is Failed -> this -} - -/** Monadic bind: the dependency that introduces a round boundary. */ -internal fun Step.flatMapStep(f: (List) -> Step): Step = when (this) { - is Done -> f(values) - is Blocked -> Blocked(requests) { resume().flatMapStep(f) } - is Failed -> this -} - -/** - * Applicative combination of independent steps that concatenates their values in order. - * - * Iterates over [steps] within a single round (no per-element native recursion); only recurses once per round via - * the [Blocked] thunk, so the recursion depth is bounded by the number of fetch rounds rather than the number of - * elements. - */ -internal fun combineConcat(steps: List>): Step { - var allDone = true - var requests = RequestMap.EMPTY - for (step in steps) { - when (step) { - is Failed -> return step - is Blocked -> { - allDone = false - requests = requests.union(step.requests) - } - is Done -> {} - } - } - if (allDone) return Done(steps.flatMap { (it as Done).values }) - return Blocked(requests) { combineConcat(steps.map { if (it is Blocked) it.resume() else it }) } -} - -/** Applicative combination that hands all resolved value lists to [f] together. */ -internal fun zipN(steps: List>, f: (List>) -> List): Step { - var allDone = true - var requests = RequestMap.EMPTY - for (step in steps) { - when (step) { - is Failed -> return step - is Blocked -> { - allDone = false - requests = requests.union(step.requests) - } - is Done -> {} - } - } - if (allDone) return Done(f(steps.map { (it as Done).values })) - return Blocked(requests) { zipN(steps.map { if (it is Blocked) it.resume() else it }, f) } -} - -internal fun zip2(a: Step, b: Step, f: (List, List) -> List): Step { - if (a is Failed) return a - if (b is Failed) return b - if (a is Done && b is Done) return Done(f(a.values, b.values)) - var requests = RequestMap.EMPTY - if (a is Blocked) requests = requests.union(a.requests) - if (b is Blocked) requests = requests.union(b.requests) - return Blocked(requests) { - zip2(if (a is Blocked) a.resume() else a, if (b is Blocked) b.resume() else b, f) - } -} diff --git a/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt b/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt deleted file mode 100644 index 495f38d5b9..0000000000 --- a/streams2/src/commonTest/kotlin/org/modelix/streams2/Streams2Test.kt +++ /dev/null @@ -1,122 +0,0 @@ -package org.modelix.streams2 - -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -/** Records every bulk call so tests can assert how many rounds happened and which keys were batched together. */ -private class RecordingSource(private val backing: Map) : IBulkExecutor { - val calls = mutableListOf>() - - override fun execute(keys: List): Map { - calls.add(keys.sorted()) - return keys.associateWith { backing.getValue(it) } - } - - val roundCount: Int get() = calls.size - val totalKeysFetched: Int get() = calls.sumOf { it.size } -} - -class Streams2Test { - private val executor = StreamExecutor() - private val data = (0..9).associateWith { "v$it" } - - @Test - fun constant_stream_does_not_touch_a_source() { - // The eager fast path: no Fetch nodes means the round loop returns immediately. - val result = executor.query(IStream.of(21).map { it * 2 }) - assertEquals(42, result) - } - - @Test - fun many_operators() { - val stream = IStream.many(1..5).map { it * 10 }.filter { it > 20 } - assertEquals(listOf(30, 40, 50), executor.queryAll(stream)) - assertEquals(120, executor.query(IStream.many(1..5).fold(0) { acc, v -> acc + v }.map { it * 8 })) - } - - @Test - fun single_fetch() { - val source = RecordingSource(data) - assertEquals("v3", executor.query(IStream.fetch(source, 3))) - assertEquals(1, source.roundCount) - assertEquals(listOf(listOf(3)), source.calls) - } - - @Test - fun independent_fetches_are_batched_into_one_round() { - val source = RecordingSource(data) - val stream = IStream.many(listOf(1, 2, 3, 4)) - .flatMap { key -> IStream.fetch(source, key) } - assertEquals(listOf("v1", "v2", "v3", "v4"), executor.queryAll(stream)) - assertEquals(1, source.roundCount) - assertEquals(listOf(listOf(1, 2, 3, 4)), source.calls) - } - - @Test - fun zip_batches_both_sides_together() { - val source = RecordingSource(data) - val combined = IStream.fetch(source, 5).zipWith(IStream.fetch(source, 6)) { a, b -> "$a+$b" } - assertEquals("v5+v6", executor.query(combined)) - assertEquals(1, source.roundCount) - assertEquals(listOf(listOf(5, 6)), source.calls) - } - - @Test - fun dependent_fetches_use_separate_rounds() { - val source = RecordingSource(data) - // The second fetch's key is derived from the first result -> it cannot be known until round 1 resolves. - val stream = IStream.fetch(source, 1).flatMapOne { first -> - val nextKey = first.removePrefix("v").toInt() + 4 // "v1" -> 5 - IStream.fetch(source, nextKey) - } - assertEquals("v5", executor.query(stream)) - assertEquals(2, source.roundCount) - assertEquals(listOf(listOf(1), listOf(5)), source.calls) - } - - @Test - fun duplicate_keys_are_fetched_once() { - val source = RecordingSource(data) - val stream = IStream.many(listOf(2, 2, 2, 7, 7)) - .flatMap { key -> IStream.fetch(source, key) } - assertEquals(listOf("v2", "v2", "v2", "v7", "v7"), executor.queryAll(stream)) - assertEquals(1, source.roundCount) - assertEquals(2, source.totalKeysFetched) // only keys 2 and 7 actually fetched - } - - @Test - fun zero_or_one_handling() { - assertEquals("default", executor.query(IStream.empty().ifEmpty { "default" })) - assertEquals(null, executor.query(IStream.empty().orNull())) - assertEquals("x", executor.query(IStream.of("x").orNull())) - assertFailsWith { - executor.query(IStream.empty().exceptionIfEmpty { IllegalStateException("empty") }) - } - } - - @Test - fun missing_key_surfaces_as_error() { - val source = RecordingSource(data) - assertFailsWith { - executor.query(IStream.fetch(source, 999)) - } - } - - @Test - fun deep_dependent_chain_is_stack_safe() { - val source = RecordingSource((0..1000).associateWith { "v$it" }) - // 1000 sequential dependent fetches -> 1000 rounds, must not overflow the stack. - var stream: IStream.One = IStream.fetch(source, 0) - repeat(1000) { - stream = stream.flatMapOne { v -> - val next = v.removePrefix("v").toInt() + 1 - IStream.fetch(source, next) - } - } - assertEquals("v1000", executor.query(stream)) - assertEquals(1001, source.roundCount) - assertTrue(source.calls.all { it.size == 1 }) - } -} From db64a4f7aae8c9543be9a917d16f1b659f0b2ab8 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 21:11:27 +0200 Subject: [PATCH 3/7] feat(streams): drive streams without an executor; batch size on the source The IStreamExecutor passed to getBlocking/getSuspending/iterate*/execute* no longer carried any batching context: fetch leaves carry their own IBulkExecutor source and the per-run Execution provides caching/dedup. Two changes remove the residual coupling. B. Batch size on the source: IBulkExecutor now declares `val batchSize` (default DEFAULT_BULK_REQUEST_BATCH_SIZE = 5000). The round driver chunks each source's keys to its own batchSize and no longer takes a batch-size parameter. BulkRequestStreamExecutor(source, batchSize) still works (it exposes the constructor batch size on the source the leaves bind to). A. Executor-less terminals: add getBlocking()/getSuspending()/iterateBlocking{}/ iterateSuspending{}/executeBlocking()/executeSuspending(); deprecate the executor-taking overloads with ReplaceWith, keeping their original behavior for compatibility. All ~180 in-repo terminal call sites migrated to the executor-less form. IStreamExecutor / IStreamExecutorProvider and BulkRequestStreamExecutor.enqueue remain (enqueue creates fetch leaves; CONTEXT is still set during BulkRequestStreamExecutor runs because ModelQL resolves the current executor via IStreamExecutor.getInstance()). Verified: streams JVM+JS compile and tests pass; datastructures and model-datastructure compile + full suites pass; model-client (JVM+JS), model-server, modelql-* compile; model-server LazyLoadingTest (batching correctness) passes. Co-Authored-By: Claude Opus 4.8 --- .../org/modelix/datastructures/btree/BTree.kt | 4 +- .../org/modelix/datastructures/BTreeTest.kt | 2 +- .../datastructures/HamtCollisionTest.kt | 52 ++++++------- .../org/modelix/datastructures/HamtTest.kt | 6 +- .../datastructures/PatriciaTrieTest.kt | 70 +++++++++--------- .../modelix/datastructures/TreeDiffTest.kt | 16 ++-- .../model/test/RandomModelChangeGenerator.kt | 2 +- .../modelix/model/client2/IModelClientV2.kt | 2 +- .../modelix/model/client2/RolesMigration.kt | 2 +- .../org/modelix/model/client2/ClientJS.kt | 2 +- .../model/client2/MutableModelTreeJsImpl.kt | 2 +- .../history/HistoryIndexNode.kt | 2 +- .../datastructures/history/HistoryQueries.kt | 8 +- .../datastructures/model/ModelTreeBuilder.kt | 4 +- .../kotlin/org/modelix/model/VersionMerger.kt | 2 +- .../kotlin/org/modelix/model/lazy/CLTree.kt | 2 +- .../org/modelix/model/lazy/CLVersion.kt | 2 +- .../model/lazy/OperationsCompressor.kt | 2 +- .../mutable/SingleThreadMutableModelTree.kt | 2 +- .../model/operations/AbstractOperation.kt | 6 +- .../modelix/model/operations/AddNewChildOp.kt | 12 +-- .../model/operations/AddNewChildSubtreeOp.kt | 20 ++--- .../modelix/model/operations/BulkUpdateOp.kt | 2 +- .../modelix/model/operations/DeleteNodeOp.kt | 4 +- .../modelix/model/operations/MoveNodeOp.kt | 12 +-- .../modelix/model/operations/SetConceptOp.kt | 2 +- .../modelix/model/operations/SetPropertyOp.kt | 2 +- .../model/operations/SetReferenceOp.kt | 2 +- .../kotlin/DuplicateImportConflictTest.kt | 10 +-- .../src/commonTest/kotlin/HamtTest.kt | 28 +++---- .../kotlin/HistoryIndexNodeAttributesTest.kt | 6 +- .../src/commonTest/kotlin/HistoryIndexTest.kt | 26 +++---- .../src/commonTest/kotlin/ObjectDiffTest.kt | 6 +- .../kotlin/RandomTreeChangeGenerator.kt | 2 +- .../kotlin/VersionAttributesTest.kt | 4 +- .../model/server/handlers/LionwebApiImpl.kt | 5 +- .../server/handlers/RepositoriesManager.kt | 12 +-- .../model/server/ReplicatedRepositoryTest.kt | 2 +- .../modelix/mps/gitimport/GitExportTest.kt | 6 +- .../modelix/mps/gitimport/GitImportTest.kt | 12 +-- .../org/modelix/mps/sync3/BindingWorker.kt | 2 +- .../org/modelix/mps/sync3/ProjectSyncTest.kt | 6 +- streams-redesign.md | 30 +++++++- streams/README.md | 15 ++-- .../streams/BulkRequestStreamExecutor.kt | 44 ++++++++--- .../org/modelix/streams/IStreamExecutor.kt | 73 ++++++++++++++++++- .../modelix/streams/SimpleStreamExecutor.kt | 14 ++-- .../kotlin/org/modelix/streams/StreamImpl.kt | 24 +++--- .../org/modelix/streams/engine/StepEngine.kt | 14 ++-- .../modelix/streams/BlockingStreamExecutor.kt | 8 +- 50 files changed, 355 insertions(+), 240 deletions(-) diff --git a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt index 1468b726d9..779de04738 100644 --- a/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt +++ b/datastructures/src/commonMain/kotlin/org/modelix/datastructures/btree/BTree.kt @@ -13,10 +13,10 @@ data class BTree(val root: BTreeNode) : IStreamExecutorProvider by r fun validate() { graph.getStreamExecutor().query { root.validate(true) - check(root.getEntries().toList().getBlocking(graph).map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getBlocking(graph)) { + check(root.getEntries().toList().getBlocking().map { it.key }.toSet().size == root.getEntries().map { it.key }.count().getBlocking()) { "duplicate entries: $root" } - check(root.getEntries().map { it.key }.toList().getBlocking(graph).sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getBlocking(graph)) { + check(root.getEntries().map { it.key }.toList().getBlocking().sortedWith(root.config.keyConfiguration) == root.getEntries().map { it.key }.toList().getBlocking()) { "not sorted: $this" } IStream.of(Unit) diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt index 86c85a6eda..f8cba8b748 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/BTreeTest.kt @@ -72,7 +72,7 @@ class BTreeTest { assertEquals( (100L..200L).map { it to it * 2 }, - tree.getAll((100L..200L)).toList().getBlocking(tree), + tree.getAll((100L..200L)).toList().getBlocking(), ) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt index 1e2734877f..0b8f727218 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtCollisionTest.kt @@ -27,19 +27,19 @@ class HamtCollisionTest { ) var tree: HamtTree = HamtTree(HamtInternalNode.createEmpty(config)) - tree = tree.put(0b000000000, "a").getBlocking(tree) - tree = tree.put(0b000000001, "b").getBlocking(tree) - tree = tree.put(0b000000011, "c").getBlocking(tree) - tree = tree.put(0b100000000, "d").getBlocking(tree) - tree = tree.put(0b100000001, "e").getBlocking(tree) - tree = tree.put(0b100000011, "f").getBlocking(tree) - - assertEquals("a", tree.get(0b000000000).getBlocking(tree)) - assertEquals("b", tree.get(0b000000001).getBlocking(tree)) - assertEquals("c", tree.get(0b000000011).getBlocking(tree)) - assertEquals("d", tree.get(0b100000000).getBlocking(tree)) - assertEquals("e", tree.get(0b100000001).getBlocking(tree)) - assertEquals("f", tree.get(0b100000011).getBlocking(tree)) + tree = tree.put(0b000000000, "a").getBlocking() + tree = tree.put(0b000000001, "b").getBlocking() + tree = tree.put(0b000000011, "c").getBlocking() + tree = tree.put(0b100000000, "d").getBlocking() + tree = tree.put(0b100000001, "e").getBlocking() + tree = tree.put(0b100000011, "f").getBlocking() + + assertEquals("a", tree.get(0b000000000).getBlocking()) + assertEquals("b", tree.get(0b000000001).getBlocking()) + assertEquals("c", tree.get(0b000000011).getBlocking()) + assertEquals("d", tree.get(0b100000000).getBlocking()) + assertEquals("e", tree.get(0b100000001).getBlocking()) + assertEquals("f", tree.get(0b100000011).getBlocking()) } /** @@ -60,22 +60,22 @@ class HamtCollisionTest { ) var tree: HamtTree = HamtTree(HamtInternalNode.createEmpty(config)) - tree = tree.put(0b00, "a").getBlocking(tree) - tree = tree.put(0b01, "b").getBlocking(tree) - tree = tree.put(0b10, "c").getBlocking(tree) - tree = tree.put(0b11, "d").getBlocking(tree) + tree = tree.put(0b00, "a").getBlocking() + tree = tree.put(0b01, "b").getBlocking() + tree = tree.put(0b10, "c").getBlocking() + tree = tree.put(0b11, "d").getBlocking() - assertEquals("a", tree.get(0b00).getBlocking(tree)) - assertEquals("b", tree.get(0b01).getBlocking(tree)) - assertEquals("c", tree.get(0b10).getBlocking(tree)) - assertEquals("d", tree.get(0b11).getBlocking(tree)) + assertEquals("a", tree.get(0b00).getBlocking()) + assertEquals("b", tree.get(0b01).getBlocking()) + assertEquals("c", tree.get(0b10).getBlocking()) + assertEquals("d", tree.get(0b11).getBlocking()) - tree = tree.put(0b01, "changed").getBlocking(tree) + tree = tree.put(0b01, "changed").getBlocking() - assertEquals("a", tree.get(0b00).getBlocking(tree)) - assertEquals("changed", tree.get(0b01).getBlocking(tree)) - assertEquals("c", tree.get(0b10).getBlocking(tree)) - assertEquals("d", tree.get(0b11).getBlocking(tree)) + assertEquals("a", tree.get(0b00).getBlocking()) + assertEquals("changed", tree.get(0b01).getBlocking()) + assertEquals("c", tree.get(0b10).getBlocking()) + assertEquals("d", tree.get(0b11).getBlocking()) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt index f3c058ba32..bae666724e 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtTest.kt @@ -33,7 +33,7 @@ class HamtTest { if (expected.isNotEmpty()) { val key = expected.keys.random(rand) // println("remove $key") - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() expected.remove(key) } } @@ -41,13 +41,13 @@ class HamtTest { val key = "k" + rand.nextInt(keyRange).toString() val value = "v" + rand.nextInt(valueRange).toString() // println("insert $key -> $value") - tree = tree.put(key, value).getBlocking(tree) + tree = tree.put(key, value).getBlocking() expected[key] = value } } for (entry in expected.entries) { - assertEquals(entry.value, tree.get(entry.key).getBlocking(tree), "for key ${entry.key}") + assertEquals(entry.value, tree.get(entry.key).getBlocking(), "for key ${entry.key}") } } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt index 4d7156aed3..1a582f7722 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/PatriciaTrieTest.kt @@ -25,10 +25,10 @@ class PatriciaTrieTest { fun assertTree() { val expectedEntries = addedEntries.associateWith { values[it]!! } - val actualEntries = tree.getAll().toList().getBlocking(tree).toMap() + val actualEntries = tree.getAll().toList().getBlocking().toMap() assertEquals(expectedEntries, actualEntries) - val expectedTree = addedEntries.fold(initialTree) { acc, it -> acc.put(it, values[it]!!).getBlocking(tree) } + val expectedTree = addedEntries.fold(initialTree) { acc, it -> acc.put(it, values[it]!!).getBlocking() } assertEquals(expectedTree, tree) assertEquals(expectedTree.asObject().getHash(), tree.asObject().getHash()) @@ -39,13 +39,13 @@ class PatriciaTrieTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } @@ -56,55 +56,55 @@ class PatriciaTrieTest { @Test fun `slice with shorter prefix and single entry`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("abcdef", "1").getBlocking(tree) + tree = tree.put("abcdef", "1").getBlocking() - assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) + assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) } @Test fun `slice with shorter prefix and two entries`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("abcdef", "1").getBlocking(tree) - tree = tree.put("abcdeg", "2").getBlocking(tree) + tree = tree.put("abcdef", "1").getBlocking() + tree = tree.put("abcdeg", "2").getBlocking() - assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("2", tree.slice("abc").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + assertEquals("1", tree.slice("abc").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("2", tree.slice("abc").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test fun `slice with prefix between two entries`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() - assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) + assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) } @Test fun `slice with one before and two after`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) - tree = tree.put("abcdeg", "3").getBlocking(tree) + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() + tree = tree.put("abcdeg", "3").getBlocking() - assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("3", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + assertEquals(null, tree.slice("abcd").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("3", tree.slice("abcd").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test fun `slice with two before and two after at existing split`() { var tree: PatriciaTrie = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) - tree = tree.put("a", "0").getBlocking(tree) - tree = tree.put("ab", "1").getBlocking(tree) - tree = tree.put("abcdef", "2").getBlocking(tree) - tree = tree.put("abcdeg", "3").getBlocking(tree) - - assertEquals(null, tree.slice("ab").flatMapZeroOrOne { it.get("a") }.getBlocking(tree)) - assertEquals("1", tree.slice("ab").flatMapZeroOrOne { it.get("ab") }.getBlocking(tree)) - assertEquals("2", tree.slice("ab").flatMapZeroOrOne { it.get("abcdef") }.getBlocking(tree)) - assertEquals("3", tree.slice("ab").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking(tree)) + tree = tree.put("a", "0").getBlocking() + tree = tree.put("ab", "1").getBlocking() + tree = tree.put("abcdef", "2").getBlocking() + tree = tree.put("abcdeg", "3").getBlocking() + + assertEquals(null, tree.slice("ab").flatMapZeroOrOne { it.get("a") }.getBlocking()) + assertEquals("1", tree.slice("ab").flatMapZeroOrOne { it.get("ab") }.getBlocking()) + assertEquals("2", tree.slice("ab").flatMapZeroOrOne { it.get("abcdef") }.getBlocking()) + assertEquals("3", tree.slice("ab").flatMapZeroOrOne { it.get("abcdeg") }.getBlocking()) } @Test @@ -127,16 +127,16 @@ class PatriciaTrieTest { "abcdC", ) for (key in initialKeys) { - tree = tree.put(key, "value of $key").getBlocking(tree) + tree = tree.put(key, "value of $key").getBlocking() } assertEquals( initialKeys.associateWith { "value of $it" }, - tree.getAll().toList().getBlocking(tree).toMap(), + tree.getAll().toList().getBlocking().toMap(), ) for (key in initialKeys) { - assertEquals("value of $key", tree.get(key).getBlocking(tree)) + assertEquals("value of $key", tree.get(key).getBlocking()) } var replacementTree = PatriciaTrie.withStrings(IObjectGraph.FREE_FLOATING) @@ -151,16 +151,16 @@ class PatriciaTrieTest { "abcdB93", ) for (key in replacementKeys) { - replacementTree = replacementTree.put(key, "new value of $key").getBlocking(tree) + replacementTree = replacementTree.put(key, "new value of $key").getBlocking() } - tree = tree.replaceSlice("abcdB", replacementTree).getBlocking(tree) + tree = tree.replaceSlice("abcdB", replacementTree).getBlocking() assertEquals( initialKeys.filterNot { it.startsWith("abcdB") }.map { it to "value of $it" } .plus(replacementKeys.map { it to "new value of $it" }) .sortedBy { it.first } .joinToString("\n") { "${it.first} -> ${it.second}" }, - tree.getAll().toList().getBlocking(tree).sortedBy { it.first }.joinToString("\n") { "${it.first} -> ${it.second}" }, + tree.getAll().toList().getBlocking().sortedBy { it.first }.joinToString("\n") { "${it.first} -> ${it.second}" }, ) } } diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt index b2e4a88ae3..b25d5f3f60 100644 --- a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/TreeDiffTest.kt @@ -29,7 +29,7 @@ class TreeDiffTest { val restoringGraph = FullyLoadedObjectGraph() val initialDiff = tree.asObject().getDescendantsAndSelf() .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree).toMap() + .toList().getBlocking().toMap() val restoringConfig = PatriciaTrieConfig( restoringGraph, StringDataTypeConfiguration(), @@ -40,11 +40,11 @@ class TreeDiffTest { fun assertTree() { val diff = tree.asObject().objectDiff(restoredTree.asObject()) .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree).toMap() + .toList().getBlocking().toMap() restoredTree = PatriciaTrie(restoringGraph.loadObjects(tree.asObject().getHash(), restoringDeserializer, diff)) val expectedEntries = addedEntries.associateWith { values[it]!! } - val actualEntries = restoredTree.getAll().toList().getBlocking(tree).toMap() + val actualEntries = restoredTree.getAll().toList().getBlocking().toMap() assertEquals(expectedEntries, actualEntries) } @@ -53,13 +53,13 @@ class TreeDiffTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } @@ -85,7 +85,7 @@ class TreeDiffTest { for (previousTree in (1..5).map { previousTrees.random(rand) }) { val diff = tree.asObject().objectDiff(previousTree.asObject()) .map { it.getHash() to it.data.serialize() } - .toList().getBlocking(tree) + .toList().getBlocking() val duplicateObjects = diff.groupBy { it.first }.filter { it.value.size > 1 }.map { it.value.first() } assertEquals(emptyList(), duplicateObjects) @@ -98,13 +98,13 @@ class TreeDiffTest { val key = removedEntries.random(rand) removedEntries.remove(key) addedEntries.add(key) - tree = tree.put(key, values[key]!!).getBlocking(tree) + tree = tree.put(key, values[key]!!).getBlocking() assertTree() } else { val key = addedEntries.random(rand) removedEntries.add(key) addedEntries.remove(key) - tree = tree.remove(key).getBlocking(tree) + tree = tree.remove(key).getBlocking() assertTree() } } diff --git a/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt b/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt index 44b7a2831a..3bcd23d9e8 100644 --- a/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt +++ b/model-api/src/commonMain/kotlin/org/modelix/model/test/RandomModelChangeGenerator.kt @@ -100,7 +100,7 @@ class RandomModelChangeGenerator(val rootNode: INode, private val rand: Random) .map { it.asRegularNode() } .filter(condition) .firstOrNull() - .getBlocking(rootNode.asAsyncNode()) + .getBlocking() } } diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt index c285142e80..30f64ea956 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/IModelClientV2.kt @@ -153,5 +153,5 @@ interface IModelClientV2 { @DelicateModelixApi suspend fun IModelClientV2.diffAsMutationParameters(repositoryId: RepositoryId, newVersion: ObjectHash, oldVersion: ObjectHash): List> { val version = lazyLoadVersion(repositoryId, newVersion.toString()) as CLVersion - return version.historyAsMutationParameters(oldVersion).toList().getSuspending(version.graph) + return version.historyAsMutationParameters(oldVersion).toList().getSuspending() } diff --git a/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt b/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt index 3776392a1e..c5685794ff 100644 --- a/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt +++ b/model-client/src/commonMain/kotlin/org/modelix/model/client2/RolesMigration.kt @@ -110,7 +110,7 @@ suspend fun IModelClientV2.migrateRoles( }, ), ).andThenUnit() - }.drainAll().executeBlocking(oldTree) + }.drainAll().executeBlocking() } push(branch, newVersion, oldVersion) diff --git a/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt b/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt index 6378dd0b42..759fd48201 100644 --- a/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt +++ b/model-client/src/jsMain/kotlin/org/modelix/model/client2/ClientJS.kt @@ -511,7 +511,7 @@ internal class ClientJSImpl(private val modelClient: ModelClientV2) : ClientJS { version.historyAsMutationParameters(ObjectHash(oldVersion)) .map { it.toJS() } .toList() - .getSuspending(version.graph) + .getSuspending() .toTypedArray() } } diff --git a/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt b/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt index 8f6cfa4cb9..6927dfa1a8 100644 --- a/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt +++ b/model-client/src/jsMain/kotlin/org/modelix/model/client2/MutableModelTreeJsImpl.kt @@ -66,7 +66,7 @@ internal class ChangeListener(private val tree: IMutableModelTree, private val c } override fun treeChanged(oldTree: IGenericModelTree, newTree: IGenericModelTree) { - newTree.getChanges(oldTree, false).iterateBlocking(newTree) { + newTree.getChanges(oldTree, false).iterateBlocking { when (it) { is ConceptChangedEvent -> changeCallback(ConceptChanged(nodeIdToInode(it.nodeId))) is ContainmentChangedEvent -> changeCallback(ContainmentChanged(nodeIdToInode(it.nodeId))) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt index 6b40d6807b..4a0e2551e6 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryIndexNode.kt @@ -126,7 +126,7 @@ sealed class HistoryIndexNode : IObjectData { } fun of(version1: Object, version2: Object): HistoryIndexNode { - return of(version1).asObject(version1.graph).merge(of(version2).asObject(version2.graph)).getBlocking(version1.graph).data + return of(version1).asObject(version1.graph).merge(of(version2).asObject(version2.graph)).getBlocking().data } /** diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt index 334ae2a54f..48cd2c18ca 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/history/HistoryQueries.kt @@ -28,7 +28,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : val interval = delay / 2 runUntilLimit { - index.data.splitAtInterval(EquidistantIntervalsSpec(interval).withTimeRangeFilter(timeRange)).iterateSuspending(index.graph) { node -> + index.data.splitAtInterval(EquidistantIntervalsSpec(interval).withTimeRangeFilter(timeRange)).iterateSuspending { node -> if (previousMinTime - node.maxTime >= delay) { if (sessions.lastIndex >= pagination.asRange().last) throw LimitReached() sessions += HistoryInterval( @@ -76,7 +76,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : var previousIntervalId: Long = Long.MAX_VALUE runUntilLimit { - index.data.splitAtInterval(intervalsSpec).iterateSuspending(index.graph) { node -> + index.data.splitAtInterval(intervalsSpec).iterateSuspending { node -> val intervalId = intervalsSpec.getIntervalIndex(node.maxTime) check(intervalId <= previousIntervalId) if (intervalId == previousIntervalId) { @@ -114,7 +114,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : pagination: PaginationParameters, ): List { val index: Object = historyIndex() - val inTimeRange = if (timeRange == null) index else index.getRange(timeRange).orNull().getSuspending(index.graph) + val inTimeRange = if (timeRange == null) index else index.getRange(timeRange).orNull().getSuspending() if (inTimeRange == null) return emptyList() return inTimeRange.data.getRange(pagination.asRange()) .flatMapOrdered { it.getAllVersionsReversed() } @@ -129,7 +129,7 @@ class HistoryQueries(val historyIndex: suspend () -> Object) : ) } .toList() - .getSuspending(index.graph) + .getSuspending() } override suspend fun splitAt(splitPoints: List): List { diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt index 89b17d9e2f..8f93c83968 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/datastructures/model/ModelTreeBuilder.kt @@ -42,7 +42,7 @@ abstract class ModelTreeBuilder private constructor(protected val common return HamtInternalNode.createEmpty(config) .put(root.data.id, root.ref, common.graph) .orNull() - .getBlocking(common.graph)!! + .getBlocking()!! .let { HamtTree(it) } .autoResolveValues() .asModelTree(common.treeId, common.storeRoleIds) @@ -64,7 +64,7 @@ abstract class ModelTreeBuilder private constructor(protected val common ) return PatriciaTrie(config) .put(root.data.id, root.ref) - .getBlocking(common.graph) + .getBlocking() .autoResolveValues() .asModelTree(common.treeId, common.storeRoleIds) } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt index fb779ab997..89d4099259 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/VersionMerger.kt @@ -107,7 +107,7 @@ constructor(private val idGenerator: IIdGenerator?) { } } } + transaction.tree.getChildren(transaction.tree.getRootNodeId(), ITree.DETACHED_NODES_LINK) - .toList().getBlocking(transaction.tree) + .toList().getBlocking() .map { DeleteNodeOp(it).apply(mutableTree) } mergedVersion = CLVersion.builder() diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt index 6ba9bb25dc..9f9df9a5a9 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLTree.kt @@ -57,7 +57,7 @@ private fun createNewTreeData( HamtInternalNode.createEmpty(config) .put(root.id, graph.fromCreated(root), graph) .orNull() - .getBlocking(graph)!!, + .getBlocking()!!, ), trieWithNodeRefIds = null, usesRoleIds = useRoleIds, diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt index 92403c3cf6..d8fedbf381 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/CLVersion.kt @@ -320,7 +320,7 @@ class CLVersion(val obj: Object) : IVersion { } fun loadFromHash(hash: ObjectHash, graph: IObjectGraph): CLVersion { - return requestFromHash(hash, graph).getBlocking(graph) + return requestFromHash(hash, graph).getBlocking() } fun requestFromHash(hash: ObjectHash, graph: IObjectGraph): IStream.One { diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt index 9bb8e44ea1..d87629c22f 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/lazy/OperationsCompressor.kt @@ -56,7 +56,7 @@ class OperationsCompressor(val resultTree: Object) { } for (id in createdNodes) { - if (!resultTree.data.getModelTree().containsNode(id).getBlocking(resultTree.graph)) { + if (!resultTree.data.getModelTree().containsNode(id).getBlocking()) { throw RuntimeException("Tree expected to contain node $id") } } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt index 8e99eaba66..63a2fe27b9 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/mutable/SingleThreadMutableModelTree.kt @@ -58,7 +58,7 @@ class SingleThreadMutableModelTree( override fun mutate(parameters: MutationParameters) { val oldTree = tree - val newTree = tree.mutate(parameters).getBlocking(tree) + val newTree = tree.mutate(parameters).getBlocking() tree = newTree for (listener in listeners) { listener.treeChanged(oldTree, newTree) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt index 19d6742631..f8e1456527 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AbstractOperation.kt @@ -29,8 +29,8 @@ sealed class AbstractOperation : IOperation { } protected fun getNodePosition(tree: IModelTree, nodeId: INodeReference): PositionInRole { - val (parent, role) = requireNotNull(tree.getContainment(nodeId).getBlocking(tree)) { "Node has no parent: $nodeId" } - val index = tree.getChildren(parent, role).toList().getBlocking(tree).indexOf(nodeId) + val (parent, role) = requireNotNull(tree.getContainment(nodeId).getBlocking()) { "Node has no parent: $nodeId" } + val index = tree.getChildren(parent, role).toList().getBlocking().indexOf(nodeId) return PositionInRole(RoleInNode(parent, role), index) } @@ -41,7 +41,7 @@ sealed class AbstractOperation : IOperation { protected fun getDetachedNodesEndPosition(tree: IModelTree): PositionInRole { val detachedRole = RoleInNode(tree.getRootNodeId(), ITree.DETACHED_NODES_LINK) - val index = tree.getChildren(detachedRole.nodeId.toGlobal(tree.getId()), detachedRole.role).count().getBlocking(tree) + val index = tree.getChildren(detachedRole.nodeId.toGlobal(tree.getId()), detachedRole.role).count().getBlocking() return PositionInRole(detachedRole, index) } } diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt index da63ac666b..0992c3da2b 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildOp.kt @@ -63,20 +63,20 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept } override fun captureIntend(tree: IModelTree): IOperationIntend { - val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree) + val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking() return Intend(CapturedInsertPosition(position.index, children.toList())) } inner class Intend(val capturedPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - val parentExists = tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking(tree) - val childExists = IStream.many(childIdsAndConcepts).flatMap { tree.containsNode(it.first) }.toList().getBlocking(tree) + val parentExists = tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking() + val childExists = IStream.many(childIdsAndConcepts).flatMap { tree.containsNode(it.first) }.toList().getBlocking() val targetPosition = if (parentExists) { val newIndex = if (position.index < 0) { position.index } else { capturedPosition.findIndex( - tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree), + tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(), position.index, ) } @@ -91,7 +91,7 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept // handle already existing child IDs childIdsAndConcepts.zip(childExists).asReversed().flatMap { (idAndConcept, exists) -> val (childId, childConcept) = idAndConcept - val currentContainment = tree.getContainment(childId).getBlocking(tree) + val currentContainment = tree.getContainment(childId).getBlocking() // If the containment is correct, ignore the index to avoid unnecessary changes. As part of the // conflict resolution algorithm we are allowed to make any decision. There are no right or wrong @@ -105,7 +105,7 @@ open class AddNewChildrenOp(val position: PositionInRole, val childIdsAndConcept } if (exists) { - if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking(tree)) { + if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking()) { emptyList() } else { listOf(MoveNodeOp(childId, targetPosition)) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt index 12603c82d0..e0b3c30b62 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/AddNewChildSubtreeOp.kt @@ -28,15 +28,15 @@ class AddNewChildSubtreeOp( } override fun apply(tree: IMutableModelTree): IAppliedOperation { - decompress().iterateBlocking(resultTreeHash.graph) { it.apply(tree) } + decompress().iterateBlocking { it.apply(tree) } return Applied() } fun decompress(): IStream.Many { val resultTree = getResultTree() return resultTree.getDescendants(childId.toGlobal(resultTree.getId()), true).flatMapOrdered { node -> - val parent = resultTree.getParent(node).getBlocking(resultTree)!! - val roleInParent = resultTree.getRoleInParent(node).getBlocking(resultTree)!! + val parent = resultTree.getParent(node).getBlocking()!! + val roleInParent = resultTree.getRoleInParent(node).getBlocking()!! val pos = PositionInRole( parent, roleInParent, @@ -45,8 +45,8 @@ class AddNewChildSubtreeOp( decompressNode(resultTree, node, pos, false) } + resultTree.getDescendants(childId.toGlobal(resultTree.getId()), true).flatMapOrdered { node -> - val parent = resultTree.getParent(node).getBlocking(resultTree)!! - val roleInParent = resultTree.getRoleInParent(node).getBlocking(resultTree)!! + val parent = resultTree.getParent(node).getBlocking()!! + val roleInParent = resultTree.getRoleInParent(node).getBlocking()!! val pos = PositionInRole( parent, roleInParent, @@ -64,7 +64,7 @@ class AddNewChildSubtreeOp( SetReferenceOp(node, role, target) } } else { - IStream.of(AddNewChildOp(position!!, node, tree.getConceptReference(node).getBlocking(tree))) + + IStream.of(AddNewChildOp(position!!, node, tree.getConceptReference(node).getBlocking())) + tree.getProperties(node).map { (property, value) -> SetPropertyOp(node, property, value) } } } @@ -82,20 +82,20 @@ class AddNewChildSubtreeOp( .getDescendants(childId.toGlobal(resultTree.getId()), true) .map { DeleteNodeOp(it) } .toList() - .getBlocking(resultTree.asObject().graph) + .getBlocking() .asReversed() } } override fun captureIntend(tree: IModelTree): IOperationIntend { - val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree) + val children = tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking() return Intend(CapturedInsertPosition(position.index, children)) } inner class Intend(val capturedPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - if (tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { - val newIndex = capturedPosition.findIndex(tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking(tree)) + if (tree.containsNode(position.nodeId.toGlobal(tree.getId())).getBlocking()) { + val newIndex = capturedPosition.findIndex(tree.getChildren(position.nodeId.toGlobal(tree.getId()), position.role).toList().getBlocking()) return listOf(withPosition(position.withIndex(newIndex))) } else { return listOf(withPosition(getDetachedNodesEndPosition(tree))) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt index 3e8048d673..7c037f2f50 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/BulkUpdateOp.kt @@ -56,7 +56,7 @@ class BulkUpdateOp( override fun restoreIntend(tree: IModelTree): List { // The intended change is to put the model into the given state. Any concurrent change can just be // overwritten as long as the subtree root as the starting point still exists. - return if (tree.containsNode(subtreeRootId.toGlobal(tree.getId())).getBlocking(tree)) { + return if (tree.containsNode(subtreeRootId.toGlobal(tree.getId())).getBlocking()) { listOf(getOriginalOp()) } else { listOf(NoOp()) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt index a6ba48a1bf..9170779a89 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/DeleteNodeOp.kt @@ -33,8 +33,8 @@ class DeleteNodeOp(val childId: INodeReference) : AbstractOperation(), IOperatio } override fun restoreIntend(tree: IModelTree): List { - if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking(tree)) return listOf(NoOp()) - val allChildren = tree.getChildren(childId.toGlobal(tree.getId())).toList().getBlocking(tree) + if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking()) return listOf(NoOp()) + val allChildren = tree.getChildren(childId.toGlobal(tree.getId())).toList().getBlocking() if (allChildren.isNotEmpty()) { val targetPos = getDetachedNodesEndPosition(tree) return allChildren diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt index 34049a6045..30b24d4886 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/MoveNodeOp.kt @@ -44,24 +44,24 @@ class MoveNodeOp(val childId: INodeReference, val targetPosition: PositionInRole override fun captureIntend(tree: IModelTree): IOperationIntend { val capturedTargetPosition = CapturedInsertPosition( targetPosition.index, - tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(tree), + tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(), ) return Intend(capturedTargetPosition) } inner class Intend(val capturedTargetPosition: CapturedInsertPosition) : IOperationIntend { override fun restoreIntend(tree: IModelTree): List { - if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking(tree)) return emptyList() + if (!tree.containsNode(childId.toGlobal(tree.getId())).getBlocking()) return emptyList() val newSourcePosition = getNodePosition(tree, childId.toGlobal(tree.getId())) - if (!tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { + if (!tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking()) { return listOf( withPos(getDetachedNodesEndPosition(tree)), ) } - if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking(tree)) return emptyList() - val newTargetPosition = if (tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking(tree)) { + if (tree.getAncestors(targetPosition.nodeId.toGlobal(tree.getId()), false).contains(childId.toGlobal(tree.getId())).getBlocking()) return emptyList() + val newTargetPosition = if (tree.containsNode(targetPosition.nodeId.toGlobal(tree.getId())).getBlocking()) { val newTargetIndex = capturedTargetPosition.findIndex( - tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(tree), + tree.getChildren(targetPosition.nodeId.toGlobal(tree.getId()), targetPosition.role).toList().getBlocking(), targetPosition.index, ) targetPosition.withIndex(newTargetIndex) diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt index 5e146d1739..d93c9baa4f 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetConceptOp.kt @@ -31,7 +31,7 @@ class SetConceptOp(val nodeId: INodeReference, val concept: ConceptReference?) : override fun getOriginalOp(): IOperation = this override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt index f4961df74f..1aeddd62d7 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetPropertyOp.kt @@ -21,7 +21,7 @@ class SetPropertyOp(val nodeId: INodeReference, val role: IPropertyReference, va } override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(nodeId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt index 23780ce220..1673410378 100644 --- a/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt +++ b/model-datastructure/src/commonMain/kotlin/org/modelix/model/operations/SetReferenceOp.kt @@ -22,7 +22,7 @@ class SetReferenceOp(val sourceId: INodeReference, val role: IReferenceLinkRefer } override fun restoreIntend(tree: IModelTree): List { - return if (tree.containsNode(sourceId.toGlobal(tree.getId())).getBlocking(tree)) listOf(this) else listOf(NoOp()) + return if (tree.containsNode(sourceId.toGlobal(tree.getId())).getBlocking()) listOf(this) else listOf(NoOp()) } override fun captureIntend(tree: IModelTree) = this diff --git a/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt b/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt index 09d67a65ab..693ef60861 100644 --- a/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt +++ b/model-datastructure/src/commonTest/kotlin/DuplicateImportConflictTest.kt @@ -144,20 +144,20 @@ class DuplicateImportConflictTest : TreeTestBase() { } private fun assertSameTree(tree1: IGenericModelTree, tree2: IGenericModelTree) { - val changes = tree2.getChanges(tree1, false).toList().getBlocking(tree2) + val changes = tree2.getChanges(tree1, false).toList().getBlocking() for (changeEvent in changes) { when (changeEvent) { is ChildrenChangedEvent -> { - val children1 = tree1.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking(tree1) - val children2 = tree2.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking(tree2) + val children1 = tree1.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking() + val children2 = tree2.getChildren(changeEvent.nodeId, changeEvent.role).toList().getBlocking() println(changeEvent) println(" children1: $children1") println(" children2: $children2") } is ReferenceChangedEvent -> { - val target1 = tree1.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking(tree1) - val target2 = tree2.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking(tree2) + val target1 = tree1.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking() + val target2 = tree2.getReferenceTarget(changeEvent.nodeId, changeEvent.role).getBlocking() println(changeEvent) println(" target1: $target1") println(" target2: $target2") diff --git a/model-datastructure/src/commonTest/kotlin/HamtTest.kt b/model-datastructure/src/commonTest/kotlin/HamtTest.kt index 0a746912d7..f3e8331f07 100644 --- a/model-datastructure/src/commonTest/kotlin/HamtTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HamtTest.kt @@ -35,25 +35,25 @@ class HamtTest { // add entry val key = rand.nextInt(1000).toLong() val value = rand.nextLong() - hamt = hamt.put(key, createEntry(value, graph)).getBlocking(hamt) + hamt = hamt.put(key, createEntry(value, graph)).getBlocking() expectedMap[key] = value } else { val keys: List = ArrayList(expectedMap.keys) val key = keys[rand.nextInt(keys.size)] if (rand.nextBoolean()) { // remove entry - hamt = hamt.remove(key).getBlocking(hamt) + hamt = hamt.remove(key).getBlocking() expectedMap.remove(key) } else { // replace entry val value = rand.nextLong() - hamt = hamt.put(key, createEntry(value, graph)).getBlocking(hamt) + hamt = hamt.put(key, createEntry(value, graph)).getBlocking() expectedMap[key] = value } } storeCache.clearCache() for ((key, value) in expectedMap) { - assertEquals(value, hamt.get(key).flatMapZeroOrOne { it.resolve() }.getBlocking(hamt)!!.data.id) + assertEquals(value, hamt.get(key).flatMapZeroOrOne { it.resolve() }.getBlocking()!!.data.id) } } } @@ -83,17 +83,17 @@ class HamtTest { valueConfig = ObjectReferenceDataTypeConfiguration(graph, CPNode), ) var hamt = HamtTree(HamtInternalNode.createEmpty(config)) - var getId = { e: IStream.ZeroOrOne> -> e.flatMapZeroOrOne { it.resolve() }.getBlocking(hamt)!!.data.id } + var getId = { e: IStream.ZeroOrOne> -> e.flatMapZeroOrOne { it.resolve() }.getBlocking()!!.data.id } - hamt = hamt.put(965L, createEntry(-6579471327666419615, graph)).getBlocking(hamt) - hamt = hamt.put(949L, createEntry(4912341421267007347, graph)).getBlocking(hamt) + hamt = hamt.put(965L, createEntry(-6579471327666419615, graph)).getBlocking() + hamt = hamt.put(949L, createEntry(4912341421267007347, graph)).getBlocking() assertEquals(4912341421267007347, getId(hamt.get(949L))) - hamt = hamt.put(260L, createEntry(4166750678024106842, graph)).getBlocking(hamt) + hamt = hamt.put(260L, createEntry(4166750678024106842, graph)).getBlocking() assertEquals(4166750678024106842, getId(hamt.get(260L))) - hamt = hamt.put(794L, createEntry(5492533034562136353, graph)).getBlocking(hamt) - hamt = hamt.put(104L, createEntry(-6505928823483070382, graph)).getBlocking(hamt) - hamt = hamt.put(47L, createEntry(3122507882718949737, graph)).getBlocking(hamt) - hamt = hamt.put(693L, createEntry(-2086105010854963537, graph)).getBlocking(hamt) + hamt = hamt.put(794L, createEntry(5492533034562136353, graph)).getBlocking() + hamt = hamt.put(104L, createEntry(-6505928823483070382, graph)).getBlocking() + hamt = hamt.put(47L, createEntry(3122507882718949737, graph)).getBlocking() + hamt = hamt.put(693L, createEntry(-2086105010854963537, graph)).getBlocking() storeCache.clearCache() // assertEquals(69239088, (hamt!!.getData() as CPHamtInternal).bitmap) // assertEquals(6, (hamt!!.getData() as CPHamtInternal).children.count()) @@ -130,8 +130,8 @@ class HamtTest { for (i in 1..10) { var map = emptyMap - entries.entries.shuffled(rand).forEach { map = map.put(it.key, it.value).getBlocking(map) } - keysToRemove.forEach { map = map.remove(it).getBlocking(map) } + entries.entries.shuffled(rand).forEach { map = map.put(it.key, it.value).getBlocking() } + keysToRemove.forEach { map = map.remove(it).getBlocking() } val hash = map.asObject().getHashString() if (i == 1) { expectedHash = hash diff --git a/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt b/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt index c8b3768788..3ce1c3c405 100644 --- a/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HistoryIndexNodeAttributesTest.kt @@ -67,7 +67,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val node = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(AttributeValuesAggregation.of("staging", "prod"), node.data.attributes.getValues("env")) assertEquals(AttributeValuesAggregation.of("1"), node.data.attributes.getValues("run")) } @@ -80,7 +80,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val node = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(AttributeValuesAggregation.of("a", "b"), node.data.attributes["env"]) assertEquals(AttributeValuesAggregation.of("1"), node.data.attributes["run"]) } @@ -104,7 +104,7 @@ class HistoryIndexNodeAttributesTest { val graph = v1.graph val original = HistoryIndexNode.of(v1.obj).asObject(graph) .merge(HistoryIndexNode.of(v2.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() val serialized = original.data.serialize() // Range node serializes attributes in field index 8 (0-based, LEVEL1-delimited). // Extract and deserialize just that field rather than the full node, because the full diff --git a/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt b/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt index 3bdf3d3b10..ef65cd7e02 100644 --- a/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt +++ b/model-datastructure/src/commonTest/kotlin/HistoryIndexTest.kt @@ -67,7 +67,7 @@ class HistoryIndexTest { val graph = version1.graph val history1 = HistoryIndexNode.of(version1.obj, version2.obj).asObject(graph) val history2 = HistoryIndexNode.of(version3.obj).asObject(graph) - val history = history1.merge(history2).getBlocking(graph) + val history = history1.merge(history2).getBlocking() assertEquals(3, history.size) assertEquals(3, history.height) } @@ -82,7 +82,7 @@ class HistoryIndexTest { val history = HistoryIndexNode.of(version1.obj, version2.obj).asObject(graph) .merge(HistoryIndexNode.of(version3.obj).asObject(graph)) .merge(HistoryIndexNode.of(version4.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(4, history.size) assertEquals(4, history.height) } @@ -99,7 +99,7 @@ class HistoryIndexTest { .merge(HistoryIndexNode.of(version3.obj).asObject(graph)) .merge(HistoryIndexNode.of(version4.obj).asObject(graph)) .merge(HistoryIndexNode.of(version5.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals(5, history.size) assertEquals(4, history.height) } @@ -135,16 +135,16 @@ class HistoryIndexTest { .merge(HistoryIndexNode.of(version4b.obj).asObject(graph)) .merge(HistoryIndexNode.of(version5b.obj).asObject(graph)) .merge(HistoryIndexNode.of(version6b.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() val history = historyA .merge(historyB) .merge(HistoryIndexNode.of(version7.obj).asObject(graph)) .merge(HistoryIndexNode.of(version8.obj).asObject(graph)) - .getBlocking(graph) + .getBlocking() assertEquals( listOf(version1, version2, version3a, version3b, version4a, version4b, version5a, version5b, version6b, version7, version8).map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.size) assertEquals(5, history.height) @@ -158,12 +158,12 @@ class HistoryIndexTest { } val graph = versions.first().graph val history = versions.drop(1).fold(HistoryIndexNode.of(versions.first().asObject()).asObject(graph)) { acc, it -> - acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking(graph) + acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking() } assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.height) } @@ -176,12 +176,12 @@ class HistoryIndexTest { } val graph = versions.first().graph val history = versions.drop(1).shuffled(Random(78234554)).fold(HistoryIndexNode.of(versions.first().asObject()).asObject(graph)) { acc, it -> - acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking(graph) + acc.merge(HistoryIndexNode.of(it.asObject()).asObject(graph)).getBlocking() } assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(13, history.height) } @@ -198,7 +198,7 @@ class HistoryIndexTest { assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(11, history.height) } @@ -215,7 +215,7 @@ class HistoryIndexTest { assertEquals(versions.size.toLong(), history.size) assertEquals( versions.map { it.getObjectHash() }, - history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(graph), + history.data.getAllVersions().map { it.getHash() }.toList().getBlocking(), ) assertEquals(14, history.height) } @@ -228,6 +228,6 @@ class HistoryIndexTest { val centerIndex = versions.size / 2 return buildHistory(versions.subList(0, centerIndex)) .merge(buildHistory(versions.subList(centerIndex, versions.size))) - .getBlocking(graph) + .getBlocking() } } diff --git a/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt b/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt index 6b8b969c9f..682d03cabf 100644 --- a/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt +++ b/model-datastructure/src/commonTest/kotlin/ObjectDiffTest.kt @@ -87,9 +87,9 @@ class ObjectDiffTest { newTree = changeGenerator2.applyRandomChange(newTree, null) } - val diff = newTree.getTreeObject().objectDiff(initialTree.getTreeObject()).toList().getBlocking(store1) - val initialObjects = initialTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking(store1) - val newObjects = newTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking(store1) + val diff = newTree.getTreeObject().objectDiff(initialTree.getTreeObject()).toList().getBlocking() + val initialObjects = initialTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking() + val newObjects = newTree.getTreeObject().getDescendantsAndSelf().toList().getBlocking() val unnecessaryObjects = (diff.associateBy { it.getHashString() } - newObjects.map { it.getHashString() }.toSet()).values.toSet() assertEquals(emptySet(), unnecessaryObjects) diff --git a/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt b/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt index 578a87ea63..084a623d74 100644 --- a/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt +++ b/model-datastructure/src/commonTest/kotlin/RandomTreeChangeGenerator.kt @@ -295,7 +295,7 @@ private class TransactionAdapter(val transaction: IGenericMutableModelTree.Write override fun getChildren(parentId: Long, role: String?): Iterable { return transaction.tree.getChildren(parentId.translate(), IChildLinkReference.fromString(role)) - .map { it.translate() }.toList().getBlocking(transaction.tree) + .map { it.translate() }.toList().getBlocking() } override fun getAllChildren(parentId: Long): Iterable { diff --git a/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt b/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt index c4dffce19b..661cf18daa 100644 --- a/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt +++ b/model-datastructure/src/commonTest/kotlin/VersionAttributesTest.kt @@ -29,7 +29,7 @@ class VersionAttributesTest { val restoredVersion = graph2.loadObjects( rootHash = originalVersion.getObjectHash(), rootDeserializer = CPVersion, - receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(graph1), + receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(), ).let { CLVersion(it) } assertEquals(mapOf(key to value), restoredVersion.getAttributes()) @@ -55,7 +55,7 @@ class VersionAttributesTest { val restoredVersion = graph2.loadObjects( rootHash = originalVersion.getObjectHash(), rootDeserializer = CPVersion, - receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(graph1), + receivedObjects = originalVersion.asObject().getDescendantsAndSelf().toMap({ it.getHash() }, { it.data.serialize() }).getBlocking(), ).let { CLVersion(it) } assertEquals(attrs, restoredVersion.getAttributes()) diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt index 675bba3ab9..38ecf48f86 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/LionwebApiImpl.kt @@ -91,10 +91,7 @@ class LionwebApiImpl(val repoManager: IRepositoriesManager) : LionwebApi() { // TODO use an index to find the nodes by their foreign ID (aka original ID) if (foreignIds.isNotEmpty()) { -// version.obj.graph.getStreamExecutor().iterateSuspending({ -// version.tree.nodesMap.getEntries().flatMap { it.second.resolve() } -// .filter { foreignIds.contains(it.data.getPropertyValue(NodeData.ID_PROPERTY_KEY)) } -// }) { +// version.obj.graph.getStreamExecutor().iterateSuspending { // modelixIds.add(it.data.id) // } } diff --git a/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt b/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt index 3bc0af6084..24d4a83ebe 100644 --- a/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt +++ b/model-server/src/main/kotlin/org/modelix/model/server/handlers/RepositoriesManager.kt @@ -361,14 +361,14 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { // Deleting the root node isn't allowed val mainTree = newVersion.getModelTree() - check(mainTree.containsNode(mainTree.getRootNodeId()).getBlocking(mainTree)) + check(mainTree.containsNode(mainTree.getRootNodeId()).getBlocking()) // ensure there are no missing objects // newVersion.graph.getStreamExecutor().iterate({ newVersion.diff(oldVersion) }) { } if (oldVersion != null) { // If the object diff is buggy, client and server will skip over the same objects. // The model diff should also iterate over all new objects and is used for additional validation. - newVersion.getModelTree().getChanges(oldVersion.getModelTree(), false).iterateBlocking(newVersion.getModelTree()) { } + newVersion.getModelTree().getChanges(oldVersion.getModelTree(), false).iterateBlocking { } } // TODO check invariants of the model (consistent parent-child relations, single root, containment cycles) @@ -380,7 +380,7 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { check(newVersion.operations.count() == 0) } else { val mutableTree = baseVersion.getModelTree().asMutableSingleThreaded() - newVersion.operationsAsStream().iterateBlocking(mainTree) { op -> + newVersion.operationsAsStream().iterateBlocking { op -> op.apply(mutableTree) } check(mutableTree.getTransaction().tree.getHash() == newVersion.getModelTree().getHash()) { @@ -438,8 +438,8 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { val newElement = HistoryIndexNode.of(version.obj).asObject(graph) val newIndex = when (parentIndices.size) { 0 -> newElement - 1 -> parentIndices.single().merge(newElement).getBlocking(graph) - 2 -> parentIndices[0].merge(parentIndices[1]).merge(newElement).getBlocking(graph) + 1 -> parentIndices.single().merge(newElement).getBlocking() + 2 -> parentIndices[0].merge(parentIndices[1]).merge(newElement).getBlocking() else -> error("impossible") } newIndex.write() @@ -613,7 +613,7 @@ class RepositoriesManager(val stores: StoreManager) : IRepositoriesManager { val version = CLVersion.loadFromHash(versionHash, sourceStore) // Use diff with empty list to get all objects reachable from this version - version.diff(emptyList()).iterateBlocking(sourceStore) { obj -> + version.diff(emptyList()).iterateBlocking { obj -> reachableHashes.add(obj.getHashString()) } } diff --git a/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt b/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt index 4bd2434066..19a1827c92 100644 --- a/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt +++ b/model-server/src/test/kotlin/org/modelix/model/server/ReplicatedRepositoryTest.kt @@ -454,7 +454,7 @@ private fun getChildren(modelTree: IMutableModelTree): SortedSet getChildren(it.tree) } private fun getChildren(modelTree: IGenericModelTree): SortedSet = - modelTree.getChildren(modelTree.getRootNodeId()).toList().getBlocking(modelTree) + modelTree.getChildren(modelTree.getRootNodeId()).toList().getBlocking() .toSortedSet(compareBy { it.serialize() }) private fun IMutableModelTree.treeHash(): String = runRead { t -> t.tree.asObject().getHashString() } diff --git a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt index 068f24843e..1a924c5fec 100644 --- a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt +++ b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitExportTest.kt @@ -95,7 +95,7 @@ class GitExportTest : MPSTestBase() { val reimportedVersion = client.pull(modelixBranchReimport, null) // check that no changes got lost during the round-trip - val diff = reimportedVersion.getModelTree().getChanges(modifiedVersion.getModelTree(), false).toList().getSuspending(reimportedVersion.asObject().graph) + val diff = reimportedVersion.getModelTree().getChanges(modifiedVersion.getModelTree(), false).toList().getSuspending() val changes = ArrayList() for (event in diff) { when (event) { @@ -105,9 +105,9 @@ class GitExportTest : MPSTestBase() { is NodeRemovedEvent -> TODO() is ChildrenChangedEvent -> TODO() is PropertyChangedEvent -> { - changes += modifiedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending(modifiedVersion.asObject().graph) + + changes += modifiedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending() + " -> " + - reimportedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending(reimportedVersion.asObject().graph) + reimportedVersion.getModelTree().getProperty(event.nodeId, event.role).getSuspending() } is ReferenceChangedEvent -> TODO() } diff --git a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt index 18e2c19370..cb517b8479 100644 --- a/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt +++ b/mps-git-import-plugin/src/test/kotlin/org/modelix/mps/gitimport/GitImportTest.kt @@ -583,7 +583,7 @@ class GitImportTest : MPSTestBase() { for (version in latestVersion.historyAsSequence()) { for (parentVersion in version.getParentVersions()) { val baseObjects = parentVersion.getModelTree().asObject().getDescendantsAndSelf().map { it.getHash() }.toList() - .getBlocking(parentVersion.asObject().graph) + .getBlocking() val baseObjectsSet = baseObjects.toSet() // no object is returned twice @@ -602,13 +602,13 @@ class GitImportTest : MPSTestBase() { it } .toList() - .getBlocking(parentVersion.asObject().graph) + .getBlocking() // also the delta itself doesn't contain duplicate objects val duplicateObjects: List> = deltaObjects.groupBy { it.getHash() }.filter { it.value.size > 1 }.map { it.value.first() } if (duplicateObjects.isNotEmpty()) { // place breakpoint here for debugging - version.diff(parentVersion, filter).toList().getBlocking(parentVersion.asObject().graph) + version.diff(parentVersion, filter).toList().getBlocking() } assertEquals(emptyList>(), duplicateObjects) @@ -617,12 +617,12 @@ class GitImportTest : MPSTestBase() { if (unnecessaryObjects.isNotEmpty()) { // just for debugging parentVersion.getModelTree().asObject().getDescendantsAndSelf().toList() - .getBlocking(parentVersion.asObject().graph).joinToString("\n") { it.toString() } + .getBlocking().joinToString("\n") { it.toString() } .let { File("oldVersion.txt").writeText(it) } version.getModelTree().asObject().getDescendantsAndSelf().toList() - .getBlocking(parentVersion.asObject().graph).joinToString("\n") { it.toString() } + .getBlocking().joinToString("\n") { it.toString() } .let { File("newVersion.txt").writeText(it) } - version.diff(parentVersion, filter).toList().getBlocking(parentVersion.asObject().graph) + version.diff(parentVersion, filter).toList().getBlocking() } assertEquals(emptyList>(), unnecessaryObjects) } diff --git a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt index 2edaf21d16..7ae6bcfb86 100644 --- a/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt +++ b/mps-sync-plugin3/src/main/kotlin/org/modelix/mps/sync3/BindingWorker.kt @@ -446,7 +446,7 @@ class BindingWorker( val node = sourceModel.tryResolveNode(nodeId) ?: return invalidationTree.invalidate(node, false) } - newTree.getChanges(baseVersion.getModelTree(), changesOnly = true).iterateSuspending(newTree.asObject().graph) { event -> + newTree.getChanges(baseVersion.getModelTree(), changesOnly = true).iterateSuspending { event -> when (event) { is ContainmentChangedEvent, is NodeRemovedEvent, is NodeAddedEvent -> { // There will be a ChildrenChangedEvent that indirectly handles these cases. diff --git a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt index 8f8e6be9e9..c287e51f69 100644 --- a/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt +++ b/mps-sync-plugin3/src/test/kotlin/org/modelix/mps/sync3/ProjectSyncTest.kt @@ -198,7 +198,7 @@ class ProjectSyncTest : ProjectSyncTestBase() { assertNotEquals(version2.getObjectHash(), version3.getObjectHash()) // version4 is the result of merging version3 into version2 - val diff = version4.getModelTree().getChanges(version2.getModelTree(), false).toList().getBlocking(version4.getModelTree()) + val diff = version4.getModelTree().getChanges(version2.getModelTree(), false).toList().getBlocking() assertEquals(emptyList>(), diff) assertEquals(version2.getTreeReference().getHash(), version4.getTreeReference().getHash()) @@ -291,8 +291,8 @@ class ProjectSyncTest : ProjectSyncTestBase() { assertEquals(1, changes.size) val change = changes.single() as PropertyChangedEvent assertEquals(MPSProperty(nameProperty).getUID(), change.role.getUID()) - assertEquals("MyClass", version1.getModelTree().getProperty(change.nodeId, change.role).getBlocking(version1.getModelTree())) - assertEquals("Changed", version2.getModelTree().getProperty(change.nodeId, change.role).getBlocking(version1.getModelTree())) + assertEquals("MyClass", version1.getModelTree().getProperty(change.nodeId, change.role).getBlocking()) + assertEquals("Changed", version2.getModelTree().getProperty(change.nodeId, change.role).getBlocking()) } fun `test descendants of new node are synchronized`() = runChangeInMpsTest { classNode -> diff --git a/streams-redesign.md b/streams-redesign.md index 74eab38abe..fde756a3af 100644 --- a/streams-redesign.md +++ b/streams-redesign.md @@ -163,7 +163,30 @@ These were latent couplings to a transitive dependency, not uses of the `streams --- -## 5. Behavioral tradeoffs +## 5. Follow-up: the executor is no longer required to run a stream + +Because batching is now structural — a fetch leaf carries its own `IBulkExecutor` source — a `Step` is fully +self-contained. The `IStreamExecutor` passed to the terminal operations no longer carries any batching context; it +only supplied a batch size and (for `BulkRequestStreamExecutor`) set `IStreamExecutor.CONTEXT`. Two changes removed +that residual coupling: + +- **Batch size moved onto the source.** `IBulkExecutor` now declares `val batchSize` (default + `DEFAULT_BULK_REQUEST_BATCH_SIZE = 5000`); the driver chunks each source's keys to *its own* batch size. The driver + no longer takes a batch-size parameter. `BulkRequestStreamExecutor(source, batchSize)` keeps working — it exposes + the constructor batch size on the source the leaves bind to. +- **Executor-less terminals.** New `getBlocking()` / `getSuspending()` / `iterateBlocking { }` / + `iterateSuspending { }` / `executeBlocking()` / `executeSuspending()` drive a fresh execution directly. The + `*(executor: IStreamExecutor | IStreamExecutorProvider)` overloads are `@Deprecated(ReplaceWith(...))` and keep their + original behavior for compatibility. + +All in-repo call sites (~180, mostly `getBlocking`) were migrated to the executor-less form. `IStreamExecutor` / +`IStreamExecutorProvider` and `BulkRequestStreamExecutor.enqueue` remain — `enqueue` is still how fetch leaves are +created, and `IStreamExecutor.CONTEXT` is still set during `BulkRequestStreamExecutor` runs because ModelQL resolves +the "current executor" via `IStreamExecutor.getInstance()`. + +--- + +## 6. Behavioral tradeoffs These follow from the "no incremental emission" decision and are intentional. They change performance characteristics, not results. @@ -178,10 +201,13 @@ not results. 3. **`take` / `skip` operate on materialized results** — they do not prune upstream fetches. 4. **`SimpleStreamExecutor` now batches** per source/round — strictly fewer round-trips than before. -## 6. Known limitations / future work +## 7. Known limitations / future work - **Within-round stack safety.** The round driver trampolines across `Blocked` (the common fetch-dependent case). A pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode `Step` as a stack-safe free monad (explicit interpreter loop) if needed. - **Optional streaming `iterate*`** — see tradeoff #1. - **Restore `cached()` memoization** if ModelQL recompute cost proves material. +- **Retire the executor entirely.** With the executor no longer required to run a stream (§5), `IStreamExecutor` / + `IStreamExecutorProvider` could be removed over time — the remaining users are `enqueue` (fetch-leaf creation) and + ModelQL's `getInstance()`-based "current executor" lookup, both of which can be reworked. diff --git a/streams/README.md b/streams/README.md index 801983bf3a..c8054a5e99 100644 --- a/streams/README.md +++ b/streams/README.md @@ -52,7 +52,7 @@ scheduler and nothing allocated. The driver (`Execution.drive` / `driveSuspending`) is a loop where each iteration is one batch round: -1. issue **one bulk call per data source** (chunked to `batchSize`), and run any async leaves; +1. issue **one bulk call per data source** (chunked to that source's `IBulkExecutor.batchSize`), and run any async leaves; 2. fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds); 3. `resume()` and repeat until `Done`. @@ -60,7 +60,10 @@ The loop is the **trampoline** that keeps fetch-dependent chains stack-safe rega Batching is **structural**: a fetch leaf carries its own data source, so the driver groups fetches per source per round regardless of which executor runs it. `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to -its `IBulkExecutor`. +its `IBulkExecutor`. Because the source is carried by the stream, the **batch size lives on `IBulkExecutor`** (not on +the executor), and **terminal operations don't need an executor**: `getBlocking()` / `getSuspending()` / +`iterateBlocking { }` / `iterateSuspending { }` / `executeBlocking()` drive the self-contained stream directly. The +executor-taking overloads are `@Deprecated` and kept for compatibility. ## Public API @@ -68,9 +71,11 @@ Cardinality is encoded in the type: `IStream.Many` (0+), `IStream.ZeroOrOne` (0. `IStream.One` (exactly 1), and `IStream.Completable` (completion, no value). - Builders: `IStream.of`, `empty`, `many`, `zip`, `fromFlow`, `singleFromCoroutine`, … -- Execution: `IStreamExecutor` (`query`, `iterate`, suspending variants), `SimpleStreamExecutor`, - `BulkRequestStreamExecutor`, `IExecutableStream`. -- Batchable source: `IBulkExecutor` (`execute` + `executeSuspending`). +- Terminals (no executor needed): `getBlocking()`, `getSuspending()`, `iterateBlocking { }`, + `iterateSuspending { }`, `executeBlocking()`. The `*(executor)` overloads remain, deprecated. +- Executors (still available; `BulkRequestStreamExecutor` also provides `enqueue`): `IStreamExecutor`, + `SimpleStreamExecutor`, `BulkRequestStreamExecutor`, `IExecutableStream`. +- Batchable source: `IBulkExecutor` (`execute` + `executeSuspending` + `batchSize`). ## Layout diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt index 583614a95d..79e8410748 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt @@ -6,48 +6,68 @@ import org.modelix.streams.engine.drive import org.modelix.streams.engine.driveSuspending import org.modelix.streams.engine.fetchStep +/** Default maximum number of keys requested in a single bulk call. */ +const val DEFAULT_BULK_REQUEST_BATCH_SIZE: Int = 5000 + interface IBulkExecutor { fun execute(keys: List): Map suspend fun executeSuspending(keys: List): Map + + /** + * Maximum number of keys to request in a single [execute]/[executeSuspending] call. Each batch round chunks this + * source's pending keys to this size. The batch size is a property of the data source itself, so it applies + * regardless of which executor drives the stream. + */ + val batchSize: Int get() = DEFAULT_BULK_REQUEST_BATCH_SIZE } /** - * Executor that batches the individual fetches enqueued via [enqueue] into bulk calls against [bulkExecutor]. + * Executor that batches the individual fetches enqueued via [enqueue] into bulk calls against the wrapped source. * - * Batching is structural: [enqueue] returns a stream backed by a fetch leaf bound to [bulkExecutor], and the round - * driver groups all fetches reachable in a round (across independent stream branches) into a single - * [IBulkExecutor.execute] call, chunked to [batchSize]. Dependent fetches (reached through `flatMap`) fall into later - * rounds. This replaces the previous Reaktive subscribe-collect-batch mechanism. + * Since the [Step] engine batches structurally (per source per round) and the batch size now lives on the + * [IBulkExecutor] itself, this type is no longer required for batching to work — any executor (e.g. + * [SimpleStreamExecutor]) drives the same fetch leaves with the same batching. It is retained for API compatibility + * and as the holder of [enqueue]. */ class BulkRequestStreamExecutor( - private val bulkExecutor: IBulkExecutor, - val batchSize: Int = 5000, + bulkExecutor: IBulkExecutor, + val batchSize: Int = DEFAULT_BULK_REQUEST_BATCH_SIZE, ) : IStreamExecutor, IStreamExecutorProvider { + // Honor the batch size passed to this constructor by exposing it on the source the fetch leaves are bound to. + private val source: IBulkExecutor = + if (bulkExecutor.batchSize == batchSize) { + bulkExecutor + } else { + object : IBulkExecutor by bulkExecutor { + override val batchSize: Int get() = this@BulkRequestStreamExecutor.batchSize + } + } + override fun getStreamExecutor(): IStreamExecutor = this @Suppress("UNCHECKED_CAST") fun enqueue(key: K): IStream.ZeroOrOne = - StreamImpl { execution -> fetchStep(execution, bulkExecutor as IBulkExecutor, key) as Step } + StreamImpl { execution -> fetchStep(execution, source as IBulkExecutor, key) as Step } override fun query(body: () -> IStream.One): T { return IStreamExecutor.CONTEXT.computeWith(this) { val execution = Execution() - execution.drive(body().asStep(execution), batchSize).single() + execution.drive(body().asStep(execution)).single() } } override suspend fun querySuspending(body: suspend () -> IStream.One): T { return IStreamExecutor.CONTEXT.runInCoroutine(this) { val execution = Execution() - execution.driveSuspending(body().asStep(execution), batchSize).single() + execution.driveSuspending(body().asStep(execution)).single() } } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { IStreamExecutor.CONTEXT.computeWith(this) { val execution = Execution() - execution.drive(streamProvider().asStep(execution), batchSize).forEach(visitor) + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } } @@ -57,7 +77,7 @@ class BulkRequestStreamExecutor( ) { IStreamExecutor.CONTEXT.runInCoroutine(this) { val execution = Execution() - execution.driveSuspending(streamProvider().asStep(execution), batchSize).forEach { visitor(it) } + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt index 26aaf5887f..946e332646 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt @@ -1,6 +1,9 @@ package org.modelix.streams import org.modelix.kotlin.utils.ContextValue +import org.modelix.streams.engine.Execution +import org.modelix.streams.engine.drive +import org.modelix.streams.engine.driveSuspending /** * There reason that there are three different types of implementations if they all have different execution semantics. @@ -88,26 +91,90 @@ fun IStreamExecutor.asProvider(): IStreamExecutorProvider = SimpleStreamExecutor // fun IStream.One.getSynchronous(executor: IStreamExecutorProvider): T = getSynchronous(executor.getStreamExecutor()) // fun IStream.One.getSynchronous(executor: IStreamExecutor): T = executor.query { this } +// Terminal operations. The stream carries its own data source(s), so no executor is required to run it: each call +// drives a fresh execution, batching fetches per source per round (chunked to IBulkExecutor.batchSize). The +// executor-taking overloads are retained for compatibility and delegate to these. + +fun IStream.One.getBlocking(): T { + val execution = Execution() + return execution.drive(asStep(execution)).single() +} + +fun IStream.ZeroOrOne.getBlocking(): T? { + val execution = Execution() + return execution.drive(asStep(execution)).firstOrNull() +} + +suspend fun IStream.One.getSuspending(): T { + val execution = Execution() + return execution.driveSuspending(asStep(execution)).single() +} + +suspend fun IStream.ZeroOrOne.getSuspending(): T? { + val execution = Execution() + return execution.driveSuspending(asStep(execution)).firstOrNull() +} + +fun IStream.Many.iterateBlocking(visitor: (T) -> Unit) { + val execution = Execution() + execution.drive(asStep(execution)).forEach(visitor) +} + +suspend fun IStream.Many.iterateSuspending(visitor: suspend (T) -> Unit) { + val execution = Execution() + execution.driveSuspending(asStep(execution)).forEach { visitor(it) } +} + +fun IStream.Completable.executeBlocking() { + val execution = Execution() + execution.drive(asStep(execution)) +} + +suspend fun IStream.Completable.executeSuspending() { + val execution = Execution() + execution.driveSuspending(asStep(execution)) +} + +private const val EXECUTOR_DEPRECATION = "The executor is no longer required; the stream carries its own data source(s)." + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.One.getBlocking(executor: IStreamExecutorProvider): T = getBlocking(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.One.getBlocking(executor: IStreamExecutor): T = executor.query { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.ZeroOrOne.getBlocking(executor: IStreamExecutorProvider): T? = getBlocking(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getBlocking()")) fun IStream.ZeroOrOne.getBlocking(executor: IStreamExecutor): T? = executor.query { this.orNull() } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.One.getSuspending(executor: IStreamExecutorProvider): T = getSuspending(executor.getStreamExecutor()) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.One.getSuspending(executor: IStreamExecutor): T = executor.querySuspending { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutorProvider): T? = getSuspending(executor.getStreamExecutor()) -suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutor): T? = executor.querySuspending { this.orNull() } -// fun IStream.Many.iterateSynchronous(executor: IStreamExecutorProvider, visitor: (T) -> Unit): Unit = iterateSynchronous(executor.getStreamExecutor(), visitor) -// fun IStream.Many.iterateSynchronous(executor: IStreamExecutor, visitor: (T) -> Unit): Unit = executor.iterate({ this }, visitor) +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("getSuspending()")) +suspend fun IStream.ZeroOrOne.getSuspending(executor: IStreamExecutor): T? = executor.querySuspending { this.orNull() } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateBlocking(visitor)")) fun IStream.Many.iterateBlocking(executor: IStreamExecutorProvider, visitor: (T) -> Unit): Unit = iterateBlocking(executor.getStreamExecutor(), visitor) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateBlocking(visitor)")) fun IStream.Many.iterateBlocking(executor: IStreamExecutor, visitor: (T) -> Unit): Unit = executor.iterate({ this }, visitor) +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("executeBlocking()")) fun IStream.Completable.executeBlocking(executor: IStreamExecutorProvider): Unit = executor.execute { this } + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("executeSuspending()")) suspend fun IStream.Completable.executeSuspending(executor: IStreamExecutorProvider): Unit = executor.executeSuspending { this } +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateSuspending(visitor)")) suspend fun IStream.Many.iterateSuspending(executor: IStreamExecutorProvider, visitor: suspend (T) -> Unit): Unit = iterateSuspending(executor.getStreamExecutor(), visitor) + +@Deprecated(EXECUTOR_DEPRECATION, ReplaceWith("iterateSuspending(visitor)")) suspend fun IStream.Many.iterateSuspending(executor: IStreamExecutor, visitor: suspend (T) -> Unit): Unit = executor.iterateSuspending({ this }, visitor) diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt index 38f9018a81..28bdd72e94 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt @@ -5,24 +5,24 @@ import org.modelix.streams.engine.drive import org.modelix.streams.engine.driveSuspending /** - * Default executor. Drives streams with no batch-size limit. Fetches embedded in a stream (via - * [BulkRequestStreamExecutor.enqueue]) are still batched per source per round, so this is safe to use even when a - * stream contains data requests; it simply doesn't impose a maximum batch size. + * Default executor. Fetches embedded in a stream (via [BulkRequestStreamExecutor.enqueue]) are batched per source per + * round, chunked to each source's [IBulkExecutor.batchSize], so this is safe to use even when a stream contains data + * requests. */ object SimpleStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { val execution = Execution() - return execution.drive(body().asStep(execution), Int.MAX_VALUE).single() + return execution.drive(body().asStep(execution)).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { val execution = Execution() - return execution.driveSuspending(body().asStep(execution), Int.MAX_VALUE).single() + return execution.driveSuspending(body().asStep(execution)).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { val execution = Execution() - execution.drive(streamProvider().asStep(execution), Int.MAX_VALUE).forEach(visitor) + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } override suspend fun iterateSuspending( @@ -30,6 +30,6 @@ object SimpleStreamExecutor : IStreamExecutor { visitor: suspend (T) -> Unit, ) { val execution = Execution() - execution.driveSuspending(streamProvider().asStep(execution), Int.MAX_VALUE).forEach { visitor(it) } + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt index ebb6f0fc81..1f78898fc5 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt @@ -43,12 +43,12 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna override fun asFlow(): Flow = flow { val execution = Execution() - for (value in execution.driveSuspending(build(execution), Int.MAX_VALUE)) emit(value) + for (value in execution.driveSuspending(build(execution))) emit(value) } override fun asSequence(): Sequence { val execution = Execution() - return execution.drive(build(execution), Int.MAX_VALUE).asSequence() + return execution.drive(build(execution)).asSequence() } override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } @@ -173,20 +173,20 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna @DelicateModelixApi override fun iterateBlocking(visitor: (E) -> Unit) { val execution = Execution() - execution.drive(build(execution), Int.MAX_VALUE).forEach(visitor) + execution.drive(build(execution)).forEach(visitor) } @DelicateModelixApi override suspend fun iterateSuspending(visitor: suspend (E) -> Unit) { val execution = Execution() - execution.driveSuspending(build(execution), Int.MAX_VALUE).forEach { visitor(it) } + execution.driveSuspending(build(execution)).forEach { visitor(it) } } @Suppress("UNCHECKED_CAST") @DelicateModelixApi override fun getBlocking(): E { val execution = Execution() - val values = execution.drive(build(execution), Int.MAX_VALUE) + val values = execution.drive(build(execution)) return (if (values.isEmpty()) null else values.first()) as E } @@ -194,7 +194,7 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna @DelicateModelixApi override suspend fun getSuspending(): E { val execution = Execution() - val values = execution.driveSuspending(build(execution), Int.MAX_VALUE) + val values = execution.driveSuspending(build(execution)) return (if (values.isEmpty()) null else values.first()) as E } } @@ -208,12 +208,12 @@ internal class CompletableImpl(val build: (Execution) -> Step) : IStreamIn override fun asFlow(): Flow = flow { val execution = Execution() - for (value in execution.driveSuspending(build(execution), Int.MAX_VALUE)) emit(value) + for (value in execution.driveSuspending(build(execution))) emit(value) } override fun asSequence(): Sequence { val execution = Execution() - return execution.drive(build(execution), Int.MAX_VALUE).asSequence() + return execution.drive(build(execution)).asSequence() } override fun toList(): IStream.One> = StreamImpl { execution -> build(execution).mapValues { listOf(it) } } @@ -236,25 +236,25 @@ internal class CompletableImpl(val build: (Execution) -> Step) : IStreamIn @DelicateModelixApi override fun iterateBlocking(visitor: (Any?) -> Unit) { val execution = Execution() - execution.drive(build(execution), Int.MAX_VALUE).forEach(visitor) + execution.drive(build(execution)).forEach(visitor) } @DelicateModelixApi override suspend fun iterateSuspending(visitor: suspend (Any?) -> Unit) { val execution = Execution() - execution.driveSuspending(build(execution), Int.MAX_VALUE).forEach { visitor(it) } + execution.driveSuspending(build(execution)).forEach { visitor(it) } } @DelicateModelixApi override fun executeBlocking() { val execution = Execution() - execution.drive(build(execution), Int.MAX_VALUE) + execution.drive(build(execution)) } @DelicateModelixApi override suspend fun executeSuspending() { val execution = Execution() - execution.driveSuspending(build(execution), Int.MAX_VALUE) + execution.driveSuspending(build(execution)) } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt index 27f2c75c91..f1174f9f93 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt @@ -194,11 +194,11 @@ internal fun asyncStep( // --------------------------------------------------------------------------------------------------------------------- /** - * Drives a step to completion blocking. Each loop iteration is one round: one bulk call per source (chunked to - * [batchSize]) plus any async leaves, then resume. The loop is the trampoline that keeps fetch-dependent chains - * stack-safe regardless of depth. + * Drives a step to completion blocking. Each loop iteration is one round: one bulk call per source (chunked to that + * source's [IBulkExecutor.batchSize]) plus any async leaves, then resume. The loop is the trampoline that keeps + * fetch-dependent chains stack-safe regardless of depth. */ -internal fun Execution.drive(initial: Step, batchSize: Int): List { +internal fun Execution.drive(initial: Step): List { var step = initial while (true) { when (val current = step) { @@ -206,7 +206,7 @@ internal fun Execution.drive(initial: Step, batchSize: Int): List { is Failed -> throw current.cause is Blocked -> { for ((source, keys) in current.pending.fetches) { - for (chunk in keys.chunked(batchSize)) { + for (chunk in keys.chunked(source.batchSize)) { @Suppress("UNCHECKED_CAST") val results = source.execute(chunk) as Map fillFetch(source, chunk.toSet(), results) @@ -221,7 +221,7 @@ internal fun Execution.drive(initial: Step, batchSize: Int): List { } } -internal suspend fun Execution.driveSuspending(initial: Step, batchSize: Int): List { +internal suspend fun Execution.driveSuspending(initial: Step): List { var step = initial while (true) { when (val current = step) { @@ -229,7 +229,7 @@ internal suspend fun Execution.driveSuspending(initial: Step, batchSize: is Failed -> throw current.cause is Blocked -> { for ((source, keys) in current.pending.fetches) { - for (chunk in keys.chunked(batchSize)) { + for (chunk in keys.chunked(source.batchSize)) { @Suppress("UNCHECKED_CAST") val results = source.executeSuspending(chunk) as Map fillFetch(source, chunk.toSet(), results) diff --git a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt index ca52ce54a0..b548fd8f3b 100644 --- a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt +++ b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt @@ -12,17 +12,17 @@ import org.modelix.streams.engine.driveSuspending object BlockingStreamExecutor : IStreamExecutor { override fun query(body: () -> IStream.One): T { val execution = Execution() - return execution.drive(body().asStep(execution), Int.MAX_VALUE).single() + return execution.drive(body().asStep(execution)).single() } override suspend fun querySuspending(body: suspend () -> IStream.One): T { val execution = Execution() - return execution.driveSuspending(body().asStep(execution), Int.MAX_VALUE).single() + return execution.driveSuspending(body().asStep(execution)).single() } override fun iterate(streamProvider: () -> IStream.Many, visitor: (T) -> Unit) { val execution = Execution() - execution.drive(streamProvider().asStep(execution), Int.MAX_VALUE).forEach(visitor) + execution.drive(streamProvider().asStep(execution)).forEach(visitor) } override suspend fun iterateSuspending( @@ -30,6 +30,6 @@ object BlockingStreamExecutor : IStreamExecutor { visitor: suspend (T) -> Unit, ) { val execution = Execution() - execution.driveSuspending(streamProvider().asStep(execution), Int.MAX_VALUE).forEach { visitor(it) } + execution.driveSuspending(streamProvider().asStep(execution)).forEach { visitor(it) } } } From 051474ed4401aa4c15596d04132e12bde0c1d340 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 21:18:55 +0200 Subject: [PATCH 4/7] fix(streams): implement cached() multicast memoization cached() was a no-op, which broke ModelQLClientTest.testCaching: a query step consumed by multiple branches (with a side-effecting setProperty) was evaluated once per consumer instead of once. Implement real multicast in the engine: cached() resolves its inner stream through a shared MemoCell stored per-Execution and keyed by a stable token. Every consumer gets a view of the same cell, and the cell's inner step is advanced at most once per round, so the underlying work and any side effects run exactly once and the result is shared. Pending.union now dedupes async leaves by token so a memoized leaf shared across branches isn't run twice in a round. Verified: new CachedStreamTest (side-effect-once across consumers, including via zip); ModelQLClientTest passes (16/16, incl. testCaching); datastructures, model-datastructure, modelql-core/-untyped suites pass. Co-Authored-By: Claude Opus 4.8 --- streams-redesign.md | 7 ++- streams/README.md | 4 +- .../kotlin/org/modelix/streams/StreamImpl.kt | 8 ++- .../org/modelix/streams/engine/StepEngine.kt | 38 +++++++++++- .../org/modelix/streams/CachedStreamTest.kt | 59 +++++++++++++++++++ 5 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt diff --git a/streams-redesign.md b/streams-redesign.md index fde756a3af..c59c2746a5 100644 --- a/streams-redesign.md +++ b/streams-redesign.md @@ -196,8 +196,10 @@ not results. server-side iterations (e.g. walking all objects) this raises peak memory. *If this matters for a hot path, the clean fix is to add genuine per-round streaming to just the `iterate*` drivers without disturbing the rest of the engine.* -2. **`cached()` is currently a no-op.** Fetch-level dedup (the expensive part) is handled by the per-run cache; only - pure-recompute memoization is lost. +2. **`cached()` multicasts (evaluates once per run).** A stream consumed by multiple branches is resolved once via a + shared memo cell (`MemoCell`) that the round driver advances at most once per round, so work *and side effects* are + not duplicated and the result is shared. ModelQL depends on this for shared/side-effecting query steps (see + `ModelQLClientTest.testCaching`). 3. **`take` / `skip` operate on materialized results** — they do not prune upstream fetches. 4. **`SimpleStreamExecutor` now batches** per source/round — strictly fewer round-trips than before. @@ -207,7 +209,6 @@ not results. pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode `Step` as a stack-safe free monad (explicit interpreter loop) if needed. - **Optional streaming `iterate*`** — see tradeoff #1. -- **Restore `cached()` memoization** if ModelQL recompute cost proves material. - **Retire the executor entirely.** With the executor no longer required to run a stream (§5), `IStreamExecutor` / `IStreamExecutorProvider` could be removed over time — the remaining users are `enqueue` (fetch-leaf creation) and ModelQL's `getInstance()`-based "current executor" lookup, both of which can be reworked. diff --git a/streams/README.md b/streams/README.md index c8054a5e99..3c2245c000 100644 --- a/streams/README.md +++ b/streams/README.md @@ -99,7 +99,9 @@ These follow from the engine resolving each query fully (no incremental emission 1. `iterate` / `iterateSuspending` fully materialize before visiting — higher peak memory for very large iterations. The clean fix, if a hot path needs it, is per-round streaming in just the `iterate*` drivers. -2. `cached()` is currently a no-op; fetch-level dedup (the expensive part) is handled by the per-run cache. +2. `cached()` multicasts: a stream consumed by multiple branches is evaluated once per run (via a shared memo cell + advanced at most once per round), so side effects and work aren't duplicated. ModelQL relies on this for shared + query steps. 3. `take` / `skip` operate on materialized results (don't prune upstream fetches). 4. Within-round stack safety covers fetch-dependent chains (the common case). A pathological deep *pure* `flatMap` chain that never blocks would still recurse; the fix is to encode `Step` as a stack-safe free monad if needed. diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt index 1f78898fc5..eedd6f4795 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt @@ -17,6 +17,7 @@ import org.modelix.streams.engine.drive import org.modelix.streams.engine.driveSuspending import org.modelix.streams.engine.flatMapStep import org.modelix.streams.engine.mapValues +import org.modelix.streams.engine.memoStep import org.modelix.streams.engine.recover import org.modelix.streams.engine.zipN @@ -168,7 +169,12 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna override fun orNull(): IStream.One = StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) listOf(null) else listOf(values.single()) } } - override fun cached(): IStream.One = this + @Suppress("UNCHECKED_CAST") + override fun cached(): IStream.One { + val self = this + val token = Any() + return StreamImpl { execution -> memoStep(execution, token) { self.build(execution) as Step } as Step } + } @DelicateModelixApi override fun iterateBlocking(visitor: (E) -> Unit) { diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt index f1174f9f93..ac33ad3d2e 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt @@ -41,7 +41,12 @@ internal class Pending private constructor( val fetches = HashMap, MutableSet>() for ((source, keys) in this.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) for ((source, keys) in other.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) - return Pending(fetches, this.asyncActions + other.asyncActions) + // Dedupe async leaves by token so a memoized leaf shared across branches runs only once per round. + val seen = HashSet() + val actions = ArrayList(this.asyncActions.size + other.asyncActions.size) + for (action in this.asyncActions) if (seen.add(action.token)) actions.add(action) + for (action in other.asyncActions) if (seen.add(action.token)) actions.add(action) + return Pending(fetches, actions) } companion object { @@ -146,6 +151,10 @@ private val MISSING = Any() internal class Execution { private val fetchCaches = HashMap, HashMap>() private val asyncResults = HashMap>() + private val memoCells = HashMap() + + /** Returns the shared memo cell for [token], initializing it (running [init] once) on first access. */ + fun memoCell(token: Any, init: () -> Step): MemoCell = memoCells.getOrPut(token) { MemoCell(init()) } @Suppress("UNCHECKED_CAST") private fun cacheFor(source: IBulkExecutor<*, *>) = @@ -189,6 +198,33 @@ internal fun asyncStep( } } +/** + * Shared, single-advancing progress of a memoized stream. Every reference to a cached stream produces a *view* of the + * same cell; the cell's inner step is advanced at most once per round (regardless of how many views resume it), so the + * underlying computation — and any side effects in it — runs exactly once and its result is multicast to all + * consumers. This is the [org.modelix.streams.IStream.One.cached] mechanism. + */ +internal class MemoCell(var step: Step) { + fun advanceFrom(snapshot: Step) { + if (step === snapshot && snapshot is Blocked) { + step = snapshot.resume() + } + } + + fun view(): Step = when (val snapshot = step) { + is Done -> snapshot + is Failed -> snapshot + is Blocked -> Blocked(snapshot.pending) { + advanceFrom(snapshot) + view() + } + } +} + +/** A leaf that resolves the inner stream of a [org.modelix.streams.IStream.One.cached] call once, shared per run. */ +internal fun memoStep(execution: Execution, token: Any, init: () -> Step): Step = + execution.memoCell(token, init).view() + // --------------------------------------------------------------------------------------------------------------------- // Drivers // --------------------------------------------------------------------------------------------------------------------- diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt new file mode 100644 index 0000000000..d2863cbb54 --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/CachedStreamTest.kt @@ -0,0 +1,59 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Verifies the multicast semantics of [IStream.One.cached]: a stream consumed by multiple downstream branches is + * evaluated exactly once (including any side effects), and the single result is shared. This is what ModelQL relies on + * for shared/memoized query steps (see ModelQLClientTest.testCaching). + */ +class CachedStreamTest { + private val executor = SimpleStreamExecutor + + @Test + fun `cached one is evaluated once across multiple consumers`() { + var evaluations = 0 + val cached = IStream.of(Unit).map { + evaluations++ + 42 + }.cached() + + val result = executor.query { + cached.map { it + 1000 }.plus(cached.map { it + 2000 }).toList() + } + + assertEquals(listOf(1042, 2042), result) + assertEquals(1, evaluations) // the side effect ran exactly once, not once per consumer + } + + @Test + fun `without cached the side effect runs per consumer`() { + var evaluations = 0 + val notCached = IStream.of(Unit).map { + evaluations++ + 42 + } + + val result = executor.query { + notCached.map { it + 1000 }.plus(notCached.map { it + 2000 }).toList() + } + + assertEquals(listOf(1042, 2042), result) + assertEquals(2, evaluations) // confirms the distinction: each consumer re-evaluates + } + + @Test + fun `cached one shared via zip evaluates once`() { + var evaluations = 0 + val cached = IStream.of(7).map { + evaluations++ + it + }.cached() + + val result = executor.query { cached.zipWith(cached) { a, b -> a + b } } + + assertEquals(14, result) + assertEquals(1, evaluations) + } +} From 23790a0fcb704df03486a500ebc2c6beef409f4c Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 22:22:08 +0200 Subject: [PATCH 5/7] ci: drop removed --merge-runs flag from sarif-multitool merge sarif-multitool 5.x removed --merge-runs; merging runs by tool is now the default merge behavior, so the flag caused an unknown-option error. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index bdec818fb8..e7e3b043b5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,7 @@ jobs: node-version-file: '.nvmrc' - name: Combine SARIF files run: | - npx @microsoft/sarif-multitool merge --merge-runs --output-file merged.sarif $(find . -iname '*.sarif*') + npx @microsoft/sarif-multitool merge --output-file merged.sarif $(find . -iname '*.sarif*') env: # Disables globalization support. # This makes the @microsoft/sarif-multitool work without ICU package installed. From 30f7a442761497be23ee5c9b28f22887c83438ee Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sat, 13 Jun 2026 22:36:30 +0200 Subject: [PATCH 6/7] feat(streams): add common Many operators Add frequently-needed operators to IStream as extension functions composed from existing engine primitives (no StreamImpl changes): boolean reductions (any/all/none), filterNot/filterIsInstance/ filterIndexed, first/last accessors, mapIndexed, collection conversions (toSet/groupBy/associateBy/associateWith/toMap), sorting, distinctBy, reduce/sum/sumOf/min-maxBy, startWith/endWith, joinToString. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../kotlin/org/modelix/streams/IStream.kt | 145 ++++++++++++++++++ .../modelix/streams/StreamExtensionsTests.kt | 94 ++++++++++++ 2 files changed, 239 insertions(+) diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt index 339b9eab93..a8b11ded51 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStream.kt @@ -4,6 +4,8 @@ import kotlinx.coroutines.flow.Flow import org.modelix.streams.IStream.Many import org.modelix.streams.IStream.One import org.modelix.streams.IStream.OneOrMany +import org.modelix.streams.IStream.ZeroOrOne +import kotlin.jvm.JvmName interface IStream { fun convert(converter: IStreamBuilder): IStream @@ -156,3 +158,146 @@ fun IStream.Many.onEach(action: (T) -> Unit): IStream.Many = map { action(it) it } + +// --- Boolean reductions --- + +/** Emits `true` if at least one element satisfies [predicate], `false` otherwise (and for an empty stream). */ +fun IStream.Many.any(predicate: (T) -> Boolean): IStream.One = filter(predicate).isNotEmpty() + +/** Emits `true` if every element satisfies [predicate]. An empty stream emits `true`. */ +fun IStream.Many.all(predicate: (T) -> Boolean): IStream.One = filter { !predicate(it) }.isEmpty() + +/** Emits `true` if no element satisfies [predicate]. An empty stream emits `true`. */ +fun IStream.Many.none(predicate: (T) -> Boolean): IStream.One = filter(predicate).isEmpty() + +/** Emits `true` if the stream contains no elements. */ +fun IStream.Many<*>.none(): IStream.One = isEmpty() + +// --- Filtering --- + +/** Keeps the elements that do *not* satisfy [predicate]. */ +fun IStream.Many.filterNot(predicate: (T) -> Boolean): IStream.Many = filter { !predicate(it) } + +/** Keeps only elements that are instances of [R]. */ +inline fun IStream.Many<*>.filterIsInstance(): IStream.Many = mapNotNull { it as? R } + +/** Keeps the elements for which [predicate] (receiving the element's index) returns `true`. */ +fun IStream.Many.filterIndexed(predicate: (Int, T) -> Boolean): IStream.Many = + withIndex().filter { predicate(it.index, it.value) }.map { it.value } + +/** Emits at most [count] elements counted from the start of the stream. Alias for [IStream.Many.skip]. */ +fun IStream.Many.drop(count: Int): IStream.Many = skip(count.toLong()) + +// --- Element access --- + +/** Emits the first element, or fails with [NoSuchElementException] if the stream is empty. */ +fun IStream.Many.first(): IStream.One = + firstOrEmpty().exceptionIfEmpty { NoSuchElementException("Stream is empty") } + +/** Emits the first element satisfying [predicate], or fails with [NoSuchElementException] if there is none. */ +fun IStream.Many.first(predicate: (T) -> Boolean): IStream.One = filter(predicate).first() + +/** Emits the first element satisfying [predicate], or `null` if there is none. */ +fun IStream.Many.firstOrNull(predicate: (T) -> Boolean): IStream.One = filter(predicate).firstOrNull() + +/** Emits the last element, or fails with [NoSuchElementException] if the stream is empty. */ +fun IStream.Many.last(): IStream.One = toList().map { it.last() } + +/** Emits the last element, or `null` if the stream is empty. */ +fun IStream.Many.lastOrNull(): IStream.One = toList().map { it.lastOrNull() } + +// --- Indexed / element-wise mapping --- + +/** Maps each element together with its index. */ +fun IStream.Many.mapIndexed(mapper: (Int, T) -> R): IStream.Many = + withIndex().map { mapper(it.index, it.value) } + +// --- Collection conversions --- + +/** Collects all elements into a [Set]. */ +fun IStream.Many.toSet(): IStream.One> = toList().map { it.toSet() } + +/** Groups the elements by the key returned by [keySelector]. */ +fun IStream.Many.groupBy(keySelector: (T) -> K): IStream.One>> = + toList().map { it.groupBy(keySelector) } + +/** Groups the elements by [keySelector], mapping each grouped element through [valueSelector]. */ +fun IStream.Many.groupBy(keySelector: (T) -> K, valueSelector: (T) -> V): IStream.One>> = + toList().map { it.groupBy(keySelector, valueSelector) } + +/** Associates each element with the key produced by [keySelector]. Later duplicates win. */ +fun IStream.Many.associateBy(keySelector: (T) -> K): IStream.One> = toMap(keySelector) { it } + +/** Associates each element (used as the key) with the value produced by [valueSelector]. */ +fun IStream.Many.associateWith(valueSelector: (T) -> V): IStream.One> = toMap({ it }, valueSelector) + +/** Collects a stream of pairs into a [Map]. Later duplicate keys win. */ +fun IStream.Many>.toMap(): IStream.One> = toMap({ it.first }, { it.second }) + +// --- Sorting --- + +fun > IStream.Many.sorted(): IStream.Many = toList().flatMapIterable { it.sorted() } +fun > IStream.Many.sortedDescending(): IStream.Many = toList().flatMapIterable { it.sortedDescending() } +fun > IStream.Many.sortedBy(selector: (T) -> R): IStream.Many = + toList().flatMapIterable { it.sortedBy(selector) } +fun > IStream.Many.sortedByDescending(selector: (T) -> R): IStream.Many = + toList().flatMapIterable { it.sortedByDescending(selector) } +fun IStream.Many.sortedWith(comparator: Comparator): IStream.Many = + toList().flatMapIterable { it.sortedWith(comparator) } + +// --- distinct / dedup --- + +/** Removes elements that share a key (as returned by [selector]) with an earlier element. */ +fun IStream.Many.distinctBy(selector: (T) -> K): IStream.Many = + toList().flatMapIterable { it.distinctBy(selector) } + +// --- Numeric and general reductions --- + +/** Reduces the stream with [operation], starting from the first element. Fails on an empty stream. */ +fun IStream.Many.reduce(operation: (acc: T, T) -> T): IStream.One = toList().map { it.reduce(operation) } + +/** Sums the [Int] values produced by [selector]. */ +fun IStream.Many.sumOf(selector: (T) -> Int): IStream.One = fold(0) { acc, e -> acc + selector(e) } + +@JvmName("sumInt") +fun IStream.Many.sum(): IStream.One = fold(0) { acc, e -> acc + e } + +@JvmName("sumLong") +fun IStream.Many.sum(): IStream.One = fold(0L) { acc, e -> acc + e } + +@JvmName("sumDouble") +fun IStream.Many.sum(): IStream.One = fold(0.0) { acc, e -> acc + e } + +/** Emits the element yielding the largest value by [selector], or `null` if the stream is empty. */ +fun > IStream.Many.maxByOrNull(selector: (T) -> R): IStream.One = + toList().map { it.maxByOrNull(selector) } + +/** Emits the element yielding the smallest value by [selector], or `null` if the stream is empty. */ +fun > IStream.Many.minByOrNull(selector: (T) -> R): IStream.One = + toList().map { it.minByOrNull(selector) } + +/** Emits the largest element according to [comparator], or `null` if the stream is empty. */ +fun IStream.Many.maxWithOrNull(comparator: Comparator): IStream.One = + toList().map { it.maxWithOrNull(comparator) } + +/** Emits the smallest element according to [comparator], or `null` if the stream is empty. */ +fun IStream.Many.minWithOrNull(comparator: Comparator): IStream.One = + toList().map { it.minWithOrNull(comparator) } + +// --- Prepending / appending single elements --- + +/** Emits [value] before all elements of this stream. */ +fun IStream.Many.startWith(value: T): IStream.Many = IStream.of(value).concat(this) + +/** Emits [value] after all elements of this stream. */ +fun IStream.Many.endWith(value: T): IStream.OneOrMany = concat(IStream.of(value)) + +// --- String joining --- + +/** Joins all elements into a single [String]. */ +fun IStream.Many.joinToString( + separator: CharSequence = ", ", + prefix: CharSequence = "", + postfix: CharSequence = "", + transform: ((T) -> CharSequence)? = null, +): IStream.One = toList().map { it.joinToString(separator, prefix, postfix, transform = transform) } diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt index 6538c7483a..33d31a0070 100644 --- a/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt +++ b/streams/src/commonTest/kotlin/org/modelix/streams/StreamExtensionsTests.kt @@ -44,4 +44,98 @@ class StreamExtensionsTests { assertEquals(10, executor.query { IStream.many(1..4).fold(0) { acc, v -> acc + v } }) assertEquals(4, executor.query { IStream.many(1..4).count() }) } + + @Test + fun `any all none`() { + assertEquals(true, executor.query { IStream.many(1..4).any { it > 3 } }) + assertEquals(false, executor.query { IStream.many(1..4).any { it > 4 } }) + assertEquals(true, executor.query { IStream.many(1..4).all { it > 0 } }) + assertEquals(false, executor.query { IStream.many(1..4).all { it > 1 } }) + assertEquals(true, executor.query { IStream.many(1..4).none { it > 4 } }) + assertEquals(true, executor.query { IStream.many(emptyList()).all { it > 0 } }) + assertEquals(true, executor.query { IStream.many(emptyList()).none() }) + } + + @Test + fun `filterNot and filterIsInstance`() { + assertEquals(listOf(1, 3), executor.query { IStream.many(1..4).filterNot { it % 2 == 0 }.toList() }) + assertEquals( + listOf("a", "b"), + executor.query { IStream.many(listOf("a", 1, "b", 2)).filterIsInstance().toList() }, + ) + } + + @Test + fun `first and last`() { + assertEquals(1, executor.query { IStream.many(1..4).first() }) + assertEquals(2, executor.query { IStream.many(1..4).first { it % 2 == 0 } }) + assertEquals(null, executor.query { IStream.many(1..4).firstOrNull { it > 4 } }) + assertEquals(4, executor.query { IStream.many(1..4).last() }) + assertEquals(null, executor.query { IStream.many(emptyList()).lastOrNull() }) + } + + @Test + fun `mapIndexed and filterIndexed`() { + assertEquals( + listOf("0:a", "1:b"), + executor.query { IStream.many(listOf("a", "b")).mapIndexed { i, v -> "$i:$v" }.toList() }, + ) + assertEquals( + listOf("a", "c"), + executor.query { IStream.many(listOf("a", "b", "c", "d")).filterIndexed { i, _ -> i % 2 == 0 }.toList() }, + ) + } + + @Test + fun `collection conversions`() { + assertEquals(setOf(1, 2, 3), executor.query { IStream.many(listOf(1, 2, 2, 3)).toSet() }) + assertEquals( + mapOf(0 to listOf(2, 4), 1 to listOf(1, 3)), + executor.query { IStream.many(1..4).groupBy { it % 2 } }, + ) + assertEquals( + mapOf(1 to "a", 2 to "bb"), + executor.query { IStream.many(listOf("a", "bb")).associateBy { it.length } }, + ) + assertEquals( + mapOf("a" to 1, "bb" to 2), + executor.query { IStream.many(listOf("a", "bb")).associateWith { it.length } }, + ) + assertEquals( + mapOf(1 to "a", 2 to "b"), + executor.query { IStream.many(listOf(1 to "a", 2 to "b")).toMap() }, + ) + } + + @Test + fun `sorting and distinctBy`() { + assertEquals(listOf(1, 2, 3, 4), executor.query { IStream.many(listOf(3, 1, 4, 2)).sorted().toList() }) + assertEquals(listOf(4, 3, 2, 1), executor.query { IStream.many(listOf(3, 1, 4, 2)).sortedDescending().toList() }) + assertEquals( + listOf("a", "bb", "ccc"), + executor.query { IStream.many(listOf("ccc", "a", "bb")).sortedBy { it.length }.toList() }, + ) + assertEquals( + listOf("a", "bb"), + executor.query { IStream.many(listOf("a", "bb", "cc", "d")).distinctBy { it.length }.toList() }, + ) + } + + @Test + fun `reductions`() { + assertEquals(24, executor.query { IStream.many(1..4).reduce { acc, v -> acc * v } }) + assertEquals(10, executor.query { IStream.many(1..4).sumOf { it } }) + assertEquals(10, executor.query { IStream.many(1..4).sum() }) + assertEquals(10L, executor.query { IStream.many(listOf(1L, 2L, 3L, 4L)).sum() }) + assertEquals("ccc", executor.query { IStream.many(listOf("a", "bb", "ccc")).maxByOrNull { it.length } }) + assertEquals("a", executor.query { IStream.many(listOf("a", "bb", "ccc")).minByOrNull { it.length } }) + assertEquals(null, executor.query { IStream.many(emptyList()).maxByOrNull { it.length } }) + } + + @Test + fun `startWith endWith and joinToString`() { + assertEquals(listOf(0, 1, 2, 3), executor.query { IStream.many(1..3).startWith(0).toList() }) + assertEquals(listOf(1, 2, 3, 4), executor.query { IStream.many(1..3).endWith(4).toList() }) + assertEquals("[1, 2, 3]", executor.query { IStream.many(1..3).joinToString(prefix = "[", postfix = "]") }) + } } From fbe71a6dc8ac0c092d885033b0e5523af8a6ed32 Mon Sep 17 00:00:00 2001 From: Sascha Lisson Date: Sun, 14 Jun 2026 08:27:42 +0200 Subject: [PATCH 7/7] perf(streams): depth-first bounded request frontier in the round engine Rework the Step engine from a Blocked/Pending/resume lockstep model to a lazily-evaluated node graph (Done/FetchStep/MapStep/FlatMapStep/ZipStep/ FanStep/AsyncStep/Recover/OnError/MemoStep) driven by a depth-first walk. Each round collects the first-unmet request of every pending branch into a shared pool capped at the source's batchSize, and FanStep builds its children lazily (left-to-right, dropped once resolved). The peak request frontier is therefore <= batchSize regardless of how wide a tree level is, restoring the property the old BulkRequestStreamExecutor.RequestQueue.sendNextBatch provided (a depth-first traversal that bounds the live request set). Sibling requests still share a round when they fit under the cap, so applicative batching is preserved; an unbounded cap collapses to one round per level. Combinators evaluate eagerly when their inputs are already Done, so a synchronous prefix and its side effects run at build time in build order. Some call sites depend on this (e.g. HamtLeafNode.getChanges sets a var in one synchronous stream and reads it in a later deferZeroOrOne); fully-lazy evaluation broke tree diffing until eager-on-Done was restored. The public IStream API and combinator signatures are unchanged; only flatMapOrdered/flatMapZeroOrOne switch to the new lazy fanOut, and the driver functions became Execution members. Tests: FrontierSizeTest asserts the frontier stays within batchSize on a wide+deep tree (widest level 16384, cap 1000) and collapses to one round per level when uncapped. HamtGetChangesTest adds direct coverage of the HAMT diff (guards the eager-evaluation dependency). Validated across streams (JVM + JS), datastructures, model-datastructure, modelql-core/untyped/typed, model-api, and bulk-model-sync-lib. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../datastructures/HamtGetChangesTest.kt | 43 ++ streams-redesign.md | 39 +- streams/README.md | 50 +- .../streams/BulkRequestStreamExecutor.kt | 2 - .../org/modelix/streams/IStreamExecutor.kt | 2 - .../modelix/streams/SimpleStreamExecutor.kt | 2 - .../kotlin/org/modelix/streams/StreamImpl.kt | 7 +- .../org/modelix/streams/engine/StepEngine.kt | 524 ++++++++++-------- .../org/modelix/streams/FrontierSizeTest.kt | 91 +++ .../modelix/streams/BlockingStreamExecutor.kt | 2 - 10 files changed, 510 insertions(+), 252 deletions(-) create mode 100644 datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt create mode 100644 streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt diff --git a/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt new file mode 100644 index 0000000000..5459d0964d --- /dev/null +++ b/datastructures/src/commonTest/kotlin/org/modelix/datastructures/HamtGetChangesTest.kt @@ -0,0 +1,43 @@ +package org.modelix.datastructures + +import org.modelix.datastructures.hamt.HamtInternalNode +import org.modelix.datastructures.hamt.HamtNode +import org.modelix.datastructures.hamt.HamtTree +import org.modelix.datastructures.objects.IObjectGraph +import org.modelix.datastructures.objects.StringDataTypeConfiguration +import org.modelix.streams.getBlocking +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Direct coverage of [org.modelix.datastructures.IPersistentMap.getChanges] on the HAMT. Guards that a value change is + * reported as [EntryChangedEvent] (not add/remove): this relies on the streams engine evaluating a synchronous prefix + * eagerly, which `HamtLeafNode.getChanges` depends on via a `var` set by one stream and read by a later deferred one. + */ +class HamtGetChangesTest { + private fun newTree(): HamtTree { + val config = HamtNode.Config( + graph = IObjectGraph.FREE_FLOATING, + keyConfig = StringDataTypeConfiguration(), + valueConfig = StringDataTypeConfiguration(), + ) + return HamtTree(HamtInternalNode.createEmpty(config)) + } + + @Test + fun single_entry_value_change_is_a_change_not_add() { + val old = newTree().put("k1", "a").getBlocking() + val new = old.put("k1", "b").getBlocking() + val changes = new.getChanges(old, changesOnly = false).toList().getBlocking() + assertEquals(listOf(EntryChangedEvent("k1", "a", "b")), changes) + } + + @Test + fun one_changed_among_several() { + var old = newTree() + for (i in 0 until 20) old = old.put("k$i", "v$i").getBlocking() + val new = old.put("k7", "changed").getBlocking() + val changes = new.getChanges(old, changesOnly = false).toList().getBlocking() + assertEquals(listOf(EntryChangedEvent("k7", "v7", "changed")), changes) + } +} diff --git a/streams-redesign.md b/streams-redesign.md index c59c2746a5..3bd96155f1 100644 --- a/streams-redesign.md +++ b/streams-redesign.md @@ -51,7 +51,11 @@ Prior art for this pattern: Haxl (Haskell), ZIO Query / `ZQuery` (Scala), Stitch ### `Step` — the intermediate representation A stream is described lazily and interpreted in **rounds**. The core type -([`streams/.../engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)): +([`streams/.../engine/StepEngine.kt`](streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt)). + +> **Note:** this section describes the *first* cut. The `Blocked`/`Pending`/`resume` representation below was later +> replaced by a lazily-evaluated node graph to restore a depth-first bounded request frontier — see §6a for the +> current model. The applicative/monadic split and round-based batching described here are unchanged. ```kotlin sealed interface Step @@ -203,11 +207,38 @@ not results. 3. **`take` / `skip` operate on materialized results** — they do not prune upstream fetches. 4. **`SimpleStreamExecutor` now batches** per source/round — strictly fewer round-trips than before. +## 6a. Depth-first, bounded request frontier + +The first cut of the round driver expanded the *entire* breadth of a traversal level per round: it materialized a +`Step` per child (the applicative `combineConcat` over a node's children) and fetched the whole level at once. That +lost a property of the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` — a depth-first traversal that kept +the live request frontier bounded — so peak memory grew with the *width* of a tree level rather than its depth. + +The engine was reworked to restore it. `Step` is now a lazily-evaluated node graph (`Done` / `FetchStep` / `MapStep` / +`FlatMapStep` / `ZipStep` / `FanStep` / `AsyncStep` / `Recover`/`OnError`/`MemoStep`) instead of `Blocked` + +`resume`. The driver walks the graph each round collecting the first-unmet request of every pending branch into a +shared pool, but: + +- it **stops adding requests once the pool reaches the source's `batchSize`** (the cap), and +- **`FanStep` builds its children lazily** — left-to-right, dropped once resolved — so live *unresolved* children also + stay within the cap. + +Combinators still evaluate eagerly when their inputs are already `Done`, so a synchronous prefix (and its side effects) +runs at build time in build order — several call sites depend on this (e.g. `HamtLeafNode.getChanges` sets a `var` in +one synchronous stream and reads it in a later `deferZeroOrOne`). Only blocking children are deferred and bounded. + +The result: **peak request frontier `≤ batchSize`** regardless of level width, while sibling requests that fit under +the cap still share one round (batching preserved); an unbounded cap collapses to one round per level. Verified by +`FrontierSizeTest` (a wide+deep tree whose widest level far exceeds the cap) and `HamtGetChangesTest` / +`BulkRequestBatchingTest` (batching + eager-evaluation semantics). + +The result list itself is still fully materialized — tradeoff #1 (streaming `iterate*`) is unchanged and orthogonal. + ## 7. Known limitations / future work -- **Within-round stack safety.** The round driver trampolines across `Blocked` (the common fetch-dependent case). A - pathological deep *pure* `flatMap` chain that never blocks would still recurse natively; the fix is to encode `Step` - as a stack-safe free monad (explicit interpreter loop) if needed. +- **Within-round stack safety.** The driver trampolines across rounds (the common fetch-dependent case), and the + per-round walk recurses on traversal depth (bounded in practice) — matching the previous engine. A pathological deep + *pure* `flatMap` chain that never blocks would still recurse natively; the fix is an explicit-stack walk if needed. - **Optional streaming `iterate*`** — see tradeoff #1. - **Retire the executor entirely.** With the executor no longer required to run a stream (§5), `IStreamExecutor` / `IStreamExecutorProvider` could be removed over time — the remaining users are `enqueue` (fetch-leaf creation) and diff --git a/streams/README.md b/streams/README.md index 3c2245c000..dbb0d60399 100644 --- a/streams/README.md +++ b/streams/README.md @@ -32,31 +32,44 @@ The engine ([`engine/StepEngine.kt`](src/commonMain/kotlin/org/modelix/streams/e stream lazily and interprets it in **rounds**: ```kotlin -sealed interface Step -class Done(val values: List) // fully resolved -class Blocked(val pending: Pending, val resume: () -> Step) // needs a round of work first -class Failed(val cause: Throwable) +sealed class Step // a node in a lazily-evaluated computation +class Done(val values: List) // fully resolved +class FetchStep(source, key) // a bulk-fetch leaf +class MapStep / FlatMapStep / ZipStep / FanStep // transform / dependency / applicative join / fan-out +// + AsyncStep, RecoverStep, OnErrorStep, MemoStep ``` -`Pending` is the deduplicated work for one round: bulk fetches grouped by data source, plus async leaves. The -combinators encode the applicative/monadic split: +A `Step` is a node the driver walks. The combinators encode the applicative/monadic split: - `flatMapStep` (monadic) — defers the continuation into the next round. -- `combineConcat` / `zipN` / `zip2` (applicative) — **union** the pending work of independent branches into the same - round. That union *is* the batch. +- `combineConcat` / `zipN` / `fanOut` (applicative) — let the requests of independent branches be collected into the + same round. That *is* the batch. -A subtree with no `Blocked` resolves straight to `Done`: the **synchronous fast path** for local data, with no -scheduler and nothing allocated. +Combinators evaluate **eagerly when their inputs are already `Done`** (the synchronous fast path for local data: no +node allocated, side effects in `map`/`flatMap` lambdas run at build time in build order). A lazy node is built only +when an input would block on a request. ## Execution The driver (`Execution.drive` / `driveSuspending`) is a loop where each iteration is one batch round: -1. issue **one bulk call per data source** (chunked to that source's `IBulkExecutor.batchSize`), and run any async leaves; -2. fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds); -3. `resume()` and repeat until `Done`. +1. **walk** the live node graph, collecting the first-unmet request of every pending branch into a shared pool; +2. issue **one bulk call per data source** (chunked to that source's `IBulkExecutor.batchSize`), run any async leaves, + and fill the per-run cache (so each key is fetched at most once — dedup within *and* across rounds); +3. walk again, until the root resolves. -The loop is the **trampoline** that keeps fetch-dependent chains stack-safe regardless of depth. +The loop is the **trampoline** across rounds, so fetch-dependent depth costs rounds, not stack. + +### Depth-first, bounded frontier + +The walk does **not** expand the whole breadth of a traversal at once. It stops adding requests to the round once the +pool reaches the source's `batchSize` (the cap), and `FanStep` children are built lazily — instantiated left-to-right +and dropped once resolved — so the number of live *unresolved* children never exceeds the cap either. Once a request +resolves, the next walk descends deeper before expanding shallower siblings that didn't fit. The result is a +depth-first traversal whose **peak request frontier is `≤ batchSize`**, independent of how wide a tree level is — the +property the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` provided. Sibling requests still share a round +whenever they fit under the cap (the common case), so batching is preserved; with a cap above any level the behaviour +collapses to one round per level. (See `FrontierSizeTest`.) Batching is **structural**: a fetch leaf carries its own data source, so the driver groups fetches per source per round regardless of which executor runs it. `BulkRequestStreamExecutor.enqueue(key)` is simply a fetch leaf bound to @@ -97,11 +110,12 @@ src/commonMain/kotlin/org/modelix/streams/ These follow from the engine resolving each query fully (no incremental emission): -1. `iterate` / `iterateSuspending` fully materialize before visiting — higher peak memory for very large iterations. - The clean fix, if a hot path needs it, is per-round streaming in just the `iterate*` drivers. +1. `iterate` / `iterateSuspending` fully materialize the *result* before visiting — higher peak memory for very large + iterations. Note this is independent of the request frontier, which **is** bounded (see "Depth-first, bounded + frontier" above): the fetch working set stays `≤ batchSize` even though the result list does not. The clean fix for + the result side, if a hot path needs it, is per-round streaming in just the `iterate*` drivers. 2. `cached()` multicasts: a stream consumed by multiple branches is evaluated once per run (via a shared memo cell - advanced at most once per round), so side effects and work aren't duplicated. ModelQL relies on this for shared - query steps. + resolved once and reused), so side effects and work aren't duplicated. ModelQL relies on this for shared query steps. 3. `take` / `skip` operate on materialized results (don't prune upstream fetches). 4. Within-round stack safety covers fetch-dependent chains (the common case). A pathological deep *pure* `flatMap` chain that never blocks would still recurse; the fix is to encode `Step` as a stack-safe free monad if needed. diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt index 79e8410748..e6b6e33d91 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/BulkRequestStreamExecutor.kt @@ -2,8 +2,6 @@ package org.modelix.streams import org.modelix.streams.engine.Execution import org.modelix.streams.engine.Step -import org.modelix.streams.engine.drive -import org.modelix.streams.engine.driveSuspending import org.modelix.streams.engine.fetchStep /** Default maximum number of keys requested in a single bulk call. */ diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt index 946e332646..e5a5da3314 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/IStreamExecutor.kt @@ -2,8 +2,6 @@ package org.modelix.streams import org.modelix.kotlin.utils.ContextValue import org.modelix.streams.engine.Execution -import org.modelix.streams.engine.drive -import org.modelix.streams.engine.driveSuspending /** * There reason that there are three different types of implementations if they all have different execution semantics. diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt index 28bdd72e94..178224805a 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/SimpleStreamExecutor.kt @@ -1,8 +1,6 @@ package org.modelix.streams import org.modelix.streams.engine.Execution -import org.modelix.streams.engine.drive -import org.modelix.streams.engine.driveSuspending /** * Default executor. Fetches embedded in a stream (via [BulkRequestStreamExecutor.enqueue]) are batched per source per diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt index eedd6f4795..dad9d00e96 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/StreamImpl.kt @@ -13,8 +13,7 @@ import org.modelix.streams.engine.Step import org.modelix.streams.engine.asyncStep import org.modelix.streams.engine.combineConcat import org.modelix.streams.engine.doOnError -import org.modelix.streams.engine.drive -import org.modelix.streams.engine.driveSuspending +import org.modelix.streams.engine.fanOut import org.modelix.streams.engine.flatMapStep import org.modelix.streams.engine.mapValues import org.modelix.streams.engine.memoStep @@ -61,7 +60,7 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna StreamImpl { execution -> build(execution).mapValues { it.map(mapper) } } override fun flatMapOrdered(mapper: (E) -> IStream.Many): IStream.Many = - StreamImpl { execution -> build(execution).flatMapStep { values -> combineConcat(values.map { mapper(it).asStep(execution) }) } } + StreamImpl { execution -> build(execution).flatMapStep { values -> fanOut(values) { mapper(it).asStep(execution) } } } override fun concat(other: IStream.Many): IStream.Many = StreamImpl { execution -> combineConcat(listOf(build(execution), other.asStep(execution))) } @@ -161,7 +160,7 @@ internal class StreamImpl(val build: (Execution) -> Step) : IStreamInterna StreamImpl { execution -> build(execution).flatMapStep { values -> mapper(values.single()).asStep(execution) } } override fun flatMapZeroOrOne(mapper: (E) -> IStream.ZeroOrOne): IStream.ZeroOrOne = - StreamImpl { execution -> build(execution).flatMapStep { values -> combineConcat(values.map { mapper(it).asStep(execution) }) } } + StreamImpl { execution -> build(execution).flatMapStep { values -> fanOut(values) { mapper(it).asStep(execution) } } } override fun exceptionIfEmpty(exception: () -> Throwable): IStream.One = StreamImpl { execution -> build(execution).mapValues { values -> if (values.isEmpty()) throw exception() else values } } diff --git a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt index ac33ad3d2e..d65a71de5c 100644 --- a/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt +++ b/streams/src/commonMain/kotlin/org/modelix/streams/engine/StepEngine.kt @@ -6,280 +6,368 @@ import org.modelix.streams.IBulkExecutor /** * The round-based interpreter that backs every [org.modelix.streams.IStream] instance. * - * A [Step] produces zero or more values. Evaluation proceeds in *rounds*: a step is [Done], [Failed], or [Blocked] - * on a set of pending work ([Pending]) that must be resolved before it can continue. The combinators encode the - * applicative/monadic split that makes bulk-request batching automatic: - * - applicative composition ([combineConcat], [zipN], [zip2]) unions the pending work of independent branches into - * the *same* round — that is the batch; + * A [Step] is a node in a lazily-evaluated computation that produces zero or more values. Evaluation proceeds in + * *rounds*: the driver walks the live node graph, collecting the data requests it depends on, resolves them with one + * bulk call per source, and walks again — until the root resolves. The combinators encode the applicative/monadic split + * that makes bulk-request batching automatic: + * - applicative composition ([combineConcat], [zipN], [fanOut]) lets the independent requests of sibling branches be + * collected into the *same* round — that is the batch; * - monadic composition ([flatMapStep]) introduces a dependency, pushing the right-hand side into a *later* round. * - * A subtree with no pending work resolves straight to [Done] without allocating anything — the synchronous fast path. + * ### Depth-first, bounded frontier + * + * The driver does not expand the whole breadth of a traversal at once. A single walk collects the first-unmet request + * of every pending branch into a shared pool, but **stops adding requests once the pool reaches the source's + * [IBulkExecutor.batchSize]** (the cap). Fan-out children ([FanStep]) are built lazily, left-to-right, and dropped once + * resolved, so the number of live *unresolved* child nodes never exceeds the cap either. Once a branch's request + * resolves, the next walk descends one level deeper into it before expanding the breadth of shallower siblings that did + * not fit under the cap. The result is a depth-first traversal whose peak request frontier is `<= batchSize`, + * regardless of how wide a tree level is — the property the old `BulkRequestStreamExecutor.RequestQueue.sendNextBatch` + * provided. Independent sibling requests still share a round whenever they fit under the cap (the common case), so + * applicative batching is preserved; with a cap above any level the behaviour collapses to a single round per level. + * + * Errors are not represented as nodes: an exception thrown while resolving a node propagates natively and is caught by + * an enclosing [recover]/[doOnError]. A fetch that throws inside the driver propagates out of [drive] uncaught, exactly + * as before. */ -internal sealed interface Step - -internal class Done(val values: List) : Step +internal sealed class Step { + /** Memoized resolved value list; `null` until this node is fully resolved within the current execution. */ + var cached: List? = null +} -internal class Blocked(val pending: Pending, val resume: () -> Step) : Step +/** An already-resolved node. */ +internal class Done(val values: List) : Step() -internal class Failed(val cause: Throwable) : Step +/** A fetch leaf with zero-or-one semantics: a `null`/absent value resolves to an empty list. */ +internal class FetchStep(val source: IBulkExecutor, val key: Any?) : Step() /** A leaf that produces its values via a (potentially suspending) computation rather than a bulk fetch. */ -internal class AsyncAction( +internal class AsyncStep( val token: Any, val produceBlocking: () -> List, val produceSuspending: suspend () -> List, -) - -/** The deduplicated set of work pending for a single round: bulk fetches grouped by source, plus async leaves. */ -internal class Pending private constructor( - val fetches: Map, Set>, - val asyncActions: List, -) { - fun union(other: Pending): Pending { - if (this === EMPTY) return other - if (other === EMPTY) return this - val fetches = HashMap, MutableSet>() - for ((source, keys) in this.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) - for ((source, keys) in other.fetches) fetches.getOrPut(source) { HashSet() }.addAll(keys) - // Dedupe async leaves by token so a memoized leaf shared across branches runs only once per round. - val seen = HashSet() - val actions = ArrayList(this.asyncActions.size + other.asyncActions.size) - for (action in this.asyncActions) if (seen.add(action.token)) actions.add(action) - for (action in other.asyncActions) if (seen.add(action.token)) actions.add(action) - return Pending(fetches, actions) - } - - companion object { - val EMPTY = Pending(emptyMap(), emptyList()) +) : Step() - @Suppress("UNCHECKED_CAST") - fun fetch(source: IBulkExecutor<*, *>, key: Any?): Pending = - Pending(mapOf((source as IBulkExecutor) to setOf(key)), emptyList()) +/** Linear transform of the resolved value list of [src]. */ +internal class MapStep(val src: Step, val f: (List) -> List) : Step() - fun async(action: AsyncAction): Pending = Pending(emptyMap(), listOf(action)) - } +/** Monadic dependency: once [src] resolves, the values pick the next step. */ +internal class FlatMapStep(val src: Step, val f: (List) -> Step) : Step() { + var inner: Step? = null } -internal fun Step.mapValues(f: (List) -> List): Step = when (this) { - is Done -> Done(f(values)) - is Blocked -> Blocked(pending) { resume().mapValues(f) } - is Failed -> this -} +/** Applicative join: all parts resolve, then their value lists are handed to [f] together. */ +internal class ZipStep(val parts: List>, val f: (List>) -> List) : Step() -internal fun Step.flatMapStep(f: (List) -> Step): Step = when (this) { - is Done -> f(values) - is Blocked -> Blocked(pending) { resume().flatMapStep(f) } - is Failed -> this +/** + * Lazily-built ordered concatenation of [size] children. The only place a breadth frontier is born. Children are + * instantiated on demand via [makeChild] (left-to-right) and dropped once resolved, so the live unresolved children are + * bounded by the round cap. + */ +internal class FanStep(val size: Int, val makeChild: (Int) -> Step) : Step() { + val childResults: Array?> = arrayOfNulls(size) + val childNodes: Array?> = arrayOfNulls(size) } -/** Recover from a failure (or an exception thrown while resuming) by producing replacement values. */ -internal fun Step.recover(handler: (Throwable) -> List): Step = when (this) { - is Done -> this - is Failed -> Done(handler(cause)) - is Blocked -> Blocked(pending) { - try { - resume().recover(handler) - } catch (ex: Throwable) { - Done(handler(ex)) - } - } -} +/** Recovers from an exception thrown while resolving [src] by producing replacement values. */ +internal class RecoverStep(val src: Step, val handler: (Throwable) -> List) : Step() -/** Run [consumer] before a failure (or thrown exception) propagates. */ -internal fun Step.doOnError(consumer: (Throwable) -> Unit): Step = when (this) { - is Done -> this - is Failed -> { - consumer(cause) - this - } - is Blocked -> Blocked(pending) { - try { - resume().doOnError(consumer) - } catch (ex: Throwable) { - consumer(ex) - throw ex - } - } -} +/** Runs [consumer] before an exception thrown while resolving [src] propagates. */ +internal class OnErrorStep(val src: Step, val consumer: (Throwable) -> Unit) : Step() -/** Applicative combination concatenating the values of independent steps in order, batching their pending work. */ -internal fun combineConcat(steps: List>): Step { - var pending = Pending.EMPTY - var allDone = true - for (step in steps) { - when (step) { - is Failed -> return step - is Blocked -> { - allDone = false - pending = pending.union(step.pending) - } - is Done -> {} - } - } - if (allDone) return Done(steps.flatMap { (it as Done).values }) - return Blocked(pending) { combineConcat(steps.map { if (it is Blocked) it.resume() else it }) } -} - -/** Applicative combination handing all resolved value lists to [f] together. */ -internal fun zipN(steps: List>, f: (List>) -> List): Step { - var pending = Pending.EMPTY - var allDone = true - for (step in steps) { - when (step) { - is Failed -> return step - is Blocked -> { - allDone = false - pending = pending.union(step.pending) - } - is Done -> {} - } - } - if (allDone) return Done(f(steps.map { (it as Done).values })) - return Blocked(pending) { zipN(steps.map { if (it is Blocked) it.resume() else it }, f) } -} +/** A view onto a shared, single-resolution node (the [org.modelix.streams.IStream.One.cached] mechanism). */ +internal class MemoStep(val token: Any, val init: () -> Step) : Step() // --------------------------------------------------------------------------------------------------------------------- -// Per-run state and leaves +// Combinators (the API surface used by StreamImpl; signatures are kept stable) // --------------------------------------------------------------------------------------------------------------------- -private val MISSING = Any() +// Each combinator evaluates *eagerly* when its inputs are already resolved (`Done`), exactly as the previous engine +// did, so the synchronous prefix of a computation — including any side effects in `map`/`flatMap` lambdas — runs at +// build time and in build order. Some call sites rely on this (e.g. a `var` set by one synchronous stream and read by a +// later `deferZeroOrOne`). Only when an input would block on a request is a lazy node built and deferred to the driver. -/** - * Per-query state shared by every [Step] built for a single execution. Holds the fetch cache so that each key is - * fetched at most once across the whole traversal (dedup within and across rounds), and the results of async leaves. - */ -internal class Execution { - private val fetchCaches = HashMap, HashMap>() - private val asyncResults = HashMap>() - private val memoCells = HashMap() +internal fun Step.mapValues(f: (List) -> List): Step = + if (this is Done) Done(f(values)) else MapStep(this, f) - /** Returns the shared memo cell for [token], initializing it (running [init] once) on first access. */ - fun memoCell(token: Any, init: () -> Step): MemoCell = memoCells.getOrPut(token) { MemoCell(init()) } +internal fun Step.flatMapStep(f: (List) -> Step): Step = + if (this is Done) f(values) else FlatMapStep(this, f) - @Suppress("UNCHECKED_CAST") - private fun cacheFor(source: IBulkExecutor<*, *>) = - fetchCaches.getOrPut(source as IBulkExecutor) { HashMap() } +internal fun Step.recover(handler: (Throwable) -> List): Step = + if (this is Done) this else RecoverStep(this, handler) - fun isFetched(source: IBulkExecutor<*, *>, key: Any?): Boolean = cacheFor(source).containsKey(key) +internal fun Step.doOnError(consumer: (Throwable) -> Unit): Step = + if (this is Done) this else OnErrorStep(this, consumer) - fun fetchedValue(source: IBulkExecutor<*, *>, key: Any?): Any? { - val value = cacheFor(source)[key] - return if (value === MISSING) null else value - } +/** Applicative concatenation of the values of independent, already-built steps in order. */ +internal fun combineConcat(steps: List>): Step = + if (steps.all { it is Done }) Done(steps.flatMap { (it as Done).values }) else FanStep(steps.size) { steps[it] } - fun fillFetch(source: IBulkExecutor, keys: Set, results: Map) { - val cache = cacheFor(source) - for (key in keys) cache[key] = if (results.containsKey(key)) results[key] else MISSING - } - - fun hasAsync(token: Any): Boolean = asyncResults.containsKey(token) - fun asyncResult(token: Any): List = asyncResults.getValue(token) - fun fillAsync(token: Any, values: List) { asyncResults[token] = values } -} +/** Applicative combination handing all resolved value lists to [f] together. */ +internal fun zipN(steps: List>, f: (List>) -> List): Step = + if (steps.all { it is Done }) Done(f(steps.map { (it as Done).values })) else ZipStep(steps, f) -/** A fetch leaf with zero-or-one semantics: a `null`/absent value resolves to an empty step. */ -internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: Any?): Step { - if (execution.isFetched(source, key)) { - val value = execution.fetchedValue(source, key) - return if (value == null) Done(emptyList()) else Done(listOf(value)) +/** + * Applicative concatenation of children built lazily from [items] — the breadth-bounded fan-out. + * + * The leading run of children that resolve synchronously (to a [Done]) is built and concatenated eagerly, preserving + * the build-time evaluation of a synchronous prefix. At the first child that would block on a request, the rest of the + * children become a lazy [FanStep]: instantiated on demand and dropped once resolved, so the live unresolved children + * stay bounded by the round cap. A wide traversal whose children are all fetches has an empty eager prefix and is + * therefore fully lazy. + */ +internal fun fanOut(items: List, makeChild: (A) -> Step): Step { + if (items.isEmpty()) return Done(emptyList()) + val prefix = ArrayList() + var i = 0 + var firstBlocking: Step? = null + while (i < items.size) { + val child = makeChild(items[i]) + if (child is Done) { + prefix.addAll(child.values) + i++ + } else { + firstBlocking = child + break + } } - return Blocked(Pending.fetch(source, key)) { fetchStep(execution, source, key) } + if (firstBlocking == null) return Done(prefix) // every child was synchronous + val startIndex = i + val alreadyBuilt = firstBlocking + val lazyTail = FanStep(items.size - startIndex) { idx -> + if (idx == 0) alreadyBuilt else makeChild(items[startIndex + idx]) + } + return if (prefix.isEmpty()) lazyTail else FanStep(2) { idx -> if (idx == 0) Done(prefix) else lazyTail } } +// --------------------------------------------------------------------------------------------------------------------- +// Leaves +// --------------------------------------------------------------------------------------------------------------------- + +/** A fetch leaf. The [execution] is unused at build time; the cache is consulted by the driver while it walks. */ +@Suppress("UNUSED_PARAMETER") +internal fun fetchStep(execution: Execution, source: IBulkExecutor, key: Any?): Step = + FetchStep(source, key) + +@Suppress("UNUSED_PARAMETER") internal fun asyncStep( execution: Execution, token: Any, produceBlocking: () -> List, produceSuspending: suspend () -> List, -): Step { - if (execution.hasAsync(token)) return Done(execution.asyncResult(token)) - return Blocked(Pending.async(AsyncAction(token, produceBlocking, produceSuspending))) { - asyncStep(execution, token, produceBlocking, produceSuspending) - } -} +): Step = AsyncStep(token, produceBlocking, produceSuspending) + +@Suppress("UNUSED_PARAMETER") +internal fun memoStep(execution: Execution, token: Any, init: () -> Step): Step = MemoStep(token, init) + +// --------------------------------------------------------------------------------------------------------------------- +// Per-run state and the depth-first, bounded-frontier driver +// --------------------------------------------------------------------------------------------------------------------- + +private val MISSING = Any() /** - * Shared, single-advancing progress of a memoized stream. Every reference to a cached stream produces a *view* of the - * same cell; the cell's inner step is advanced at most once per round (regardless of how many views resume it), so the - * underlying computation — and any side effects in it — runs exactly once and its result is multicast to all - * consumers. This is the [org.modelix.streams.IStream.One.cached] mechanism. + * Per-query state shared by every [Step] built for a single execution. Holds the fetch cache (each key fetched at most + * once across the whole traversal), async results, and shared memo cells, plus the mutable state of the current round. */ -internal class MemoCell(var step: Step) { - fun advanceFrom(snapshot: Step) { - if (step === snapshot && snapshot is Blocked) { - step = snapshot.resume() - } - } +internal class Execution { + private val fetchCaches = HashMap, HashMap>() + private val asyncResults = HashMap>() + private val memoCells = HashMap>() - fun view(): Step = when (val snapshot = step) { - is Done -> snapshot - is Failed -> snapshot - is Blocked -> Blocked(snapshot.pending) { - advanceFrom(snapshot) - view() - } + // Mutable state of the round currently being collected; reset by [newRound]. + private var pendingFetches = LinkedHashMap, LinkedHashSet>() + private var pendingAsync = ArrayList>() + private val seenAsyncTokens = HashSet() + private var pendingCount = 0 + private var cap = Int.MAX_VALUE + + private fun cacheFor(source: IBulkExecutor) = fetchCaches.getOrPut(source) { HashMap() } + + private fun memoCell(token: Any, init: () -> Step): Step = memoCells.getOrPut(token, init) + + private fun newRound() { + pendingFetches = LinkedHashMap() + pendingAsync = ArrayList() + seenAsyncTokens.clear() + pendingCount = 0 + cap = Int.MAX_VALUE } -} -/** A leaf that resolves the inner stream of a [org.modelix.streams.IStream.One.cached] call once, shared per run. */ -internal fun memoStep(execution: Execution, token: Any, init: () -> Step): Step = - execution.memoCell(token, init).view() + private fun roundIsFull(): Boolean = pendingCount >= cap -// --------------------------------------------------------------------------------------------------------------------- -// Drivers -// --------------------------------------------------------------------------------------------------------------------- + /** Registers a fetch demand for the current round, honoring the per-source batch-size cap. */ + private fun addFetch(source: IBulkExecutor, key: Any?) { + val set = pendingFetches.getOrPut(source) { LinkedHashSet() } + if (key in set) return + cap = if (cap == Int.MAX_VALUE) source.batchSize else minOf(cap, source.batchSize) + if (pendingCount >= cap) return + set.add(key) + pendingCount++ + } -/** - * Drives a step to completion blocking. Each loop iteration is one round: one bulk call per source (chunked to that - * source's [IBulkExecutor.batchSize]) plus any async leaves, then resume. The loop is the trampoline that keeps - * fetch-dependent chains stack-safe regardless of depth. - */ -internal fun Execution.drive(initial: Step): List { - var step = initial - while (true) { - when (val current = step) { - is Done -> return current.values - is Failed -> throw current.cause - is Blocked -> { - for ((source, keys) in current.pending.fetches) { - for (chunk in keys.chunked(source.batchSize)) { - @Suppress("UNCHECKED_CAST") - val results = source.execute(chunk) as Map - fillFetch(source, chunk.toSet(), results) + private fun addAsync(step: AsyncStep<*>) { + if (seenAsyncTokens.add(step.token)) pendingAsync.add(step) + } + + /** + * Returns the node's resolved value list, or `null` if it is still blocked on a request that was registered for the + * current round. Recurses on the node graph; the recursion depth is the traversal depth (bounded in practice), + * matching the recursion characteristics of the previous engine. + */ + private fun expand(step: Step<*>): List? { + step.cached?.let { return it } + val result: List? = when (step) { + is Done -> step.values + is FetchStep -> { + val cache = cacheFor(step.source) + if (cache.containsKey(step.key)) { + val v = cache[step.key] + if (v === MISSING) emptyList() else listOf(v) + } else { + addFetch(step.source, step.key) + null + } + } + is AsyncStep -> { + if (asyncResults.containsKey(step.token)) { + asyncResults.getValue(step.token) + } else { + addAsync(step) + null + } + } + is MapStep<*, *> -> { + @Suppress("UNCHECKED_CAST") + expand(step.src)?.let { (step.f as (List) -> List)(it) } + } + is FlatMapStep<*, *> -> { + val sv = expand(step.src) + if (sv == null) { + null + } else { + @Suppress("UNCHECKED_CAST") + val typed = step as FlatMapStep + val inner = typed.inner ?: typed.f(sv).also { typed.inner = it } + expand(inner) + } + } + is ZipStep<*, *> -> { + var allResolved = true + val partValues = ArrayList>(step.parts.size) + for (part in step.parts) { + val pv = expand(part) + if (pv == null) allResolved = false else if (allResolved) partValues.add(pv) + } + if (allResolved) { + @Suppress("UNCHECKED_CAST") + (step.f as (List>) -> List)(partValues) + } else { + null + } + } + is FanStep<*> -> { + @Suppress("UNCHECKED_CAST") + val fan = step as FanStep + val out = ArrayList() + var allResolved = true + var i = 0 + while (i < fan.size) { + val done = fan.childResults[i] + if (done != null) { + if (allResolved) out.addAll(done) + i++ + continue + } + val child = fan.childNodes[i] ?: fan.makeChild(i).also { fan.childNodes[i] = it } + val cv = expand(child) + if (cv != null) { + fan.childResults[i] = cv + fan.childNodes[i] = null // free the resolved child node + if (allResolved) out.addAll(cv) + } else { + allResolved = false + // Keep scanning siblings to batch their requests into this same round, until the cap is hit. + if (roundIsFull()) break } + i++ } - for (action in current.pending.asyncActions) { - fillAsync(action.token, action.produceBlocking()) + if (allResolved) out else null + } + is RecoverStep<*> -> { + try { + expand(step.src) + } catch (ex: Throwable) { + @Suppress("UNCHECKED_CAST") + (step.handler as (Throwable) -> List)(ex) } - step = current.resume() } + is OnErrorStep<*> -> { + try { + expand(step.src) + } catch (ex: Throwable) { + step.consumer(ex) + throw ex + } + } + is MemoStep<*> -> expand(memoCell(step.token, step.init)) } + if (result != null) step.cached = result + return result } -} -internal suspend fun Execution.driveSuspending(initial: Step): List { - var step = initial - while (true) { - when (val current = step) { - is Done -> return current.values - is Failed -> throw current.cause - is Blocked -> { - for ((source, keys) in current.pending.fetches) { - for (chunk in keys.chunked(source.batchSize)) { - @Suppress("UNCHECKED_CAST") - val results = source.executeSuspending(chunk) as Map - fillFetch(source, chunk.toSet(), results) - } + /** + * Drives a step to completion, blocking. Each loop iteration is one round: collect the bounded request frontier, + * resolve it (one bulk call per source, chunked to that source's [IBulkExecutor.batchSize], plus any async leaves), + * then walk again. The loop is the trampoline across rounds; fetch-dependent depth costs rounds, not stack. + */ + fun drive(initial: Step): List { + while (true) { + newRound() + val resolved = expand(initial) + if (resolved != null) { + @Suppress("UNCHECKED_CAST") + return resolved as List + } + check(pendingFetches.isNotEmpty() || pendingAsync.isNotEmpty()) { + "stream made no progress: root unresolved but no request was demanded" + } + for ((source, keys) in pendingFetches) { + for (chunk in keys.chunked(source.batchSize)) { + fillFetch(source, chunk, source.execute(chunk)) } - for (action in current.pending.asyncActions) { - fillAsync(action.token, action.produceSuspending()) + } + for (action in pendingAsync) { + asyncResults[action.token] = action.produceBlocking() + } + } + } + + suspend fun driveSuspending(initial: Step): List { + while (true) { + newRound() + val resolved = expand(initial) + if (resolved != null) { + @Suppress("UNCHECKED_CAST") + return resolved as List + } + check(pendingFetches.isNotEmpty() || pendingAsync.isNotEmpty()) { + "stream made no progress: root unresolved but no request was demanded" + } + for ((source, keys) in pendingFetches) { + for (chunk in keys.chunked(source.batchSize)) { + fillFetch(source, chunk, source.executeSuspending(chunk)) } - yield() - step = current.resume() } + for (action in pendingAsync) { + asyncResults[action.token] = action.produceSuspending() + } + yield() } } -} -private fun Set.chunked(size: Int): List> = - if (this.size <= size) listOf(this.toList()) else this.toList().chunked(size) + private fun fillFetch(source: IBulkExecutor, keys: List, results: Map) { + val cache = cacheFor(source) + for (key in keys) cache[key] = if (results.containsKey(key)) results[key] else MISSING + } +} diff --git a/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt b/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt new file mode 100644 index 0000000000..0f8e8817b1 --- /dev/null +++ b/streams/src/commonTest/kotlin/org/modelix/streams/FrontierSizeTest.kt @@ -0,0 +1,91 @@ +package org.modelix.streams + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * A synthetic content-addressed tree exposed as an [IBulkExecutor]. The node id is the key; the value is the + * comma-separated list of child ids. Nodes are laid out as a complete [branching]-ary tree in BFS order + * (root = 0, children of i are i*branching+1 .. i*branching+branching), so children can be computed on the fly + * without materializing the whole tree. + * + * Crucially it records, per bulk round, how many keys were requested together. With a [batchSize] larger than any + * level, each [execute] call corresponds to exactly one engine round, so [maxRoundSize] is the peak fetch frontier — + * the quantity the old `BulkRequestStreamExecutor.sendNextBatch` depth-first traversal kept bounded. + */ +private class TreeSource(val branching: Int, val depth: Int) : IBulkExecutor { + val nodeCount: Int = ((pow(branching, depth + 1) - 1) / (branching - 1)) + val widestLevel: Int = pow(branching, depth) + + var maxRoundSize: Int = 0 + private set + var roundCount: Int = 0 + private set + var totalFetched: Int = 0 + private set + + override fun execute(keys: List): Map { + roundCount++ + totalFetched += keys.size + if (keys.size > maxRoundSize) maxRoundSize = keys.size + return keys.associateWith { id -> + (1..branching).map { id * branching + it }.filter { it < nodeCount }.joinToString(",") + } + } + + override suspend fun executeSuspending(keys: List): Map = execute(keys) + + companion object { + private fun pow(base: Int, exp: Int): Int { + var r = 1 + repeat(exp) { r *= base } + return r + } + } +} + +class FrontierSizeTest { + + /** Builds `getDescendants(id)` the same shape the real tree uses: of(id) + children.flatMap { descendants }. */ + private fun descendants(executor: BulkRequestStreamExecutor, id: Int): IStream.Many { + val childIds: IStream.Many = executor.enqueue(id).orNull().flatMapOrdered { csv -> + if (csv.isNullOrEmpty()) IStream.many(emptyList()) else IStream.many(csv.split(",").map { it.toInt() }) + } + return IStream.of(id).concat(childIds.flatMapOrdered { descendants(executor, it) }) + } + + @Test + fun frontier_is_bounded_by_batch_size_on_wide_deep_tree() { + val source = TreeSource(branching = 4, depth = 7) // 21845 nodes, widest level 16384 + val batchSize = 1000 + val executor = BulkRequestStreamExecutor(source, batchSize = batchSize) + + val count = executor.query { descendants(executor, 0).count() } + + // The traversal is correct and does no redundant work... + assertEquals(source.nodeCount, count) + assertEquals(source.nodeCount, source.totalFetched) + // ...and the peak request frontier stays within the batch size, even though the widest level is far larger. + // This is the depth-first bound the old sendNextBatch provided; the new round-based engine now preserves it. + assertTrue(source.widestLevel > batchSize, "test should exercise a level wider than the cap") + assertTrue( + source.maxRoundSize <= batchSize, + "peak frontier ${source.maxRoundSize} must stay within batchSize $batchSize (widest level ${source.widestLevel})", + ) + } + + @Test + fun unbounded_batch_size_batches_each_level_into_one_round() { + // With a batch size above any level, no frontier is trimmed: each tree level is fetched in a single round, so + // applicative batching is fully preserved when memory is not a constraint. (depth + 1 = 8 rounds.) + val source = TreeSource(branching = 4, depth = 7) + val executor = BulkRequestStreamExecutor(source, batchSize = Int.MAX_VALUE) + + val count = executor.query { descendants(executor, 0).count() } + + assertEquals(source.nodeCount, count) + assertEquals(8, source.roundCount) + assertEquals(source.widestLevel, source.maxRoundSize) + } +} diff --git a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt index b548fd8f3b..cafd2fb4e2 100644 --- a/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt +++ b/streams/src/jvmMain/kotlin/org/modelix/streams/BlockingStreamExecutor.kt @@ -1,8 +1,6 @@ package org.modelix.streams import org.modelix.streams.engine.Execution -import org.modelix.streams.engine.drive -import org.modelix.streams.engine.driveSuspending /** * JVM executor that always drives streams to completion blocking, resolving async leaves (e.g. flows) via