From 2bc24abe30c2a62ddc4398745672ff40832d35a2 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 15:02:28 +0100 Subject: [PATCH 01/15] wip --- .donna/project/core/top_level_architecture.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.donna/project/core/top_level_architecture.md b/.donna/project/core/top_level_architecture.md index b3a1b18..8ce56f2 100644 --- a/.donna/project/core/top_level_architecture.md +++ b/.donna/project/core/top_level_architecture.md @@ -26,3 +26,7 @@ The code is separated by layers/subsystems into subpackages: - `donna.primitives` — code that implements basic building blocks for Donna's behavior: concrete implementations of various classes from the `donna.machine`. - `donna.lib` — module that contains constructed primitives to be used in donna artifacts by referencing them by python import path. Like `donna.lib.workflow`, `donna.lib.goto`, etc. - `donna.artifacts` — artifacts that are distributed with Donna itself: specifications of how it works, predefined workflows, etc. + +## Data structures + +- Do not use `dataclass` for data structures. Use `donna.core.entities.BaseEntity` (subclass of the `pydantic.BaseModel`) instead. From ee0460672c44190186b9c5b21a250de3a27b0d55 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 15:24:19 +0100 Subject: [PATCH 02/15] wip --- .donna/project/core/top_level_architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.donna/project/core/top_level_architecture.md b/.donna/project/core/top_level_architecture.md index 8ce56f2..2df018c 100644 --- a/.donna/project/core/top_level_architecture.md +++ b/.donna/project/core/top_level_architecture.md @@ -29,4 +29,4 @@ The code is separated by layers/subsystems into subpackages: ## Data structures -- Do not use `dataclass` for data structures. Use `donna.core.entities.BaseEntity` (subclass of the `pydantic.BaseModel`) instead. +- Do not use `dataclass` for data structures. Use `donna.core.entities.BaseEntity` (subclass of the `pydantic.BaseModel`) for complex data structures and Python classes with `__slots__` for very simple ones (like cache keys). From 23f367f7bb6e1b7613ff1591b0271d3317e5e9c3 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 15:46:33 +0100 Subject: [PATCH 03/15] wip --- .donna/project/core/top_level_architecture.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.donna/project/core/top_level_architecture.md b/.donna/project/core/top_level_architecture.md index 2df018c..d51d337 100644 --- a/.donna/project/core/top_level_architecture.md +++ b/.donna/project/core/top_level_architecture.md @@ -30,3 +30,7 @@ The code is separated by layers/subsystems into subpackages: ## Data structures - Do not use `dataclass` for data structures. Use `donna.core.entities.BaseEntity` (subclass of the `pydantic.BaseModel`) for complex data structures and Python classes with `__slots__` for very simple ones (like cache keys). + +## Autotests + +- No autotests in the project for now. From 38ac92800556135e0c2d99ce2f7b37c21b9f86cd Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 17:06:01 +0100 Subject: [PATCH 04/15] wip: smth works --- donna/cli/commands/artifacts.py | 38 ++-- donna/cli/commands/journal.py | 10 +- donna/cli/utils.py | 13 +- donna/context/__init__.py | 3 + donna/context/artifacts.py | 204 ++++++++++++++++++++++ donna/context/context.py | 55 ++++++ donna/context/primitives.py | 53 ++++++ donna/context/state.py | 39 +++++ donna/context/value_scope.py | 24 +++ donna/machine/artifacts.py | 24 +-- donna/machine/journal.py | 21 +-- donna/machine/sessions.py | 23 +-- donna/machine/state.py | 21 +-- donna/machine/tasks.py | 26 ++- donna/primitives/operations/run_script.py | 6 - donna/workspaces/worlds/base.py | 3 + donna/workspaces/worlds/filesystem.py | 9 + donna/workspaces/worlds/python.py | 9 + 18 files changed, 452 insertions(+), 129 deletions(-) create mode 100644 donna/context/__init__.py create mode 100644 donna/context/artifacts.py create mode 100644 donna/context/context.py create mode 100644 donna/context/primitives.py create mode 100644 donna/context/state.py create mode 100644 donna/context/value_scope.py diff --git a/donna/cli/commands/artifacts.py b/donna/cli/commands/artifacts.py index ac5bccf..96d3726 100644 --- a/donna/cli/commands/artifacts.py +++ b/donna/cli/commands/artifacts.py @@ -16,14 +16,13 @@ TagOption, ) from donna.cli.utils import cells_cli +from donna.context.context import context from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result from donna.domain.ids import FullArtifactIdPattern from donna.machine import journal as machine_journal -from donna.machine.sessions import load_state from donna.protocol.cell_shortcuts import operation_succeeded from donna.protocol.cells import Cell -from donna.workspaces import artifacts as world_artifacts from donna.workspaces import tmp as world_tmp artifacts_cli = typer.Typer() @@ -44,20 +43,7 @@ def _parse_slug_with_extension(value: str) -> Result[tuple[str, str], ErrorsList def _log_artifact_operation(message: str) -> None: - state_result = load_state() - - if state_result.is_err(): - # log nothing if we have no session state - return - - state = state_result.unwrap() - - machine_journal.add( - message=message, - current_task_id=str(state.current_task.id) if state.current_task else None, - current_work_unit_id=None, - current_operation_id=None, - ) + machine_journal.add(message=message) def _log_operation_on_artifacts(message: str, pattern: FullArtifactIdPattern, tags: TagOption | None) -> None: @@ -79,7 +65,7 @@ def list( ) -> Iterable[Cell]: _log_operation_on_artifacts("List artifacts", pattern, tags) - artifacts = world_artifacts.list_artifacts(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, tags=tags).unwrap() return [artifact.node().status() for artifact in artifacts] @@ -92,7 +78,7 @@ def view( ) -> Iterable[Cell]: _log_operation_on_artifacts("View artifacts", pattern, tags) - artifacts = world_artifacts.list_artifacts(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, tags=tags).unwrap() return [artifact.node().info() for artifact in artifacts] @@ -105,12 +91,12 @@ def view( @cells_cli def fetch(id: FullArtifactIdArgument, output: OutputPathOption = None) -> Iterable[Cell]: if output is None: - extension = world_artifacts.artifact_file_extension(id).unwrap() + extension = context().artifacts.file_extension(id).unwrap() output = world_tmp.file_for_artifact(id, extension) _log_artifact_operation(f"Fetch artifact `{id}` to '{output}'") - world_artifacts.fetch_artifact(id, output).unwrap() + context().artifacts.fetch(id, output).unwrap() return [ operation_succeeded(f"Artifact `{id}` fetched to '{output}'", artifact_id=str(id), output_path=str(output)) @@ -155,7 +141,7 @@ def update( _log_artifact_operation(f"Update artifact `{id}` from '{input_display}'") - world_artifacts.update_artifact(id, input_path, extension=extension).unwrap() + context().artifacts.update(id, input_path, extension=extension).unwrap() return [ operation_succeeded( f"Artifact `{id}` updated from '{input_display}'", @@ -170,7 +156,7 @@ def update( def copy(source_id: FullArtifactIdArgument, target_id: FullArtifactIdArgument) -> Iterable[Cell]: _log_artifact_operation(f"Copy artifact from `{source_id}` to `{target_id}`") - world_artifacts.copy_artifact(source_id, target_id).unwrap() + context().artifacts.copy(source_id, target_id).unwrap() return [ operation_succeeded( f"Artifact `{source_id}` copied to `{target_id}`", @@ -185,7 +171,7 @@ def copy(source_id: FullArtifactIdArgument, target_id: FullArtifactIdArgument) - def move(source_id: FullArtifactIdArgument, target_id: FullArtifactIdArgument) -> Iterable[Cell]: _log_artifact_operation(f"Move artifact from `{source_id}` to `{target_id}`") - world_artifacts.move_artifact(source_id, target_id).unwrap() + context().artifacts.move(source_id, target_id).unwrap() return [ operation_succeeded( f"Artifact `{source_id}` moved to `{target_id}`", @@ -203,11 +189,11 @@ def remove( ) -> Iterable[Cell]: _log_operation_on_artifacts("Remove artifacts", pattern, tags) - artifacts = world_artifacts.list_artifacts(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, tags=tags).unwrap() cells: builtins.list[Cell] = [] for artifact in artifacts: - world_artifacts.remove_artifact(artifact.id).unwrap() + context().artifacts.remove(artifact.id).unwrap() cells.append(operation_succeeded(f"Artifact `{artifact.id}` removed", artifact_id=str(artifact.id))) return cells @@ -221,7 +207,7 @@ def validate( ) -> Iterable[Cell]: # noqa: CCR001 _log_operation_on_artifacts("Validate artifacts", pattern, tags) - artifacts = world_artifacts.list_artifacts(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, tags=tags).unwrap() errors = [] diff --git a/donna/cli/commands/journal.py b/donna/cli/commands/journal.py index 3a14cc5..4cca9bb 100644 --- a/donna/cli/commands/journal.py +++ b/donna/cli/commands/journal.py @@ -6,7 +6,6 @@ from donna.cli.application import app from donna.cli.utils import cells_cli, output_cells from donna.machine import journal as machine_journal -from donna.machine.sessions import load_state from donna.protocol.cell_shortcuts import operation_succeeded from donna.protocol.cells import Cell from donna.protocol.modes import get_cell_formatter @@ -19,14 +18,7 @@ def write( message: str = typer.Argument(..., help="Single-line message to append to journal (newlines are not allowed)."), ) -> Iterable[Cell]: - state = load_state().unwrap() - - machine_journal.add( - message=message, - current_task_id=str(state.current_task.id) if state.current_task else None, - current_work_unit_id=None, - current_operation_id=None, - ).unwrap() + machine_journal.add(message=message).unwrap() return [operation_succeeded("Journal record appended.")] diff --git a/donna/cli/utils.py b/donna/cli/utils.py index 5acea8a..b617ae1 100644 --- a/donna/cli/utils.py +++ b/donna/cli/utils.py @@ -27,22 +27,12 @@ def output_cells(cells: Iterable[Cell]) -> None: def _write_errors_to_journal(errors: ErrorsList) -> None: from donna.machine import journal as machine_journal - from donna.machine import sessions as machine_sessions - - state_result = machine_sessions.load_state() - current_task_id = None - if state_result.is_ok(): - state = state_result.unwrap() - current_task_id = str(state.current_task.id) if state.current_task else None for error in errors: message = f"Error: {error.node().journal_message()} [{error.code}]" machine_journal.add( message=message, - current_task_id=current_task_id, - current_work_unit_id=None, - current_operation_id=None, actor_id="donna", ) @@ -80,6 +70,8 @@ def _is_workspace_init_command() -> bool: def try_initialize_donna(project_dir: pathlib.Path | None, protocol: Mode) -> None: + from donna.context import Context, set_context + if _is_workspace_init_command(): workspace_config.protocol.set(protocol) if project_dir is not None: @@ -89,6 +81,7 @@ def try_initialize_donna(project_dir: pathlib.Path | None, protocol: Mode) -> No result = initialize_runtime(root_dir=project_dir, protocol=protocol) if result.is_ok(): + set_context(Context()) return errors = result.unwrap_err() diff --git a/donna/context/__init__.py b/donna/context/__init__.py new file mode 100644 index 0000000..1bb672b --- /dev/null +++ b/donna/context/__init__.py @@ -0,0 +1,3 @@ +from donna.context.context import Context, context, reset_context, set_context + +__all__ = ("Context", "context", "set_context", "reset_context") diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py new file mode 100644 index 0000000..bf518b7 --- /dev/null +++ b/donna/context/artifacts.py @@ -0,0 +1,204 @@ +import time +from pathlib import Path +from typing import TYPE_CHECKING + +from donna.core.errors import ErrorsList +from donna.core.result import Err, Ok, Result, unwrap_to_error +from donna.domain.ids import FullArtifactId, FullArtifactIdPattern, FullArtifactSectionId +from donna.machine.artifacts import Artifact, ArtifactSection +from donna.workspaces.templates import RenderMode + +if TYPE_CHECKING: + from donna.workspaces.artifacts import ArtifactRenderContext + + +class _ArtifactCacheValue: + __slots__ = ("view_artifact", "analysis_artifact", "loaded_at_ms") + + def __init__( + self, + view_artifact: Artifact | None, + analysis_artifact: Artifact | None, + loaded_at_ms: int, + ) -> None: + self.view_artifact = view_artifact + self.analysis_artifact = analysis_artifact + self.loaded_at_ms = loaded_at_ms + + +class ArtifactsCache: + __slots__ = ("_cache",) + + def __init__(self) -> None: + self._cache: dict[FullArtifactId, _ArtifactCacheValue] = {} + + @staticmethod + def _now_ms() -> int: + return time.time_ns() // 1_000_000 + + @staticmethod + def _default_render_context() -> "ArtifactRenderContext": + from donna.workspaces.artifacts import ArtifactRenderContext + + return ArtifactRenderContext(primary_mode=RenderMode.view) + + @staticmethod + def _context_slot_name(render_context: "ArtifactRenderContext") -> str: + if render_context.primary_mode == RenderMode.view: + return "view_artifact" + + if render_context.primary_mode == RenderMode.analysis: + return "analysis_artifact" + + return "" + + @unwrap_to_error + def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: int) -> Result[bool, ErrorsList]: + from donna.workspaces.config import config + + world = config().get_world(full_id.world_id).unwrap() + return Ok(world.has_artifact_changed(full_id.artifact_id, since=loaded_at_ms).unwrap()) + + def invalidate(self, full_id: FullArtifactId) -> None: + self._cache.pop(full_id, None) + + @unwrap_to_error + def load( # noqa: CCR001 + self, full_id: FullArtifactId, render_context: "ArtifactRenderContext | None" = None + ) -> Result[Artifact, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + if render_context is None: + render_context = self._default_render_context() + + if render_context.primary_mode == RenderMode.execute: + return Ok(workspace_artifacts.load_artifact(full_id, render_context).unwrap()) + + cached = self._cache.get(full_id) + cache_slot = self._context_slot_name(render_context) + since = cached.loaded_at_ms if cached is not None else 0 + cache_stale = self._is_cache_stale(full_id, since).unwrap() + + if cached is not None and cache_slot: + cached_artifact = getattr(cached, cache_slot) + + if cached_artifact is not None and not cache_stale: + return Ok(cached_artifact) + + artifact = workspace_artifacts.load_artifact(full_id, render_context).unwrap() + + loaded_at_ms = self._now_ms() + + if cached is None: + cached = _ArtifactCacheValue(view_artifact=None, analysis_artifact=None, loaded_at_ms=loaded_at_ms) + self._cache[full_id] = cached + else: + cached.loaded_at_ms = loaded_at_ms + + if render_context.primary_mode == RenderMode.view: + cached.view_artifact = artifact + elif render_context.primary_mode == RenderMode.analysis: + cached.analysis_artifact = artifact + + return Ok(artifact) + + @unwrap_to_error + def resolve_section( + self, + target_id: FullArtifactSectionId, + render_context: "ArtifactRenderContext | None" = None, + ) -> Result[ArtifactSection, ErrorsList]: + artifact = self.load(target_id.full_artifact_id, render_context).unwrap() + return Ok(artifact.get_section(target_id.local_id).unwrap()) + + @unwrap_to_error + def update( + self, + full_id: FullArtifactId, + input_path: Path, + extension: str | None = None, + ) -> Result[None, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + workspace_artifacts.update_artifact(full_id, input_path, extension=extension).unwrap() + self.invalidate(full_id) + return Ok(None) + + @unwrap_to_error + def copy(self, source_id: FullArtifactId, target_id: FullArtifactId) -> Result[None, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + workspace_artifacts.copy_artifact(source_id, target_id).unwrap() + self.invalidate(target_id) + return Ok(None) + + @unwrap_to_error + def move(self, source_id: FullArtifactId, target_id: FullArtifactId) -> Result[None, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + workspace_artifacts.move_artifact(source_id, target_id).unwrap() + self.invalidate(source_id) + self.invalidate(target_id) + return Ok(None) + + @unwrap_to_error + def remove(self, full_id: FullArtifactId) -> Result[None, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + workspace_artifacts.remove_artifact(full_id).unwrap() + self.invalidate(full_id) + return Ok(None) + + @unwrap_to_error + def file_extension(self, full_id: FullArtifactId) -> Result[str, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + return Ok(workspace_artifacts.artifact_file_extension(full_id).unwrap()) + + @unwrap_to_error + def fetch(self, full_id: FullArtifactId, output: Path) -> Result[None, ErrorsList]: + from donna.workspaces import artifacts as workspace_artifacts + + workspace_artifacts.fetch_artifact(full_id, output).unwrap() + return Ok(None) + + @unwrap_to_error + def list( # noqa: CCR001 + self, + pattern: FullArtifactIdPattern, + render_context: "ArtifactRenderContext | None" = None, + tags: list[str] | None = None, + ) -> Result[list[Artifact], ErrorsList]: + from donna.workspaces.config import config + + if render_context is None: + render_context = self._default_render_context() + + tag_filters = tags or [] + + artifacts: list[Artifact] = [] + errors: ErrorsList = [] + + for world in reversed(config().worlds_instances): + for artifact_id in world.list_artifacts(pattern): + full_id = FullArtifactId((world.id, artifact_id)) + artifact_result = self.load(full_id, render_context) + if artifact_result.is_err(): + errors.extend(artifact_result.unwrap_err()) + continue + + artifact = artifact_result.unwrap() + if tag_filters: + primary_result = artifact.primary_section() + if primary_result.is_err(): + continue + primary = primary_result.unwrap() + if not all(tag in primary.tags for tag in tag_filters): + continue + + artifacts.append(artifact) + + if errors: + return Err(errors) + + return Ok(artifacts) diff --git a/donna/context/context.py b/donna/context/context.py new file mode 100644 index 0000000..5883d51 --- /dev/null +++ b/donna/context/context.py @@ -0,0 +1,55 @@ +import contextvars + +from donna.context.artifacts import ArtifactsCache +from donna.context.primitives import PrimitivesCache +from donna.context.state import StateCache +from donna.context.value_scope import ValueScope +from donna.domain.ids import FullArtifactSectionId, WorkUnitId + + +class Context: + __slots__ = ( + "_artifacts", + "_state", + "_primitives", + "current_work_unit_id", + "current_operation_id", + ) + + def __init__(self) -> None: + self._artifacts = ArtifactsCache() + self._state = StateCache() + self._primitives = PrimitivesCache() + self.current_work_unit_id: ValueScope[WorkUnitId] = ValueScope() + self.current_operation_id: ValueScope[FullArtifactSectionId] = ValueScope() + + @property + def artifacts(self) -> ArtifactsCache: + return self._artifacts + + @property + def state(self) -> StateCache: + return self._state + + @property + def primitives(self) -> PrimitivesCache: + return self._primitives + + +_context_var: contextvars.ContextVar[Context | None] = contextvars.ContextVar("donna_machine_context", default=None) + + +def set_context(new_context: Context) -> contextvars.Token[Context | None]: + return _context_var.set(new_context) + + +def reset_context(token: contextvars.Token[Context | None]) -> None: + _context_var.reset(token) + + +def context() -> Context: + current = _context_var.get() + if current is None: + raise RuntimeError("Donna machine context is not initialized") + + return current diff --git a/donna/context/primitives.py b/donna/context/primitives.py new file mode 100644 index 0000000..4af6172 --- /dev/null +++ b/donna/context/primitives.py @@ -0,0 +1,53 @@ +import importlib +from typing import TYPE_CHECKING + +from donna.core.errors import ErrorsList +from donna.core.result import Err, Ok, Result, unwrap_to_error +from donna.domain.ids import PythonImportPath +from donna.machine import errors as machine_errors + +if TYPE_CHECKING: + from donna.machine.primitives import Primitive + + +class PrimitivesCache: + __slots__ = ("_cache",) + + def __init__(self) -> None: + self._cache: dict[PythonImportPath, "Primitive"] = {} + + @unwrap_to_error + def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", ErrorsList]: # noqa: CCR001 + from donna.machine.primitives import Primitive + + if isinstance(primitive_id, PythonImportPath): + import_path = primitive_id + else: + import_path = PythonImportPath.parse(primitive_id).unwrap() + + cached = self._cache.get(import_path) + if cached is not None: + return Ok(cached) + + import_path_str = str(import_path) + + if "." not in import_path_str: + return Err([machine_errors.PrimitiveInvalidImportPath(import_path=import_path_str)]) + + module_path, primitive_name = import_path_str.rsplit(".", maxsplit=1) + + try: + module = importlib.import_module(module_path) + except ModuleNotFoundError: + return Err([machine_errors.PrimitiveModuleNotImportable(module_path=module_path)]) + + try: + primitive = getattr(module, primitive_name) + except AttributeError: + return Err([machine_errors.PrimitiveNotAvailable(import_path=import_path_str, module_path=module_path)]) + + if not isinstance(primitive, Primitive): + return Err([machine_errors.PrimitiveNotPrimitive(import_path=import_path_str)]) + + self._cache[import_path] = primitive + return Ok(primitive) diff --git a/donna/context/state.py b/donna/context/state.py new file mode 100644 index 0000000..dbac8b5 --- /dev/null +++ b/donna/context/state.py @@ -0,0 +1,39 @@ +from typing import TYPE_CHECKING + +from donna.core.errors import ErrorsList +from donna.core.result import Err, Ok, Result, unwrap_to_error +from donna.machine import errors as machine_errors + +if TYPE_CHECKING: + from donna.machine.state import ConsistentState + + +class StateCache: + __slots__ = ("_session_state",) + + def __init__(self) -> None: + self._session_state: ConsistentState | None = None + + @unwrap_to_error + def load(self) -> Result["ConsistentState", ErrorsList]: + from donna.machine.state import ConsistentState + from donna.workspaces import utils as workspace_utils + + if self._session_state is not None: + return Ok(self._session_state) + + content = workspace_utils.session_world().unwrap().read_state("state.json").unwrap() + if content is None: + return Err([machine_errors.SessionStateNotInitialized()]) + + state = ConsistentState.from_json(content.decode("utf-8")) + self._session_state = state + return Ok(state) + + @unwrap_to_error + def save(self, state: "ConsistentState") -> Result[None, ErrorsList]: + from donna.workspaces import utils as workspace_utils + + workspace_utils.session_world().unwrap().write_state("state.json", state.to_json().encode("utf-8")).unwrap() + self._session_state = state + return Ok(None) diff --git a/donna/context/value_scope.py b/donna/context/value_scope.py new file mode 100644 index 0000000..beac9bf --- /dev/null +++ b/donna/context/value_scope.py @@ -0,0 +1,24 @@ +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Generic, TypeVar + +V = TypeVar("V") + + +class ValueScope(Generic[V]): + __slots__ = ("_value",) + + def __init__(self, initial: V | None = None) -> None: + self._value: V | None = initial + + def get(self) -> V | None: + return self._value + + @contextmanager + def scope(self, value: V | None) -> Iterator[None]: + previous = self._value + self._value = value + try: + yield + finally: + self._value = previous diff --git a/donna/machine/artifacts.py b/donna/machine/artifacts.py index 919c2ac..3c531dc 100644 --- a/donna/machine/artifacts.py +++ b/donna/machine/artifacts.py @@ -1,11 +1,11 @@ -from typing import TYPE_CHECKING, Any, Union +from typing import Any import pydantic from donna.core.entities import BaseEntity from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error -from donna.domain.ids import ArtifactSectionId, FullArtifactId, FullArtifactSectionId, PythonImportPath +from donna.domain.ids import ArtifactSectionId, FullArtifactId, PythonImportPath from donna.machine.errors import ( ArtifactPrimarySectionMissing, ArtifactSectionNotFound, @@ -14,9 +14,6 @@ from donna.protocol.cells import Cell from donna.protocol.nodes import Node -if TYPE_CHECKING: - from donna.workspaces.artifacts import ArtifactRenderContext - class ArtifactSectionConfig(BaseEntity): id: ArtifactSectionId @@ -71,7 +68,7 @@ def primary_section(self) -> Result[ArtifactSection, ErrorsList]: return Ok(primary_sections[0]) def validate_artifact(self) -> Result[None, ErrorsList]: # noqa: CCR001 - from donna.machine.primitives import resolve_primitive + from donna.context.context import context primary_sections = self._primary_sections() @@ -88,7 +85,7 @@ def validate_artifact(self) -> Result[None, ErrorsList]: # noqa: CCR001 ) for section in self.sections: - primitive_result = resolve_primitive(section.kind) + primitive_result = context().primitives.resolve(section.kind) if primitive_result.is_err(): errors.extend(primitive_result.unwrap_err()) continue @@ -187,16 +184,3 @@ def status(self) -> Cell: section_primary=self._section.primary, **self._section.meta.cells_meta(), ) - - -@unwrap_to_error -def resolve( - target_id: FullArtifactSectionId, render_context: Union["ArtifactRenderContext", None] = None -) -> Result[ArtifactSection, ErrorsList]: - from donna.workspaces import artifacts as world_artifacts - - artifact = world_artifacts.load_artifact(target_id.full_artifact_id, render_context).unwrap() - - section = artifact.get_section(target_id.local_id).unwrap() - - return Ok(section) diff --git a/donna/machine/journal.py b/donna/machine/journal.py index 31f09f6..589e24c 100644 --- a/donna/machine/journal.py +++ b/donna/machine/journal.py @@ -1,7 +1,6 @@ import datetime import json from collections.abc import Iterable -from typing import Any import pydantic @@ -71,26 +70,24 @@ def smart_agent_id() -> str: @unwrap_to_error -def add( +def add( # noqa: CCR001 message: str, - current_task_id: str | None, - current_work_unit_id: str | None, - current_operation_id: str | None, - **kwargs: Any, + actor_id: str | None = None, ) -> Result[JournalRecord, ErrorsList]: + from donna.context.context import context from donna.protocol.utils import instant_output_journal if message_has_newlines(message): return Err([machine_errors.JournalMessageContainsNewlines()]) - if "actor_id" in kwargs: - actor_id = kwargs["actor_id"] - else: + if actor_id is None: actor_id = smart_agent_id() - parsed_task_id = TaskId(current_task_id) if current_task_id is not None else None - parsed_work_unit_id = WorkUnitId(current_work_unit_id) if current_work_unit_id is not None else None - parsed_operation_id = FullArtifactSectionId(current_operation_id) if current_operation_id is not None else None + ctx = context() + state = ctx.state.load().unwrap() + parsed_task_id: TaskId | None = state.current_task.id if state.current_task else None + parsed_work_unit_id: WorkUnitId | None = ctx.current_work_unit_id.get() + parsed_operation_id: FullArtifactSectionId | None = ctx.current_operation_id.get() record = JournalRecord( timestamp=now(), diff --git a/donna/machine/sessions.py b/donna/machine/sessions.py index eef8de8..f15e5ac 100644 --- a/donna/machine/sessions.py +++ b/donna/machine/sessions.py @@ -1,6 +1,7 @@ import functools from typing import Callable, ParamSpec +from donna.context.context import context from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import ActionRequestId, FullArtifactId, FullArtifactSectionId @@ -10,23 +11,18 @@ from donna.machine.state import ConsistentState, MutableState from donna.protocol.cell_shortcuts import operation_succeeded from donna.protocol.cells import Cell -from donna.workspaces import artifacts from donna.workspaces import tmp as world_tmp from donna.workspaces import utils as workspace_utils @unwrap_to_error def load_state() -> Result[ConsistentState, ErrorsList]: - content = workspace_utils.session_world().unwrap().read_state("state.json").unwrap() - if content is None: - return Err([machine_errors.SessionStateNotInitialized()]) - - return Ok(ConsistentState.from_json(content.decode("utf-8"))) + return Ok(context().state.load().unwrap()) @unwrap_to_error def _save_state(state: ConsistentState) -> Result[None, ErrorsList]: - workspace_utils.session_world().unwrap().write_state("state.json", state.to_json().encode("utf-8")).unwrap() + context().state.save(state).unwrap() return Ok(None) @@ -68,15 +64,10 @@ def start() -> Result[list[Cell], ErrorsList]: workspace_utils.session_world().unwrap().initialize(reset=True) machine_journal.reset().unwrap() - - machine_journal.add( - message="Started new session.", - current_task_id=None, - current_work_unit_id=None, - current_operation_id=None, - ).unwrap() _save_state(MutableState.build().freeze()).unwrap() + machine_journal.add(message="Started new session.").unwrap() + return Ok([operation_succeeded("Started new session.")]) @@ -117,7 +108,7 @@ def details() -> Result[list[Cell], ErrorsList]: @unwrap_to_error def start_workflow(artifact_id: FullArtifactId) -> Result[list[Cell], ErrorsList]: # noqa: CCR001 static_state = load_state().unwrap() - workflow = artifacts.load_artifact(artifact_id).unwrap() + workflow = context().artifacts.load(artifact_id).unwrap() primary_section = workflow.primary_section().unwrap() mutator = static_state.mutator() mutator.start_workflow(workflow.id.to_full_local(primary_section.id)).unwrap() @@ -131,7 +122,7 @@ def _validate_operation_transition( state: MutableState, request_id: ActionRequestId, next_operation_id: FullArtifactSectionId ) -> Result[None, ErrorsList]: operation_id = state.get_action_request(request_id).unwrap().operation_id - workflow = artifacts.load_artifact(operation_id.full_artifact_id).unwrap() + workflow = context().artifacts.load(operation_id.full_artifact_id).unwrap() operation = workflow.get_section(operation_id.local_id).unwrap() assert isinstance(operation.meta, OperationMeta) diff --git a/donna/machine/state.py b/donna/machine/state.py index 6b6e47c..8badb4d 100644 --- a/donna/machine/state.py +++ b/donna/machine/state.py @@ -4,6 +4,7 @@ import pydantic +from donna.context.context import context from donna.core.entities import BaseEntity from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error @@ -17,7 +18,6 @@ from donna.machine import errors as machine_errors from donna.machine import journal as machine_journal from donna.machine.action_requests import ActionRequest -from donna.machine.artifacts import resolve from donna.machine.changes import ( Change, ChangeAddTask, @@ -131,9 +131,6 @@ def add_action_request(self, action_request: ActionRequest) -> None: machine_journal.add( actor_id="donna", message=f"Request agent action `{full_request.title}`", - current_task_id=str(self.current_task.id) if self.current_task else None, - current_work_unit_id=None, - current_operation_id=None, ).unwrap() self.action_requests.append(full_request) @@ -171,9 +168,6 @@ def complete_action_request( action_request = self.get_action_request(request_id).unwrap() machine_journal.add( message=f"Complete agent action `{action_request.title}`", - current_task_id=str(current_task.id), - current_work_unit_id=None, - current_operation_id=None, ).unwrap() changes = [ @@ -185,13 +179,10 @@ def complete_action_request( @unwrap_to_error def start_workflow(self, full_operation_id: FullArtifactSectionId) -> Result[None, ErrorsList]: - workflow = resolve(full_operation_id).unwrap() + workflow = context().artifacts.resolve_section(full_operation_id).unwrap() machine_journal.add( message=f"Start workflow `{workflow.title}`", - current_task_id=str(self.current_task.id) if self.current_task else None, - current_work_unit_id=None, - current_operation_id=None, ).unwrap() changes = [ChangeAddTask(operation_id=full_operation_id)] @@ -201,13 +192,10 @@ def start_workflow(self, full_operation_id: FullArtifactSectionId) -> Result[Non def finish_workflow(self, task_id: TaskId) -> None: task = self.current_task assert task is not None - workflow = resolve(task.workflow_id).unwrap() + workflow = context().artifacts.resolve_section(task.workflow_id).unwrap() machine_journal.add( message=f"Finish workflow `{workflow.title}`", - current_task_id=str(self.current_task.id) if self.current_task else None, - current_work_unit_id=None, - current_operation_id=None, ).unwrap() changes = [ChangeRemoveTask(task_id=task_id)] @@ -220,7 +208,8 @@ def execute_next_work_unit(self) -> Result[None, ErrorsList]: current_task = self.current_task assert current_task is not None - changes = next_work_unit.run(current_task).unwrap() + with context().current_work_unit_id.scope(next_work_unit.id): + changes = next_work_unit.run(current_task).unwrap() changes.append(ChangeRemoveWorkUnit(work_unit_id=next_work_unit.id)) self.apply_changes(changes) diff --git a/donna/machine/tasks.py b/donna/machine/tasks.py index 3a10d8f..45da554 100644 --- a/donna/machine/tasks.py +++ b/donna/machine/tasks.py @@ -53,9 +53,8 @@ def build( @unwrap_to_error def run(self, task: Task) -> Result[list["Change"], ErrorsList]: - from donna.machine import artifacts as machine_artifacts + from donna.context.context import context from donna.machine import journal as machine_journal - from donna.machine.primitives import resolve_primitive from donna.workspaces.artifacts import ArtifactRenderContext from donna.workspaces.templates import RenderMode @@ -64,17 +63,16 @@ def run(self, task: Task) -> Result[list["Change"], ErrorsList]: current_task=task, current_work_unit=self, ) - operation = machine_artifacts.resolve(self.operation_id, render_context).unwrap() - operation_kind = resolve_primitive(operation.kind).unwrap() - - machine_journal.add( - actor_id="donna", - message=operation.title, - current_task_id=str(task.id), - current_work_unit_id=str(self.id), - current_operation_id=str(self.operation_id), - ).unwrap() - - changes = operation_kind.execute_section(task, self, operation).unwrap() + ctx = context() + with ctx.current_operation_id.scope(self.operation_id): + operation = ctx.artifacts.resolve_section(self.operation_id, render_context).unwrap() + operation_kind = ctx.primitives.resolve(operation.kind).unwrap() + + machine_journal.add( + actor_id="donna", + message=operation.title, + ).unwrap() + + changes = operation_kind.execute_section(task, self, operation).unwrap() return Ok(changes) diff --git a/donna/primitives/operations/run_script.py b/donna/primitives/operations/run_script.py index c856a71..917ca7f 100644 --- a/donna/primitives/operations/run_script.py +++ b/donna/primitives/operations/run_script.py @@ -164,9 +164,6 @@ def execute_section( machine_journal.add( actor_id="donna", message=f"Run script `{operation.title}`", - current_task_id=str(task.id), - current_work_unit_id=str(unit.id), - current_operation_id=unit.operation_id, ).unwrap() stdout, stderr, exit_code = _run_script( @@ -181,9 +178,6 @@ def execute_section( f"Script finished `{operation.title}`, exit code: {exit_code}, " f"has stdout: {bool(stdout)}, has stderr: {bool(stderr)}`" ), - current_task_id=str(task.id), - current_work_unit_id=str(unit.id), - current_operation_id=unit.operation_id, ).unwrap() changes: list["Change"] = [] diff --git a/donna/workspaces/worlds/base.py b/donna/workspaces/worlds/base.py index 6d1949f..161f22d 100644 --- a/donna/workspaces/worlds/base.py +++ b/donna/workspaces/worlds/base.py @@ -32,6 +32,9 @@ def fetch( # noqa: E704 @abstractmethod def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: ... # noqa: E704 + @abstractmethod + def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: ... # noqa: E704 + @abstractmethod def update( # noqa: E704 self, artifact_id: ArtifactId, content: bytes, extension: str diff --git a/donna/workspaces/worlds/filesystem.py b/donna/workspaces/worlds/filesystem.py index 827f5a2..fc5fc2f 100644 --- a/donna/workspaces/worlds/filesystem.py +++ b/donna/workspaces/worlds/filesystem.py @@ -121,6 +121,15 @@ def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: return Ok(path.read_bytes()) + @unwrap_to_error + def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: + path = self._resolve_artifact_file(artifact_id).unwrap() + + if path is None: + return Ok(True) + + return Ok((path.stat().st_mtime_ns // 1_000_000) > since) + def update(self, artifact_id: ArtifactId, content: bytes, extension: str) -> Result[None, ErrorsList]: if self.readonly: return Err([world_errors.WorldReadonly(world_id=self.id)]) diff --git a/donna/workspaces/worlds/python.py b/donna/workspaces/worlds/python.py index 2c67153..4f7b9e2 100644 --- a/donna/workspaces/worlds/python.py +++ b/donna/workspaces/worlds/python.py @@ -140,6 +140,15 @@ def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: # return Ok(resource_path.read_bytes()) + @unwrap_to_error + def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: + resource_path = self._resolve_artifact_file(artifact_id).unwrap() + + if resource_path is None: + return Ok(True) + + return Ok(False) + def update(self, artifact_id: ArtifactId, content: bytes, extension: str) -> Result[None, ErrorsList]: return Err([world_errors.WorldReadonly(world_id=self.id)]) From ba9d10a9a9067b0804f582b20c44132632a8ab73 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:04:33 +0100 Subject: [PATCH 05/15] wip --- donna/context/artifacts.py | 57 ++++++++++++++-------- donna/workspaces/artifacts.py | 64 ++----------------------- donna/workspaces/worlds/base.py | 20 +++++--- donna/workspaces/worlds/filesystem.py | 68 ++++++++++++--------------- donna/workspaces/worlds/python.py | 63 +++++++++++-------------- 5 files changed, 111 insertions(+), 161 deletions(-) diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py index bf518b7..5c66f88 100644 --- a/donna/context/artifacts.py +++ b/donna/context/artifacts.py @@ -10,17 +10,20 @@ if TYPE_CHECKING: from donna.workspaces.artifacts import ArtifactRenderContext + from donna.workspaces.worlds.base import RawArtifact class _ArtifactCacheValue: - __slots__ = ("view_artifact", "analysis_artifact", "loaded_at_ms") + __slots__ = ("raw_artifact", "view_artifact", "analysis_artifact", "loaded_at_ms") def __init__( self, + raw_artifact: "RawArtifact", view_artifact: Artifact | None, analysis_artifact: Artifact | None, loaded_at_ms: int, ) -> None: + self.raw_artifact = raw_artifact self.view_artifact = view_artifact self.analysis_artifact = analysis_artifact self.loaded_at_ms = loaded_at_ms @@ -62,38 +65,52 @@ def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: int) -> Result[ def invalidate(self, full_id: FullArtifactId) -> None: self._cache.pop(full_id, None) + @staticmethod + @unwrap_to_error + def _load_raw_artifact(full_id: FullArtifactId) -> Result["RawArtifact", ErrorsList]: + from donna.workspaces.config import config + + world = config().get_world(full_id.world_id).unwrap() + return Ok(world.fetch(full_id.artifact_id).unwrap()) + + @unwrap_to_error + def _get_cache_value(self, full_id: FullArtifactId) -> Result[_ArtifactCacheValue, ErrorsList]: + cached = self._cache.get(full_id) + since = cached.loaded_at_ms if cached is not None else 0 + cache_stale = self._is_cache_stale(full_id, since).unwrap() + + if cached is not None and not cache_stale: + return Ok(cached) + + raw_artifact = self._load_raw_artifact(full_id).unwrap() + refreshed = _ArtifactCacheValue( + raw_artifact=raw_artifact, + view_artifact=None, + analysis_artifact=None, + loaded_at_ms=self._now_ms(), + ) + self._cache[full_id] = refreshed + return Ok(refreshed) + @unwrap_to_error def load( # noqa: CCR001 self, full_id: FullArtifactId, render_context: "ArtifactRenderContext | None" = None ) -> Result[Artifact, ErrorsList]: - from donna.workspaces import artifacts as workspace_artifacts - if render_context is None: render_context = self._default_render_context() + cached = self._get_cache_value(full_id).unwrap() + if render_context.primary_mode == RenderMode.execute: - return Ok(workspace_artifacts.load_artifact(full_id, render_context).unwrap()) + return Ok(cached.raw_artifact.render(full_id, render_context).unwrap()) - cached = self._cache.get(full_id) cache_slot = self._context_slot_name(render_context) - since = cached.loaded_at_ms if cached is not None else 0 - cache_stale = self._is_cache_stale(full_id, since).unwrap() - - if cached is not None and cache_slot: + if cache_slot: cached_artifact = getattr(cached, cache_slot) - - if cached_artifact is not None and not cache_stale: + if cached_artifact is not None: return Ok(cached_artifact) - artifact = workspace_artifacts.load_artifact(full_id, render_context).unwrap() - - loaded_at_ms = self._now_ms() - - if cached is None: - cached = _ArtifactCacheValue(view_artifact=None, analysis_artifact=None, loaded_at_ms=loaded_at_ms) - self._cache[full_id] = cached - else: - cached.loaded_at_ms = loaded_at_ms + artifact = cached.raw_artifact.render(full_id, render_context).unwrap() if render_context.primary_mode == RenderMode.view: cached.view_artifact = artifact diff --git a/donna/workspaces/artifacts.py b/donna/workspaces/artifacts.py index 5f8cf57..6c4d62e 100644 --- a/donna/workspaces/artifacts.py +++ b/donna/workspaces/artifacts.py @@ -3,8 +3,7 @@ from donna.core.entities import BaseEntity from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error -from donna.domain.ids import FullArtifactId, FullArtifactIdPattern, WorldId -from donna.machine.artifacts import Artifact +from donna.domain.ids import FullArtifactId, WorldId from donna.machine.tasks import Task, WorkUnit from donna.workspaces import errors from donna.workspaces.config import config @@ -105,10 +104,10 @@ def artifact_file_extension(full_id: FullArtifactId) -> Result[str, ErrorsList]: @unwrap_to_error def fetch_artifact(full_id: FullArtifactId, output: pathlib.Path) -> Result[None, ErrorsList]: world = config().get_world(full_id.world_id).unwrap() - content = world.fetch_source(full_id.artifact_id).unwrap() + raw_artifact = world.fetch(full_id.artifact_id).unwrap() with output.open("wb") as f: - f.write(content) + f.write(raw_artifact.get_bytes()) return Ok(None) @@ -223,7 +222,8 @@ def copy_artifact(source_id: FullArtifactId, target_id: FullArtifactId) -> Resul ] ) - content_bytes = source_world.fetch_source(source_id.artifact_id).unwrap() + source_raw_artifact = source_world.fetch(source_id.artifact_id).unwrap() + content_bytes = source_raw_artifact.get_bytes() source_extension = source_world.file_extension_for(source_id.artifact_id).unwrap() if not source_extension: @@ -268,57 +268,3 @@ def remove_artifact(full_id: FullArtifactId) -> Result[None, ErrorsList]: world.remove(full_id.artifact_id).unwrap() return Ok(None) - - -@unwrap_to_error -def load_artifact( - full_id: FullArtifactId, render_context: ArtifactRenderContext | None = None -) -> Result[Artifact, ErrorsList]: - if render_context is None: - render_context = ArtifactRenderContext(primary_mode=RenderMode.view) - - world = config().get_world(full_id.world_id).unwrap() - return Ok(world.fetch(full_id.artifact_id, render_context).unwrap()) - - -def list_artifacts( # noqa: CCR001 - pattern: FullArtifactIdPattern, - render_context: ArtifactRenderContext | None = None, - tags: list[str] | None = None, -) -> Result[list[Artifact], ErrorsList]: - if render_context is None: - render_context = ArtifactRenderContext(primary_mode=RenderMode.view) - - tag_filters = tags or [] - - artifacts: list[Artifact] = [] - errors: ErrorsList = [] - - for world in reversed(config().worlds_instances): - for artifact_id in world.list_artifacts(pattern): - full_id = FullArtifactId((world.id, artifact_id)) - artifact_result = load_artifact(full_id, render_context) - if artifact_result.is_err(): - errors.extend(artifact_result.unwrap_err()) - continue - artifact = artifact_result.unwrap() - if tag_filters and not _artifact_matches_tags(artifact, tag_filters): - continue - artifacts.append(artifact) - - if errors: - return Err(errors) - - return Ok(artifacts) - - -def _artifact_matches_tags(artifact: Artifact, tags: list[str]) -> bool: - if not tags: - return True - - primary_result = artifact.primary_section() - if primary_result.is_err(): - return False - - primary = primary_result.unwrap() - return all(tag in primary.tags for tag in tags) diff --git a/donna/workspaces/worlds/base.py b/donna/workspaces/worlds/base.py index 161f22d..9404034 100644 --- a/donna/workspaces/worlds/base.py +++ b/donna/workspaces/worlds/base.py @@ -7,7 +7,7 @@ from donna.core.entities import BaseEntity from donna.core.errors import ErrorsList from donna.core.result import Result -from donna.domain.ids import ArtifactId, FullArtifactIdPattern, WorldId +from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern, WorldId from donna.machine.artifacts import Artifact from donna.machine.primitives import Primitive @@ -16,6 +16,17 @@ from donna.workspaces.config import WorldConfig +class RawArtifact(BaseEntity, ABC): + source_id: str + + @abstractmethod + def get_bytes(self) -> bytes: ... # noqa: E704 + + @abstractmethod + def render(self, full_id: FullArtifactId, render_context: "ArtifactRenderContext") -> Result[Artifact, ErrorsList]: + pass + + class World(BaseEntity, ABC): id: WorldId readonly: bool = True @@ -25,12 +36,7 @@ class World(BaseEntity, ABC): def has(self, artifact_id: ArtifactId) -> bool: ... # noqa: E704 @abstractmethod - def fetch( # noqa: E704 - self, artifact_id: ArtifactId, render_context: "ArtifactRenderContext" - ) -> Result[Artifact, ErrorsList]: ... - - @abstractmethod - def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: ... # noqa: E704 + def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: ... # noqa: E704 @abstractmethod def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: ... # noqa: E704 diff --git a/donna/workspaces/worlds/filesystem.py b/donna/workspaces/worlds/filesystem.py index fc5fc2f..3811db0 100644 --- a/donna/workspaces/worlds/filesystem.py +++ b/donna/workspaces/worlds/filesystem.py @@ -9,15 +9,32 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern -from donna.machine.artifacts import Artifact from donna.workspaces import errors as world_errors -from donna.workspaces.artifacts import ArtifactRenderContext from donna.workspaces.artifacts_discovery import ArtifactListingNode, list_artifacts_by_pattern +from donna.workspaces.worlds.base import RawArtifact from donna.workspaces.worlds.base import World as BaseWorld from donna.workspaces.worlds.base import WorldConstructor if TYPE_CHECKING: - from donna.workspaces.config import SourceConfigValue, WorldConfig + from donna.machine.artifacts import Artifact + from donna.workspaces.artifacts import ArtifactRenderContext + from donna.workspaces.config import WorldConfig + + +class FilesystemRawArtifact(RawArtifact): + path: pathlib.Path + + def get_bytes(self) -> bytes: + return self.path.read_bytes() + + @unwrap_to_error + def render( + self, full_id: FullArtifactId, render_context: "ArtifactRenderContext" + ) -> Result["Artifact", ErrorsList]: + from donna.workspaces.config import config + + source_config = config().get_source_config(self.source_id).unwrap() + return Ok(source_config.construct_artifact_from_bytes(full_id, self.get_bytes(), render_context).unwrap()) class World(BaseWorld): @@ -60,26 +77,6 @@ def _resolve_artifact_file(self, artifact_id: ArtifactId) -> Result[pathlib.Path return Ok(matches[0]) - def _get_source_by_filename( - self, artifact_id: ArtifactId, filename: str - ) -> Result["SourceConfigValue", ErrorsList]: - from donna.workspaces.config import config - - extension = pathlib.Path(filename).suffix - source_config = config().find_source_for_extension(extension) - if source_config is None: - return Err( - [ - world_errors.UnsupportedArtifactSourceExtension( - artifact_id=artifact_id, - world_id=self.id, - extension=extension, - ) - ] - ) - - return Ok(source_config) - def has(self, artifact_id: ArtifactId) -> bool: resolve_result = self._resolve_artifact_file(artifact_id) if resolve_result.is_err(): @@ -88,38 +85,31 @@ def has(self, artifact_id: ArtifactId) -> bool: return resolve_result.unwrap() is not None @unwrap_to_error - def fetch(self, artifact_id: ArtifactId, render_context: ArtifactRenderContext) -> Result[Artifact, ErrorsList]: + def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: path = self._resolve_artifact_file(artifact_id).unwrap() if path is None: return Err([world_errors.ArtifactNotFound(artifact_id=artifact_id, world_id=self.id)]) - content_bytes = path.read_bytes() - full_id = FullArtifactId((self.id, artifact_id)) - - extension = pathlib.Path(path.name).suffix from donna.workspaces.config import config - source_config = config().find_source_for_extension(extension) + source_config = config().find_source_for_extension(path.suffix) if source_config is None: return Err( [ world_errors.UnsupportedArtifactSourceExtension( artifact_id=artifact_id, world_id=self.id, - extension=extension, + extension=path.suffix, ) ] ) - return Ok(source_config.construct_artifact_from_bytes(full_id, content_bytes, render_context).unwrap()) - - @unwrap_to_error - def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: - path = self._resolve_artifact_file(artifact_id).unwrap() - if path is None: - return Err([world_errors.ArtifactNotFound(artifact_id=artifact_id, world_id=self.id)]) - - return Ok(path.read_bytes()) + return Ok( + FilesystemRawArtifact( + source_id=source_config.kind, + path=path, + ) + ) @unwrap_to_error def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: diff --git a/donna/workspaces/worlds/python.py b/donna/workspaces/worlds/python.py index 4f7b9e2..3ca927f 100644 --- a/donna/workspaces/worlds/python.py +++ b/donna/workspaces/worlds/python.py @@ -7,15 +7,32 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern, WorldId -from donna.machine.artifacts import Artifact from donna.workspaces import errors as world_errors -from donna.workspaces.artifacts import ArtifactRenderContext from donna.workspaces.artifacts_discovery import ArtifactListingNode, list_artifacts_by_pattern +from donna.workspaces.worlds.base import RawArtifact from donna.workspaces.worlds.base import World as BaseWorld from donna.workspaces.worlds.base import WorldConstructor if TYPE_CHECKING: - from donna.workspaces.config import SourceConfigValue, WorldConfig + from donna.machine.artifacts import Artifact + from donna.workspaces.artifacts import ArtifactRenderContext + from donna.workspaces.config import WorldConfig + + +class PythonRawArtifact(RawArtifact): + content: bytes + + def get_bytes(self) -> bytes: + return self.content + + @unwrap_to_error + def render( + self, full_id: FullArtifactId, render_context: "ArtifactRenderContext" + ) -> Result["Artifact", ErrorsList]: + from donna.workspaces.config import config + + source_config = config().get_source_config(self.source_id).unwrap() + return Ok(source_config.construct_artifact_from_bytes(full_id, self.get_bytes(), render_context).unwrap()) class Python(BaseWorld): @@ -79,26 +96,6 @@ def _resolve_artifact_file( return Ok(matches[0]) - def _get_source_by_filename( - self, artifact_id: ArtifactId, filename: str - ) -> Result["SourceConfigValue", ErrorsList]: - from donna.workspaces.config import config - - extension = pathlib.Path(filename).suffix - source_config = config().find_source_for_extension(extension) - if source_config is None: - return Err( - [ - world_errors.UnsupportedArtifactSourceExtension( - artifact_id=artifact_id, - world_id=self.id, - extension=extension, - ) - ] - ) - - return Ok(source_config) - def has(self, artifact_id: ArtifactId) -> bool: resolve_result = self._resolve_artifact_file(artifact_id) if resolve_result.is_err(): @@ -107,14 +104,11 @@ def has(self, artifact_id: ArtifactId) -> bool: return resolve_result.unwrap() is not None @unwrap_to_error - def fetch(self, artifact_id: ArtifactId, render_context: ArtifactRenderContext) -> Result[Artifact, ErrorsList]: + def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: # noqa: CCR001 resource_path = self._resolve_artifact_file(artifact_id).unwrap() if resource_path is None: return Err([world_errors.ArtifactNotFound(artifact_id=artifact_id, world_id=self.id)]) - content_bytes = resource_path.read_bytes() - full_id = FullArtifactId((self.id, artifact_id)) - extension = pathlib.Path(resource_path.name).suffix from donna.workspaces.config import config @@ -130,15 +124,12 @@ def fetch(self, artifact_id: ArtifactId, render_context: ArtifactRenderContext) ] ) - return Ok(source_config.construct_artifact_from_bytes(full_id, content_bytes, render_context).unwrap()) - - @unwrap_to_error - def fetch_source(self, artifact_id: ArtifactId) -> Result[bytes, ErrorsList]: # noqa: CCR001 - resource_path = self._resolve_artifact_file(artifact_id).unwrap() - if resource_path is None: - return Err([world_errors.ArtifactNotFound(artifact_id=artifact_id, world_id=self.id)]) - - return Ok(resource_path.read_bytes()) + return Ok( + PythonRawArtifact( + source_id=source_config.kind, + content=resource_path.read_bytes(), + ) + ) @unwrap_to_error def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: From e56ddb5e024458e3645033a1ed7a89aaf945e29b Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:34:14 +0100 Subject: [PATCH 06/15] wip --- donna/context/artifacts.py | 52 +++++++++++++++++++++-------------- donna/context/entity_cache.py | 31 +++++++++++++++++++++ donna/context/primitives.py | 31 +++++++++++++++++---- donna/context/state.py | 49 ++++++++++++++++++++++++++++----- donna/workspaces/config.py | 1 + 5 files changed, 130 insertions(+), 34 deletions(-) create mode 100644 donna/context/entity_cache.py diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py index 5c66f88..9a81727 100644 --- a/donna/context/artifacts.py +++ b/donna/context/artifacts.py @@ -1,7 +1,7 @@ -import time from pathlib import Path from typing import TYPE_CHECKING +from donna.context.entity_cache import TimedCache, TimedCacheValue from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import FullArtifactId, FullArtifactIdPattern, FullArtifactSectionId @@ -13,8 +13,8 @@ from donna.workspaces.worlds.base import RawArtifact -class _ArtifactCacheValue: - __slots__ = ("raw_artifact", "view_artifact", "analysis_artifact", "loaded_at_ms") +class _ArtifactCacheValue(TimedCacheValue): + __slots__ = ("raw_artifact", "view_artifact", "analysis_artifact") def __init__( self, @@ -22,23 +22,20 @@ def __init__( view_artifact: Artifact | None, analysis_artifact: Artifact | None, loaded_at_ms: int, + checked_at_ms: int, ) -> None: + super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) self.raw_artifact = raw_artifact self.view_artifact = view_artifact self.analysis_artifact = analysis_artifact - self.loaded_at_ms = loaded_at_ms -class ArtifactsCache: +class ArtifactsCache(TimedCache): __slots__ = ("_cache",) def __init__(self) -> None: self._cache: dict[FullArtifactId, _ArtifactCacheValue] = {} - @staticmethod - def _now_ms() -> int: - return time.time_ns() // 1_000_000 - @staticmethod def _default_render_context() -> "ArtifactRenderContext": from donna.workspaces.artifacts import ArtifactRenderContext @@ -62,9 +59,6 @@ def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: int) -> Result[ world = config().get_world(full_id.world_id).unwrap() return Ok(world.has_artifact_changed(full_id.artifact_id, since=loaded_at_ms).unwrap()) - def invalidate(self, full_id: FullArtifactId) -> None: - self._cache.pop(full_id, None) - @staticmethod @unwrap_to_error def _load_raw_artifact(full_id: FullArtifactId) -> Result["RawArtifact", ErrorsList]: @@ -74,24 +68,40 @@ def _load_raw_artifact(full_id: FullArtifactId) -> Result["RawArtifact", ErrorsL return Ok(world.fetch(full_id.artifact_id).unwrap()) @unwrap_to_error - def _get_cache_value(self, full_id: FullArtifactId) -> Result[_ArtifactCacheValue, ErrorsList]: - cached = self._cache.get(full_id) - since = cached.loaded_at_ms if cached is not None else 0 - cache_stale = self._is_cache_stale(full_id, since).unwrap() - - if cached is not None and not cache_stale: - return Ok(cached) - + def _refresh_cache_value(self, full_id: FullArtifactId, now_ms: int) -> Result[_ArtifactCacheValue, ErrorsList]: raw_artifact = self._load_raw_artifact(full_id).unwrap() refreshed = _ArtifactCacheValue( raw_artifact=raw_artifact, view_artifact=None, analysis_artifact=None, - loaded_at_ms=self._now_ms(), + loaded_at_ms=now_ms, + checked_at_ms=now_ms, ) self._cache[full_id] = refreshed return Ok(refreshed) + @unwrap_to_error + def _get_cache_value(self, full_id: FullArtifactId) -> Result[_ArtifactCacheValue, ErrorsList]: + cached = self._cache.get(full_id) + now_ms = self._now_ms() + + if cached is None: + return Ok(self._refresh_cache_value(full_id, now_ms).unwrap()) + + # Skip expensive world checks when cache lifetime has not elapsed yet. + if self._is_within_lifetime(cached, now_ms): + return Ok(cached) + + cache_stale = self._is_cache_stale(full_id, cached.loaded_at_ms).unwrap() + self._mark_checked(cached, now_ms) + if not cache_stale: + return Ok(cached) + + return Ok(self._refresh_cache_value(full_id, now_ms).unwrap()) + + def invalidate(self, full_id: FullArtifactId) -> None: + self._cache.pop(full_id, None) + @unwrap_to_error def load( # noqa: CCR001 self, full_id: FullArtifactId, render_context: "ArtifactRenderContext | None" = None diff --git a/donna/context/entity_cache.py b/donna/context/entity_cache.py new file mode 100644 index 0000000..7827898 --- /dev/null +++ b/donna/context/entity_cache.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import time +from abc import ABC + +from donna.workspaces.config import config + + +class TimedCacheValue: + __slots__ = ("loaded_at_ms", "checked_at_ms") + + def __init__(self, loaded_at_ms: int, checked_at_ms: int) -> None: + self.loaded_at_ms = loaded_at_ms + self.checked_at_ms = checked_at_ms + + +class TimedCache(ABC): + @staticmethod + def _now_ms() -> int: + return time.time_ns() // 1_000_000 + + @staticmethod + def _cache_lifetime_ms() -> int: + return max(0, int(config().cache_lifetime * 1000)) + + def _is_within_lifetime(self, cached: TimedCacheValue, now_ms: int) -> bool: + return (now_ms - cached.checked_at_ms) < self._cache_lifetime_ms() + + @staticmethod + def _mark_checked(cached: TimedCacheValue, now_ms: int) -> None: + cached.checked_at_ms = now_ms diff --git a/donna/context/primitives.py b/donna/context/primitives.py index 4af6172..46064ab 100644 --- a/donna/context/primitives.py +++ b/donna/context/primitives.py @@ -1,6 +1,7 @@ import importlib from typing import TYPE_CHECKING +from donna.context.entity_cache import TimedCache, TimedCacheValue from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import PythonImportPath @@ -10,11 +11,19 @@ from donna.machine.primitives import Primitive -class PrimitivesCache: +class _PrimitiveCacheValue(TimedCacheValue): + __slots__ = ("primitive",) + + def __init__(self, primitive: "Primitive", loaded_at_ms: int, checked_at_ms: int) -> None: + super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) + self.primitive = primitive + + +class PrimitivesCache(TimedCache): __slots__ = ("_cache",) def __init__(self) -> None: - self._cache: dict[PythonImportPath, "Primitive"] = {} + self._cache: dict[PythonImportPath, _PrimitiveCacheValue] = {} @unwrap_to_error def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", ErrorsList]: # noqa: CCR001 @@ -26,8 +35,9 @@ def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", E import_path = PythonImportPath.parse(primitive_id).unwrap() cached = self._cache.get(import_path) - if cached is not None: - return Ok(cached) + now_ms = self._now_ms() + if cached is not None and self._is_within_lifetime(cached, now_ms): + return Ok(cached.primitive) import_path_str = str(import_path) @@ -49,5 +59,14 @@ def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", E if not isinstance(primitive, Primitive): return Err([machine_errors.PrimitiveNotPrimitive(import_path=import_path_str)]) - self._cache[import_path] = primitive - return Ok(primitive) + if cached is not None: + previous_primitive = cached.primitive + cached.primitive = primitive + self._mark_checked(cached, now_ms) + if previous_primitive is not primitive: + cached.loaded_at_ms = now_ms + else: + cached = _PrimitiveCacheValue(primitive=primitive, loaded_at_ms=now_ms, checked_at_ms=now_ms) + self._cache[import_path] = cached + + return Ok(cached.primitive) diff --git a/donna/context/state.py b/donna/context/state.py index dbac8b5..ee321c1 100644 --- a/donna/context/state.py +++ b/donna/context/state.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING +from donna.context.entity_cache import TimedCache, TimedCacheValue from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.machine import errors as machine_errors @@ -8,32 +9,66 @@ from donna.machine.state import ConsistentState -class StateCache: +class _StateCacheValue(TimedCacheValue): + __slots__ = ("state", "state_json") + + def __init__( + self, + state: "ConsistentState", + state_json: bytes, + loaded_at_ms: int, + checked_at_ms: int, + ) -> None: + super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) + self.state = state + self.state_json = state_json + + +class StateCache(TimedCache): __slots__ = ("_session_state",) def __init__(self) -> None: - self._session_state: ConsistentState | None = None + self._session_state: _StateCacheValue | None = None @unwrap_to_error def load(self) -> Result["ConsistentState", ErrorsList]: from donna.machine.state import ConsistentState from donna.workspaces import utils as workspace_utils - if self._session_state is not None: - return Ok(self._session_state) + now_ms = self._now_ms() + cached = self._session_state + + if cached is not None and self._is_within_lifetime(cached, now_ms): + return Ok(cached.state) content = workspace_utils.session_world().unwrap().read_state("state.json").unwrap() if content is None: return Err([machine_errors.SessionStateNotInitialized()]) + if cached is not None and cached.state_json == content: + self._mark_checked(cached, now_ms) + return Ok(cached.state) + state = ConsistentState.from_json(content.decode("utf-8")) - self._session_state = state + self._session_state = _StateCacheValue( + state=state, + state_json=content, + loaded_at_ms=now_ms, + checked_at_ms=now_ms, + ) return Ok(state) @unwrap_to_error def save(self, state: "ConsistentState") -> Result[None, ErrorsList]: from donna.workspaces import utils as workspace_utils - workspace_utils.session_world().unwrap().write_state("state.json", state.to_json().encode("utf-8")).unwrap() - self._session_state = state + content = state.to_json().encode("utf-8") + workspace_utils.session_world().unwrap().write_state("state.json", content).unwrap() + now_ms = self._now_ms() + self._session_state = _StateCacheValue( + state=state, + state_json=content, + loaded_at_ms=now_ms, + checked_at_ms=now_ms, + ) return Ok(None) diff --git a/donna/workspaces/config.py b/donna/workspaces/config.py index f19adf7..05b95ca 100644 --- a/donna/workspaces/config.py +++ b/donna/workspaces/config.py @@ -103,6 +103,7 @@ class Config(BaseEntity): _sources_instances: list[SourceConfigValue] = pydantic.PrivateAttr(default_factory=list) tmp_dir: pathlib.Path = pathlib.Path("./tmp") + cache_lifetime: float = 1.0 def model_post_init(self, __context: Any) -> None: # noqa: CCR001 worlds: list[BaseWorld] = [] From 12ef68375d4d6bdc336fcc540e1569ecf9c2bb0e Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:44:23 +0100 Subject: [PATCH 07/15] wip --- donna/cli/commands/artifacts.py | 9 +++++---- donna/context/artifacts.py | 20 +++++--------------- donna/machine/sessions.py | 5 +++-- donna/machine/state.py | 13 ++++--------- donna/workspaces/artifacts.py | 7 +++++-- 5 files changed, 22 insertions(+), 32 deletions(-) diff --git a/donna/cli/commands/artifacts.py b/donna/cli/commands/artifacts.py index 96d3726..938460e 100644 --- a/donna/cli/commands/artifacts.py +++ b/donna/cli/commands/artifacts.py @@ -24,6 +24,7 @@ from donna.protocol.cell_shortcuts import operation_succeeded from donna.protocol.cells import Cell from donna.workspaces import tmp as world_tmp +from donna.workspaces.artifacts import RENDER_CONTEXT_VIEW artifacts_cli = typer.Typer() @@ -65,7 +66,7 @@ def list( ) -> Iterable[Cell]: _log_operation_on_artifacts("List artifacts", pattern, tags) - artifacts = context().artifacts.list(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, RENDER_CONTEXT_VIEW, tags=tags).unwrap() return [artifact.node().status() for artifact in artifacts] @@ -78,7 +79,7 @@ def view( ) -> Iterable[Cell]: _log_operation_on_artifacts("View artifacts", pattern, tags) - artifacts = context().artifacts.list(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, RENDER_CONTEXT_VIEW, tags=tags).unwrap() return [artifact.node().info() for artifact in artifacts] @@ -189,7 +190,7 @@ def remove( ) -> Iterable[Cell]: _log_operation_on_artifacts("Remove artifacts", pattern, tags) - artifacts = context().artifacts.list(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, RENDER_CONTEXT_VIEW, tags=tags).unwrap() cells: builtins.list[Cell] = [] for artifact in artifacts: @@ -207,7 +208,7 @@ def validate( ) -> Iterable[Cell]: # noqa: CCR001 _log_operation_on_artifacts("Validate artifacts", pattern, tags) - artifacts = context().artifacts.list(pattern, tags=tags).unwrap() + artifacts = context().artifacts.list(pattern, RENDER_CONTEXT_VIEW, tags=tags).unwrap() errors = [] diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py index 9a81727..afc6956 100644 --- a/donna/context/artifacts.py +++ b/donna/context/artifacts.py @@ -36,12 +36,6 @@ class ArtifactsCache(TimedCache): def __init__(self) -> None: self._cache: dict[FullArtifactId, _ArtifactCacheValue] = {} - @staticmethod - def _default_render_context() -> "ArtifactRenderContext": - from donna.workspaces.artifacts import ArtifactRenderContext - - return ArtifactRenderContext(primary_mode=RenderMode.view) - @staticmethod def _context_slot_name(render_context: "ArtifactRenderContext") -> str: if render_context.primary_mode == RenderMode.view: @@ -104,11 +98,10 @@ def invalidate(self, full_id: FullArtifactId) -> None: @unwrap_to_error def load( # noqa: CCR001 - self, full_id: FullArtifactId, render_context: "ArtifactRenderContext | None" = None + self, + full_id: FullArtifactId, + render_context: "ArtifactRenderContext", ) -> Result[Artifact, ErrorsList]: - if render_context is None: - render_context = self._default_render_context() - cached = self._get_cache_value(full_id).unwrap() if render_context.primary_mode == RenderMode.execute: @@ -133,7 +126,7 @@ def load( # noqa: CCR001 def resolve_section( self, target_id: FullArtifactSectionId, - render_context: "ArtifactRenderContext | None" = None, + render_context: "ArtifactRenderContext", ) -> Result[ArtifactSection, ErrorsList]: artifact = self.load(target_id.full_artifact_id, render_context).unwrap() return Ok(artifact.get_section(target_id.local_id).unwrap()) @@ -193,14 +186,11 @@ def fetch(self, full_id: FullArtifactId, output: Path) -> Result[None, ErrorsLis def list( # noqa: CCR001 self, pattern: FullArtifactIdPattern, - render_context: "ArtifactRenderContext | None" = None, + render_context: "ArtifactRenderContext", tags: list[str] | None = None, ) -> Result[list[Artifact], ErrorsList]: from donna.workspaces.config import config - if render_context is None: - render_context = self._default_render_context() - tag_filters = tags or [] artifacts: list[Artifact] = [] diff --git a/donna/machine/sessions.py b/donna/machine/sessions.py index f15e5ac..7da257e 100644 --- a/donna/machine/sessions.py +++ b/donna/machine/sessions.py @@ -13,6 +13,7 @@ from donna.protocol.cells import Cell from donna.workspaces import tmp as world_tmp from donna.workspaces import utils as workspace_utils +from donna.workspaces.artifacts import RENDER_CONTEXT_VIEW @unwrap_to_error @@ -108,7 +109,7 @@ def details() -> Result[list[Cell], ErrorsList]: @unwrap_to_error def start_workflow(artifact_id: FullArtifactId) -> Result[list[Cell], ErrorsList]: # noqa: CCR001 static_state = load_state().unwrap() - workflow = context().artifacts.load(artifact_id).unwrap() + workflow = context().artifacts.load(artifact_id, RENDER_CONTEXT_VIEW).unwrap() primary_section = workflow.primary_section().unwrap() mutator = static_state.mutator() mutator.start_workflow(workflow.id.to_full_local(primary_section.id)).unwrap() @@ -122,7 +123,7 @@ def _validate_operation_transition( state: MutableState, request_id: ActionRequestId, next_operation_id: FullArtifactSectionId ) -> Result[None, ErrorsList]: operation_id = state.get_action_request(request_id).unwrap().operation_id - workflow = context().artifacts.load(operation_id.full_artifact_id).unwrap() + workflow = context().artifacts.load(operation_id.full_artifact_id, RENDER_CONTEXT_VIEW).unwrap() operation = workflow.get_section(operation_id.local_id).unwrap() assert isinstance(operation.meta, OperationMeta) diff --git a/donna/machine/state.py b/donna/machine/state.py index 8badb4d..121812b 100644 --- a/donna/machine/state.py +++ b/donna/machine/state.py @@ -8,13 +8,7 @@ from donna.core.entities import BaseEntity from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error -from donna.domain.ids import ( - ActionRequestId, - FullArtifactSectionId, - InternalId, - TaskId, - WorkUnitId, -) +from donna.domain.ids import ActionRequestId, FullArtifactSectionId, InternalId, TaskId, WorkUnitId from donna.machine import errors as machine_errors from donna.machine import journal as machine_journal from donna.machine.action_requests import ActionRequest @@ -29,6 +23,7 @@ from donna.machine.tasks import Task, WorkUnit from donna.protocol.cells import Cell from donna.protocol.nodes import Node +from donna.workspaces.artifacts import RENDER_CONTEXT_VIEW class BaseState(BaseEntity): @@ -179,7 +174,7 @@ def complete_action_request( @unwrap_to_error def start_workflow(self, full_operation_id: FullArtifactSectionId) -> Result[None, ErrorsList]: - workflow = context().artifacts.resolve_section(full_operation_id).unwrap() + workflow = context().artifacts.resolve_section(full_operation_id, RENDER_CONTEXT_VIEW).unwrap() machine_journal.add( message=f"Start workflow `{workflow.title}`", @@ -192,7 +187,7 @@ def start_workflow(self, full_operation_id: FullArtifactSectionId) -> Result[Non def finish_workflow(self, task_id: TaskId) -> None: task = self.current_task assert task is not None - workflow = context().artifacts.resolve_section(task.workflow_id).unwrap() + workflow = context().artifacts.resolve_section(task.workflow_id, RENDER_CONTEXT_VIEW).unwrap() machine_journal.add( message=f"Finish workflow `{workflow.title}`", diff --git a/donna/workspaces/artifacts.py b/donna/workspaces/artifacts.py index 6c4d62e..f1a6b00 100644 --- a/donna/workspaces/artifacts.py +++ b/donna/workspaces/artifacts.py @@ -16,6 +16,9 @@ class ArtifactRenderContext(BaseEntity): current_work_unit: WorkUnit | None = None +RENDER_CONTEXT_VIEW = ArtifactRenderContext(primary_mode=RenderMode.view) + + class ArtifactUpdateError(errors.WorkspaceError): cell_kind: str = "artifact_update_error" artifact_id: FullArtifactId @@ -196,7 +199,7 @@ def mismatch_error(a: str, b: str) -> Result[None, ErrorsList]: if source_config is None: return Err([NoSourceForArtifactExtension(artifact_id=full_id, path=input, extension=normalized_source_suffix)]) - render_context = ArtifactRenderContext(primary_mode=RenderMode.view) + render_context = RENDER_CONTEXT_VIEW test_artifact = source_config.construct_artifact_from_bytes(full_id, content_bytes, render_context).unwrap() validation_result = test_artifact.validate_artifact() @@ -242,7 +245,7 @@ def copy_artifact(source_id: FullArtifactId, target_id: FullArtifactId) -> Resul ] ) - render_context = ArtifactRenderContext(primary_mode=RenderMode.view) + render_context = RENDER_CONTEXT_VIEW test_artifact = source_config.construct_artifact_from_bytes(target_id, content_bytes, render_context).unwrap() test_artifact.validate_artifact().unwrap() From e0290249a2dc752376624e545c720e6b7b425f05 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:47:48 +0100 Subject: [PATCH 08/15] wip --- donna/context/primitives.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/donna/context/primitives.py b/donna/context/primitives.py index 46064ab..ad34078 100644 --- a/donna/context/primitives.py +++ b/donna/context/primitives.py @@ -26,20 +26,15 @@ def __init__(self) -> None: self._cache: dict[PythonImportPath, _PrimitiveCacheValue] = {} @unwrap_to_error - def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", ErrorsList]: # noqa: CCR001 + def resolve(self, primitive_id: PythonImportPath) -> Result["Primitive", ErrorsList]: # noqa: CCR001 from donna.machine.primitives import Primitive - if isinstance(primitive_id, PythonImportPath): - import_path = primitive_id - else: - import_path = PythonImportPath.parse(primitive_id).unwrap() - - cached = self._cache.get(import_path) + cached = self._cache.get(primitive_id) now_ms = self._now_ms() if cached is not None and self._is_within_lifetime(cached, now_ms): return Ok(cached.primitive) - import_path_str = str(import_path) + import_path_str = str(primitive_id) if "." not in import_path_str: return Err([machine_errors.PrimitiveInvalidImportPath(import_path=import_path_str)]) @@ -67,6 +62,6 @@ def resolve(self, primitive_id: PythonImportPath | str) -> Result["Primitive", E cached.loaded_at_ms = now_ms else: cached = _PrimitiveCacheValue(primitive=primitive, loaded_at_ms=now_ms, checked_at_ms=now_ms) - self._cache[import_path] = cached + self._cache[primitive_id] = cached return Ok(cached.primitive) From de78a35d309610a108f6bdfecc37a758dd340799 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:51:40 +0100 Subject: [PATCH 09/15] wip --- donna/context/artifacts.py | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py index afc6956..229c8ed 100644 --- a/donna/context/artifacts.py +++ b/donna/context/artifacts.py @@ -14,20 +14,18 @@ class _ArtifactCacheValue(TimedCacheValue): - __slots__ = ("raw_artifact", "view_artifact", "analysis_artifact") + __slots__ = ("raw_artifact", "rendered_artifacts") def __init__( self, raw_artifact: "RawArtifact", - view_artifact: Artifact | None, - analysis_artifact: Artifact | None, + rendered_artifacts: dict[RenderMode, Artifact], loaded_at_ms: int, checked_at_ms: int, ) -> None: super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) self.raw_artifact = raw_artifact - self.view_artifact = view_artifact - self.analysis_artifact = analysis_artifact + self.rendered_artifacts = rendered_artifacts class ArtifactsCache(TimedCache): @@ -36,16 +34,6 @@ class ArtifactsCache(TimedCache): def __init__(self) -> None: self._cache: dict[FullArtifactId, _ArtifactCacheValue] = {} - @staticmethod - def _context_slot_name(render_context: "ArtifactRenderContext") -> str: - if render_context.primary_mode == RenderMode.view: - return "view_artifact" - - if render_context.primary_mode == RenderMode.analysis: - return "analysis_artifact" - - return "" - @unwrap_to_error def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: int) -> Result[bool, ErrorsList]: from donna.workspaces.config import config @@ -66,8 +54,7 @@ def _refresh_cache_value(self, full_id: FullArtifactId, now_ms: int) -> Result[_ raw_artifact = self._load_raw_artifact(full_id).unwrap() refreshed = _ArtifactCacheValue( raw_artifact=raw_artifact, - view_artifact=None, - analysis_artifact=None, + rendered_artifacts={}, loaded_at_ms=now_ms, checked_at_ms=now_ms, ) @@ -107,18 +94,12 @@ def load( # noqa: CCR001 if render_context.primary_mode == RenderMode.execute: return Ok(cached.raw_artifact.render(full_id, render_context).unwrap()) - cache_slot = self._context_slot_name(render_context) - if cache_slot: - cached_artifact = getattr(cached, cache_slot) - if cached_artifact is not None: - return Ok(cached_artifact) + cached_artifact = cached.rendered_artifacts.get(render_context.primary_mode) + if cached_artifact is not None: + return Ok(cached_artifact) artifact = cached.raw_artifact.render(full_id, render_context).unwrap() - - if render_context.primary_mode == RenderMode.view: - cached.view_artifact = artifact - elif render_context.primary_mode == RenderMode.analysis: - cached.analysis_artifact = artifact + cached.rendered_artifacts[render_context.primary_mode] = artifact return Ok(artifact) From c894d6b7bfa249ace2d9c054d15a1ee69e95f7fb Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:54:34 +0100 Subject: [PATCH 10/15] wip --- donna/context/artifacts.py | 11 +++++++---- donna/context/entity_cache.py | 15 ++++++++------- donna/context/primitives.py | 3 ++- donna/context/state.py | 5 +++-- donna/domain/types.py | 3 +++ donna/workspaces/worlds/base.py | 6 +++++- donna/workspaces/worlds/filesystem.py | 3 ++- donna/workspaces/worlds/python.py | 3 ++- 8 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 donna/domain/types.py diff --git a/donna/context/artifacts.py b/donna/context/artifacts.py index 229c8ed..3741263 100644 --- a/donna/context/artifacts.py +++ b/donna/context/artifacts.py @@ -5,6 +5,7 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import FullArtifactId, FullArtifactIdPattern, FullArtifactSectionId +from donna.domain.types import Milliseconds from donna.machine.artifacts import Artifact, ArtifactSection from donna.workspaces.templates import RenderMode @@ -20,8 +21,8 @@ def __init__( self, raw_artifact: "RawArtifact", rendered_artifacts: dict[RenderMode, Artifact], - loaded_at_ms: int, - checked_at_ms: int, + loaded_at_ms: Milliseconds, + checked_at_ms: Milliseconds, ) -> None: super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) self.raw_artifact = raw_artifact @@ -35,7 +36,7 @@ def __init__(self) -> None: self._cache: dict[FullArtifactId, _ArtifactCacheValue] = {} @unwrap_to_error - def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: int) -> Result[bool, ErrorsList]: + def _is_cache_stale(self, full_id: FullArtifactId, loaded_at_ms: Milliseconds) -> Result[bool, ErrorsList]: from donna.workspaces.config import config world = config().get_world(full_id.world_id).unwrap() @@ -50,7 +51,9 @@ def _load_raw_artifact(full_id: FullArtifactId) -> Result["RawArtifact", ErrorsL return Ok(world.fetch(full_id.artifact_id).unwrap()) @unwrap_to_error - def _refresh_cache_value(self, full_id: FullArtifactId, now_ms: int) -> Result[_ArtifactCacheValue, ErrorsList]: + def _refresh_cache_value( + self, full_id: FullArtifactId, now_ms: Milliseconds + ) -> Result[_ArtifactCacheValue, ErrorsList]: raw_artifact = self._load_raw_artifact(full_id).unwrap() refreshed = _ArtifactCacheValue( raw_artifact=raw_artifact, diff --git a/donna/context/entity_cache.py b/donna/context/entity_cache.py index 7827898..569ff1a 100644 --- a/donna/context/entity_cache.py +++ b/donna/context/entity_cache.py @@ -3,29 +3,30 @@ import time from abc import ABC +from donna.domain.types import Milliseconds from donna.workspaces.config import config class TimedCacheValue: __slots__ = ("loaded_at_ms", "checked_at_ms") - def __init__(self, loaded_at_ms: int, checked_at_ms: int) -> None: + def __init__(self, loaded_at_ms: Milliseconds, checked_at_ms: Milliseconds) -> None: self.loaded_at_ms = loaded_at_ms self.checked_at_ms = checked_at_ms class TimedCache(ABC): @staticmethod - def _now_ms() -> int: - return time.time_ns() // 1_000_000 + def _now_ms() -> Milliseconds: + return Milliseconds(time.time_ns() // 1_000_000) @staticmethod - def _cache_lifetime_ms() -> int: - return max(0, int(config().cache_lifetime * 1000)) + def _cache_lifetime_ms() -> Milliseconds: + return Milliseconds(max(0, int(config().cache_lifetime * 1000))) - def _is_within_lifetime(self, cached: TimedCacheValue, now_ms: int) -> bool: + def _is_within_lifetime(self, cached: TimedCacheValue, now_ms: Milliseconds) -> bool: return (now_ms - cached.checked_at_ms) < self._cache_lifetime_ms() @staticmethod - def _mark_checked(cached: TimedCacheValue, now_ms: int) -> None: + def _mark_checked(cached: TimedCacheValue, now_ms: Milliseconds) -> None: cached.checked_at_ms = now_ms diff --git a/donna/context/primitives.py b/donna/context/primitives.py index ad34078..d5a4664 100644 --- a/donna/context/primitives.py +++ b/donna/context/primitives.py @@ -5,6 +5,7 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import PythonImportPath +from donna.domain.types import Milliseconds from donna.machine import errors as machine_errors if TYPE_CHECKING: @@ -14,7 +15,7 @@ class _PrimitiveCacheValue(TimedCacheValue): __slots__ = ("primitive",) - def __init__(self, primitive: "Primitive", loaded_at_ms: int, checked_at_ms: int) -> None: + def __init__(self, primitive: "Primitive", loaded_at_ms: Milliseconds, checked_at_ms: Milliseconds) -> None: super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) self.primitive = primitive diff --git a/donna/context/state.py b/donna/context/state.py index ee321c1..c69d08d 100644 --- a/donna/context/state.py +++ b/donna/context/state.py @@ -3,6 +3,7 @@ from donna.context.entity_cache import TimedCache, TimedCacheValue from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error +from donna.domain.types import Milliseconds from donna.machine import errors as machine_errors if TYPE_CHECKING: @@ -16,8 +17,8 @@ def __init__( self, state: "ConsistentState", state_json: bytes, - loaded_at_ms: int, - checked_at_ms: int, + loaded_at_ms: Milliseconds, + checked_at_ms: Milliseconds, ) -> None: super().__init__(loaded_at_ms=loaded_at_ms, checked_at_ms=checked_at_ms) self.state = state diff --git a/donna/domain/types.py b/donna/domain/types.py new file mode 100644 index 0000000..a5b1a5c --- /dev/null +++ b/donna/domain/types.py @@ -0,0 +1,3 @@ +from typing import NewType + +Milliseconds = NewType("Milliseconds", int) diff --git a/donna/workspaces/worlds/base.py b/donna/workspaces/worlds/base.py index 9404034..1fd7d29 100644 --- a/donna/workspaces/worlds/base.py +++ b/donna/workspaces/worlds/base.py @@ -8,6 +8,7 @@ from donna.core.errors import ErrorsList from donna.core.result import Result from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern, WorldId +from donna.domain.types import Milliseconds from donna.machine.artifacts import Artifact from donna.machine.primitives import Primitive @@ -39,7 +40,10 @@ def has(self, artifact_id: ArtifactId) -> bool: ... # noqa: E704 def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: ... # noqa: E704 @abstractmethod - def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: ... # noqa: E704 + def has_artifact_changed( + self, artifact_id: ArtifactId, since: Milliseconds + ) -> Result[bool, ErrorsList]: + pass @abstractmethod def update( # noqa: E704 diff --git a/donna/workspaces/worlds/filesystem.py b/donna/workspaces/worlds/filesystem.py index 3811db0..c4a2953 100644 --- a/donna/workspaces/worlds/filesystem.py +++ b/donna/workspaces/worlds/filesystem.py @@ -9,6 +9,7 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern +from donna.domain.types import Milliseconds from donna.workspaces import errors as world_errors from donna.workspaces.artifacts_discovery import ArtifactListingNode, list_artifacts_by_pattern from donna.workspaces.worlds.base import RawArtifact @@ -112,7 +113,7 @@ def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: ) @unwrap_to_error - def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: + def has_artifact_changed(self, artifact_id: ArtifactId, since: Milliseconds) -> Result[bool, ErrorsList]: path = self._resolve_artifact_file(artifact_id).unwrap() if path is None: diff --git a/donna/workspaces/worlds/python.py b/donna/workspaces/worlds/python.py index 3ca927f..fd99909 100644 --- a/donna/workspaces/worlds/python.py +++ b/donna/workspaces/worlds/python.py @@ -7,6 +7,7 @@ from donna.core.errors import ErrorsList from donna.core.result import Err, Ok, Result, unwrap_to_error from donna.domain.ids import ArtifactId, FullArtifactId, FullArtifactIdPattern, WorldId +from donna.domain.types import Milliseconds from donna.workspaces import errors as world_errors from donna.workspaces.artifacts_discovery import ArtifactListingNode, list_artifacts_by_pattern from donna.workspaces.worlds.base import RawArtifact @@ -132,7 +133,7 @@ def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: # ) @unwrap_to_error - def has_artifact_changed(self, artifact_id: ArtifactId, since: int) -> Result[bool, ErrorsList]: + def has_artifact_changed(self, artifact_id: ArtifactId, since: Milliseconds) -> Result[bool, ErrorsList]: resource_path = self._resolve_artifact_file(artifact_id).unwrap() if resource_path is None: From 06435efbbf832fc4e29d883d72aa504be65d7783 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 19:55:16 +0100 Subject: [PATCH 11/15] wip --- donna/workspaces/worlds/python.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/donna/workspaces/worlds/python.py b/donna/workspaces/worlds/python.py index fd99909..54af009 100644 --- a/donna/workspaces/worlds/python.py +++ b/donna/workspaces/worlds/python.py @@ -134,11 +134,6 @@ def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: # @unwrap_to_error def has_artifact_changed(self, artifact_id: ArtifactId, since: Milliseconds) -> Result[bool, ErrorsList]: - resource_path = self._resolve_artifact_file(artifact_id).unwrap() - - if resource_path is None: - return Ok(True) - return Ok(False) def update(self, artifact_id: ArtifactId, content: bytes, extension: str) -> Result[None, ErrorsList]: From 9d2923d51cb0ed04b8c91ff7a825379cf4033f14 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 20:07:00 +0100 Subject: [PATCH 12/15] docs --- .donna/project/core/top_level_architecture.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.donna/project/core/top_level_architecture.md b/.donna/project/core/top_level_architecture.md index d51d337..0f49382 100644 --- a/.donna/project/core/top_level_architecture.md +++ b/.donna/project/core/top_level_architecture.md @@ -20,6 +20,7 @@ The code is separated by layers/subsystems into subpackages: - `donna.core` — code that not in the Donna's domain, but required to its functioning: domain-independent utils, basic classes for errors, exceptions and other entities, etc. - `donna.domain` — code that is required by all Donna'specific logic: ID classes, common types, etc. - `donna.machine` — code that implements the core Donna's logic — how Donna works regardless of external environments, i.e. pure domain behavior. +- `donna.context` — code that stores and provides execution-scoped runtime context for Donna's domain logic: artifact/state/primitive caches and scoped values like current actor/work unit/operation identifiers. - `donna.world` — code that implements various worlds where Donna can find and manage artifacts: from artifacts discovery to their loading, parsing, updating. Also contains code related to configuration of Donna. - `donna.protocol` — code that implements protocol via which Donna's core domain logic interacts with external environments: CLI, API, etc. Includes basic classes for information representing (for the external environments) and its formatting. - `donna.cli` — code that implements the `donna` CLI tool, its commands, arguments parsing, etc. From 6c3c6b69516446dc1e24aa986c4a3be7b3ac1f6e Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 20:09:21 +0100 Subject: [PATCH 13/15] wip --- changes/unreleased.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/unreleased.md b/changes/unreleased.md index 16e3058..5297868 100644 --- a/changes/unreleased.md +++ b/changes/unreleased.md @@ -1,4 +1,7 @@ ### Changes +- ff-57 Added context storage/cache for globally accessible entities like artifacts, primitives, and session state. + - Added `donna.machine.context` module. + - Refactored `World` interface. - gh-62 Added `donna version` command that prints the version of the tool. - gh-60 Fixed CLI artifact extension recognition for `donna artifacts update`. From a0d33ebaebb9bb61a5823b4ada0908b9cc73c559 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 20:10:14 +0100 Subject: [PATCH 14/15] wip --- donna/workspaces/worlds/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/donna/workspaces/worlds/base.py b/donna/workspaces/worlds/base.py index 1fd7d29..25e69ff 100644 --- a/donna/workspaces/worlds/base.py +++ b/donna/workspaces/worlds/base.py @@ -40,9 +40,7 @@ def has(self, artifact_id: ArtifactId) -> bool: ... # noqa: E704 def fetch(self, artifact_id: ArtifactId) -> Result[RawArtifact, ErrorsList]: ... # noqa: E704 @abstractmethod - def has_artifact_changed( - self, artifact_id: ArtifactId, since: Milliseconds - ) -> Result[bool, ErrorsList]: + def has_artifact_changed(self, artifact_id: ArtifactId, since: Milliseconds) -> Result[bool, ErrorsList]: pass @abstractmethod From e616bdd3c9e74cc1905d2dbac008972bbb9bc1d9 Mon Sep 17 00:00:00 2001 From: "Aliaksei Yaletski (Tiendil)" Date: Sat, 28 Feb 2026 20:10:49 +0100 Subject: [PATCH 15/15] chages approved --- changes/{unreleased.md => next_release.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{unreleased.md => next_release.md} (100%) diff --git a/changes/unreleased.md b/changes/next_release.md similarity index 100% rename from changes/unreleased.md rename to changes/next_release.md