From dda813302fb5302582451ccc7f189d6c5b0bde58 Mon Sep 17 00:00:00 2001 From: lorenzo Date: Fri, 19 Jun 2026 23:17:05 +0200 Subject: [PATCH 01/21] full rewrite --- .claude/DESIGN.md | 168 +++++-- .claude/ROADMAP.md | 54 +- examples/01_sync_api.py | 99 ---- examples/02_models_and_documents.py | 106 ---- examples/03_resolver_read_write.py | 187 ------- examples/04_inline_and_merge.py | 124 ----- examples/05_custom_node_types.py | 76 --- examples/06_hcs_plate_single_collection.py | 133 ----- examples/07_hcs_plate_nested.py | 136 ----- examples/README.md | 18 - examples/data/hcs_nested/A/1/0.zarr/zarr.json | 44 -- examples/data/hcs_nested/A/1/well.json | 29 -- examples/data/hcs_nested/A/2/0.zarr/zarr.json | 44 -- examples/data/hcs_nested/A/2/well.json | 29 -- examples/data/hcs_nested/B/1/0.zarr/zarr.json | 44 -- examples/data/hcs_nested/B/1/well.json | 29 -- examples/data/hcs_nested/B/2/0.zarr/zarr.json | 44 -- examples/data/hcs_nested/B/2/well.json | 29 -- examples/data/hcs_nested/collection.json | 67 --- examples/data/hcs_single/A/1/0.zarr/zarr.json | 44 -- examples/data/hcs_single/A/2/0.zarr/zarr.json | 44 -- examples/data/hcs_single/B/1/0.zarr/zarr.json | 44 -- examples/data/hcs_single/B/2/0.zarr/zarr.json | 44 -- examples/data/hcs_single/collection.json | 135 ----- examples/data/inline/collection.json | 23 - examples/data/inline/image.zarr/zarr.json | 54 -- examples/data/resolver/collection.json | 28 -- examples/data/resolver/image.zarr/zarr.json | 65 --- .../data/resolver/tables/measurements.json | 18 - examples/data/sync_api/collection.json | 22 - examples/data/sync_api/image.zarr/zarr.json | 44 -- src/ngio_collections/__init__.py | 91 +--- src/ngio_collections/_document.py | 74 +++ src/ngio_collections/_resolver.py | 285 +++++++++++ src/ngio_collections/api.py | 235 --------- src/ngio_collections/document.py | 220 --------- src/ngio_collections/models/__init__.py | 89 +--- src/ngio_collections/models/_base.py | 464 ++++++++++++++++++ src/ngio_collections/models/attributes.py | 70 --- src/ngio_collections/models/base.py | 284 ----------- src/ngio_collections/models/coordinates.py | 47 -- src/ngio_collections/models/nodes.py | 146 ------ src/ngio_collections/registry.py | 55 --- src/ngio_collections/resolver.py | 453 ----------------- src/ngio_collections/store/__init__.py | 6 +- .../store/{fsspec.py => _fsspec.py} | 4 + .../store/{local.py => _local.py} | 4 + .../store/{protocols.py => _protocols.py} | 11 +- tests/test_api.py | 379 -------------- tests/test_attributes.py | 113 ----- tests/test_document.py | 303 ------------ tests/test_imports.py | 41 -- tests/test_local_store.py | 39 -- tests/test_models_base.py | 277 ----------- tests/test_models_nodes.py | 213 -------- tests/test_reference_fixtures.py | 58 --- tests/test_registry.py | 46 -- tests/test_resolver_inline.py | 389 --------------- tests/test_resolver_read.py | 190 ------- tests/test_resolver_tree.py | 168 ------- tests/test_resolver_write.py | 148 ------ tests/test_roundtrip.py | 438 +++++++++++++++++ tests/test_state_and_create.py | 191 +++++++ 63 files changed, 1709 insertions(+), 5847 deletions(-) delete mode 100644 examples/01_sync_api.py delete mode 100644 examples/02_models_and_documents.py delete mode 100644 examples/03_resolver_read_write.py delete mode 100644 examples/04_inline_and_merge.py delete mode 100644 examples/05_custom_node_types.py delete mode 100644 examples/06_hcs_plate_single_collection.py delete mode 100644 examples/07_hcs_plate_nested.py delete mode 100644 examples/README.md delete mode 100644 examples/data/hcs_nested/A/1/0.zarr/zarr.json delete mode 100644 examples/data/hcs_nested/A/1/well.json delete mode 100644 examples/data/hcs_nested/A/2/0.zarr/zarr.json delete mode 100644 examples/data/hcs_nested/A/2/well.json delete mode 100644 examples/data/hcs_nested/B/1/0.zarr/zarr.json delete mode 100644 examples/data/hcs_nested/B/1/well.json delete mode 100644 examples/data/hcs_nested/B/2/0.zarr/zarr.json delete mode 100644 examples/data/hcs_nested/B/2/well.json delete mode 100644 examples/data/hcs_nested/collection.json delete mode 100644 examples/data/hcs_single/A/1/0.zarr/zarr.json delete mode 100644 examples/data/hcs_single/A/2/0.zarr/zarr.json delete mode 100644 examples/data/hcs_single/B/1/0.zarr/zarr.json delete mode 100644 examples/data/hcs_single/B/2/0.zarr/zarr.json delete mode 100644 examples/data/hcs_single/collection.json delete mode 100644 examples/data/inline/collection.json delete mode 100644 examples/data/inline/image.zarr/zarr.json delete mode 100644 examples/data/resolver/collection.json delete mode 100644 examples/data/resolver/image.zarr/zarr.json delete mode 100644 examples/data/resolver/tables/measurements.json delete mode 100644 examples/data/sync_api/collection.json delete mode 100644 examples/data/sync_api/image.zarr/zarr.json create mode 100644 src/ngio_collections/_document.py create mode 100644 src/ngio_collections/_resolver.py delete mode 100644 src/ngio_collections/api.py delete mode 100644 src/ngio_collections/document.py create mode 100644 src/ngio_collections/models/_base.py delete mode 100644 src/ngio_collections/models/attributes.py delete mode 100644 src/ngio_collections/models/base.py delete mode 100644 src/ngio_collections/models/coordinates.py delete mode 100644 src/ngio_collections/models/nodes.py delete mode 100644 src/ngio_collections/registry.py delete mode 100644 src/ngio_collections/resolver.py rename src/ngio_collections/store/{fsspec.py => _fsspec.py} (90%) rename src/ngio_collections/store/{local.py => _local.py} (83%) rename src/ngio_collections/store/{protocols.py => _protocols.py} (62%) delete mode 100644 tests/test_api.py delete mode 100644 tests/test_attributes.py delete mode 100644 tests/test_document.py delete mode 100644 tests/test_imports.py delete mode 100644 tests/test_local_store.py delete mode 100644 tests/test_models_base.py delete mode 100644 tests/test_models_nodes.py delete mode 100644 tests/test_reference_fixtures.py delete mode 100644 tests/test_registry.py delete mode 100644 tests/test_resolver_inline.py delete mode 100644 tests/test_resolver_read.py delete mode 100644 tests/test_resolver_tree.py delete mode 100644 tests/test_resolver_write.py create mode 100644 tests/test_roundtrip.py create mode 100644 tests/test_state_and_create.py diff --git a/.claude/DESIGN.md b/.claude/DESIGN.md index a059390..6054ad1 100644 --- a/.claude/DESIGN.md +++ b/.claude/DESIGN.md @@ -1,6 +1,7 @@ # ngio-collections-py — Preliminary Design -**Status:** draft · 2026-06-11 +**Status:** draft · 2026-06-11 · **partly superseded by the functional rewrite +(see banner)** **Context:** Greenfield successor to the `fractal-collections-tools` (/Users/locerr/Projects/Fractal/fractal-v3-prototyping/fractal-collections-tools) prototype (an implementation of the OME-NGFF RFC-8 *Collections* draft). This document @@ -9,6 +10,43 @@ this package starts from. --- +## Implementation note — functional rewrite (2026-06-19) + +The shipped package took a **functional / immutable** direction that supersedes +several specifics below. The **rationale still holds** (RFC-8 round-trip +fidelity, lazy resolution, document-granular saves, async-native core, +URL-addressed stores, mixed-store as the eventual goal, graceful degradation of +unknown types/attributes). What changed in the implementation: + +- **Single frozen node layer, not a Stored/Resolved split (§11).** Nodes are + frozen Pydantic values (`Node` / `RefNode` and the `Collection*`/`Multiscale*` + subtypes); editing returns a NEW tree and never mutates the source. There is + no `StoredNode`/`ResolvedNode` pair and no `models/nodes.py` / `resolved.py`. +- **Provenance is `PrivateAttr` on the node** (`_document`, and `_origin` on a + collapsed boundary — an `Origin`/`NodeMetaInfos` snapshot), carried only via + `model_copy`. The §5 merge rule lives in one place — `merge` / `split` in + `models._base` — and `split` inverts the merge by origin (§9.4). +- **No registry.** Node type is chosen by the `type` discriminator with a + graceful fallback to generic `Node`/`RefNode` (`build_node` / `build_ref_node` + / `build_any_node`). `NodeRegistry` / `DEFAULT_REGISTRY` / validation-context + registration (§2.6, §3.4) are not implemented. +- **No typed `attrs` view and no attribute model classes** (§3.5, §7). + `attributes` stays a raw `dict[str, JsonValue]` for round-trip fidelity; + `PlateAttribute` / `LabelObj` / `SinglescaleNode` etc. do not exist. +- **No sync facade / `ngio_collections.api`** (§5). The Resolver is async; use + `asyncio.run(...)`. +- **Resolver surface is `inline` / `create` / `save` / `delete_subtree`** (not + `open` / `children` / `resolve_tree` / `save_tree`). `MetadataDocument` is a + Protocol over one file (`content` / `store` / `url` + `deserialize_payload` / + `serialize_payload`), not a root-bearing object with `form`/`version`/`stub_path`. + +The authoritative module map is **§8 (Module layout)** and the architecture +overview is **§4**, both updated to the rewrite. Sections §2–§3, §5 (API +sketch / sync API), §7, and §11 are kept as historical design narrative; read +them through this banner. + +--- + ## 1. Goals - A faithful, round-trip-safe implementation of RFC-8 collection metadata: @@ -200,26 +238,29 @@ a stub using that document's `stub_path`. ``` ┌──────────────────────────────────────────────────────────┐ -│ models/ pure Pydantic: BaseNode, node types, │ -│ attributes, coordinates. No IO, no URLs. │ +│ models/_base.py pure Pydantic: frozen Node / RefNode │ +│ (+ Collection/Multiscale subtypes), │ +│ PathObj, the §5 merge/split rule, and │ +│ the functional edit engine. No IO. │ ├──────────────────────────────────────────────────────────┤ -│ document MetadataDocument: provenance + pure │ -│ (de)serialize of ONE metadata file │ -│ (json or zarr form). │ +│ _document.py MetadataDocument Protocol + Json/Zarr │ +│ impls: pure (de)serialize of ONE │ +│ metadata file's `ome` payload. │ ├──────────────────────────────────────────────────────────┤ -│ resolver async open / resolve / children / │ -│ resolve_tree / save / write. │ -│ URL-keyed MetadataDocument cache. │ -│ The only caller of the Store. │ +│ _resolver.py async Resolver: inline / create / │ +│ save / delete_subtree. URL-keyed │ +│ document cache. The only Store caller. │ ├──────────────────────────────────────────────────────────┤ -│ store/ ReadableStore / WritableStore protocols, │ -│ fsspec-backed default, zero-dep LocalStore.│ +│ store/ ReadableStore / WritableStore protocols│ +│ (_protocols), zero-dep LocalStore │ +│ (_local), FsspecStore skeleton (_fsspec)│ └──────────────────────────────────────────────────────────┘ - sync.py — thin synchronous facade over resolver ``` Dependency rule: each layer imports only downward. Models never import the -document layer; the document layer never imports the store. +document layer; the document layer never imports the store. Editing is +functional — every edit returns a new frozen tree; the parsed source is never +mutated. --- @@ -281,9 +322,11 @@ multiscale that lives on a read-only store). `inline()` is where the merge is materialized: when a stub is collapsed into its resolved subtree, the collapsed node carries the target root's attributes overlaid by the stub's own — **shallow, key-level, stub wins** (the stub annotates the reference; -the nearer scope overrides) — and the stub's `id`/`name`. The rule lives in -one pure function, `models.merged_attributes(stub, target_root)`, the single -home of the §5 merge. +the nearer scope overrides) — and the stub's `id`/`name`. The rule has a single +home in `models._base`: `merged_attributes(stub, target)` computes the overlay, +`merge(stub, target)` materializes the collapsed boundary node (recording an +`Origin` so the merge is invertible), and `split(node)` inverts it by origin on +write-back (§9.4). `inline()` is copy-building end to end: the input tree, the cached documents, and the resolver cache are never touched, and the result is a @@ -450,20 +493,29 @@ absolutely and local derived data relatively. ``` src/ngio_collections/ + __init__.py # the public surface (19 names): Resolver, stores + + # protocols, node/path model types + _document.py # MetadataDocument Protocol + Json/Zarr impls + _resolver.py # async Resolver (inline / create / save / delete_subtree) models/ - base.py # BaseObj, BaseNode, IdStr, Path objects, attrs view - nodes.py # CollectionNode, MultiscaleNode, SinglescaleNode - attributes.py # plate / well / acquisition / labels - coordinates.py # CoordinateSystem, CoordinateTransformation, scene - registry.py # NodeRegistry (no singletons) - document.py # MetadataDocument, parse_metadata_document, single serialize path - resolver.py # async Resolver + __init__.py # re-exports the model public subset + _base.py # BaseObj; frozen Node / RefNode (+ Collection/Multiscale + # subtypes); ZarrPath / JsonPath / PathObj; NodeState; + # the §5 merge/split rule; build_* constructors; + # the functional edit engine (update/add/remove/…) store/ - protocols.py # ReadableStore, WritableStore, StoreReadOnlyError - local.py # LocalStore (zero-dep) - fsspec.py # FsspecStore skeleton (optional dependency) + __init__.py # re-exports the store public subset + _protocols.py # ReadableStore, WritableStore, StoreReadOnlyError + _local.py # LocalStore (zero-dep) + _fsspec.py # FsspecStore skeleton (optional dependency) ``` +Every module under `models/` and `store/` is private (`_*.py`); the public +names are re-exported from each subpackage's `__init__` and from the top-level +`ngio_collections`. The merge engine, node constructors, provenance dataclasses +(`Origin` / `NodeMetaInfos`), and the document layer are intentionally NOT part +of the public surface. + --- ## 9. Open spec questions (RFC-8) @@ -488,7 +540,12 @@ Tracked here because the implementation takes a position on each: be able to override metadata on read-only targets; the merged view's `id`/`name` are likewise the stub's. Worth an RFC clarification, including whether a stub may satisfy an attribute MUST (e.g. - `coordinateSystems`) on the parent side. + `coordinateSystems`) on the parent side. **Write-back position (§11):** the + merge is invertible — *by origin, edge keeps overrides*. An edited key that + originated on the stub is written back to the parent edge (the target keeps + its original, shadowed value); every other current key — including + brand-new ones — is written to the home (target) document; a removed key + drops from both layers. --- @@ -528,3 +585,58 @@ future-work section): `gather` when frontier sizes get large). - Optional dirty tracking on top of document-granular saves. - A typed RFC-5 transformation union once that spec settles. + +--- + +## 11. Stored/Resolved node split (2026-06-18) + +The headline use case — open an inlined collection, edit it in memory, write it +back keeping the file structure and attributes correct — was blocked by +`BaseNode` wearing three hats: the on-disk wire model, the parsed/provenance +node, and (post-`inline`) the merged editing surface. The merge was lossy (a +key present on both stub and target lost the target's value) and the inlined +tree was one synthetic document, so saving it flattened the whole collection +into one file. The fix splits the node into two layers. + +- **`StoredNode`** (`models/base.py`, `models/nodes.py`) — the faithful + Pydantic mirror of one document's node (`extra="allow"`, structural + validators, `path`/ref forms, `_document`/`_parent` provenance). The + pre-split names (`BaseNode`, `CollectionNode`, …) stay as back-compat + aliases. Each stored type gets a `resolved_form` ClassVar (mirroring + `ref_form`); `None` ⇒ the generic fallback. +- **`ResolvedNode`** (`models/resolved.py`) — produced ONLY by `inline()`: a + plain (non-Pydantic) mutable working model holding private references back + into the stored layer (`_home` document, `_stored` node, `_edge` → + `EdgeRef`), with the ergonomic edit API (`attrs`, `add`, `pop`, `walk`, + `find`, `target_path`). Typed twins exist for the built-ins; custom types + fall back to the generic `ResolvedNode` (or opt into a twin via + `resolved_form`). No on-mutation validation — invariants re-apply once, at + `to_stored_root()`. + +Resolution vocabulary, made consistent: `inline()` (verb) → `ResolvedNode` +(fully-resolved result); `resolve()` / `resolve_tree()` are the lazy partial +steps that leave stubs in place (§3.3). So `inline()` reframes as +**StoredNode-tree → ResolvedNode-tree**, and write-back as +**ResolvedNode-tree → StoredNode-documents**. + +**`Resolver.save_tree(root)`** is the inverse of `inline()`: it partitions the +resolved tree by home document (each boundary node — `_edge` set — roots its +own document and is re-emitted as a path stub in its parent), rebuilds each +document via `to_stored_root` (attributes un-merged by origin per §9.4; added +nodes embedded in their parent's document; unknown `extra` keys carried through +from the cached original `StoredNode` by `model_copy`), and saves only the +documents whose serialized payload changed. A tree saved with no edits writes +nothing. **`Resolver.delete_subtree(node)`** (with a new `WritableStore.delete`) +is the destructive companion to `pop()`'s in-memory unlink: deletes the +external file(s) of the boundary nodes in a subtree (call before popping). + +Sync API: `open_collection` / `open_multiscale` now return the `ResolvedNode` +root; `write_collection_back` / `write_multiscale_back` wrap `save_tree`. The +compose-by-reference writers (`write_collection` / `write_multiscale`) keep +taking `StoredNode`s — the document-granular `save()` editing path is +unchanged. + +Partly retires §10's `write()` item: bottom-up composition (writers) and +write-back of an opened tree (`save_tree`) are now covered; restructuring by +*externalizing* an added node into its own new document stays deferred (added +nodes embed in their parent's document). diff --git a/.claude/ROADMAP.md b/.claude/ROADMAP.md index 6714dca..aa8d796 100644 --- a/.claude/ROADMAP.md +++ b/.claude/ROADMAP.md @@ -1,14 +1,26 @@ # Roadmap **Status:** revised 2026-06-11 (simplicity over completeness) · companion to -[DESIGN.md](DESIGN.md) +[DESIGN.md](DESIGN.md) · **partly superseded by the functional rewrite (see banner)** + +> **Implementation note (2026-06-19).** The local read+write story is +> implemented and green, but via the **functional / immutable rewrite** — see +> DESIGN.md's "Implementation note" banner. The milestones below are kept as a +> historical record; their *specifics* that no longer apply: the registry +> (M1: `NodeRegistry` / `DEFAULT_REGISTRY`), the structural-validator / +> attribute-class / `SinglescaleNode` model (M1, in `models/nodes.py`), the +> `Stored`/`Resolved` split (M5, `models/resolved.py`), and the sync +> `ngio_collections.api` facade (M3/M5). The real surface is a single frozen +> `Node`/`RefNode` model in `models/_base.py` and an async `Resolver` +> (`inline` / `create` / `save` / `delete_subtree`); the §5 merge rule lives in +> `merge` / `split`. Use `asyncio.run(...)` — there is no sync facade. Scope of this roadmap: a complete, round-trip-safe **local** implementation — parse, validate, navigate, edit, save on the local filesystem. Remote and mixed-store support remain the primary eventual use case (the core stays async-native for that reason), but their implementation is deferred: `FsspecStore` exists only as an interface skeleton; `RouterStore` is -design-only (DESIGN.md �6), with no code yet. Everything +design-only (DESIGN.md §6), with no code yet. Everything deferred is recorded under [Future work](#future-work) below and in DESIGN.md §10. @@ -20,8 +32,9 @@ Sequencing principles: the roadmap ends when the local write path round-trips. - CI from milestone 1, so every subsequent milestone lands gated. -Current state: structural skeleton (modules, signatures, and trivial pieces -in place; behavior stubbed), trimmed to the local-scope surface. +Current state: the full local story is implemented and tested green +(parse → inline → edit → write-back, document-granular) via the functional +rewrite; remote/mixed-store remain deferred (see Future work). --- @@ -114,7 +127,33 @@ Document-granular editing — the core value proposition. **Done when:** editing one node's attributes and saving touches exactly one file on disk, and a re-opened tree reflects the edit with everything else -byte-identical. That is also the end of this roadmap. +byte-identical. + +## M5 — Round-trip via the Stored/Resolved split (added 2026-06-18) + +The headline use case: open an inlined collection, edit it in memory, write it +back keeping the file structure and attributes correct. See DESIGN.md §11. + +- [x] Split `BaseNode` into `StoredNode` (wire/parse/serialize, `BaseNode` + alias kept) and a new `ResolvedNode` layer (`models/resolved.py`): plain + mutable working model with the `attrs` / `add` / `pop` edit API, typed twins + for the built-ins + generic fallback (opt-in via the `resolved_form` + ClassVar). +- [x] `Resolver.inline()` rebuilt as StoredNode-tree → ResolvedNode-tree, each + node retaining `_home` / `_stored` / `_edge` provenance. +- [x] `Resolver.save_tree(root)`: ResolvedNode-tree → StoredNode-documents — + attributes un-merged by origin (edge keeps overrides, DESIGN.md §9.4), added + nodes embedded in their parent's document, only changed documents rewritten, + unknown keys preserved. +- [x] `WritableStore.delete` + `Resolver.delete_subtree` (destructive companion + to `pop()`'s in-memory unlink). +- [x] Sync `write_collection_back` / `write_multiscale_back`; `open_*` return + the resolved tree. + +**Done when:** open → edit (attrs add/remove, add node, pop node) → write-back +lands each change in the right document, leaves untouched files byte-identical, +and a no-op write-back touches nothing (`tests/test_resolved_roundtrip.py`, +`examples/08_resolved_edit.py`). --- @@ -135,7 +174,10 @@ re-evaluate once M4 is done. See also DESIGN.md §10. - **`Resolver.write(node, url, stub_path=...)`** — externalizing a node into a new document (collection restructuring). Bottom-up *composition* by reference is covered 2026-06-12 by the sync writers returning reference - stubs (with `relativize` path rewriting); restructuring stays deferred. + stubs (with `relativize` path rewriting), and write-back of an opened tree + by M5's `save_tree` (2026-06-18); what remains deferred is externalizing an + *added* node into its own new document (added nodes embed in their parent's + document). - **Attribute-registry extensibility** — removed as dead code (the `attrs` view takes attribute classes directly); re-add only if a use case appears. - **Conformance suite** against the RFC-8 examples; revisit the open spec diff --git a/examples/01_sync_api.py b/examples/01_sync_api.py deleted file mode 100644 index 5ee4fe3..0000000 --- a/examples/01_sync_api.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Quickstart: the sync convenience API for scripts and notebooks. - -No ``asyncio.run()`` anywhere — that's the point. ``write_multiscale`` / -``write_collection`` emit one document each (children embedded) and return a -reference stub for it, so a collection can reference the written document -instead of embedding a copy; ``open_collection`` / ``open_multiscale`` return -the fully inlined tree. ``walk()`` / ``find()`` navigate the result. - -Run with: - - pixi run -e dev python examples/01_sync_api.py -""" - -import shutil -from pathlib import Path - -import ngio_collections as ngc - -ROOT = Path(__file__).parent / "data" / "sync_api" - - -def build_multiscale() -> ngc.MultiscaleNode: - systems = ngc.CoordinateSystemsAttribute( - [ - ngc.CoordinateSystem( - id="physical", - axes=[{"name": "y", "type": "space"}, {"name": "x", "type": "space"}], - ) - ] - ) - return ngc.MultiscaleNode( - id="image", - name="DAPI", - nodes=[ - ngc.SinglescaleNode( - id="s0", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={"coordinateTransformations": []}, - ) - ], - attributes={systems.key: systems.model_dump(mode="json", by_alias=True)}, - ) - - -def show(node: ngc.BaseNode, depth: int = 0) -> None: - print(f"{' ' * depth}{node.type} {node.id!r} attrs={list(node.attributes)}") - for child in getattr(node, "nodes", None) or []: - if isinstance(child, ngc.BaseNode): - show(child, depth + 1) - - -def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - - # A multiscale as its own zarr-form document (singlescales embedded). - # The writer hands back the reference form: {type, id, name, path}. - ref: ngc.MultiscaleRef = ngc.write_multiscale( - build_multiscale(), str(ROOT / "image.zarr") - ) - # Parent-level attributes live on the stub; they win over the target's - # attributes when the reference is inlined on read. - ref.attributes["ngio:description"] = "sync api demo" - - # The collection references the existing document instead of embedding it; - # the stub path is relativized on write ("./image.zarr"). - collection = ngc.CollectionNode( - id="experiment", - name="My Experiment", - nodes=[ref], - ) - ngc.write_collection(collection, str(ROOT / "collection.json")) - root = ngc.open_collection(str(ROOT / "collection.json")) - print("collection tree (fully inlined):") - show(root) - - # Navigation: walk() flattens the subtree (self first, depth-first) and - # find() looks a node up by id — no hand-rolled recursion needed. - print("\nwalk() — flat view of the tree:") - for node in root.walk(): - print(f" {node.type} {node.id!r}") - - s0 = root.find("s0") - assert s0 is not None - print(f"\nfind('s0') -> {s0.type} {s0.id!r} name={s0.name!r}") - - # A multiscale document can also be opened directly, and its attributes - # read through the typed attrs view. - image = ngc.open_multiscale(str(ROOT / "image.zarr")) - systems = image.attrs[ngc.CoordinateSystemsAttribute] - print(f"\nopen_multiscale: coordinate systems = {[cs.id for cs in systems.root]}") - - # the stub's path is the target URL, even on the inlined document - # this can be used for data access - print(f"\nimage.nodes[0].target_url -> {image.nodes[0].target_url}") - - -if __name__ == "__main__": - main() diff --git a/examples/02_models_and_documents.py b/examples/02_models_and_documents.py deleted file mode 100644 index 91b6866..0000000 --- a/examples/02_models_and_documents.py +++ /dev/null @@ -1,106 +0,0 @@ -"""The pure model layer: nodes, typed attributes, and document round-trips. - -No IO in this script — nodes and documents are plain Pydantic objects -(DESIGN.md §7). Shows structural validation at construction, the typed -``attrs`` view over the raw attributes dict, ``walk()`` / ``find()`` -navigation, and the ``parse_metadata_document`` / ``serialize`` round-trip. - -Run with: - - pixi run -e dev python examples/02_models_and_documents.py -""" - -import json - -from pydantic import ValidationError - -import ngio_collections as ngc -from ngio_collections.models import LabelObj - - -def build_tree() -> ngc.CollectionNode: - s0 = ngc.SinglescaleNode( - id="s0", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={ - "coordinateTransformations": [ - { - "type": "scale", - "scale": [0.65, 0.65], - "input": {"id": "s0"}, - "output": {"id": "physical"}, - } - ] - }, - ) - systems = ngc.CoordinateSystemsAttribute( - [ - ngc.CoordinateSystem( - id="physical", - axes=[{"name": "y", "type": "space"}, {"name": "x", "type": "space"}], - ) - ] - ) - image = ngc.MultiscaleNode( - id="image", - name="DAPI", - nodes=[s0], - attributes={systems.key: systems.model_dump(mode="json", by_alias=True)}, - ) - return ngc.CollectionNode(id="experiment", name="My Experiment", nodes=[image]) - - -def main() -> None: - # --- Structural rules are enforced at construction ---------------------- - try: - ngc.CollectionNode(id="c", name="c") # neither `nodes` nor `path` - except ValidationError as err: - print("validation error:", err.errors()[0]["msg"]) - - root = build_tree() - - # --- walk() / find(): flat traversal and id lookup ---------------------- - print("\nwalk:", [node.id for node in root.walk()]) - image = root.find("image") - assert isinstance(image, ngc.MultiscaleNode) - - # --- The attrs view: typed, validating reads and writes ----------------- - # Reads validate the raw JSON into the attribute model; assignment dumps - # the spec shape back into the dict. The raw dict stays the source of - # truth, so unknown attributes round-trip untouched. - systems = image.attrs[ngc.CoordinateSystemsAttribute] - print("axes:", [axis["name"] for axis in systems.root[0].axes]) - - image.attrs[ngc.LabelsAttribute] = ngc.LabelsAttribute( - label_attributes=[LabelObj(label_value=1, color=[255, 0, 0, 255])] - ) - print("labels set:", ngc.LabelsAttribute in image.attrs) - - # --- Documents: serialize and re-parse, no IO --------------------------- - # A MetadataDocument is the unit of serialization; the `ome` version - # lives on it, off the node models. - doc = ngc.MetadataDocument( - root=root, url="memory://collection.json", form="json", version="0.x" - ) - payload = doc.serialize() - print("\nserialized document:") - print(json.dumps(payload, indent=2)[:300], "...") - - reparsed = ngc.parse_metadata_document(payload, url="memory://collection.json") - assert [n.id for n in reparsed.root.walk()] == [n.id for n in root.walk()] - print("\nround-trip preserves the tree:", [n.id for n in reparsed.root.walk()]) - - # --- Graceful degradation: unknown types stay opaque -------------------- - custom = ngc.CollectionNode( - id="c", - name="c", - nodes=[{"type": "mobie:view", "id": "v1", "name": "view", "customField": 42}], - ) - view = custom.nodes[0] - print("\nunknown type parses as:", type(view).__name__) - print("extras round-trip:", view.model_dump(by_alias=True)["customField"]) - - -if __name__ == "__main__": - main() diff --git a/examples/03_resolver_read_write.py b/examples/03_resolver_read_write.py deleted file mode 100644 index 670db6d..0000000 --- a/examples/03_resolver_read_write.py +++ /dev/null @@ -1,187 +0,0 @@ -"""The async core: externalized documents, lazy reads, document-granular saves. - -Writes an RFC-8 collection with one document per externalized node: - - data/resolver/ - ├── collection.json <- root collection (stubs for the children) - ├── tables/measurements.json <- nested collection, its own document - └── image.zarr/zarr.json <- multiscale, stored in zarr.json form - -Then reads it back through the ``Resolver``: ``open()`` reads only the root -document (children stay stubs), ``resolve()`` fetches one child on demand, -``resolve_tree()`` warms the cache for the whole reachable tree, and editing -a node rewrites only its owning document on ``save()``. - -Run with: - - pixi run -e dev python examples/03_resolver_read_write.py -""" - -import asyncio -import hashlib -import shutil -from pathlib import Path - -import ngio_collections as ngc - -ROOT = Path(__file__).parent / "data" / "resolver" -VERSION = "0.x" - - -def build_image() -> ngc.MultiscaleNode: - """A multiscale image with one resolution level. - - The singlescale's path points at the array data; its scale transformation - maps it into the "physical" coordinate system declared on the multiscale. - """ - s0 = ngc.SinglescaleNode( - id="s0", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={ - "coordinateTransformations": [ - { - "type": "scale", - "scale": [1.0, 0.65, 0.65], - "input": {"id": "s0"}, - "output": {"id": "physical"}, - } - ] - }, - ) - physical = ngc.CoordinateSystem( - id="physical", - axes=[ - {"name": "z", "type": "space", "unit": "micrometer"}, - {"name": "y", "type": "space", "unit": "micrometer"}, - {"name": "x", "type": "space", "unit": "micrometer"}, - ], - ) - systems = ngc.CoordinateSystemsAttribute([physical]) - return ngc.MultiscaleNode( - id="image", - name="DAPI", - nodes=[s0], - attributes={ - systems.key: systems.model_dump( - mode="json", by_alias=True, exclude_none=True - ) - }, - ) - - -async def write_fixture(resolver: ngc.Resolver) -> None: - """One MetadataDocument per externalized node; the root references them - through path stubs. ``stub_path`` is how the parent document will - reference each child document.""" - image_doc = ngc.MetadataDocument( - root=build_image(), - url=str(ROOT / "image.zarr" / "zarr.json"), - form="zarr", - version=VERSION, - stub_path=ngc.ZarrPath(path="./image.zarr"), - ) - await resolver.save(image_doc) - - tables = ngc.CollectionNode( - id="tables", - name="Tables", - nodes=[ - # Unregistered node types are perfectly valid: readers treat them - # as opaque nodes and keep their custom fields. - {"type": "fractal:table", "id": "t1", "name": "regionprops"}, - ], - ) - tables_doc = ngc.MetadataDocument( - root=tables, - url=str(ROOT / "tables" / "measurements.json"), - form="json", - version=VERSION, - stub_path=ngc.JsonPath(path="./tables/measurements.json"), - ) - await resolver.save(tables_doc) - - root = ngc.CollectionNode( - id="my-experiment", - name="My Experiment", - nodes=[ - ngc.MultiscaleNode( - id="image", name="DAPI", path=ngc.ZarrPath(path="./image.zarr") - ), - ngc.CollectionNode( - id="tables", - name="Tables", - path=ngc.JsonPath(path="./tables/measurements.json"), - ), - ], - ) - root_doc = ngc.MetadataDocument( - root=root, url=str(ROOT / "collection.json"), form="json", version=VERSION - ) - await resolver.save(root_doc) - - -def print_tree(node: ngc.BaseNode, indent: int = 0) -> None: - stub = f" -> {node.path.path}" if node.path is not None else "" - print(f"{' ' * indent}[{node.type}] {node.id}{stub}") - for child in getattr(node, "nodes", None) or []: - if isinstance(child, ngc.BaseNode): - print_tree(child, indent + 1) - - -def snapshot() -> dict[Path, str]: - return {p: hashlib.sha256(p.read_bytes()).hexdigest() for p in ROOT.rglob("*.json")} - - -async def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - await write_fixture(ngc.Resolver(ngc.LocalStore())) - - # A fresh resolver, so open() parses the documents from disk. - resolver = ngc.Resolver(ngc.LocalStore()) - - # --- Open only reads the root document; children stay as stubs ---------- - doc = await resolver.open(str(ROOT / "collection.json")) - root = doc.root - print("After open() (lazy, one file read):") - print_tree(root) - - # --- Resolve a single child on demand ------------------------------------ - # Resolution never mutates the tree: `root` keeps its stub, the resolved - # document lives in the resolver's URL-keyed cache. - image_stub = root.find("image") - image_doc = await resolver.resolve(image_stub) - systems = image_doc.root.attrs[ngc.CoordinateSystemsAttribute] - print( - f"\nResolved {image_doc.root.id!r}: " - f"coordinate systems = {[cs.id for cs in systems.root]}" - ) - - # --- Or warm the cache for the whole reachable tree ---------------------- - # Stubs whose path points at plain Zarr data rather than an OME metadata - # document (the singlescale's `./s0` array) are skipped by default - # (`on_error="skip"`). Afterwards children() / resolve() are cache reads. - documents = await resolver.resolve_tree(doc) - print(f"\nresolve_tree() fetched {len(documents)} documents:") - for d in documents: - print(f" {d.form:>4} {d.url}") - - # children() transparently replaces stubs with their resolved roots. - for child in await resolver.children(root): - print(f"child {child.id!r}: attrs={list(child.attributes)}") - - # --- Edit one node and save: only its owning document is rewritten ------ - before = snapshot() - tables_doc = await resolver.resolve(root.find("tables")) - tables_doc.root.attributes["fractal:status"] = "validated" - await resolver.save(tables_doc) - after = snapshot() - - print("\nFiles changed by save(tables):") - for path in after: - if before[path] != after[path]: - print(f" {path.relative_to(ROOT)}") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/04_inline_and_merge.py b/examples/04_inline_and_merge.py deleted file mode 100644 index 044c566..0000000 --- a/examples/04_inline_and_merge.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Inline a resolved tree: the attribute merge materialized (DESIGN.md §5). - -The collection's `image` stub carries its own attributes -(`ngio:description`) on top of the target multiscale's -(`coordinateSystems`, `labels`). `Resolver.inline()` builds a NEW document -in which the stub is collapsed into a copy of its resolved subtree, with -the merged attributes: target root's overlaid by the stub's (stub wins) and -the stub's id/name. The originals are never touched. - -Writes stay explicit and document-granular — annotating a multiscale on a -read-only store means writing to the stub and saving only the collection -document, which this script also demonstrates. - -Run with: - - pixi run -e dev python examples/04_inline_and_merge.py -""" - -import asyncio -import hashlib -import shutil -from pathlib import Path - -import ngio_collections as ngc -from ngio_collections.models import LabelObj - -ROOT = Path(__file__).parent / "data" / "inline" -VERSION = "0.x" - - -def digest(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest()[:12] - - -async def write_fixture() -> None: - """A collection whose stub annotates an externalized multiscale.""" - resolver = ngc.Resolver(ngc.LocalStore()) - systems = ngc.CoordinateSystemsAttribute( - [ngc.CoordinateSystem(id="physical", axes=[{"name": "x", "type": "space"}])] - ) - image = ngc.MultiscaleNode( - id="image", - name="DAPI", - nodes=[ - ngc.SinglescaleNode( - id="s0", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={"coordinateTransformations": []}, - ) - ], - attributes={systems.key: systems.model_dump(mode="json", by_alias=True)}, - ) - image.attrs[ngc.LabelsAttribute] = ngc.LabelsAttribute( - label_attributes=[LabelObj(label_value=1, color=[255, 0, 0, 255])] - ) - await resolver.save( - ngc.MetadataDocument( - root=image, - url=str(ROOT / "image.zarr" / "zarr.json"), - form="zarr", - version=VERSION, - stub_path=ngc.ZarrPath(path="./image.zarr"), - ) - ) - root = ngc.CollectionNode( - id="my-experiment", - name="My Experiment", - nodes=[ - ngc.MultiscaleNode( - id="image", - name="DAPI", - path=ngc.ZarrPath(path="./image.zarr"), - attributes={"ngio:description": "stub-side annotation"}, - ) - ], - ) - await resolver.save( - ngc.MetadataDocument( - root=root, url=str(ROOT / "collection.json"), form="json", version=VERSION - ) - ) - - -async def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - await write_fixture() - - # A fresh resolver, so open() parses the documents from disk. - resolver = ngc.Resolver(ngc.LocalStore()) - doc = await resolver.open(str(ROOT / "collection.json")) - stub = doc.root.nodes[0] - - print("stub attributes: ", list(stub.attributes)) - target = (await resolver.resolve(stub)).root - print("target attributes:", list(target.attributes)) - - # The §5 merge, materialized: the stub collapsed into its resolved - # subtree, target attributes overlaid by the stub's (stub wins). - inlined = await resolver.inline(doc) - image = inlined.root.nodes[0] - print("merged attributes:", list(image.attributes)) - - # The inlined node is a real node: typed reads via the normal attrs view. - labels = image.attrs[ngc.LabelsAttribute] - print("label colors:", [label.color for label in labels.label_attributes]) - - # The originals are untouched: the parsed tree keeps its stub. - print("original stub intact:", stub.path is not None and stub.nodes is None) - - # Annotate the (possibly read-only) multiscale via the stub: only the - # collection document is rewritten, image.zarr/zarr.json is untouched. - zarr_json = ROOT / "image.zarr" / "zarr.json" - before = digest(zarr_json) - stub.attributes["ngio:reviewed"] = True - await resolver.save(doc) - print("\nsaved", doc.url) - print("image.zarr/zarr.json untouched:", digest(zarr_json) == before) - reinlined = await resolver.inline(doc) - print("merged attributes:", list(reinlined.root.nodes[0].attributes)) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/05_custom_node_types.py b/examples/05_custom_node_types.py deleted file mode 100644 index b4e3aee..0000000 --- a/examples/05_custom_node_types.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Registering custom node types (DESIGN.md §3.4). - -A third-party package subclasses ``BaseNode``, registers it under its -``type`` key in a ``NodeRegistry``, and children parse as the custom class. -Registries are plain objects, not singletons: pass one to -``parse_metadata_document(..., registry=...)`` or ``Resolver(store, -registry=...)``. Unregistered types degrade gracefully to a plain -``BaseNode`` and round-trip their extra fields untouched. - -Run with: - - pixi run -e dev python examples/05_custom_node_types.py -""" - -from typing import Literal - -import ngio_collections as ngc - - -class TableNode(ngc.BaseNode): - """A custom node type with its own typed field.""" - - type: Literal["fractal:table"] = "fractal:table" - region: str | None = None - - -def build_registry() -> ngc.NodeRegistry: - # A fresh registry starts empty: register the built-ins you need plus - # your own types (DEFAULT_REGISTRY keeps the everyday ergonomics). - registry = ngc.NodeRegistry() - registry.register("collection", ngc.CollectionNode) - registry.register("multiscale", ngc.MultiscaleNode) - registry.register("singlescale", ngc.SinglescaleNode) - registry.register("fractal:table", TableNode) - return registry - - -DATA = { - "ome": { - "version": "0.x", - "type": "collection", - "id": "experiment", - "name": "My Experiment", - "nodes": [ - { - "type": "fractal:table", - "id": "t1", - "name": "regionprops", - "region": "FOV_1", - }, - {"type": "mobie:view", "id": "v1", "name": "view", "customField": 42}, - ], - } -} - - -def main() -> None: - doc = ngc.parse_metadata_document( - DATA, url="memory://collection.json", registry=build_registry() - ) - table = doc.root.find("t1") - print(f"registered type parses as {type(table).__name__}, region={table.region!r}") - - # Unregistered types stay opaque BaseNodes; their extras round-trip. - view = doc.root.find("v1") - dumped = doc.serialize()["ome"]["nodes"][1] - print(f"unregistered type parses as {type(view).__name__}, dump={dumped}") - - # Without the registry, the custom type is opaque too (graceful - # degradation) — same document, no error, no custom field typing. - plain = ngc.parse_metadata_document(DATA, url="memory://collection.json") - print("without registry:", type(plain.root.find("t1")).__name__) - - -if __name__ == "__main__": - main() diff --git a/examples/06_hcs_plate_single_collection.py b/examples/06_hcs_plate_single_collection.py deleted file mode 100644 index 31cd2a5..0000000 --- a/examples/06_hcs_plate_single_collection.py +++ /dev/null @@ -1,133 +0,0 @@ -"""An HCS plate as a SINGLE collection document, images externalized. - -One ``collection.json`` holds the whole plate->well hierarchy inline: the -plate is a collection carrying the ``plate`` attribute, each well a child -collection carrying the ``well`` attribute. Only the multiscale images are -externalized, into per-field subdirectories the wells reference by path: - - data/hcs_single/ - ├── collection.json <- plate + wells inline, image stubs - ├── A/1/0.zarr/zarr.json <- multiscale image (field 0 of well A/1) - ├── A/2/0.zarr/zarr.json - ├── B/1/0.zarr/zarr.json - └── B/2/0.zarr/zarr.json - -Built bottom-up with the sync API: ``write_multiscale`` emits each image and -hands back a reference stub, the wells embed those stubs, and -``write_collection`` emits the one plate document — relativizing every nested -image path against it (``./A/1/0.zarr`` …). - -Run with: - - pixi run -e dev python examples/06_hcs_plate_single_collection.py -""" - -import shutil -from pathlib import Path - -import ngio_collections as ngc -from ngio_collections.models import ColumnObj, RowObj - -ROOT = Path(__file__).parent / "data" / "hcs_single" - -ROWS = ["A", "B"] -COLUMNS = ["1", "2"] - - -def build_image(row: str, col: str) -> ngc.MultiscaleNode: - """A one-level multiscale for field 0 of well ``{row}/{col}``. - - Node ids are unique per well so the whole plate stays collision-free once - every image is inlined into one tree on read. - """ - systems = ngc.CoordinateSystemsAttribute( - [ - ngc.CoordinateSystem( - id="physical", - axes=[{"name": "y", "type": "space"}, {"name": "x", "type": "space"}], - ) - ] - ) - return ngc.MultiscaleNode( - id=f"img_{row}{col}", - name="0", - nodes=[ - ngc.SinglescaleNode( - id=f"s0_{row}{col}", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={"coordinateTransformations": []}, - ) - ], - attributes={systems.key: systems.model_dump(mode="json", by_alias=True)}, - ) - - -def build_well(row: str, col: str) -> ngc.CollectionNode: - """A well collection whose single child is the externalized image stub.""" - image_ref = ngc.write_multiscale( - build_image(row, col), str(ROOT / row / col / "0.zarr") - ) - well = ngc.WellAttribute( - row=ngc.ReferenceObj(id=row), column=ngc.ReferenceObj(id=col) - ) - return ngc.CollectionNode( - id=f"well_{row}{col}", - name=f"{row}{col}", - nodes=[image_ref], - attributes={ - well.key: well.model_dump(mode="json", by_alias=True, exclude_none=True) - }, - ) - - -def show(node: ngc.BaseNode, depth: int = 0) -> None: - stub = f" -> {node.path.path}" if node.path is not None else "" - print(f"{' ' * depth}[{node.type}] {node.id} attrs={list(node.attributes)}{stub}") - for child in getattr(node, "nodes", None) or []: - if isinstance(child, ngc.BaseNode): - show(child, depth + 1) - - -def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - - # Wells (with their image stubs) stay inline; only images are externalized. - wells = [build_well(row, col) for row in ROWS for col in COLUMNS] - plate = ngc.PlateAttribute( - rows=[RowObj(id=row) for row in ROWS], - columns=[ColumnObj(id=col) for col in COLUMNS], - ) - plate_node = ngc.CollectionNode( - id="plate", - name="My Plate", - nodes=wells, - attributes={ - plate.key: plate.model_dump(mode="json", by_alias=True, exclude_none=True) - }, - ) - ngc.write_collection(plate_node, str(ROOT / "collection.json")) - - print("written files:") - for file in sorted(ROOT.rglob("*.json")): - print(f" {file.relative_to(ROOT)}") - - # Read back: open_collection inlines the image documents into the plate. - root = ngc.open_collection(str(ROOT / "collection.json")) - print("\nplate tree (fully inlined):") - show(root) - - # Navigate the flattened plate with walk() / find(). - plate_attr = root.attrs[ngc.PlateAttribute] - print( - f"\nplate: {len(plate_attr.rows)} rows x {len(plate_attr.columns)} columns, " - f"{sum(n.type == 'collection' for n in root.walk()) - 1} wells" - ) - well_b2 = root.find("well_B2") - assert well_b2 is not None - location = well_b2.attrs[ngc.WellAttribute] - print(f"well_B2 at row={location.row.id!r} column={location.column.id!r}") - - -if __name__ == "__main__": - main() diff --git a/examples/07_hcs_plate_nested.py b/examples/07_hcs_plate_nested.py deleted file mode 100644 index 19ca726..0000000 --- a/examples/07_hcs_plate_nested.py +++ /dev/null @@ -1,136 +0,0 @@ -"""An HCS plate as a fully externalized tree, one document per node. - -Mirrors the on-disk OME-Zarr plate layout: the plate at the top level, each -well its own document in a ``{row}/{col}`` subdirectory, and each image in -``{row}/{col}/{image}``. Every parent references its children by path stub: - - data/hcs_nested/ - ├── collection.json <- plate, well stubs -> ./A/1/well.json … - ├── A/1/well.json <- well A/1, image stubs -> ./0.zarr - ├── A/1/0.zarr/zarr.json <- multiscale image (field 0) - ├── A/2/well.json - ├── A/2/0.zarr/zarr.json - └── … - -Built bottom-up with the sync API: each ``write_*`` emits one document and -returns a reference stub, which the parent embeds; ``write_collection`` -relativizes the embedded stub paths against the parent's URL, so the well -document references ``./0.zarr`` and the plate references ``./A/1/well.json``. - -Run with: - - pixi run -e dev python examples/07_hcs_plate_nested.py -""" - -import shutil -from pathlib import Path - -import ngio_collections as ngc -from ngio_collections.models import ColumnObj, RowObj - -ROOT = Path(__file__).parent / "data" / "hcs_nested" - -ROWS = ["A", "B"] -COLUMNS = ["1", "2"] - - -def build_image(row: str, col: str) -> ngc.MultiscaleNode: - """A one-level multiscale for field 0 of well ``{row}/{col}``. - - Node ids stay unique across the plate so the inlined-on-read tree (every - document collapsed into one) has no id collisions. - """ - systems = ngc.CoordinateSystemsAttribute( - [ - ngc.CoordinateSystem( - id="physical", - axes=[{"name": "y", "type": "space"}, {"name": "x", "type": "space"}], - ) - ] - ) - return ngc.MultiscaleNode( - id=f"img_{row}{col}", - name="0", - nodes=[ - ngc.SinglescaleNode( - id=f"s0_{row}{col}", - name="s0", - path=ngc.ZarrPath(path="./s0"), - attributes={"coordinateTransformations": []}, - ) - ], - attributes={systems.key: systems.model_dump(mode="json", by_alias=True)}, - ) - - -def write_well(row: str, col: str) -> ngc.CollectionRef: - """Write image then well, each its own document; return the well stub.""" - image_ref = ngc.write_multiscale( - build_image(row, col), str(ROOT / row / col / "0.zarr") - ) - well = ngc.WellAttribute( - row=ngc.ReferenceObj(id=row), column=ngc.ReferenceObj(id=col) - ) - well_node = ngc.CollectionNode( - id=f"well_{row}{col}", - name=f"{row}{col}", - nodes=[image_ref], - attributes={ - well.key: well.model_dump(mode="json", by_alias=True, exclude_none=True) - }, - ) - # Writing the well relativizes the image stub against it: ./0.zarr - return ngc.write_collection(well_node, str(ROOT / row / col / "well.json")) - - -def show(node: ngc.BaseNode, depth: int = 0) -> None: - stub = f" -> {node.path.path}" if node.path is not None else "" - print(f"{' ' * depth}[{node.type}] {node.id} attrs={list(node.attributes)}{stub}") - for child in getattr(node, "nodes", None) or []: - if isinstance(child, ngc.BaseNode): - show(child, depth + 1) - - -def main() -> None: - shutil.rmtree(ROOT, ignore_errors=True) - - # Each well is its own document; the plate references them by path. - well_refs = [write_well(row, col) for row in ROWS for col in COLUMNS] - plate = ngc.PlateAttribute( - rows=[RowObj(id=row) for row in ROWS], - columns=[ColumnObj(id=col) for col in COLUMNS], - ) - plate_node = ngc.CollectionNode( - id="plate", - name="My Plate", - nodes=well_refs, - attributes={ - plate.key: plate.model_dump(mode="json", by_alias=True, exclude_none=True) - }, - ) - ngc.write_collection(plate_node, str(ROOT / "collection.json")) - - print("written files (one document per node):") - for file in sorted(ROOT.rglob("*.json")): - print(f" {file.relative_to(ROOT)}") - - # The lazy view: open() reads only the plate; wells stay stubs. - print("\nplate document alone (wells are stubs):") - show(ngc.open_collection(str(ROOT / "collection.json"), max_depth=0)) - - # The hydrated view: open_collection inlines wells and their images. - root = ngc.open_collection(str(ROOT / "collection.json")) - print("\nplate tree (fully inlined):") - show(root) - - well_a1 = root.find("well_A1") - assert well_a1 is not None - location = well_a1.attrs[ngc.WellAttribute] - print( - f"\nwell_A1 at row={location.row.id!r} column={location.column.id!r}, " - f"images={[n.id for n in well_a1.walk() if n.type == 'multiscale']}" - ) - - -if __name__ == "__main__": - main() diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 2b59af8..0000000 --- a/examples/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Examples - -Each script is self-contained (it writes its own fixture data under -`examples/data/`, which is gitignored) and runnable with: - -```bash -pixi run -e dev python examples/