From 80f2661d60af7d155ffa59a10327cd1cfefe2b71 Mon Sep 17 00:00:00 2001 From: Sergio Marcelino Date: Fri, 12 Jun 2026 16:00:30 -0300 Subject: [PATCH] feat(docs): top level docs for repository structure and sops --- .github/scripts/validate_worker.py | 11 +- .github/workflows/create-tag.yml | 2 + .github/workflows/release.yml | 2 + README.md | 21 +- coder/src/config.rs | 2 +- coder/src/functions/mod.rs | 2 +- coder/tests/integration.rs | 2 +- coder/tests/manifest.rs | 2 +- docs/README.md | 43 +++++ docs/architecture/README.md | 19 ++ docs/architecture/deploy-modes.md | 69 +++++++ docs/architecture/iii-worker-yaml.md | 79 ++++++++ docs/architecture/per-worker-architecture.md | 71 +++++++ docs/architecture/skills-and-permissions.md | 77 ++++++++ docs/architecture/testing-and-ci.md | 89 +++++++++ docs/architecture/worker-model.md | 54 ++++++ docs/sops/README.md | 21 ++ .../sops/binary-worker.md | 16 +- docs/sops/new-worker.md | 152 +++++++++++++++ docs/sops/release.md | 180 ++++++++++++++++++ email/README.md | 3 +- iii-lsp/README.md | 2 +- storage/README.md | 2 +- storage/src/config.rs | 2 +- storage/tests/integration.rs | 2 +- todo-worker-python/README.md | 6 +- todo-worker/README.md | 4 +- worker-readme.md | 2 +- 28 files changed, 904 insertions(+), 33 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/architecture/README.md create mode 100644 docs/architecture/deploy-modes.md create mode 100644 docs/architecture/iii-worker-yaml.md create mode 100644 docs/architecture/per-worker-architecture.md create mode 100644 docs/architecture/skills-and-permissions.md create mode 100644 docs/architecture/testing-and-ci.md create mode 100644 docs/architecture/worker-model.md create mode 100644 docs/sops/README.md rename binary-worker.md => docs/sops/binary-worker.md (98%) create mode 100644 docs/sops/new-worker.md create mode 100644 docs/sops/release.md diff --git a/.github/scripts/validate_worker.py b/.github/scripts/validate_worker.py index 858cb0a2..57a004d8 100644 --- a/.github/scripts/validate_worker.py +++ b/.github/scripts/validate_worker.py @@ -29,8 +29,9 @@ import _lib # noqa: E402 -# Workers the harness materialises into ./data/skills//index.md on -# boot via skills::download. Must match harness/src/skills.rs::BOOTSTRAP_NAMES. +# Workers whose skills the harness stack requires at boot, making +# skills/SKILL.md a hard PR gate. Keep in sync with what the harness +# actually bootstraps. BOOTSTRAP_WORKERS = frozenset({ "iii-directory", "shell", @@ -162,17 +163,17 @@ def soft(msg: str) -> None: if not skill_md.exists(): hard( f"{worker}/skills/SKILL.md is missing — bundled workers must ship one " - f"(see binary-worker.md)" + f"(see docs/sops/binary-worker.md)" ) elif skill_md.stat().st_size == 0: hard( f"{worker}/{skill_md.relative_to(root).as_posix()} is empty — " - f"must contain the H1 + summary (see binary-worker.md)" + f"must contain the H1 + summary (see docs/sops/binary-worker.md)" ) elif skill_md.stat().st_size > SKILL_MD_SIZE_CAP: hard( f"{worker}/{skill_md.relative_to(root).as_posix()} exceeds 256 KiB cap " - f"({skill_md.stat().st_size} bytes; see binary-worker.md)" + f"({skill_md.stat().st_size} bytes; see docs/sops/binary-worker.md)" ) for e in errs: diff --git a/.github/workflows/create-tag.yml b/.github/workflows/create-tag.yml index dd79691d..bbf67aac 100644 --- a/.github/workflows/create-tag.yml +++ b/.github/workflows/create-tag.yml @@ -1,3 +1,4 @@ +# Release SOP: docs/sops/release.md name: Create Tag on: @@ -19,6 +20,7 @@ on: - iii-lsp-vscode - image-resize - mcp + - session-manager - shell - storage bump: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69daddcb..fc371099 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,4 @@ +# Release SOP: docs/sops/release.md name: Release on: @@ -13,6 +14,7 @@ on: - 'iii-lsp/v*' - 'image-resize/v*' - 'mcp/v*' + - 'session-manager/v*' - 'shell/v*' - 'storage/v*' workflow_dispatch: diff --git a/README.md b/README.md index 61e69f93..45d66e1a 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ npx skills add iii-hq/iii --all | [`iii-lsp`](iii-lsp/) | Rust | Language Server for iii function ids, trigger configs, and worker discovery. Autocomplete / hover across JS/TS, Python, Rust. | | [`iii-lsp-vscode`](iii-lsp-vscode/) | Node | VS Code extension that embeds `iii-lsp`. | | [`image-resize`](image-resize/) | Rust | Image resize via channel I/O — JPEG/PNG/WebP with EXIF auto-orient, scale-to-fit / crop-to-fit. | -| [`llm-budget`](llm-budget/) | Rust | Workspace + agent LLM spend caps with alerts, forecast, and period rollover under `budget::*`. | | [`mcp`](mcp/) | Rust | MCP 2025-06-18 Streamable HTTP bridge — exposes iii functions tagged `mcp.expose` as MCP tools. | | [`shell`](shell/) | Rust | Unix shell + filesystem worker — `shell::exec` with allowlist/denylist/timeout/output caps and background jobs; `fs::ls`/`stat`/`mkdir`/`rm`/`chmod`/`mv`/`grep`/`sed`/`read`/`write` with host jail, denylist, and size caps. | | [`storage`](storage/) | Rust | S3-compatible object storage across AWS S3, GCS, Cloudflare R2, and a managed local rustfs backend. Streamed uploads, presigned URLs, and object change triggers. | @@ -118,11 +117,13 @@ host. ## Add a new worker -[`binary-worker.md`](binary-worker.md) is the deep dive for the Rust binary -scaffold (manifest, config, layout, function contract). -[`worker-readme.md`](worker-readme.md) covers the README contract every -worker shares (install via `iii worker add`, run via `iii start`, how-to -over reference). +Start with [`docs/sops/new-worker.md`](docs/sops/new-worker.md) — the +cross-cutting checklist (naming, required files, CI gates, release wiring). +For the inside of a Rust `deploy: binary` worker, continue with +[`docs/sops/binary-worker.md`](docs/sops/binary-worker.md). Each worker ships +a consumer `README.md` per the [`worker-readme.md`](worker-readme.md) +contract (install via `iii worker add`, quickstart, configuration). +[`docs/README.md`](docs/README.md) indexes all shared docs. ## CI @@ -138,6 +139,9 @@ The `pr-checks` job additionally enforces, per changed worker: `README.md` present, `iii.worker.yaml` valid, `tests/` non-empty, and the manifest version is greater than the version on the PR's base branch. +Full reference (discovery buckets, interface boot smoke, e2e workflows): +[`docs/architecture/testing-and-ci.md`](docs/architecture/testing-and-ci.md). + ## CD Releases are cut manually via the **Create Tag** workflow @@ -151,9 +155,14 @@ resulting `/v` tag drives a single dispatcher [`_rust-binary.yml`](.github/workflows/_rust-binary.yml). - `image` → multi-arch image to `ghcr.io//` via [`_container.yml`](.github/workflows/_container.yml). + - `bundle` → single-file archive via + [`_bundle.yml`](.github/workflows/_bundle.yml). 2. Calls `POST /publish` against the workers registry API via [`_publish-registry.yml`](.github/workflows/_publish-registry.yml). +Step-by-step (variants, troubleshooting, rollback): +[`docs/sops/release.md`](docs/sops/release.md). + ## License Apache 2.0 — see [`LICENSE`](LICENSE). diff --git a/coder/src/config.rs b/coder/src/config.rs index 9a91b208..a3e224a3 100644 --- a/coder/src/config.rs +++ b/coder/src/config.rs @@ -2,7 +2,7 @@ //! attribute so an empty YAML object still produces a fully-populated //! struct; `impl Default` mirrors those functions so the binary can fall //! back to defaults when `--config` is missing or unparseable (see -//! [`binary-worker.md`](../../binary-worker.md) §5). +//! [`binary-worker.md`](../../docs/sops/binary-worker.md) §5). use std::path::PathBuf; diff --git a/coder/src/functions/mod.rs b/coder/src/functions/mod.rs index 6727f9c0..fc450604 100644 --- a/coder/src/functions/mod.rs +++ b/coder/src/functions/mod.rs @@ -2,7 +2,7 @@ //! `main.rs` calls `register_all` after `register_worker`; each //! `register_` below uses `RegisterFunction::new_async` with typed //! `JsonSchema` request/response structs so the SDK can emit schemas for -//! tools and docs (binary-worker.md §7). +//! tools and docs (docs/sops/binary-worker.md §7). //! //! WIRE-SURFACE CATALOG — `catalog()` below is the single source of truth //! for every function's id + registration description, plus the diff --git a/coder/tests/integration.rs b/coder/tests/integration.rs index b11c0171..22734e9b 100644 --- a/coder/tests/integration.rs +++ b/coder/tests/integration.rs @@ -1,4 +1,4 @@ -//! End-to-end test (binary-worker.md §9 pattern A): spawn the `iii` engine +//! End-to-end test (docs/sops/binary-worker.md §9 pattern A): spawn the `iii` engine //! and the `coder` worker as subprocesses, drive the worker through //! `iii-sdk` as a client, then tear both down. **Self-skips** when `iii` //! is not on `PATH` so CI hosts without the engine still pass. diff --git a/coder/tests/manifest.rs b/coder/tests/manifest.rs index a2acd180..d465e8e6 100644 --- a/coder/tests/manifest.rs +++ b/coder/tests/manifest.rs @@ -1,5 +1,5 @@ //! Validates the `--manifest` subcommand the registry publish pipeline -//! relies on (binary-worker.md §11 / pattern A). +//! relies on (docs/sops/binary-worker.md §11 / pattern A). use std::process::Command; diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..b97bc4c7 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,43 @@ +# Workers documentation + +This folder collects **procedures** (SOPs) and **shared architecture** for +workers in this monorepo. Per-worker deep docs stay inside each worker folder +(e.g. [`session-manager/architecture/`](../session-manager/architecture/)). + +## I want to… + +| Goal | Read | +|---|---| +| Add a new worker to the repo | [`sops/new-worker.md`](sops/new-worker.md) | +| Scaffold a Rust `deploy: binary` worker | [`sops/binary-worker.md`](sops/binary-worker.md) | +| Ship a version to the registry | [`sops/release.md`](sops/release.md) | +| Fix a failed release | [`sops/release.md`](sops/release.md) § Troubleshooting | +| Write a consumer `README.md` | [`worker-readme.md`](../worker-readme.md) | +| Author `skills/SKILL.md` | [`DOCUMENTATION_GUIDELINES.md`](../DOCUMENTATION_GUIDELINES.md) | +| Understand `iii.worker.yaml` fields | [`architecture/iii-worker-yaml.md`](architecture/iii-worker-yaml.md) | +| Understand CI gates on a PR | [`architecture/testing-and-ci.md`](architecture/testing-and-ci.md) | +| Integrate with session-manager | [`session-manager/architecture/integration.md`](../session-manager/architecture/integration.md) | + +## Layout + +```text +docs/ +├── README.md # this file +├── sops/ # step-by-step procedures +│ ├── new-worker.md # cross-cutting onboarding +│ ├── binary-worker.md # Rust binary scaffold +│ └── release.md # cut, monitor, verify releases +└── architecture/ # shared reference (not step-by-step) + ├── worker-model.md + ├── iii-worker-yaml.md + ├── deploy-modes.md + ├── testing-and-ci.md + ├── skills-and-permissions.md + └── per-worker-architecture.md +``` + +## Related docs (repo root) + +- [`worker-readme.md`](../worker-readme.md) — consumer README contract +- [`DOCUMENTATION_GUIDELINES.md`](../DOCUMENTATION_GUIDELINES.md) — `skills/SKILL.md` authoring +- [`tech-specs/`](../tech-specs/) — product-level specs before or alongside implementation diff --git a/docs/architecture/README.md b/docs/architecture/README.md new file mode 100644 index 00000000..1daf0a82 --- /dev/null +++ b/docs/architecture/README.md @@ -0,0 +1,19 @@ +# Shared worker architecture + +Reference material for concepts that apply across workers. For step-by-step +procedures, see [`../sops/`](../sops/). + +| Document | Summary | +|---|---| +| [`worker-model.md`](worker-model.md) | Worker lifecycle, engine bus, function ids, registry discovery | +| [`iii-worker-yaml.md`](iii-worker-yaml.md) | Manifest field reference and which CI/CD jobs consume each field | +| [`deploy-modes.md`](deploy-modes.md) | `binary` / `image` / `bundle` paths through build and publish | +| [`testing-and-ci.md`](testing-and-ci.md) | PR discovery, gates, interface boot smoke, dedicated e2e workflows | +| [`skills-and-permissions.md`](skills-and-permissions.md) | `SKILL.md` lifecycle and `iii-permissions.yaml` conventions | +| [`per-worker-architecture.md`](per-worker-architecture.md) | When to add `/architecture/`; session-manager as reference | + +## Per-worker architecture + +Workers with non-trivial integration surfaces may ship their own +`/architecture/` folder. Example: +[`session-manager/architecture/`](../../session-manager/architecture/). diff --git a/docs/architecture/deploy-modes.md b/docs/architecture/deploy-modes.md new file mode 100644 index 00000000..94ca11c7 --- /dev/null +++ b/docs/architecture/deploy-modes.md @@ -0,0 +1,69 @@ +# Deploy modes + +How `iii.worker.yaml` `deploy` routes CI interface smoke and release builds. + +## Overview + +| `deploy` | Artifact | Build workflow | Typical language | +|---|---|---|---| +| `binary` | Per-target CLI archives on GitHub Release | `_rust-binary.yml` | Rust | +| `image` | `ghcr.io//:` | `_container.yml` | Node, Python | +| `bundle` | `.tar.gz` on GitHub Release | `_bundle.yml` | Node (esbuild) | + +Release dispatcher: [`release.yml`](../../.github/workflows/release.yml) reads +`deploy` from `iii.worker.yaml` via `parse_release_tag.py`. + +## Binary + +- **Build:** up to 9 cross-compiled targets (or `targets:` subset). +- **Assets:** `-.tar.gz` / `.zip` + `.sha256` checksums. +- **Publish boot:** downloads `*-x86_64-unknown-linux-gnu.tar.gz` from the + Release, runs the binary for interface collection. +- **Registry payload:** per-target download URLs resolved by + `resolve_binary_artifacts.py`. + +## Image + +- **Build:** multi-arch Docker image pushed to GHCR. +- **Publish boot:** from local source via `iii worker add ./` (the + image itself is not booted; `runtime`/`scripts.start` drive the local run). +- **Registry payload:** the built image reference (`image_tag` output). + +Templates: [`todo-worker/`](../../todo-worker/), [`todo-worker-python/`](../../todo-worker-python/). + +## Bundle + +- **Build:** esbuild single-file `index.mjs` + `iii.worker.yaml` packed into + `.tar.gz`. +- **Asset URL:** `https://github.com//releases/download//.tar.gz` +- **Publish boot:** extracts bundle, runs `node ./index.mjs`. +- **Dependencies:** bundled worker may declare in-repo deps in `iii.worker.yaml` + `dependencies`; when the bundle worker's source changes, those deps join the + CI matrix (see [`testing-and-ci.md`](testing-and-ci.md)). + +Example: [`harness/`](../../harness/). + +## Publish boot modes + +[`manifest_version.py deploy-mode`](../../.github/scripts/manifest_version.py) +selects how `_publish-registry.yml` starts the worker: + +| Mode | When | Boot | +|---|---|---| +| `release-binary` | `deploy: binary` | Download + run Linux gnu binary from Release | +| `release-bundle` | `deploy: bundle` | Extract tarball, `node ./index.mjs` | +| `iii-add` | Other deploys with `runtime` or `scripts.start` | `iii worker add ./` | +| `cargo-run` | Other deploys, Rust, no `runtime`/`scripts.start` | `cargo run` (+ `config.collect.yaml` if present) | + +## config.collect.yaml + +Workers whose default `config.yaml` spawns sidecars or requires local paths +unavailable on CI ship `config.collect.yaml` — a lighter config used only for +interface collection at publish time and in the `interface-smoke` CI job. + +Precedents: `storage`, `shell`, `coder`, `database`. + +## Related + +- Release SOP: [`../sops/release.md`](../sops/release.md) +- Testing: [`testing-and-ci.md`](testing-and-ci.md) diff --git a/docs/architecture/iii-worker-yaml.md b/docs/architecture/iii-worker-yaml.md new file mode 100644 index 00000000..83f0c2f7 --- /dev/null +++ b/docs/architecture/iii-worker-yaml.md @@ -0,0 +1,79 @@ +# iii.worker.yaml contract + +Every worker folder ships `iii.worker.yaml` at its root. CI discovery, +release routing, and registry publish all read this file. + +## Required fields + +| Field | Type | Purpose | +|---|---|---| +| `iii` | `v1` | Schema version | +| `name` | string | Folder name; git tag prefix; registry id | +| `language` | enum | `rust` \| `javascript` \| `node` \| `python` — routes CI language job | +| `deploy` | enum | `binary` \| `image` \| `bundle` — routes release build + publish | +| `manifest` | path | Version source: `Cargo.toml`, `package.json`, `pyproject.toml` | +| `description` | string | Registry + `--manifest` output | + +## Binary-specific + +| Field | Purpose | Consumer | +|---|---|---| +| `bin` | Cargo binary name (defaults to `name`) | `_rust-binary.yml`, publish boot | +| `targets` | Optional list of Rust triples to build | `_rust-binary.yml` matrix subset; `supported_targets` in manifest | + +When `targets` is omitted, all nine default triples are built: macOS +x86_64/aarch64, Windows x86_64/i686/aarch64, Linux x86_64 gnu/musl, +aarch64 gnu, and armv7 gnueabihf (the matrix lives in +[`_rust-binary.yml`](../../.github/workflows/_rust-binary.yml)). + +## Opt-outs and runtime + +| Field | Purpose | Consumer | +|---|---|---| +| `interface_smoke: false` | Skip interface boot smoke + registry publish | `ci.yml`, `release.yml` | +| `runtime` / `scripts.start` | Local boot definition; presence routes publish boot to `iii-add` for non-binary/bundle deploys | `manifest_version.py deploy-mode`, `iii worker add` | +| `scripts.install` | Build command for local install | `iii worker add` (local source) | + +## Bundle-specific + +| Field | Purpose | +|---|---| +| `dependencies` | Map of worker name → semver range for bundled sub-workers | + +Example: [`harness/iii.worker.yaml`](../../harness/iii.worker.yaml). + +## Example (minimal Rust binary) + +```yaml +iii: v1 +name: session-manager +language: rust +deploy: binary +manifest: Cargo.toml +bin: session-manager +description: Durable, reactive, branching store of typed conversation entries. +``` + +## Example (POSIX-only targets) + +```yaml +targets: + - x86_64-apple-darwin + - aarch64-apple-darwin + - x86_64-unknown-linux-gnu + - x86_64-unknown-linux-musl + - aarch64-unknown-linux-gnu + - armv7-unknown-linux-gnueabihf +``` + +See [`shell/iii.worker.yaml`](../../shell/iii.worker.yaml). + +## Validation + +- `pr-checks` parses the file via [`validate_worker.py`](../../.github/scripts/validate_worker.py). +- `parse_release_tag.py` rejects unknown `deploy` values at release time. + +## Related + +- Deploy routing: [`deploy-modes.md`](deploy-modes.md) +- Onboarding checklist: [`../sops/new-worker.md`](../sops/new-worker.md) §6 diff --git a/docs/architecture/per-worker-architecture.md b/docs/architecture/per-worker-architecture.md new file mode 100644 index 00000000..07bf3dc6 --- /dev/null +++ b/docs/architecture/per-worker-architecture.md @@ -0,0 +1,71 @@ +# Per-worker architecture docs + +When and how to add deep architecture documentation inside a worker folder. + +## Repo-wide vs per-worker + +| Location | Scope | Audience | +|---|---|---| +| [`docs/architecture/`](../architecture/) | Concepts shared by all workers | Contributors adding any worker | +| `/architecture/` | One worker's design + integration contract | Maintainers + integrators of that worker | +| [`tech-specs/`](../../tech-specs/) | Product-level specs (may predate implementation) | Design review, cross-repo readers | +| `/README.md` | Consumer how-to (install, quickstart, config) | Operators after `iii worker add` | + +Consumer READMEs are **not** architecture docs. See [`worker-readme.md`](../../worker-readme.md). + +## When to add `/architecture/` + +Add a folder when **any** of these apply: + +- Other workers or clients integrate via many function ids + trigger types + (handoff contract needed). +- Storage, event, or security model is non-obvious from the README. +- Maintainers need internals separate from integrator docs. + +Skip it when the worker is a thin tool with a handful of functions and no +reactive surface (a README + tests may suffice). + +## Recommended layout + +Follow [`session-manager/architecture/`](../../session-manager/architecture/) as +the reference: + +```text +/architecture/ +├── README.md # index: one paragraph, diagram, vocabulary, doc map +├── internals.md # maintainers: storage, pipelines, invariants +└── integration.md # consumers: function ids, triggers, permissions, sequences +``` + +| File | Write for | Content | +|---|---|---| +| `README.md` | Everyone landing here | System in one paragraph + diagram; table pointing to children | +| `internals.md` | People changing the worker | Data structures, backends, concurrency, error paths | +| `integration.md` | People calling the worker | Stable API contract; trigger subscription patterns; what *not* to do | + +## Executable spec + +BDD features (`tests/features/*.feature`) and integration tests are the +executable companion to architecture prose. Annotate scenarios with the +regression they prevent (session-manager pattern). + +## Relationship to tech-specs + +| Layer | Role | +|---|---| +| `tech-specs/2026-06-agentic/.md` | Design of record before/during build | +| `/architecture/` | As-built reference aligned with the code | +| `tests/features/` | Behavioural truth | + +When they diverge, fix the code or update architecture docs in the same PR. + +## Linking + +- Root [`README.md`](../../README.md) Modules table → worker README or + `architecture/` when present. +- [`docs/README.md`](../README.md) → per-worker integration docs for major surfaces. + +## Related + +- session-manager example: [`session-manager/architecture/README.md`](../../session-manager/architecture/README.md) +- Shared worker model: [`worker-model.md`](worker-model.md) diff --git a/docs/architecture/skills-and-permissions.md b/docs/architecture/skills-and-permissions.md new file mode 100644 index 00000000..e7b0066a --- /dev/null +++ b/docs/architecture/skills-and-permissions.md @@ -0,0 +1,77 @@ +# Skills and permissions + +How agent-facing skill docs and default permissions are managed. + +## skills/SKILL.md lifecycle + +### Authoring + +Each worker may ship `skills/SKILL.md` — a lean intent doc for agents (when to +use, boundaries, function catalogue). **Not** JSON schemas or worked examples; +those live in `iii get function info`. + +Author per [`DOCUMENTATION_GUIDELINES.md`](../../DOCUMENTATION_GUIDELINES.md). + +### PR validation + +| Case | Rule | +|---|---| +| Bootstrap workers (`shell`, `iii-directory`) | `skills/SKILL.md` **required**, non-empty, ≤ 256 KiB | +| Other workers | Optional; validated only if present | + +Bootstrap list: `BOOTSTRAP_WORKERS` in +[`validate_worker.py`](../../.github/scripts/validate_worker.py) — the workers +whose skills the harness stack requires at boot. Keep it in sync with what the +harness actually bootstraps when that set changes. + +### Publish + +On every successful release (when `interface_smoke != false`): + +1. `build_skills_payload.py` collects `skills/SKILL.md` and `skills/.md` +2. `POST /w//skills` — skipped cleanly when no markdown found + +### Out-of-band republish + +[`publish-worker-skills.yml`](../../.github/workflows/publish-worker-skills.yml) +updates skills without a version bump. Worker must be in `ALLOWED_WORKERS` +([`parse_publish_workers_input.py`](../../.github/scripts/parse_publish_workers_input.py)). + +**Drift note:** `email` ships skills but is not yet in `ALLOWED_WORKERS`. + +## iii-permissions.yaml + +Repo-root [`iii-permissions.yaml`](../../iii-permissions.yaml) defines default +agent-callable surfaces. Harness loads it via `permissions_path` and hot-reloads +on save. + +### Rule syntax + +```yaml +rules: + - '!function_id' # deny (quote required — bare ! is YAML tag syntax) + - 'worker::*' # allow glob + - function_id # allow exact match +``` + +**First match wins.** No match → `needs_approval`. + +### Conventions for new workers + +| Surface type | Default | +|---|---| +| Transcript / config mutators | Deny (`!session::append`, `!configuration::set`, …) | +| Operator / health signals | Deny (`!shell::config-status`, internal reload hooks) | +| Read-only introspection | Allow (`engine::functions::list`, `coder::read-file`, …) | +| Sensitive reads | Leave unmatched → approval-gated | + +Precedent: `session-manager` block — deny all writes and `session::store::*`; +reads stay at default approval. + +Update [`iii-permissions.yaml`](../../iii-permissions.yaml) when adding a worker +whose functions agents should or should not call without approval. + +## Related + +- New worker checklist: [`../sops/new-worker.md`](../sops/new-worker.md) §7–§8 +- session-manager integration permissions: [`session-manager/architecture/integration.md`](../../session-manager/architecture/integration.md) §2 diff --git a/docs/architecture/testing-and-ci.md b/docs/architecture/testing-and-ci.md new file mode 100644 index 00000000..95dbc141 --- /dev/null +++ b/docs/architecture/testing-and-ci.md @@ -0,0 +1,89 @@ +# Testing and CI + +How pull requests are gated for workers in this monorepo. + +**Sources of truth:** [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml), +[`.github/scripts/discover_changed_workers.py`](../../.github/scripts/discover_changed_workers.py), +[`.github/scripts/validate_worker.py`](../../.github/scripts/validate_worker.py). + +## Discovery + +`discover` job runs `discover_changed_workers.py` comparing the PR to its base. +A directory is a **worker** when it contains `iii.worker.yaml` at its root. +`docs/` is not discovered (no manifest). + +Outputs: + +| Key | Meaning | +|---|---| +| `all` | Every changed worker folder | +| `source_changed` | Workers with non-metadata file changes | +| `rust` / `node` / `python` | Language buckets from `iii.worker.yaml` | +| `vscode_changed` | `iii-lsp-vscode/` changed (special-case job) | + +**Harness fan-out:** when `harness/` changes, in-repo deps listed in +`harness/iii.worker.yaml` `dependencies` join the rust matrix (version-bump +gates still apply only to workers the PR author edited). + +**Metadata-only PRs:** if a worker's only changes match +`iii.worker.yaml`, `README.md`, `Cargo.toml`, `Cargo.lock`, `AGENTS*.md`, +version/tests/README gates downgrade to GitHub notices. + +## pr-checks (per changed worker) + +[`validate_worker.py`](../../.github/scripts/validate_worker.py): + +1. `README.md` exists and is non-empty +2. `iii.worker.yaml` parses with required fields +3. Manifest version ≥ version on base branch +4. `tests/` exists and is non-empty +5. Bootstrap workers (`shell`, `iii-directory`): `skills/SKILL.md` present, + non-empty, ≤ 256 KiB + +## Language jobs + +| Language | Lint | Test | +|---|---|---| +| Rust | `cargo fmt --check`, `clippy -D warnings` | `cargo test --all-features` | +| Node | `biome ci` | `npm test` (if `tests/` exists) | +| Python | `ruff check`, `ruff format --check` | `pytest` (if `tests/` exists) | + +Workers with `web/package.json` (e.g. `console`) pre-build the SPA before cargo +in both `rust` and `interface-smoke` jobs. + +## Interface boot smoke (Rust) + +**Why it exists:** release publish boots the worker on a clean runner with no +`data/` directory. A worker can compile and pass unit tests yet crash at publish +when SQLite parent dirs or sidecars are missing (#104 / `database/v0.2.6`). + +**Flow:** + +1. `cargo build` (default features — same as release binary) +2. Install `iii` CLI + start engine +3. Start worker from `./target/debug/` (with `--config config.collect.yaml` when shipped) +4. `collect_worker_interface.py` — 120 s wait, assert non-empty interface + +**Opt-out:** `interface_smoke: false` in `iii.worker.yaml` (e.g. `iii-lsp`). + +## Dedicated e2e workflows + +Some workers have harness-level e2e beyond unit tests: + +| Workflow | Worker | +|---|---| +| `shell-e2e.yml` | `shell` | +| `database-e2e.yml` | `database` | +| `storage-e2e.yml` | `storage` | + +Add a dedicated workflow when integration with the full harness stack is +release-blocking and too slow for the per-PR matrix. + +## Script tests + +`.github/scripts/tests/` — pytest for release/discovery helpers. Runs on every PR. + +## Related + +- New worker expectations: [`../sops/new-worker.md`](../sops/new-worker.md) §4–§5 +- Release interface collection: [`../sops/release.md`](../sops/release.md) diff --git a/docs/architecture/worker-model.md b/docs/architecture/worker-model.md new file mode 100644 index 00000000..c9dfb54e --- /dev/null +++ b/docs/architecture/worker-model.md @@ -0,0 +1,54 @@ +# Worker model + +How a worker relates to the iii engine and the workers registry. + +## Lifecycle + +A typical long-running worker: + +1. **Connect** — `iii_sdk::register_worker` (Rust) or equivalent SDK call over + WebSocket (`ws://127.0.0.1:49134` by default, overridable via `--url`). +2. **Register** — declare custom trigger types, then register functions (and + optionally subscribe to engine trigger types). +3. **Serve** — handle invocations until SIGINT/SIGTERM. +4. **Shutdown** — `iii.shutdown_async().await` (Rust) for clean disconnect. + +Binary workers support `--manifest` to print registry metadata (name, version, +`default_config`, `supported_targets`) and exit — used by the publish pipeline. + +## Engine as bus + +The iii engine is the message bus. Workers do not call each other directly; +they invoke functions via `iii.trigger('worker::namespace::function', payload)` +and subscribe to trigger types for reactive updates. + +Function ids are `::`-separated paths, conventionally prefixed by the worker's +domain: `shell::exec`, `session::append` (session-manager), `shell::fs::read`. + +## Discovery + +| Context | How workers are found | +|---|---| +| In-repo development | Folder at repo root with `iii.worker.yaml` | +| Production install | Workers registry API — `iii worker add ` | +| Runtime catalogue | `engine::workers::list`, `engine::functions::list` | + +Published workers ship a collected **interface** (functions + trigger types) +attached to the registry manifest at release time. + +## Deploy shapes + +Workers ship as one of three deploy kinds (see [`deploy-modes.md`](deploy-modes.md)): + +- **binary** — single cross-compiled CLI per target triple +- **image** — OCI container (Node/Python daemons) +- **bundle** — single-file archive (esbuild bundle for Node monorepos like harness) + +The kind is declared in `iii.worker.yaml` `deploy` and routes CI smoke + release +build jobs. + +## Related + +- Scaffold: [`../sops/binary-worker.md`](../sops/binary-worker.md) +- Manifest fields: [`iii-worker-yaml.md`](iii-worker-yaml.md) +- Release: [`../sops/release.md`](../sops/release.md) diff --git a/docs/sops/README.md b/docs/sops/README.md new file mode 100644 index 00000000..9ddcfa6a --- /dev/null +++ b/docs/sops/README.md @@ -0,0 +1,21 @@ +# Standard operating procedures + +Step-by-step guides for adding and shipping workers. On conflict with workflow +YAML, **the workflow wins** — update these docs. + +| SOP | When to use | +|---|---| +| [`new-worker.md`](new-worker.md) | First read when adding any worker — naming, repo wiring, CI, release checklist | +| [`binary-worker.md`](binary-worker.md) | Scaffolding a Rust `deploy: binary` daemon (layout, functions, triggers, tests) | +| [`release.md`](release.md) | Cutting a version, re-running a failed release, troubleshooting publish | + +## Typical flow + +1. Read [`new-worker.md`](new-worker.md) §1–§6 and pick a deploy mode. +2. Follow the language-specific scaffold: + - Rust binary → [`binary-worker.md`](binary-worker.md) + - Node container → [`todo-worker/`](../../todo-worker/) template + - Python container → [`todo-worker-python/`](../../todo-worker-python/) template + - Node bundle → [`harness/`](../../harness/) (monorepo pattern) +3. Open a PR; CI runs per [`architecture/testing-and-ci.md`](../architecture/testing-and-ci.md). +4. Ship via [`release.md`](release.md). diff --git a/binary-worker.md b/docs/sops/binary-worker.md similarity index 98% rename from binary-worker.md rename to docs/sops/binary-worker.md index 64537074..5ea2a8a1 100644 --- a/binary-worker.md +++ b/docs/sops/binary-worker.md @@ -1,18 +1,20 @@ # Binary worker how-to +Cross-cutting steps → [`new-worker.md`](new-worker.md). Shipping → [`release.md`](release.md). + This document is the **single source of truth** for building a Rust **binary** iii worker: a single cross-compiled CLI that connects to the iii engine over WebSocket, registers functions and triggers, and runs until SIGTERM. Everything you need to scaffold one is **here**—layouts, contracts, and copy-paste snippets. For the cross-cutting checklist (folder names, registry, CI, release flow) -see [`AGENTS-NEW-WORKER.md`](AGENTS-NEW-WORKER.md). This document covers the -**inside** of one such worker: the manifest, config, layout, function -registration, trigger registration, end-to-end tests, and README contract. +see [`new-worker.md`](new-worker.md). This document covers the **inside** of +one such worker: the manifest, config, layout, function registration, trigger +registration, end-to-end tests, and README contract. If your worker is not a Rust binary (`deploy: image`, or any Node/Python -worker), stop here and read [`AGENTS-NEW-WORKER.md`](AGENTS-NEW-WORKER.md) -instead — the inside of those workers looks different. +worker), stop here and read [`new-worker.md`](new-worker.md) instead — the +inside of those workers looks different. ## 1. When to use this doc @@ -814,7 +816,7 @@ without `iii` can filter them out. the registry's `readme` field on `POST /publish`. **Consumer-facing structure** (section order, tone, what belongs in README vs -reference tooling) is defined in [`worker-readme.md`](worker-readme.md). Follow +reference tooling) is defined in [`worker-readme.md`](../../worker-readme.md). Follow that guide for headings like Install, Quickstart, and Configuration. Avoid front-loading huge function/schema tables: `iii worker info`, rustdoc, @@ -1277,7 +1279,7 @@ async fn end_to_end_via_iii_sdk() { ### `/README.md` -Use [`worker-readme.md`](worker-readme.md) for section order and consumer tone. +Use [`worker-readme.md`](../../worker-readme.md) for section order and consumer tone. Minimum viable shape (use a four-backtick outer fence when the README embeds code blocks): diff --git a/docs/sops/new-worker.md b/docs/sops/new-worker.md new file mode 100644 index 00000000..90c0ee72 --- /dev/null +++ b/docs/sops/new-worker.md @@ -0,0 +1,152 @@ +# New worker onboarding + +**Sources of truth:** this checklist; language scaffolds in +[`binary-worker.md`](binary-worker.md) and the todo-worker templates; CI in +[`.github/workflows/ci.yml`](../../.github/workflows/ci.yml); release in +[`release.md`](release.md). On conflict, the workflow wins — update this doc. + +Cross-cutting SOP for adding **any** worker to this monorepo. For the inside +of a Rust `deploy: binary` daemon, continue with [`binary-worker.md`](binary-worker.md) +after §2. + +## 1. Naming and folder rules + +- **Folder name** = `iii.worker.yaml` `name` = `[package].name` (Rust) = + `[[bin]].name` = `bin:` field. +- Pattern: `^[a-z0-9][a-z0-9_-]*$` (enforced by `TAG_RE` in + [`.github/scripts/parse_release_tag.py`](../../.github/scripts/parse_release_tag.py)). +- Do **not** prefix with `iii-` unless the worker itself is named that way + (e.g. `iii-directory`, `iii-lsp`). +- Git release tags use the folder name: `/vX.Y.Z`. + +## 2. Required files by deploy mode + +Every worker needs a top-level folder with: + +| File / dir | All workers | Notes | +|---|---|---| +| `iii.worker.yaml` | yes | Registry + CI metadata | +| Version manifest | yes | `Cargo.toml`, `package.json`, or `pyproject.toml` per `manifest:` | +| `config.yaml` | yes | Operator defaults (or equivalent for container workers) | +| `tests/` (non-empty) | yes | See §5 | +| `README.md` | yes | Per [`worker-readme.md`](../../worker-readme.md) | + +Pick a scaffold by `deploy` + `language`: + +| `deploy` | `language` | Scaffold | +|---|---|---| +| `binary` | `rust` | [`binary-worker.md`](binary-worker.md) | +| `image` | `javascript` / `node` | [`todo-worker/`](../../todo-worker/) | +| `image` | `python` | [`todo-worker-python/`](../../todo-worker-python/) | +| `bundle` | `javascript` | [`harness/`](../../harness/) (monorepo bundle) | + +`iii.worker.yaml` must declare valid `deploy` (`binary` | `image` | `bundle`) +and `language` (`rust` | `javascript` | `node` | `python`). + +## 3. Repo registration + +Add a row to the **Modules** table in [`README.md`](../../README.md). Every +shipped worker has one — kind, one-line summary, link to README or architecture. + +## 4. CI expectations + +On the first PR that touches the worker, [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) +runs: + +| Job | What it enforces | +|---|---| +| `pr-checks` | `README.md` present; `iii.worker.yaml` valid; manifest version ≥ base; `tests/` non-empty | +| Language job | Rust: `fmt`, `clippy -D warnings`, `test`; Node: `biome ci`, `npm test`; Python: `ruff`, `pytest` | +| `interface-smoke` | Rust only: build from source, boot engine + worker, collect non-empty interface | + +**Before opening the PR:** + +- If the default `config.yaml` needs sidecars or local paths unavailable on a + clean runner, ship `config.collect.yaml` (lighter boot for interface + collection). See `storage/config.collect.yaml` and `shell/config.collect.yaml`. +- If the worker cannot be booted for interface collection (stdio server, no + registered functions), set `interface_smoke: false` in `iii.worker.yaml` + (e.g. `iii-lsp`). This skips interface smoke **and** registry publish at + release time. + +Metadata-only changes (`iii.worker.yaml`, `README.md`, `Cargo.toml`, `Cargo.lock`, +`AGENTS*.md`) downgrade version/tests/README gates to notices instead of hard +errors. + +## 5. Tests contract + +- `tests/` must exist and be non-empty — `pr-checks` enforces this on every + changed worker. +- **Rust binary:** pattern A (integration against lib) or pattern B (Cucumber + BDD). See [`binary-worker.md`](binary-worker.md) §9. +- **Node:** `npm test` when `tests/` exists. +- **Python:** `tests/test_*.py` runnable with `pytest`. + +Dedicated e2e workflows (`shell-e2e.yml`, `database-e2e.yml`, `storage-e2e.yml`) +are added when a worker needs harness-level integration beyond unit tests. + +## 6. Release wiring (one-time per worker) + +Before the first release, wire the worker into the CD pipeline. Forgetting a +step can fail **silently** (e.g. tag push triggers nothing). + +| # | Location | Action | +|---|---|---| +| 1 | [`.github/workflows/create-tag.yml`](../../.github/workflows/create-tag.yml) | Add worker to `inputs.worker.options` | +| 2 | [`.github/workflows/release.yml`](../../.github/workflows/release.yml) | Add `'/v*'` to `on.push.tags` | +| 3 | [`.github/scripts/parse_publish_workers_input.py`](../../.github/scripts/parse_publish_workers_input.py) | Add to `ALLOWED_WORKERS` **only if** the worker ships `skills/` and you want out-of-band skills publishing via [`publish-worker-skills.yml`](../../.github/workflows/publish-worker-skills.yml) | +| 4 | [`.github/scripts/validate_worker.py`](../../.github/scripts/validate_worker.py) | Add to `BOOTSTRAP_WORKERS` **only if** the harness stack requires this worker's skill at boot — makes `skills/SKILL.md` a hard PR gate (currently `shell`, `iii-directory`) | + +**Worked example:** `session-manager` — added to `create-tag.yml` options and +`release.yml` tag patterns. No `BOOTSTRAP_WORKERS` entry (not harness-bootstrapped). + +**Fallback:** if the tag pattern is missing, run **Release** manually via +`workflow_dispatch` with the tag input until §6 row 2 is fixed. + +**Known drift:** `email` ships `skills/SKILL.md` but is not in `ALLOWED_WORKERS` +today — add it when enabling out-of-band skills publish for that worker. + +## 7. Agent permissions + +Decide default agent-callable surfaces in [`iii-permissions.yaml`](../../iii-permissions.yaml). +Rules are first-match-wins: + +- **Deny** (`'!function_id'`) for transcript-mutating, config-mutating, or + operator-only functions. Precedent: the `session-manager` block (deny all + `session::store::*` and write paths; reads stay at `needs_approval` default). +- **Allow** bare strings or globs for read-only introspection agents need by + default (`engine::functions::list`, `directory::registry::workers::list`, …). + +Update permissions when the worker exposes functions agents should or should not +call without approval. + +## 8. Skills + +Ship `skills/SKILL.md` when agents should discover **when** to use the worker +(intent, boundaries, function catalogue — not JSON schemas). Author per +[`DOCUMENTATION_GUIDELINES.md`](../../DOCUMENTATION_GUIDELINES.md). + +- **Bootstrap workers** (`shell`, `iii-directory`): `skills/SKILL.md` is + **required** (≤ 256 KiB) — the harness stack expects these skills at boot. +- **On release:** skills are auto-uploaded via `POST /w//skills` when + markdown is present; skipped cleanly when absent. +- **Out-of-band:** [`publish-worker-skills.yml`](../../.github/workflows/publish-worker-skills.yml) + re-publishes skills without a version bump (worker must be in `ALLOWED_WORKERS`). + +## 9. First release + +Follow [`release.md`](release.md). For a brand-new worker, prefer +**registry tag `next`** on the first publish so `iii worker add` users on +`latest` are not surprised. + +Recommended preflight: + +```bash +# Rust binary — from the worker folder +cargo fmt --all -- --check +cargo clippy --all-targets --all-features -- -D warnings +cargo test --all-features +./target/debug/ --manifest | jq . +``` + +Then run **Create Tag** on `main` with the worker selected. diff --git a/docs/sops/release.md b/docs/sops/release.md new file mode 100644 index 00000000..28559055 --- /dev/null +++ b/docs/sops/release.md @@ -0,0 +1,180 @@ +# Worker release + +**Sources of truth:** +[`.github/workflows/create-tag.yml`](../../.github/workflows/create-tag.yml), +[`.github/workflows/release.yml`](../../.github/workflows/release.yml), +[`.github/workflows/_rust-binary.yml`](../../.github/workflows/_rust-binary.yml), +[`.github/workflows/_container.yml`](../../.github/workflows/_container.yml), +[`.github/workflows/_bundle.yml`](../../.github/workflows/_bundle.yml), +[`.github/workflows/_publish-registry.yml`](../../.github/workflows/_publish-registry.yml), +[`.github/scripts/parse_release_tag.py`](../../.github/scripts/parse_release_tag.py). +On conflict, the workflow wins — update this doc. + +Operational SOP for cutting a worker version, monitoring the pipeline, and +verifying registry publish. One-time wiring is in [`new-worker.md`](new-worker.md) §6. + +## Prerequisites + +- **Branch:** Create Tag requires `main` (preflight enforces it). +- **Access:** GitHub Actions on this repo; org secrets configured (do not paste + values): + - `III_CI_APP_ID` / `III_CI_APP_PRIVATE_KEY` — bot commit + tag push in Create Tag + - `WORKERS_REGISTRY_API_KEY` — `POST /publish` and `POST /w//skills` +- **Worker wired:** `create-tag.yml` options + `release.yml` tag pattern (see + [`new-worker.md`](new-worker.md) §6). +- **Local green:** lint + tests for the worker; Rust binary: `--manifest` JSON valid. + +## Standard release (happy path) + +### 1. Create Tag + +Actions → **Create Tag**: + +| Input | Meaning | +|---|---| +| Worker | Folder name (must be in workflow options) | +| Bump | `patch` / `minor` / `major` | +| Registry tag | `latest` or `next` — channel for `iii worker add` resolution | + +The workflow: + +1. Bumps version in the worker manifest (`Cargo.toml`, `package.json`, …). +2. Commits `chore(): bump to vX.Y.Z` to `main`. +3. Creates and pushes an **annotated** tag `/vX.Y.Z` with + `registry-tag: ` in the tag message. + +### 2. Release pipeline + +Tag push triggers **Release** (`release.yml`): + +```mermaid +flowchart LR + createTag[Create Tag] -->|"tag worker/vX.Y.Z"| setupJob[setup] + setupJob --> ghRelease[create GitHub Release] + ghRelease --> buildBinary["binary: _rust-binary.yml"] + ghRelease --> buildImage["image: _container.yml"] + ghRelease --> buildBundle["bundle: _bundle.yml"] + buildBinary --> publishJob[_publish-registry.yml] + buildImage --> publishJob + buildBundle --> publishJob + publishJob --> postPublish["POST /publish + skills"] +``` + +| Stage | Job | Output | +|---|---|---| +| setup | Parse tag + `iii.worker.yaml`; detect web bundle / smoke opt-out | worker, version, deploy, targets, … | +| create-release | GitHub Release shell | Release page for the tag | +| binary-build | `_rust-binary.yml` | Per-target `.tar.gz` / `.zip` + `.sha256` on the Release | +| container-build | `_container.yml` | Multi-arch image at `ghcr.io//` | +| bundle-build | `_bundle.yml` | `.tar.gz` + `.sha256` on the Release | +| publish | `_publish-registry.yml` | Registry manifest + optional skills upload | + +`deploy` from `iii.worker.yaml` selects exactly one build job. + +### 3. What publish does + +[`_publish-registry.yml`](../../.github/workflows/_publish-registry.yml): + +1. Boots a clean `iii` engine (`workers: []`). +2. Starts the **released artifact** (mode from `manifest_version.py deploy-mode`): + - `release-binary` — downloads Linux gnu tarball from the GitHub Release + - `release-bundle` — extracts `.tar.gz`, runs `node ./index.mjs` + - `iii-add` — `iii worker add ./` (non-binary/bundle deploys with `runtime`/`scripts.start`, e.g. image workers) + - `cargo-run` — `cargo run` from source (remaining Rust workers) +3. Uses `config.collect.yaml` when present (sidecar-free boot). +4. Collects function + trigger interface (120 s timeout); asserts non-empty. +5. Resolves release assets into `payload.json`. +6. `POST /publish` to `https://api.workers.iii.dev`. +7. `POST /w//skills` when `skills/*.md` exist (skipped when absent). + +Workers with `interface_smoke: false` skip the entire publish job. + +### 4. Registry tag semantics + +| Channel | Typical use | +|---|---| +| `latest` | Default; what most `iii worker add` installs resolve | +| `next` | Pre-release / risky channel; safer for first publish | + +The channel is stored in the **annotated tag message** (`registry-tag:`). +`release.yml` refetches the annotated tag for this reason. Lightweight tags +lose the channel and default to `latest`. + +## Variants + +### Re-run a failed release + +Actions → **Release** → `workflow_dispatch` → enter the existing tag +(e.g. `session-manager/v0.1.0`). No new tag or version bump needed. +Concurrency group `release-${{ github.ref }}` serializes per tag. + +### Prerelease + +Create Tag cannot produce prerelease suffixes. Push a manual **annotated** tag: + +```text +/vX.Y.Z-beta.1 +``` + +With tag message including `registry-tag: next`. Marks the GitHub Release as +prerelease; still builds and publishes (unless `interface_smoke: false`). + +### Dry run + +Tag shape: `/vX.Y.Z-dry-run.1` (parsed by `parse_release_tag.py`). + +- Runs the full build matrix +- Skips GitHub Release asset upload and registry publish +- Useful to validate a new worker's pipeline before `v0.1.0` + +### Skills-only update + +Actions → **Publish worker skills** — worker must be in +`ALLOWED_WORKERS` ([`parse_publish_workers_input.py`](../../.github/scripts/parse_publish_workers_input.py)). +No version bump; updates skill markdown on the registry channel you pick. + +### Not via this pipeline + +`iii-lsp-vscode` uses [`release-lsp.yml`](../../.github/workflows/release-lsp.yml) +(VS Code extension packaging, separate tag pattern `iii-lsp-vscode/v*`). + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| Tag pushed, nothing ran | Missing `'/v*'` in `release.yml` | Add pattern (§6 in `new-worker.md`); dispatch Release manually meanwhile | +| setup fails | Invalid tag shape or bad `iii.worker.yaml` | Tag must match `worker/vVERSION`; `deploy` must be `binary`\|`image`\|`bundle` | +| binary-build fails on one target | Cross-compile issue | Consider `targets:` subset in `iii.worker.yaml` (`shell` is POSIX-only); other targets still upload (`fail-fast: false`) | +| interface collection times out | Worker crashes on clean runner (#104 class) | Ship `config.collect.yaml`; check `iii-engine.log`, `worker-.log` in the job | +| Worker exits before collection | Missing parent dir, sidecar, bad default path | Same as above; reproduce locally with no `data/` dir | +| artifact resolution 404 | Build job didn't upload for that target | Check GitHub Release assets for the tag | +| publish HTTP non-200 | Registry rejection or bad payload | Response body printed in job log; verify `WORKERS_REGISTRY_API_KEY` | +| publish skipped entirely | `interface_smoke: false` | Expected for stdio/discovery-only workers | + +On failure, publish dumps `iii-engine.log` and `worker-.log` (last 200 lines). + +## Rollback + +There is **no unpublish**. Recovery: + +1. Fix the issue on `main`. +2. Cut a new patch via Create Tag (registry `latest` moves forward). +3. Use `registry-tag: next` when uncertain before promoting to `latest`. + +GitHub Release assets for the bad version remain (immutable history). + +## Post-release verification + +On a clean host: + +```bash +iii worker add +iii worker info +``` + +Confirm: + +- Version matches the tag you cut. +- Function and trigger types match expectations. +- GitHub Release has complete assets (per-target archives + `.sha256` for + binary/bundle deploys). diff --git a/email/README.md b/email/README.md index 8514c158..ef2ed13c 100644 --- a/email/README.md +++ b/email/README.md @@ -209,7 +209,8 @@ laptops without a running engine still pass `@pure` scenarios. ### Verification before publishing -The full preflight checklist for binary workers (`workers/binary-worker.md`): +The full preflight checklist for binary workers +([`docs/sops/binary-worker.md`](../docs/sops/binary-worker.md)): ```bash cargo fmt --all -- --check diff --git a/iii-lsp/README.md b/iii-lsp/README.md index e421fbef..3a5b5bf7 100644 --- a/iii-lsp/README.md +++ b/iii-lsp/README.md @@ -64,4 +64,4 @@ Configure the client to launch the `iii-lsp` binary over stdio for the supported ## See also - [iii-lsp-vscode/README.md](../iii-lsp-vscode/README.md) — VS Code extension that wraps this binary. -- [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md) — monorepo conventions and release flow. +- [docs/sops/new-worker.md](../docs/sops/new-worker.md) — monorepo conventions and release flow. diff --git a/storage/README.md b/storage/README.md index f54d7a1c..974eabf7 100644 --- a/storage/README.md +++ b/storage/README.md @@ -285,7 +285,7 @@ is env-var-gated — see `tests/e2e/run-tests.sh` for the orchestrator. ### Verification before publishing The full preflight checklist for binary workers -(`workers/binary-worker.md` §11): +([`docs/sops/binary-worker.md`](../docs/sops/binary-worker.md) §11): ```bash cargo fmt --all -- --check diff --git a/storage/src/config.rs b/storage/src/config.rs index 9f81e749..13f3b545 100644 --- a/storage/src/config.rs +++ b/storage/src/config.rs @@ -404,7 +404,7 @@ pub fn redact_secret(s: &str) -> String { /// Load and validate a `WorkerConfig` from `path`. /// /// Free-function alias for [`WorkerConfig::from_file`] that returns -/// [`anyhow::Result`], matching the binary-worker spec (`workers/binary-worker.md` +/// [`anyhow::Result`], matching the binary-worker spec (`docs/sops/binary-worker.md` /// §5). Errors carry the file path as context. pub fn load_config(path: &str) -> anyhow::Result { WorkerConfig::from_file(path) diff --git a/storage/tests/integration.rs b/storage/tests/integration.rs index 726a57e6..06c4ef0b 100644 --- a/storage/tests/integration.rs +++ b/storage/tests/integration.rs @@ -1,4 +1,4 @@ -//! End-to-end test (binary-worker.md §9 pattern A): spawn the `iii` engine +//! End-to-end test (docs/sops/binary-worker.md §9 pattern A): spawn the `iii` engine //! and the `storage` worker as subprocesses, drive the worker through //! `iii-sdk` as a client, then tear both down. **Self-skips** when `iii` is //! not on `PATH` so CI hosts without the engine still pass. diff --git a/todo-worker-python/README.md b/todo-worker-python/README.md index 0672a9de..e0d5efdd 100644 --- a/todo-worker-python/README.md +++ b/todo-worker-python/README.md @@ -1,6 +1,6 @@ # todo-worker-python -Quickstart CRUD todo worker built on the Python [iii SDK](https://github.com/iii-hq/iii). Mirrors the Node-based [todo-worker](../todo-worker/) one-for-one and is the canonical "Python container" template referenced in [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md). +Quickstart CRUD todo worker built on the Python [iii SDK](https://github.com/iii-hq/iii). Mirrors the Node-based [todo-worker](../todo-worker/) one-for-one and is the canonical Python container template in this repo. If you're scaffolding a new Python worker, start by copying this directory and trimming what you don't need. @@ -67,9 +67,9 @@ The image inherits `III_URL=ws://localhost:49134`; override at runtime with `-e ## Tests -A `tests/` directory is not yet present. Per [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md) §5, Python workers should ship `tests/test_*.py` runnable with `pytest`. Adding one is the next step before this worker can be released through the standard `pr-checks` flow. +A `tests/` directory is not yet present. Per [docs/sops/new-worker.md](../docs/sops/new-worker.md) §5, Python workers should ship `tests/test_*.py` runnable with `pytest`. Adding one is the next step before this worker can be released through the standard `pr-checks` flow. ## See also - [todo-worker/README.md](../todo-worker/README.md) — same API, Node SDK. -- [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md) — full checklist for adding a new worker. +- [docs/sops/new-worker.md](../docs/sops/new-worker.md) — full checklist for adding a new worker. diff --git a/todo-worker/README.md b/todo-worker/README.md index c5904968..0409611c 100644 --- a/todo-worker/README.md +++ b/todo-worker/README.md @@ -1,6 +1,6 @@ # todo-worker -Quickstart CRUD todo worker built on the Node.js [iii SDK](https://github.com/iii-hq/iii). It exposes a tiny in-memory todos service over five HTTP routes and is the canonical "Node container" template referenced in [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md). +Quickstart CRUD todo worker built on the Node.js [iii SDK](https://github.com/iii-hq/iii). It exposes a tiny in-memory todos service over five HTTP routes and is the canonical Node container template in this repo. If you're scaffolding a new Node worker, start by copying this directory and trimming what you don't need. @@ -71,4 +71,4 @@ The image inherits `III_URL=ws://localhost:49134`; override at runtime with `-e ## See also - [todo-worker-python/README.md](../todo-worker-python/README.md) — same API, Python SDK. -- [AGENTS-NEW-WORKER.md](../AGENTS-NEW-WORKER.md) — full checklist for adding a new worker. +- [docs/sops/new-worker.md](../docs/sops/new-worker.md) — full checklist for adding a new worker. diff --git a/worker-readme.md b/worker-readme.md index dbda5da5..a8fe2c0e 100644 --- a/worker-readme.md +++ b/worker-readme.md @@ -69,7 +69,7 @@ iii worker add ```` Where `` is the value of `iii.worker.yaml.name` (which equals the -folder name; see [`AGENTS-NEW-WORKER.md`](AGENTS-NEW-WORKER.md) §1). +folder name; see [`docs/sops/new-worker.md`](docs/sops/new-worker.md) §1). That is the whole user-facing install — no source build, no `sudo install`, no `--manifest | jq` verification step. `iii worker add` fetches the binary, writes a config block into `~/.iii/config.yaml`,