diff --git a/.github/scripts/build_skills_payload.py b/.github/scripts/build_skills_payload.py index 500091e7..a9a130d3 100644 --- a/.github/scripts/build_skills_payload.py +++ b/.github/scripts/build_skills_payload.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 """Build the POST /w//skills payload from a worker directory. -Walks ``/skill.md`` and ``/skills/**/*.md`` and produces the JSON -body expected by the workers-registry endpoint. Skill paths map to keys as: +Walks ``/skills/SKILL.md`` (and legacy top-of-tree paths) plus +``/skills/**/*.md`` and produces the JSON body expected by the +workers-registry endpoint. Skill paths map to keys as: - /skill.md -> "index.md" - /skills/index.md -> "index.md" (override; warn if both exist) - /skills/.md -> "skills/.md" + /skills/SKILL.md -> "index.md" + /skills/index.md -> "index.md" (legacy fallback) + /skill.md -> "index.md" (legacy fallback) + /skills/.md -> "skills/.md" (except SKILL.md / index.md) If no non-empty markdown is found the script writes ``skip=true`` to ``$GITHUB_OUTPUT`` (so the calling workflow can gate the POST step off) and @@ -25,38 +27,59 @@ KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._/\-]*\.md$", re.IGNORECASE) +def _read_nonempty(path: pathlib.Path) -> str | None: + body = path.read_text(encoding="utf-8") + return body if body.strip() else None + + +def _resolve_top_skill( + worker_root: pathlib.Path, +) -> tuple[str | None, pathlib.Path | None]: + """Return ``(index.md body, winning path)`` from the top-of-tree candidates. + + Resolution order: ``skills/SKILL.md``, then legacy ``skills/index.md``, then + legacy ``skill.md``. When multiple candidates exist, a GitHub Actions + warning is emitted and the highest-priority file wins. + """ + leaves_dir = worker_root / "skills" + candidates: list[tuple[str, pathlib.Path]] = [ + ("skills/SKILL.md", leaves_dir / "SKILL.md"), + ("skills/index.md", leaves_dir / "index.md"), + ("skill.md", worker_root / "skill.md"), + ] + present = [(label, path) for label, path in candidates if path.is_file()] + if not present: + return None, None + + winner_label, winner_path = present[0] + for label, _ in present[1:]: + print( + f"::warning::{worker_root.name}: both {label} and " + f"{winner_label} present; using {winner_label} as the top-of-tree." + ) + return _read_nonempty(winner_path), winner_path + + def collect_skills(worker_root: pathlib.Path) -> dict[str, str]: """Return a ``{payload-key: markdown-body}`` map for one worker directory. - The top-of-tree resolution order is ``skills/index.md`` then ``skill.md``; - if both exist, a GitHub Actions warning is emitted and the nested one wins - (this matches ``iii-directory``'s on-disk convention). Empty bodies are - skipped silently so blank placeholder files don't end up in the registry. + The worker overview is always published as registry key ``index.md``, + sourced from ``skills/SKILL.md`` when present. Empty bodies are skipped + silently so blank placeholder files don't end up in the registry. """ skills: dict[str, str] = {} leaves_dir = worker_root / "skills" + skills_skill = leaves_dir / "SKILL.md" skills_index = leaves_dir / "index.md" - intro = worker_root / "skill.md" - - if skills_index.is_file(): - body = skills_index.read_text(encoding="utf-8") - if body.strip(): - skills["index.md"] = body - if intro.is_file(): - print( - f"::warning::{worker_root.name}: both skill.md and " - "skills/index.md present; using skills/index.md as the " - "top-of-tree." - ) - elif intro.is_file(): - body = intro.read_text(encoding="utf-8") - if body.strip(): - skills["index.md"] = body + + top_body, _ = _resolve_top_skill(worker_root) + if top_body is not None: + skills["index.md"] = top_body if leaves_dir.is_dir(): for path in sorted(leaves_dir.rglob("*.md")): - if path == skills_index: + if path in (skills_skill, skills_index): continue rel = path.relative_to(worker_root).as_posix() if not KEY_RE.match(rel): diff --git a/.github/scripts/validate_worker.py b/.github/scripts/validate_worker.py index 80710f2d..858cb0a2 100644 --- a/.github/scripts/validate_worker.py +++ b/.github/scripts/validate_worker.py @@ -6,8 +6,8 @@ 2. iii.worker.yaml parses and has required fields + valid enum values. 3. The manifest version on this ref is greater than or equal to on --base-ref. 4. tests/ exists and is non-empty. - 5. For workers in BOOTSTRAP_WORKERS, skill.md exists, is non-empty, and is - within the 256 KiB cap — the harness bootstraps these onto disk via + 5. For workers in BOOTSTRAP_WORKERS, skills/SKILL.md exists, is non-empty, + and is within the 256 KiB cap — the harness bootstraps these onto disk via iii-directory on first boot; a missing or oversized file breaks the chat surface's orientation. @@ -153,22 +153,25 @@ def soft(msg: str) -> None: elif not any(tests_dir.iterdir()): soft(f"{worker}/tests/ is empty") - # 5. Bundled workers must ship skill.md within the size cap. + # 5. Bundled workers must ship skills/SKILL.md within the size cap. if worker in BOOTSTRAP_WORKERS: - skill_md = root / "skill.md" + skill_md = root / "skills" / "SKILL.md" + legacy_skill_md = root / "skill.md" + if not skill_md.exists() and legacy_skill_md.exists(): + skill_md = legacy_skill_md if not skill_md.exists(): hard( - f"{worker}/skill.md is missing — bundled workers must ship one " + f"{worker}/skills/SKILL.md is missing — bundled workers must ship one " f"(see binary-worker.md)" ) elif skill_md.stat().st_size == 0: hard( - f"{worker}/skill.md is empty — must contain the H1 + summary " - f"(see binary-worker.md)" + f"{worker}/{skill_md.relative_to(root).as_posix()} is empty — " + f"must contain the H1 + summary (see binary-worker.md)" ) elif skill_md.stat().st_size > SKILL_MD_SIZE_CAP: hard( - f"{worker}/skill.md exceeds 256 KiB cap " + f"{worker}/{skill_md.relative_to(root).as_posix()} exceeds 256 KiB cap " f"({skill_md.stat().st_size} bytes; see binary-worker.md)" ) diff --git a/console/README.md b/console/README.md index c70bc8e1..f6a47846 100644 --- a/console/README.md +++ b/console/README.md @@ -207,4 +207,4 @@ cd web && pnpm typecheck && pnpm lint ## License -Apache 2.0 — see [LICENSE](../LICENSE). +Apache 2.0 — see [LICENSE](https://github.com/iii-hq/workers/blob/main/LICENSE). diff --git a/database/README.md b/database/README.md index 73272b5a..31f4bf58 100644 --- a/database/README.md +++ b/database/README.md @@ -236,4 +236,4 @@ A few operations are no-ops on certain drivers. They emit a `tracing::warn!` rat ## License -MIT. +Apache 2.0 — see [LICENSE](https://github.com/iii-hq/workers/blob/main/LICENSE). diff --git a/image-resize/README.md b/image-resize/README.md index e7da92c6..dd95c04b 100644 --- a/image-resize/README.md +++ b/image-resize/README.md @@ -63,4 +63,4 @@ Benchmarks on a standard worker instance (2 vCPU, 512MB): ## License -Apache 2.0 \ No newline at end of file +Apache 2.0 — see [LICENSE](https://github.com/iii-hq/workers/blob/main/LICENSE). \ No newline at end of file diff --git a/shell/ARCHITECTURE.md b/shell/ARCHITECTURE.md index f5191f00..c01632cf 100644 --- a/shell/ARCHITECTURE.md +++ b/shell/ARCHITECTURE.md @@ -1,6 +1,6 @@ # shell — architecture and operator notes -The published `README.md` and `skill.md` for this worker are rendered from `docs/`. This file holds the operator/contributor material that does not belong in the published surfaces — full configuration table, threat model, wire shapes for the streaming functions, troubleshooting, tests, and deferred work. +The published `README.md` and `skills/SKILL.md` for this worker are hand-maintained. This file holds the operator/contributor material that does not belong in the published surfaces: full configuration table, threat model, wire shapes for the streaming functions, troubleshooting, tests, and deferred work. ## Build and wire-up diff --git a/shell/README.md b/shell/README.md index 31187961..6bcb0225 100644 --- a/shell/README.md +++ b/shell/README.md @@ -1,190 +1,122 @@ - - # shell -Unix shell and filesystem worker on the iii bus. Every agent that needs to touch the OS (run a build, read a file, list a directory, call a CLI) goes through `shell::*` and `shell::fs::*`, so allowlists, timeouts, output caps, and a host-root jail live in one place. Both surfaces accept an optional `target` field that forwards the call into a live `iii-sandbox` microVM, so the same allowlist policy gates host and sandbox execution. +Run allowlisted Unix commands, background jobs, and structured filesystem operations from the iii engine, on the host or forwarded into a sandbox microVM. ## Install -```bash +```sh iii worker add shell ``` -`iii worker add` fetches the binary, writes a config block into the engine's `config.yaml`, and the engine starts the worker on the next `iii worker start`. - -For sandbox-targeted execution and `shell::fs::*` forwarding, install [`iii-sandbox`](../iii-sandbox); `iii worker add shell` does not currently pull it in. For surfacing `shell::*` to LLM agents, pair with [`skills`](../skills): +Sandbox-targeted execution and `shell::fs::*` forwarding need the `iii-sandbox` worker; `iii worker add shell` does not pull it in. To surface `shell::*` to LLM agents, pair with `iii-directory`: -```bash +```sh iii worker add iii-sandbox -iii worker add skills -``` - -## Quickstart - -```rust -use iii_sdk::{register_worker, InitOptions, TriggerRequest}; -use serde_json::json; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let worker = register_worker("ws://localhost:49134", InitOptions::default()); - - let result = worker - .trigger(TriggerRequest { - function_id: "shell::exec".into(), - payload: json!({ - "command": "echo", - "args": ["hello"], - }), - action: None, - timeout_ms: Some(5_000), - }) - .await?; - - println!("{result:#?}"); - Ok(()) -} +iii worker add iii-directory ``` -```typescript -import { registerWorker } from 'iii-sdk' +## Configure -const worker = registerWorker('ws://localhost:49134') +Settings load from a YAML file passed with `--config ` (default `./config.yaml`). The worker refuses to start unless `fs.host_root` is set, or `fs.allow_unjailed: true` is explicitly opted in, because an unset root exposes the whole host filesystem behind only the advisory denylist. -const result = await worker.trigger({ - function_id: 'shell::exec', - payload: { command: 'echo', args: ['hello'] }, -}) +```yaml +max_timeout_ms: 30000 # hard cap; per-call timeout_ms is clamped to this +default_timeout_ms: 10000 # applied when the caller omits timeout_ms +max_output_bytes: 1048576 # 1 MiB; stdout/stderr past this set *_truncated +inherit_env: false # when false, only allowed_env keys are forwarded +allowed_env: [PATH, HOME, LANG, LC_ALL, TERM] + +# exec gate. argv[0] is matched by basename or exact path; an empty +# allowlist means open. denylist_patterns are advisory regex over +# argv.join(" "), a tripwire for honest mistakes only. +allowlist: [ls, cat, pwd, echo, grep, wc, head, tail, sort, uniq, cut, date] +denylist_patterns: + - "rm\\s+-rf\\s+/" + - "mkfs" -console.log(result) -``` +max_concurrent_jobs: 16 # exec_bg past this is rejected +job_retention_secs: 3600 # finished jobs pruned after this -```python -from iii import register_worker +fs: + host_root: /tmp # jail root for shell::fs::*; required (see above) + allow_unjailed: false # opt-in to running with host_root unset + max_read_bytes: 16777216 # 0 = unlimited + max_write_bytes: 16777216 # 0 = unlimited + denylist_paths: [/etc/passwd, /etc/shadow] -worker = register_worker("ws://localhost:49134") +sandbox: + enabled: true # false -> every target: sandbox call returns S210 +``` -result = worker.trigger({ - "function_id": "shell::exec", - "payload": {"command": "echo", "args": ["hello"]}, -}) +Host `shell::exec` is not a security boundary: any allowlisted interpreter (`sh`, `node`, `python3`) can construct a denylisted token at runtime and bypass the regex. Run untrusted input with `target: { kind: "sandbox", sandbox_id }`, which forwards through the `iii-sandbox` microVM. The allowlist and denylist still apply on top of either backend. -print(result) -``` +## Quick start -The example calls `shell::exec` on the host. The same payload retargets at a microVM with `target: { "kind": "sandbox", "sandbox_id": "" }`. Other entry points: `shell::exec_bg`, `shell::status`, `shell::kill`, `shell::list`, plus the `shell::fs::*` family (`ls`, `stat`, `read`, `write`, `grep`, `sed`, `mkdir`, `rm`, `chmod`, `mv`). +```ts +import { registerWorker } from 'iii-sdk' -## Configuration +const iii = registerWorker(process.env.III_URL ?? 'ws://127.0.0.1:49134') -```yaml -max_timeout_ms: 30000 -default_timeout_ms: 10000 -max_output_bytes: 1048576 -working_dir: null -inherit_env: false -allowed_env: - - PATH - - HOME - - LANG - - LC_ALL - - TERM -# Default allowlist is intentionally read-only. Tools that can shell out -# (git hooks, curl -o/file://, find -exec, awk system(), sed e/-i, cargo -# build.rs, node -e, python3 -c, npm run, env ) are left out on -# purpose — add them per deployment after you've decided on the threat -# model. This worker is NOT a sandbox. Use `printenv` for read-only env -# inspection; `env` is excluded because `env ` execs arbitrary -# programs while passing argv[0]=="env" through the allowlist gate. -allowlist: - - ls - - cat - - pwd - - echo - - grep - - wc - - head - - tail - - sort - - uniq - - cut - - date - - whoami - - hostname - - which - - jq - - uname - - df - - du - - ps - - printenv - - basename - - dirname -# Denylist patterns are advisory, not a security boundary. They run as -# regex against `argv.join(" ")`, so a caller invoking an allowlisted -# shell or interpreter (sh, node, python, etc.) can bypass any pattern -# by constructing the forbidden token at runtime — variables, eval, -# IFS tricks, base64, etc. Treat these as a tripwire for honest -# mistakes; the actual security boundary is the sandbox backend. -denylist_patterns: - - "rm\\s+-rf\\s+/" - - ":\\(\\)\\s*\\{\\s*:\\|" - - "mkfs" - - "dd\\s+if=" - - "shutdown" - - "reboot" - - "/etc/passwd" - - "/etc/shadow" - # Sub-execution / write escapes for commonly-added tools - - "\\bfind\\b[^|;&]*-exec(dir)?\\b" - - "\\bawk\\b[^|;&]*system\\s*\\(" - - "\\bsed\\b[^|;&]*(-i\\b|\\be\\b)" - - "\\bcurl\\b[^|;&]*(file://|-o\\s|--output-dir\\b|-F\\s+@)" - - "\\bgit\\b[^|;&]*(--upload-pack|--receive-pack|core\\.pager|core\\.hooksPath|GIT_SSH_COMMAND)" - - "\\b(node|python3?)\\b[^|;&]*\\s-(e|c)\\b" - - "\\bnpm\\b[^|;&]*\\brun\\b" -max_concurrent_jobs: 16 -job_retention_secs: 3600 +const result = await iii.trigger({ + function_id: 'shell::exec', + payload: { command: 'echo', args: ['hello'] }, +}) -fs: - # SET host_root to a directory you intend to expose to shell::fs::*. - # When unset, the worker refuses to start unless allow_unjailed is true - # (because the alternative is "the entire filesystem is reachable - # behind only the advisory denylist", which is rarely intended). - # - # Default is /tmp: exists on every Unix host, is writable, and contains - # only ephemeral data. Operators should point this at the workspace - # they actually intend the shell worker to manage. - host_root: /tmp - allow_unjailed: false - max_read_bytes: 16777216 - max_write_bytes: 16777216 - denylist_paths: - - /etc/passwd - - /etc/shadow - -# When enabled is true, callers can target a live sandbox via the -# top-level `target` field on shell::exec, shell::exec_bg, and every -# shell::fs::* request. When false, every sandbox-targeted call -# returns S210 ("sandbox target disabled in config") regardless of -# whether iii-sandbox itself is running. -sandbox: - enabled: true +console.log(result) ``` -## Additional Resources - -- [Changing a path's permissions](skills/chmod.md) -- [Running a one-shot command in the foreground](skills/exec.md) -- [Spawning a long-running command as a background job](skills/exec_bg.md) -- [Searching a directory tree with regex](skills/grep.md) -- [Terminating a running background job](skills/kill.md) -- [Surveying current background jobs](skills/list.md) -- [Listing a directory inside the jail](skills/ls.md) -- [Creating a directory inside the jail](skills/mkdir.md) -- [Renaming or moving a path inside the jail](skills/mv.md) -- [Streaming a file's bytes through a channel](skills/read.md) -- [Removing a path inside the jail](skills/rm.md) -- [Find-and-replace across files](skills/sed.md) -- [Reading a single path's metadata](skills/stat.md) -- [Polling a background job to completion](skills/status.md) -- [Streaming bytes into a file](skills/write.md) +The example runs on the host. The same payload retargets at a microVM with `target: { kind: 'sandbox', sandbox_id: '' }`. The other entry points are `shell::exec_bg`, `shell::status`, `shell::kill`, `shell::list`, plus the `shell::fs::*` family (`ls`, `stat`, `read`, `write`, `grep`, `sed`, `mkdir`, `rm`, `chmod`, `mv`). + +## Functions + +| Function | Purpose | +|---|---| +| `shell::exec` | Run an allowlisted command in the foreground; returns stdout, stderr, exit code, and timing. Blocks until exit or timeout. | +| `shell::exec_bg` | Spawn an allowlisted command as a background job; returns `{ job_id, argv }` immediately. Host-targeted jobs ignore `timeout_ms` (end via `shell::kill` or natural exit); sandbox jobs honor it. | +| `shell::status` | Fetch one job's full record: state, exit code, and captured stdout/stderr. `not_found` means the id never existed or aged out past `job_retention_secs`. | +| `shell::list` | Enumerate current jobs as lightweight summaries; argv, stdout, and stderr are redacted. | +| `shell::kill` | Terminate a running background job by `job_id`. Sandbox jobs cannot be hard-killed: the record flips to `killed` but the in-VM process runs until its `timeout_ms` (or `sandbox::stop`). | +| `shell::fs::ls` | List a directory's entries with structured metadata. | +| `shell::fs::stat` | Read one path's metadata (size, mode, symlink flag). | +| `shell::fs::mkdir` | Create a directory, optionally with missing parents. | +| `shell::fs::rm` | Remove a file or directory, optionally recursive. | +| `shell::fs::chmod` | Change a path's mode, and optionally its uid/gid. | +| `shell::fs::mv` | Rename or move one path within the jail. | +| `shell::fs::grep` | Recursive regex search across a tree; returns structured matches. Keys are singular `include_glob`/`exclude_glob`; the case flag is `ignore_case`. | +| `shell::fs::sed` | Regex find-and-replace across one file or many. | +| `shell::fs::write` | Stream bytes into a file through an SDK channel; writes via a temp file and renames atomically. No inline `content` field. | +| `shell::fs::read` | Stream a file's bytes out through an SDK channel. For an inline read on the web surface, use the `harness::fs::read_inline` wrapper instead. | + +Every `shell::fs::*` call accepts the same optional `target` as `exec`, so host and sandbox share one wire shape. + +## Errors + +Returned error bodies carry a stable `code` field. Allowlist and denylist rejections come back as a plain message (`command '' not in allowlist`, `command matches denylist: `) rather than an S-code. + +| Code | Meaning | +|---|---| +| `S200` | In-VM execution failure on a sandbox target. | +| `S210` | Invalid request: non-absolute path, empty command or pattern, bad octal mode, malformed payload, or `sandbox.enabled: false` on a sandbox-targeted call. | +| `S211` | Path not found. | +| `S212` | Wrong file type for the operation (for example, a file where a directory was expected). | +| `S213` | Path already exists. | +| `S214` | Directory not empty (non-recursive `rm`). | +| `S215` | Path escapes `host_root`, hits `fs.denylist_paths`, or permission denied. | +| `S216` | Generic shell-internal failure: host spawn error, channel error, or a bad engine response. | +| `S217` | Invalid regex passed to `grep`/`sed`. | +| `S218` | `fs.max_read_bytes` / `fs.max_write_bytes` cap exceeded. | +| `S300` | Sandbox VM boot failed (needs a virtualization host: Apple Silicon or `/dev/kvm`). | + +## Troubleshooting + +- **`fs.host_root is unset ... refusing to start unjailed`**: set `fs.host_root` to a directory, or set `fs.allow_unjailed: true`. +- **`command '' not in allowlist`**: the basename of `argv[0]` is not in a non-empty `allowlist`. Add it, or empty the list to allow anything. +- **`S215 path escapes host_root` on a path inside the jail**: a symlink in the path resolves outside the jail. Resolve it yourself, or move the target inside `host_root`. +- **`S300` on a sandbox target**: the host cannot boot microVMs. Sandbox execution requires Apple Silicon or `/dev/kvm`. +- **Worker never connects**: the engine is not running or not bound on the configured `--url`. Start the engine first; the default WebSocket port is 49134. + +For the threat model, streaming wire shapes, and contributor build steps, see [ARCHITECTURE.md](ARCHITECTURE.md). + +## License + +Apache 2.0 — see [LICENSE](https://github.com/iii-hq/workers/blob/main/LICENSE). diff --git a/shell/config.yaml b/shell/config.yaml index 490bdcce..eecc1d95 100644 --- a/shell/config.yaml +++ b/shell/config.yaml @@ -10,9 +10,9 @@ allowed_env: - LC_ALL - TERM # Default allowlist is intentionally read-only. Tools that can shell out -# (git hooks, curl -o/file://, find -exec, awk system(), sed e/-i, cargo -# build.rs, node -e, python3 -c, npm run, env ) are left out on -# purpose — add them per deployment after you've decided on the threat +# (git hooks, curl -o/file://, find -exec, awk system(), sed e/-i, Rust +# build-script hooks, node -e, python3 -c, npm run, env ) are left out on +# purpose; add them per deployment after you've decided on the threat # model. This worker is NOT a sandbox. Use `printenv` for read-only env # inspection; `env` is excluded because `env ` execs arbitrary # programs while passing argv[0]=="env" through the allowlist gate. @@ -43,9 +43,9 @@ allowlist: # Denylist patterns are advisory, not a security boundary. They run as # regex against `argv.join(" ")`, so a caller invoking an allowlisted # shell or interpreter (sh, node, python, etc.) can bypass any pattern -# by constructing the forbidden token at runtime — variables, eval, -# IFS tricks, base64, etc. Treat these as a tripwire for honest -# mistakes; the actual security boundary is the sandbox backend. +# by constructing the forbidden token at runtime (variables, eval, +# IFS tricks, base64, etc.). Treat these as a tripwire for honest +# mistakes; the security boundary is the sandbox backend. denylist_patterns: - "rm\\s+-rf\\s+/" - ":\\(\\)\\s*\\{\\s*:\\|" @@ -74,7 +74,7 @@ fs: # # Default is /tmp: exists on every Unix host, is writable, and contains # only ephemeral data. Operators should point this at the workspace - # they actually intend the shell worker to manage. + # they intend the shell worker to manage. host_root: /tmp allow_unjailed: false max_read_bytes: 16777216 diff --git a/shell/docs/companions.md b/shell/docs/companions.md deleted file mode 100644 index 63c6e942..00000000 --- a/shell/docs/companions.md +++ /dev/null @@ -1,6 +0,0 @@ -For sandbox-targeted execution and `shell::fs::*` forwarding, install [`iii-sandbox`](../iii-sandbox); `iii worker add shell` does not currently pull it in. For surfacing `shell::*` to LLM agents, pair with [`skills`](../skills): - -```bash -iii worker add iii-sandbox -iii worker add skills -``` diff --git a/shell/docs/intro.md b/shell/docs/intro.md deleted file mode 100644 index c83f469d..00000000 --- a/shell/docs/intro.md +++ /dev/null @@ -1,5 +0,0 @@ -Unix shell and filesystem worker on the iii bus. Every agent that needs to touch the OS (run a build, read a file, list a directory, call a CLI) goes through `shell::*` and `shell::fs::*`, so allowlists, timeouts, output caps, and a host-root jail live in one place. Both surfaces accept an optional `target` field that forwards the call into a live `iii-sandbox` microVM, so the same allowlist policy gates host and sandbox execution. - - -Host-targeted `shell::exec` is not an isolation boundary. The denylist is a regex tripwire on `argv.join(" ")`. A caller running an allowlisted interpreter (`sh`, `node`, `python3`) can construct any forbidden token at runtime and bypass it. For untrusted input, pass `target: { kind: "sandbox", sandbox_id }` so the call forwards into a microVM. Prefer `shell::fs::ls`, `shell::fs::stat`, and `shell::fs::grep` over `exec`-ing the same tools; the fs backends stay in-process, respect the jail, and return structured results. - diff --git a/shell/docs/leaves/chmod.md b/shell/docs/leaves/chmod.md deleted file mode 100644 index d31bd185..00000000 --- a/shell/docs/leaves/chmod.md +++ /dev/null @@ -1,15 +0,0 @@ -# Changing a path's permissions - -## When to use - -- Marking a freshly-written script executable after `shell::fs::write`. -- Locking down a generated config file (`"0600"`). -- Recursively normalising permissions across a generated tree. - -## Notes - -- `mode` is an octal string (e.g. `"0644"`, `"0755"`). `uid`/`gid` are optional integers; pass them to also `chown`. -- `updated` is the count of paths whose mode or owner the call mutated: 1 for a single-path call, more under `recursive: true`. -- Symlinks are not followed; chmod operates on the link itself. Recursive walks skip symlink entries so `chmod(2)`/`chown(2)` cannot deref to a target outside the walk root. -- `uid` / `gid` are advisory: the host backend refuses to chown without sufficient privileges; the call returns the trigger `Err` rather than silently skipping. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/docs/leaves/exec.md b/shell/docs/leaves/exec.md deleted file mode 100644 index 0e1e1134..00000000 --- a/shell/docs/leaves/exec.md +++ /dev/null @@ -1,16 +0,0 @@ -# Running a one-shot command in the foreground - -## When to use - -- The command finishes well under the timeout cap and the caller can block until completion. -- One-shot probes: `ls`, `cat`, `pwd`, `git status`, `wc`, `head`. -- Anything where blocking until completion is fine for the calling turn. - -## Notes - -- `command` is a program name as a string; arguments go in the `args` array. Sending `command: ["sh", "-lc", "..."]` returns a per-field deserialiser error rather than a misleading "missing 'command'". -- `timeout_ms` is clamped to `max_timeout_ms` (default 30s); negative or non-numeric values fall back to `default_timeout_ms` (default 10s). -- Output is buffered up to `max_output_bytes` (default 1 MiB). Past that, `stdout_truncated` and `stderr_truncated` flip to `true` and the rest is dropped; narrow the command (e.g. `head -n 100`) rather than asking for more bytes. -- Allowlist matches `argv[0]` by basename or exact path; an empty allowlist means open. Denylist regex runs over `argv.join(" ")`. -- `target: sandbox` returns `S300` if the host cannot boot microVMs (Apple Silicon or `/dev/kvm` required). Host-side spawn errors come back as `S216` with a `host exec:` prefix. -- Prefer `shell::fs::ls`, `shell::fs::stat`, and `shell::fs::grep` over `exec`-ing `ls`/`stat`/`grep`/`rg`; the fs backends stay in-process and respect the jail. diff --git a/shell/docs/leaves/exec_bg.md b/shell/docs/leaves/exec_bg.md deleted file mode 100644 index 53685fc9..00000000 --- a/shell/docs/leaves/exec_bg.md +++ /dev/null @@ -1,16 +0,0 @@ -# Spawning a long-running command as a background job - -## When to use - -- Builds, long greps, watchers; anything that does not fit inside `max_timeout_ms`. -- Keeping the calling turn responsive while the command runs. -- "Run cargo build and report when it is done": pair with `shell::status` polling. - -## Notes - -- Same payload shape as `shell::exec`. The same per-field deserialisers catch wrong-type fields up front. -- Allowlist and denylist gate the spawn the same way; a blocked argv comes back as the trigger `Err` and the job is never inserted into the table. -- Host-targeted background jobs **ignore** `timeout_ms` (preserves the unbounded host-bg semantic). Only `shell::kill` or natural exit ends them. -- Sandbox-targeted background jobs honour `timeout_ms`; the value is clamped through `cfg.resolve_timeout` and forwarded to `sandbox::exec`. -- Concurrency cap is `cfg.max_concurrent_jobs` (default 16). Past the cap, the call returns `Err` and the spawned child is killed before the trigger response. -- Sandbox-backed jobs cannot be hard-killed: `shell::kill` flips the record to `killed` immediately, but the in-VM process keeps running until its `timeout_ms` expires (or `sandbox::stop` tears the VM down). diff --git a/shell/docs/leaves/grep.md b/shell/docs/leaves/grep.md deleted file mode 100644 index 06a8856b..00000000 --- a/shell/docs/leaves/grep.md +++ /dev/null @@ -1,16 +0,0 @@ -# Searching a directory tree with regex - -## When to use - -- Where a caller would otherwise reach for `rg` or `grep -rn`: faster and structured. -- Pre-step before `shell::fs::sed` to preview what would be rewritten. -- Cross-file search inside a worker without spawning a process. - -## Notes - -- `include_glob` and `exclude_glob` are arrays of glob strings (e.g. `["**/*.rs"]`, `["**/target/**"]`). The keys are singular `_glob`, not plural `_globs`. -- The flag is `ignore_case`, not `case_insensitive`. -- Multiline mode is off by default; encode `(?m)` inside the pattern when multiline anchoring is needed. -- When `max_matches` is hit, `truncated: true` is set and the walk stops. Tighten the pattern or narrow `path` instead of bumping the cap blindly. -- Binary files are skipped automatically. The wire `FsMatch` has no `column` field; the legacy `file` alias is accepted on the deserialiser side but the response always renders as `path`. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/docs/leaves/kill.md b/shell/docs/leaves/kill.md deleted file mode 100644 index f7cf947e..00000000 --- a/shell/docs/leaves/kill.md +++ /dev/null @@ -1,15 +0,0 @@ -# Terminating a running background job - -## When to use - -- Cancelling a runaway build or long process spawned by `shell::exec_bg`. -- Cleaning up before re-issuing a corrected command. -- Reaping at the end of an orchestration before unregistering the worker. - -## Notes - -- There is no `signal` field on the wire. Host kills go through tokio's `Child::start_kill`, a hard SIGKILL on Unix. -- A non-running job returns `killed: false` with `reason: "not running"`. That is not an error. -- Sandbox-backed jobs cannot be hard-killed because `sandbox::exec` has no cancel hook. The record flips to `killed` and `finished_at_ms` is stamped immediately so `shell::status` and `shell::list` reflect cancellation, but the in-VM process keeps running until its `timeout_ms` expires. The response carries a `reason` explaining this. -- For real cancellation of a sandbox job, set a tight `timeout_ms` on the original `exec_bg`, or call `sandbox::stop` to tear down the VM. -- `not_found` (the trigger `Err`) means the `job_id` either never existed or aged out of retention. diff --git a/shell/docs/leaves/list.md b/shell/docs/leaves/list.md deleted file mode 100644 index 4807894c..00000000 --- a/shell/docs/leaves/list.md +++ /dev/null @@ -1,13 +0,0 @@ -# Surveying current background jobs - -## When to use - -- "What background jobs are running right now?" probes. -- Driving a dashboard or status surface over current shell activity. -- Pre-cleanup audit before a worker shutdown. - -## Notes - -- The response carries `JobSummary` only; `argv`, `stdout`, and `stderr` are deliberately omitted. The JOBS map is process-wide and any caller could otherwise read another caller's command line and captured output (which may embed credentials). -- For full records (including `stdout`/`stderr`), call `shell::status` with the `job_id`. The random UUID acts as an unguessable capability. -- Terminated jobs stay listed for `cfg.job_retention_secs` (default 3600s) past their `finished_at_ms`. After that the periodic janitor removes them and the list no longer returns them. diff --git a/shell/docs/leaves/ls.md b/shell/docs/leaves/ls.md deleted file mode 100644 index 59809ada..00000000 --- a/shell/docs/leaves/ls.md +++ /dev/null @@ -1,14 +0,0 @@ -# Listing a directory inside the jail - -## When to use - -- Enumerating filenames before reading; cheaper than `shell::fs::grep` when only names are needed. -- Confirming a path is a directory before recursing. -- Building a file picker or orchestration call that wants a structured listing. - -## Notes - -- `path` must be absolute. Anything outside `cfg.fs.host_root` is refused; the path denylist (`cfg.fs.denylist_paths`) refuses inside the jail too. -- Symlinks are not followed; an entry's `is_dir` reflects the link itself and `is_symlink` is `true`. -- `mode` is an octal string (e.g. `"0755"`); `mtime` is epoch **seconds**, not milliseconds. `JobRecord.*_at_ms` are ms but `FsEntry.mtime` is seconds. -- The wire shape mirrors the engine's `sandbox::fs::ls`, so host and sandbox targets are interchangeable from the caller's point of view. diff --git a/shell/docs/leaves/mkdir.md b/shell/docs/leaves/mkdir.md deleted file mode 100644 index 13465139..00000000 --- a/shell/docs/leaves/mkdir.md +++ /dev/null @@ -1,13 +0,0 @@ -# Creating a directory inside the jail - -## When to use - -- Pre-creating a directory tree before streaming files into it with `shell::fs::write`. -- Ensure-path-exists idiom: `parents: true` and ignore the result. -- Bootstrapping a sandbox layout by retargeting the call. - -## Notes - -- The flag is `parents`, not `recursive`. Both backends accept the same name. -- `created` is always `true` on success, including the idempotent case where `parents: true` and the directory already existed. There is no `created: false` branch; failure returns the trigger `Err` (e.g. `S211` for a missing parent without `parents: true`). -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/docs/leaves/mv.md b/shell/docs/leaves/mv.md deleted file mode 100644 index 08c94f5a..00000000 --- a/shell/docs/leaves/mv.md +++ /dev/null @@ -1,15 +0,0 @@ -# Renaming or moving a path inside the jail - -## When to use - -- Renaming in place inside the same parent directory. -- Moving across directories within the jail. -- Atomic publish of a generated file from a temp name to its final location. - -## Notes - -- Both `src` and `dst` are absolute and must be inside the same jail. -- Implementation is `rename(2)` with a fallback to copy + unlink when `src` and `dst` cross filesystems. The fallback is not atomic on its own; a crash after the copy and before the unlink leaves both copies on disk. -- Without `overwrite: true` the call refuses if `dst` exists and returns the trigger `Err`. -- This is `mv` for one path. There is no batch or glob form; loop in the caller. -- Same jail and denylist rules as `shell::fs::ls` for both `src` and `dst`. diff --git a/shell/docs/leaves/read.md b/shell/docs/leaves/read.md deleted file mode 100644 index f0c5d570..00000000 --- a/shell/docs/leaves/read.md +++ /dev/null @@ -1,13 +0,0 @@ -# Streaming a file's bytes through a channel - -## When to use - -- A peer worker wants the bytes of a file without an inline copy in the trigger response. -- Streaming a large file into another channel (for example, uploading into a sandbox) without pinning a buffer in the orchestrator. -- Reading a binary that would not survive JSON encoding inline. - -## Notes - -- Most LLM tool surfaces want bytes inlined into a tool result rather than a channel handle. For the harness web surface, the inline wrapper `harness::fs::read_inline` drives `shell::fs::read`, drains the channel, and returns the legacy `{ content: [{ text }], details: { size, truncated, bytes_read } }` envelope. Use the wrapper from the browser; reach for `shell::fs::read` directly only when you actually want the streaming channel. -- When `cfg.fs.max_read_bytes > 0` and the file's size exceeds it, the read is rejected before any bytes flow with `S218 file size N exceeds max_read_bytes M`. The default of `0` means no cap, which is unusual; operators typically pin a value. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/docs/leaves/rm.md b/shell/docs/leaves/rm.md deleted file mode 100644 index f56f16b1..00000000 --- a/shell/docs/leaves/rm.md +++ /dev/null @@ -1,14 +0,0 @@ -# Removing a path inside the jail - -## When to use - -- Tearing down a generated artefact after use. -- Resetting a workspace directory before re-bootstrapping it. -- Dropping a temp file once a flow no longer needs it. - -## Notes - -- Symlinks are removed by themselves, not their targets. `recursive` does not change that. -- There is no trash bin; this is `unlink(2)` / `rmdir(2)`. Confirm caller intent before invoking. -- `removed` is always `true` on success. Missing paths return `Err(S211)`; non-empty directories without `recursive: true` return `Err(S214)`; permission errors and other I/O failures come back as the trigger `Err` with the corresponding `FsError` shape. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/docs/leaves/sed.md b/shell/docs/leaves/sed.md deleted file mode 100644 index b2db0e6d..00000000 --- a/shell/docs/leaves/sed.md +++ /dev/null @@ -1,17 +0,0 @@ -# Find-and-replace across files - -## When to use - -- Multi-file rewrite where `shell::fs::grep` is the read-side preview. -- Single-file regex replace that would otherwise reach for `sed -i`. -- Templating step over a generated tree. - -## Notes - -- Supply either `files: ["/abs/a", …]` for an explicit list or `path` (a directory) for a recursive walk; or `path` to a single file for a one-off rewrite. -- `pattern` is a regex when `regex: true` (default). `replacement` supports `$1`, `$2`, … capture-group backreferences. Set `regex: false` for a literal pattern. -- `first_only: true` rewrites only the first match per file; the default rewrites every match. -- There is no line-anchor flag; encode it inside the pattern (`(?m)^…`). -- Per-file results carry `success: false` plus an `error` string for files that failed to rewrite (permission denied, regex compilation error). `total_replacements` sums across files. -- Approval policy mirrors `shell::fs::write`: set per-run by the orchestrator's `approval_required` array, not hardcoded into the function. -- When unsure about the regex, run `shell::fs::grep` with the same pattern first. diff --git a/shell/docs/leaves/stat.md b/shell/docs/leaves/stat.md deleted file mode 100644 index 91603936..00000000 --- a/shell/docs/leaves/stat.md +++ /dev/null @@ -1,13 +0,0 @@ -# Reading a single path's metadata - -## When to use - -- Confirming a file exists and checking its size before deciding whether to read it whole. -- Distinguishing a regular file from a symlink without dereferencing it. -- Pre-flight check for a write target's existing mode before `chmod`-ing. - -## Notes - -- A missing path returns the trigger `Err` (FsError); there is no soft-not-found envelope. Wrap the call when probing optionally. -- Symlinks are not followed; `is_symlink` reports the link itself. To get the target's metadata, follow up with another `stat` on the resolved path. -- Same jail and denylist rules as `shell::fs::ls`. `mode` is an octal string and `mtime` is epoch seconds. diff --git a/shell/docs/leaves/status.md b/shell/docs/leaves/status.md deleted file mode 100644 index be84b16e..00000000 --- a/shell/docs/leaves/status.md +++ /dev/null @@ -1,14 +0,0 @@ -# Polling a background job to completion - -## When to use - -- Polling a job spawned by `shell::exec_bg` until it leaves `running`. -- Fetching captured `stdout`/`stderr` once a job has terminated, before retention expires. -- Diagnosing a job that exited with a non-zero `exit_code`. - -## Notes - -- `not_found` (the trigger `Err`) means the `job_id` either never existed or aged out of `cfg.job_retention_secs` (default 1 hour after termination). Do not retry; re-run `shell::exec_bg` if the work still needs doing. -- Per-stream output is bounded by `cfg.max_output_bytes` (default 1 MiB). Once the cap is hit on a stream, the corresponding `*_truncated` flag stays `true` and new bytes are dropped while the job keeps running. -- Use `shell::list` for a lightweight overview of every job. `shell::status` returns the full record, including potentially large captured buffers, so it costs more per call. -- Sandbox-backed jobs that were `shell::kill`-ed flip to `killed` immediately even though the in-VM process may still be running. Their final stdout/stderr arrive on the late `sandbox::exec` response and are not applied if the record is already `killed`. diff --git a/shell/docs/leaves/write.md b/shell/docs/leaves/write.md deleted file mode 100644 index bfa76628..00000000 --- a/shell/docs/leaves/write.md +++ /dev/null @@ -1,15 +0,0 @@ -# Streaming bytes into a file - -## When to use - -- Persisting a generated artefact to disk inside the jail. -- Streaming a remote download or generated stream straight into a file without an intermediate buffer. -- Bootstrapping files into a sandbox by retargeting with `target: { kind: "sandbox", sandbox_id }`. - -## Notes - -- The wire payload does not accept raw `content: string` or `content_b64`. The caller opens a channel via the SDK, passes the `ContentRef` here, then writes bytes into the channel and closes it. -- When `cfg.fs.max_write_bytes > 0` and the streamed total exceeds the cap, the write is aborted mid-stream with `S218`. The default of `0` means no cap. -- Per-chunk idle timeout is 30s. A caller that opens a write but never sends data and never closes the channel is aborted with `S216 channel idle for 30s, aborting write` so a parked writer cannot leak the temp file. -- The worker writes through a temp file and renames atomically. On crash mid-stream, the temp file is unlinked by `TempGuard`. -- Approval policy is not hardcoded into this function. Whether a turn requires approval before a write lands is set per-run by the orchestrator's `approval_required` array. diff --git a/shell/docs/quickstart.md b/shell/docs/quickstart.md deleted file mode 100644 index c5366df0..00000000 --- a/shell/docs/quickstart.md +++ /dev/null @@ -1,52 +0,0 @@ -```rust -use iii_sdk::{register_worker, InitOptions, TriggerRequest}; -use serde_json::json; - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let worker = register_worker("ws://localhost:49134", InitOptions::default()); - - let result = worker - .trigger(TriggerRequest { - function_id: "shell::exec".into(), - payload: json!({ - "command": "echo", - "args": ["hello"], - }), - action: None, - timeout_ms: Some(5_000), - }) - .await?; - - println!("{result:#?}"); - Ok(()) -} -``` - -```typescript -import { registerWorker } from 'iii-sdk' - -const worker = registerWorker('ws://localhost:49134') - -const result = await worker.trigger({ - function_id: 'shell::exec', - payload: { command: 'echo', args: ['hello'] }, -}) - -console.log(result) -``` - -```python -from iii import register_worker - -worker = register_worker("ws://localhost:49134") - -result = worker.trigger({ - "function_id": "shell::exec", - "payload": {"command": "echo", "args": ["hello"]}, -}) - -print(result) -``` - -The example calls `shell::exec` on the host. The same payload retargets at a microVM with `target: { "kind": "sandbox", "sandbox_id": "" }`. Other entry points: `shell::exec_bg`, `shell::status`, `shell::kill`, `shell::list`, plus the `shell::fs::*` family (`ls`, `stat`, `read`, `write`, `grep`, `sed`, `mkdir`, `rm`, `chmod`, `mv`). diff --git a/shell/skill.md b/shell/skill.md deleted file mode 100644 index 7963fd0e..00000000 --- a/shell/skill.md +++ /dev/null @@ -1,32 +0,0 @@ - - -# shell - -Unix shell and filesystem worker on the iii bus. Every agent that needs to touch the OS (run a build, read a file, list a directory, call a CLI) goes through `shell::*` and `shell::fs::*`, so allowlists, timeouts, output caps, and a host-root jail live in one place. Both surfaces accept an optional `target` field that forwards the call into a live `iii-sandbox` microVM, so the same allowlist policy gates host and sandbox execution. - -Host-targeted `shell::exec` is not an isolation boundary. The denylist is a regex tripwire on `argv.join(" ")`. A caller running an allowlisted interpreter (`sh`, `node`, `python3`) can construct any forbidden token at runtime and bypass it. For untrusted input, pass `target: { kind: "sandbox", sandbox_id }` so the call forwards into a microVM. Prefer `shell::fs::ls`, `shell::fs::stat`, and `shell::fs::grep` over `exec`-ing the same tools; the fs backends stay in-process, respect the jail, and return structured results. - -For sandbox-targeted execution and `shell::fs::*` forwarding, install [`iii-sandbox`](../iii-sandbox); `iii worker add shell` does not currently pull it in. For surfacing `shell::*` to LLM agents, pair with [`skills`](../skills): - -```bash -iii worker add iii-sandbox -iii worker add skills -``` - -## Additional Resources - -- [Changing a path's permissions](skills/chmod.md) -- [Running a one-shot command in the foreground](skills/exec.md) -- [Spawning a long-running command as a background job](skills/exec_bg.md) -- [Searching a directory tree with regex](skills/grep.md) -- [Terminating a running background job](skills/kill.md) -- [Surveying current background jobs](skills/list.md) -- [Listing a directory inside the jail](skills/ls.md) -- [Creating a directory inside the jail](skills/mkdir.md) -- [Renaming or moving a path inside the jail](skills/mv.md) -- [Streaming a file's bytes through a channel](skills/read.md) -- [Removing a path inside the jail](skills/rm.md) -- [Find-and-replace across files](skills/sed.md) -- [Reading a single path's metadata](skills/stat.md) -- [Polling a background job to completion](skills/status.md) -- [Streaming bytes into a file](skills/write.md) diff --git a/shell/skills/SKILL.md b/shell/skills/SKILL.md new file mode 100644 index 00000000..6acebc9d --- /dev/null +++ b/shell/skills/SKILL.md @@ -0,0 +1,90 @@ +--- +name: shell +tags: shell, exec, filesystem, jobs, sandbox +description: >- + Run Unix commands and structured filesystem ops from the iii engine: allowlisted + exec, background jobs, and a host-jailed fs (ls/stat/mkdir/rm/chmod/mv/grep/sed/ + read/write), all forwardable into a sandbox microVM. +--- + +# shell + +The shell worker is the single door every agent uses to touch the OS: run a +build, call a CLI, read a file, list a directory. Routing it all through +`shell::*` and `shell::fs::*` keeps allowlists, denylists, timeouts, output +caps, and a host-root jail in one enforceable place. Both surfaces take an +optional `target` field that forwards the call into a live `iii-sandbox` +microVM, so one allowlist policy gates host and sandbox execution alike. + +Host-targeted `shell::exec` is not an isolation boundary. The denylist is a +regex tripwire on `argv.join(" ")`, and an allowlisted interpreter (`sh`, +`node`, `python3`) can construct any forbidden token at runtime to bypass it. +Run untrusted input with `target: { kind: "sandbox", sandbox_id }`. Prefer the +`shell::fs::*` backends over `exec`-ing `ls`/`stat`/`grep`/`rg`: they stay +in-process, respect the jail, and return structured results. + +Sandbox forwarding (and `shell::fs::*` into a VM) requires the `iii-sandbox` +worker; `iii worker add shell` does not pull it in. To surface `shell::*` to LLM +agents, pair with the `skills` worker. + +## When to Use + +- Run a one-shot command and block for its full output: `git status`, `wc`, + `head`, a quick compile probe (`shell::exec`). +- Kick off long work (build, watcher, wide grep) without blocking the turn, + then poll for completion (`shell::exec_bg` + `shell::status`). +- Survey or terminate in-flight background jobs (`shell::list`, `shell::kill`). +- List, stat, or read files with structured output instead of shelling out to + `ls`/`stat`/`cat` (`shell::fs::ls`, `shell::fs::stat`, `shell::fs::read`). +- Search or rewrite across a tree without spawning `rg`/`sed` + (`shell::fs::grep`, `shell::fs::sed`). +- Create, move, remove, or re-permission paths inside the jail + (`shell::fs::mkdir`, `shell::fs::mv`, `shell::fs::rm`, `shell::fs::chmod`). +- Persist a generated artefact, or bootstrap files into a sandbox, by streaming + bytes to a path (`shell::fs::write` with a `target`). + +## Boundaries + +- Host `shell::exec` is not a security sandbox: the denylist is bypassable by + any allowlisted interpreter. Run untrusted commands with `target: sandbox` + (needs `iii-sandbox`). +- `shell::fs::*` is jailed to `cfg.fs.host_root` and refuses denylisted paths; + paths must be absolute and symlinks are never followed. +- Sandbox-backed background jobs cannot be hard-killed: `shell::kill` flips the + record but the in-VM process runs until its `timeout_ms` (or `sandbox::stop`). +- Not for inlining file bytes into an LLM tool result: `shell::fs::read`/ + `write` move bytes over channels; use the `harness` worker's + `harness::fs::read_inline` wrapper for inline reads on the web surface. +- No batch or glob form for single-path ops (`mv`, `rm`, `stat`, …); loop in the + caller. +- Not a package manager, editor, or migration tool; for SQL use the `database` + worker. + +## Functions + +- `shell::exec`: run an allowlisted command in the foreground and return its + stdout, stderr, exit code, and timing; blocks until exit or timeout. +- `shell::exec_bg`: spawn an allowlisted command as a background job and return + a `job_id` immediately. Host-targeted jobs ignore `timeout_ms` (end via + `shell::kill` or natural exit); sandbox jobs honor it. +- `shell::status`: fetch one job's full record: state, exit code, and captured + stdout/stderr. +- `shell::list`: enumerate current jobs as lightweight summaries (no argv, + stdout, or stderr). +- `shell::kill`: terminate a running background job by `job_id`. +- `shell::fs::ls`: list a directory's entries with structured metadata. +- `shell::fs::stat`: read one path's metadata (size, mode, symlink flag). +- `shell::fs::mkdir`: create a directory, optionally with missing parents. +- `shell::fs::rm`: remove a file or directory, optionally recursive. +- `shell::fs::chmod`: change a path's mode, and optionally its uid/gid. +- `shell::fs::mv`: rename or move one path within the jail. +- `shell::fs::grep`: recursive regex search across a tree, returning structured + matches. +- `shell::fs::sed`: regex find-and-replace across one file or many. +- `shell::fs::write`: stream bytes into a file via a channel; writes through a + temp file and renames atomically. +- `shell::fs::read`: stream a file's bytes out through a channel. + +Every `shell::fs::*` call accepts the same optional `target` as `exec`, so host +and sandbox share one wire shape; reads and writes move bytes over SDK channels +rather than inlining them. diff --git a/shell/skills/chmod.md b/shell/skills/chmod.md deleted file mode 100644 index 428479a7..00000000 --- a/shell/skills/chmod.md +++ /dev/null @@ -1,17 +0,0 @@ - - -# Changing a path's permissions - -## When to use - -- Marking a freshly-written script executable after `shell::fs::write`. -- Locking down a generated config file (`"0600"`). -- Recursively normalising permissions across a generated tree. - -## Notes - -- `mode` is an octal string (e.g. `"0644"`, `"0755"`). `uid`/`gid` are optional integers; pass them to also `chown`. -- `updated` is the count of paths whose mode or owner the call mutated: 1 for a single-path call, more under `recursive: true`. -- Symlinks are not followed; chmod operates on the link itself. Recursive walks skip symlink entries so `chmod(2)`/`chown(2)` cannot deref to a target outside the walk root. -- `uid` / `gid` are advisory: the host backend refuses to chown without sufficient privileges; the call returns the trigger `Err` rather than silently skipping. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/skills/exec.md b/shell/skills/exec.md deleted file mode 100644 index 077ebd61..00000000 --- a/shell/skills/exec.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# Running a one-shot command in the foreground - -## When to use - -- The command finishes well under the timeout cap and the caller can block until completion. -- One-shot probes: `ls`, `cat`, `pwd`, `git status`, `wc`, `head`. -- Anything where blocking until completion is fine for the calling turn. - -## Notes - -- `command` is a program name as a string; arguments go in the `args` array. Sending `command: ["sh", "-lc", "..."]` returns a per-field deserialiser error rather than a misleading "missing 'command'". -- `timeout_ms` is clamped to `max_timeout_ms` (default 30s); negative or non-numeric values fall back to `default_timeout_ms` (default 10s). -- Output is buffered up to `max_output_bytes` (default 1 MiB). Past that, `stdout_truncated` and `stderr_truncated` flip to `true` and the rest is dropped; narrow the command (e.g. `head -n 100`) rather than asking for more bytes. -- Allowlist matches `argv[0]` by basename or exact path; an empty allowlist means open. Denylist regex runs over `argv.join(" ")`. -- `target: sandbox` returns `S300` if the host cannot boot microVMs (Apple Silicon or `/dev/kvm` required). Host-side spawn errors come back as `S216` with a `host exec:` prefix. -- Prefer `shell::fs::ls`, `shell::fs::stat`, and `shell::fs::grep` over `exec`-ing `ls`/`stat`/`grep`/`rg`; the fs backends stay in-process and respect the jail. diff --git a/shell/skills/exec_bg.md b/shell/skills/exec_bg.md deleted file mode 100644 index e69d51ff..00000000 --- a/shell/skills/exec_bg.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# Spawning a long-running command as a background job - -## When to use - -- Builds, long greps, watchers; anything that does not fit inside `max_timeout_ms`. -- Keeping the calling turn responsive while the command runs. -- "Run cargo build and report when it is done": pair with `shell::status` polling. - -## Notes - -- Same payload shape as `shell::exec`. The same per-field deserialisers catch wrong-type fields up front. -- Allowlist and denylist gate the spawn the same way; a blocked argv comes back as the trigger `Err` and the job is never inserted into the table. -- Host-targeted background jobs **ignore** `timeout_ms` (preserves the unbounded host-bg semantic). Only `shell::kill` or natural exit ends them. -- Sandbox-targeted background jobs honour `timeout_ms`; the value is clamped through `cfg.resolve_timeout` and forwarded to `sandbox::exec`. -- Concurrency cap is `cfg.max_concurrent_jobs` (default 16). Past the cap, the call returns `Err` and the spawned child is killed before the trigger response. -- Sandbox-backed jobs cannot be hard-killed: `shell::kill` flips the record to `killed` immediately, but the in-VM process keeps running until its `timeout_ms` expires (or `sandbox::stop` tears the VM down). diff --git a/shell/skills/grep.md b/shell/skills/grep.md deleted file mode 100644 index 3f5402cc..00000000 --- a/shell/skills/grep.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# Searching a directory tree with regex - -## When to use - -- Where a caller would otherwise reach for `rg` or `grep -rn`: faster and structured. -- Pre-step before `shell::fs::sed` to preview what would be rewritten. -- Cross-file search inside a worker without spawning a process. - -## Notes - -- `include_glob` and `exclude_glob` are arrays of glob strings (e.g. `["**/*.rs"]`, `["**/target/**"]`). The keys are singular `_glob`, not plural `_globs`. -- The flag is `ignore_case`, not `case_insensitive`. -- Multiline mode is off by default; encode `(?m)` inside the pattern when multiline anchoring is needed. -- When `max_matches` is hit, `truncated: true` is set and the walk stops. Tighten the pattern or narrow `path` instead of bumping the cap blindly. -- Binary files are skipped automatically. The wire `FsMatch` has no `column` field; the legacy `file` alias is accepted on the deserialiser side but the response always renders as `path`. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/skills/kill.md b/shell/skills/kill.md deleted file mode 100644 index 59df1434..00000000 --- a/shell/skills/kill.md +++ /dev/null @@ -1,17 +0,0 @@ - - -# Terminating a running background job - -## When to use - -- Cancelling a runaway build or long process spawned by `shell::exec_bg`. -- Cleaning up before re-issuing a corrected command. -- Reaping at the end of an orchestration before unregistering the worker. - -## Notes - -- There is no `signal` field on the wire. Host kills go through tokio's `Child::start_kill`, a hard SIGKILL on Unix. -- A non-running job returns `killed: false` with `reason: "not running"`. That is not an error. -- Sandbox-backed jobs cannot be hard-killed because `sandbox::exec` has no cancel hook. The record flips to `killed` and `finished_at_ms` is stamped immediately so `shell::status` and `shell::list` reflect cancellation, but the in-VM process keeps running until its `timeout_ms` expires. The response carries a `reason` explaining this. -- For real cancellation of a sandbox job, set a tight `timeout_ms` on the original `exec_bg`, or call `sandbox::stop` to tear down the VM. -- `not_found` (the trigger `Err`) means the `job_id` either never existed or aged out of retention. diff --git a/shell/skills/list.md b/shell/skills/list.md deleted file mode 100644 index 026034d0..00000000 --- a/shell/skills/list.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Surveying current background jobs - -## When to use - -- "What background jobs are running right now?" probes. -- Driving a dashboard or status surface over current shell activity. -- Pre-cleanup audit before a worker shutdown. - -## Notes - -- The response carries `JobSummary` only; `argv`, `stdout`, and `stderr` are deliberately omitted. The JOBS map is process-wide and any caller could otherwise read another caller's command line and captured output (which may embed credentials). -- For full records (including `stdout`/`stderr`), call `shell::status` with the `job_id`. The random UUID acts as an unguessable capability. -- Terminated jobs stay listed for `cfg.job_retention_secs` (default 3600s) past their `finished_at_ms`. After that the periodic janitor removes them and the list no longer returns them. diff --git a/shell/skills/ls.md b/shell/skills/ls.md deleted file mode 100644 index 0c11658f..00000000 --- a/shell/skills/ls.md +++ /dev/null @@ -1,16 +0,0 @@ - - -# Listing a directory inside the jail - -## When to use - -- Enumerating filenames before reading; cheaper than `shell::fs::grep` when only names are needed. -- Confirming a path is a directory before recursing. -- Building a file picker or orchestration call that wants a structured listing. - -## Notes - -- `path` must be absolute. Anything outside `cfg.fs.host_root` is refused; the path denylist (`cfg.fs.denylist_paths`) refuses inside the jail too. -- Symlinks are not followed; an entry's `is_dir` reflects the link itself and `is_symlink` is `true`. -- `mode` is an octal string (e.g. `"0755"`); `mtime` is epoch **seconds**, not milliseconds. `JobRecord.*_at_ms` are ms but `FsEntry.mtime` is seconds. -- The wire shape mirrors the engine's `sandbox::fs::ls`, so host and sandbox targets are interchangeable from the caller's point of view. diff --git a/shell/skills/mkdir.md b/shell/skills/mkdir.md deleted file mode 100644 index a6e91693..00000000 --- a/shell/skills/mkdir.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Creating a directory inside the jail - -## When to use - -- Pre-creating a directory tree before streaming files into it with `shell::fs::write`. -- Ensure-path-exists idiom: `parents: true` and ignore the result. -- Bootstrapping a sandbox layout by retargeting the call. - -## Notes - -- The flag is `parents`, not `recursive`. Both backends accept the same name. -- `created` is always `true` on success, including the idempotent case where `parents: true` and the directory already existed. There is no `created: false` branch; failure returns the trigger `Err` (e.g. `S211` for a missing parent without `parents: true`). -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/skills/mv.md b/shell/skills/mv.md deleted file mode 100644 index f4d9dfcf..00000000 --- a/shell/skills/mv.md +++ /dev/null @@ -1,17 +0,0 @@ - - -# Renaming or moving a path inside the jail - -## When to use - -- Renaming in place inside the same parent directory. -- Moving across directories within the jail. -- Atomic publish of a generated file from a temp name to its final location. - -## Notes - -- Both `src` and `dst` are absolute and must be inside the same jail. -- Implementation is `rename(2)` with a fallback to copy + unlink when `src` and `dst` cross filesystems. The fallback is not atomic on its own; a crash after the copy and before the unlink leaves both copies on disk. -- Without `overwrite: true` the call refuses if `dst` exists and returns the trigger `Err`. -- This is `mv` for one path. There is no batch or glob form; loop in the caller. -- Same jail and denylist rules as `shell::fs::ls` for both `src` and `dst`. diff --git a/shell/skills/read.md b/shell/skills/read.md deleted file mode 100644 index 937d2b6a..00000000 --- a/shell/skills/read.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Streaming a file's bytes through a channel - -## When to use - -- A peer worker wants the bytes of a file without an inline copy in the trigger response. -- Streaming a large file into another channel (for example, uploading into a sandbox) without pinning a buffer in the orchestrator. -- Reading a binary that would not survive JSON encoding inline. - -## Notes - -- Most LLM tool surfaces want bytes inlined into a tool result rather than a channel handle. For the harness web surface, the inline wrapper `harness::fs::read_inline` drives `shell::fs::read`, drains the channel, and returns the legacy `{ content: [{ text }], details: { size, truncated, bytes_read } }` envelope. Use the wrapper from the browser; reach for `shell::fs::read` directly only when you actually want the streaming channel. -- When `cfg.fs.max_read_bytes > 0` and the file's size exceeds it, the read is rejected before any bytes flow with `S218 file size N exceeds max_read_bytes M`. The default of `0` means no cap, which is unusual; operators typically pin a value. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/skills/rm.md b/shell/skills/rm.md deleted file mode 100644 index 3499889e..00000000 --- a/shell/skills/rm.md +++ /dev/null @@ -1,16 +0,0 @@ - - -# Removing a path inside the jail - -## When to use - -- Tearing down a generated artefact after use. -- Resetting a workspace directory before re-bootstrapping it. -- Dropping a temp file once a flow no longer needs it. - -## Notes - -- Symlinks are removed by themselves, not their targets. `recursive` does not change that. -- There is no trash bin; this is `unlink(2)` / `rmdir(2)`. Confirm caller intent before invoking. -- `removed` is always `true` on success. Missing paths return `Err(S211)`; non-empty directories without `recursive: true` return `Err(S214)`; permission errors and other I/O failures come back as the trigger `Err` with the corresponding `FsError` shape. -- Same jail and denylist rules as `shell::fs::ls`. diff --git a/shell/skills/sed.md b/shell/skills/sed.md deleted file mode 100644 index 8b58c5f9..00000000 --- a/shell/skills/sed.md +++ /dev/null @@ -1,19 +0,0 @@ - - -# Find-and-replace across files - -## When to use - -- Multi-file rewrite where `shell::fs::grep` is the read-side preview. -- Single-file regex replace that would otherwise reach for `sed -i`. -- Templating step over a generated tree. - -## Notes - -- Supply either `files: ["/abs/a", …]` for an explicit list or `path` (a directory) for a recursive walk; or `path` to a single file for a one-off rewrite. -- `pattern` is a regex when `regex: true` (default). `replacement` supports `$1`, `$2`, … capture-group backreferences. Set `regex: false` for a literal pattern. -- `first_only: true` rewrites only the first match per file; the default rewrites every match. -- There is no line-anchor flag; encode it inside the pattern (`(?m)^…`). -- Per-file results carry `success: false` plus an `error` string for files that failed to rewrite (permission denied, regex compilation error). `total_replacements` sums across files. -- Approval policy mirrors `shell::fs::write`: set per-run by the orchestrator's `approval_required` array, not hardcoded into the function. -- When unsure about the regex, run `shell::fs::grep` with the same pattern first. diff --git a/shell/skills/stat.md b/shell/skills/stat.md deleted file mode 100644 index eaf3c3f0..00000000 --- a/shell/skills/stat.md +++ /dev/null @@ -1,15 +0,0 @@ - - -# Reading a single path's metadata - -## When to use - -- Confirming a file exists and checking its size before deciding whether to read it whole. -- Distinguishing a regular file from a symlink without dereferencing it. -- Pre-flight check for a write target's existing mode before `chmod`-ing. - -## Notes - -- A missing path returns the trigger `Err` (FsError); there is no soft-not-found envelope. Wrap the call when probing optionally. -- Symlinks are not followed; `is_symlink` reports the link itself. To get the target's metadata, follow up with another `stat` on the resolved path. -- Same jail and denylist rules as `shell::fs::ls`. `mode` is an octal string and `mtime` is epoch seconds. diff --git a/shell/skills/status.md b/shell/skills/status.md deleted file mode 100644 index 3697a622..00000000 --- a/shell/skills/status.md +++ /dev/null @@ -1,16 +0,0 @@ - - -# Polling a background job to completion - -## When to use - -- Polling a job spawned by `shell::exec_bg` until it leaves `running`. -- Fetching captured `stdout`/`stderr` once a job has terminated, before retention expires. -- Diagnosing a job that exited with a non-zero `exit_code`. - -## Notes - -- `not_found` (the trigger `Err`) means the `job_id` either never existed or aged out of `cfg.job_retention_secs` (default 1 hour after termination). Do not retry; re-run `shell::exec_bg` if the work still needs doing. -- Per-stream output is bounded by `cfg.max_output_bytes` (default 1 MiB). Once the cap is hit on a stream, the corresponding `*_truncated` flag stays `true` and new bytes are dropped while the job keeps running. -- Use `shell::list` for a lightweight overview of every job. `shell::status` returns the full record, including potentially large captured buffers, so it costs more per call. -- Sandbox-backed jobs that were `shell::kill`-ed flip to `killed` immediately even though the in-VM process may still be running. Their final stdout/stderr arrive on the late `sandbox::exec` response and are not applied if the record is already `killed`. diff --git a/shell/skills/write.md b/shell/skills/write.md deleted file mode 100644 index 49ff63e0..00000000 --- a/shell/skills/write.md +++ /dev/null @@ -1,17 +0,0 @@ - - -# Streaming bytes into a file - -## When to use - -- Persisting a generated artefact to disk inside the jail. -- Streaming a remote download or generated stream straight into a file without an intermediate buffer. -- Bootstrapping files into a sandbox by retargeting with `target: { kind: "sandbox", sandbox_id }`. - -## Notes - -- The wire payload does not accept raw `content: string` or `content_b64`. The caller opens a channel via the SDK, passes the `ContentRef` here, then writes bytes into the channel and closes it. -- When `cfg.fs.max_write_bytes > 0` and the streamed total exceeds the cap, the write is aborted mid-stream with `S218`. The default of `0` means no cap. -- Per-chunk idle timeout is 30s. A caller that opens a write but never sends data and never closes the channel is aborted with `S216 channel idle for 30s, aborting write` so a parked writer cannot leak the temp file. -- The worker writes through a temp file and renames atomically. On crash mid-stream, the temp file is unlinked by `TempGuard`. -- Approval policy is not hardcoded into this function. Whether a turn requires approval before a write lands is set per-run by the orchestrator's `approval_required` array.