From e50db79b60cfee2327fb2f0ed1cab1c4b1490a75 Mon Sep 17 00:00:00 2001 From: i-trytoohard <193449657+i-trytoohard@users.noreply.github.com> Date: Sat, 23 May 2026 23:21:50 +0530 Subject: [PATCH 1/2] test: add isolated onboarding harness for published @aoagents/ao MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a local end-to-end onboarding test for a PUBLISHED npm version of ao: npm install -g → ao start → dashboard serves → ao stop, fully isolated from any real ao install/runtime on the machine. - scripts/test-onboarding.sh: configurable (--version latest|nightly|exact, --mode fresh|coexist, --port, --keep). Isolates via temp npm --prefix, sandbox HOME, auto free ports, runtime: process (no shared tmux), throwaway git repo, AO_CALLER_TYPE=agent, single temp root cleaned on trap EXIT. coexist mode snapshots real running.json + tmux + listening ports before/after and fails if perturbed. - skills/onboarding-test/SKILL.md (+ README entry): lets any agent onboarding-test its own PR/version, documenting both modes, isolation guarantees, and cleanup. Run locally / on demand (not gated per-PR — the published package, not the PR's source, is what it exercises). Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/test-onboarding.sh | 354 ++++++++++++++++++++++++++++++++ skills/README.md | 1 + skills/onboarding-test/SKILL.md | 116 +++++++++++ 3 files changed, 471 insertions(+) create mode 100755 scripts/test-onboarding.sh create mode 100644 skills/onboarding-test/SKILL.md diff --git a/scripts/test-onboarding.sh b/scripts/test-onboarding.sh new file mode 100755 index 0000000000..4587696a8f --- /dev/null +++ b/scripts/test-onboarding.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +# +# test-onboarding.sh — End-to-end onboarding test for a PUBLISHED @aoagents/ao. +# +# Installs the published npm package into a throwaway prefix, runs `ao start` +# against a sandboxed HOME/config, verifies the dashboard serves, then runs +# `ao stop` — all FULLY ISOLATED from any real ao install/runtime on the box. +# +# Two modes: +# --mode fresh Canonical clean onboarding (clean CI / new machine). +# --mode coexist Machine already runs ao. Proves ZERO change to the real +# install: snapshots real running.json + tmux + listening +# ports BEFORE/AFTER and fails loudly if anything moved. +# +# Isolation guarantees (all enforced by this script): +# - npm install -g into a TEMP --prefix; never touches real global modules. +# - Sandbox HOME (temp) so ~/.agent-orchestrator is fully separate; the +# sandbox `ao stop` only ever sees the sandbox running.json. +# - Auto-allocated free dashboard + terminal WS ports (never 3001/14801). +# - runtime: process (no shared tmux server) so the test cannot see/kill +# real tmux sessions. (tmux equivalent — see SKILL.md.) +# - Throwaway git repo to onboard. +# - Non-interactive: AO_CALLER_TYPE=agent + --no-orchestrator. +# - Everything under ONE temp root, auto-removed on EXIT (trap). +# +# Usage: +# scripts/test-onboarding.sh [--version ] \ +# [--mode ] \ +# [--port ] [--keep] [-h|--help] +# +# Exit code 0 = pass, non-zero = fail. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Args +# --------------------------------------------------------------------------- +VERSION="latest" +MODE="fresh" +FIXED_PORT="" +KEEP=0 + +usage() { + sed -n '2,/^set -euo/p' "$0" | sed 's/^#\{0,1\} \{0,1\}//; /^set -euo/d' + exit "${1:-0}" +} + +while [ $# -gt 0 ]; do + case "$1" in + --version) VERSION="${2:?--version needs a value}"; shift 2 ;; + --version=*) VERSION="${1#*=}"; shift ;; + --mode) MODE="${2:?--mode needs a value}"; shift 2 ;; + --mode=*) MODE="${1#*=}"; shift ;; + --port) FIXED_PORT="${2:?--port needs a value}"; shift 2 ;; + --port=*) FIXED_PORT="${1#*=}"; shift ;; + --keep) KEEP=1; shift ;; + -h|--help) usage 0 ;; + *) echo "Unknown argument: $1" >&2; usage 1 ;; + esac +done + +case "$MODE" in + fresh|coexist) ;; + *) echo "Invalid --mode: $MODE (want fresh|coexist)" >&2; exit 2 ;; +esac + +# Map version aliases to an npm spec (latest/nightly are dist-tags). +PKG_SPEC="@aoagents/ao@${VERSION}" + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- +if [ -t 1 ]; then + RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[1;33m'; BLUE=$'\033[0;34m'; NC=$'\033[0m' +else + RED=""; GREEN=""; YELLOW=""; BLUE=""; NC="" +fi +step() { echo "${BLUE}▶ $*${NC}"; } +ok() { echo "${GREEN}✓ $*${NC}"; } +warn() { echo "${YELLOW}! $*${NC}"; } +die() { echo "${RED}✗ $*${NC}" >&2; exit 1; } + +# Portable timeout: GNU `timeout` (Linux) or `gtimeout` (macOS+coreutils); +# falls back to running without a wall-clock cap when neither exists. +TIMEOUT_BIN="" +if command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout" +elif command -v gtimeout >/dev/null 2>&1; then TIMEOUT_BIN="gtimeout"; fi +run_timeout() { # secs cmd... + local secs="$1"; shift + if [ -n "$TIMEOUT_BIN" ]; then "$TIMEOUT_BIN" "$secs" "$@"; else "$@"; fi +} + +# --------------------------------------------------------------------------- +# Capture the REAL environment BEFORE we sandbox anything. +# --------------------------------------------------------------------------- +REAL_HOME="$HOME" +REAL_AO_DIR="$REAL_HOME/.agent-orchestrator" +REAL_RUNNING_JSON="$REAL_AO_DIR/running.json" + +# --------------------------------------------------------------------------- +# Sandbox layout — ONE temp root for trivial cleanup. +# --------------------------------------------------------------------------- +ROOT="$(mktemp -d "${TMPDIR:-/tmp}/ao-onboarding.XXXXXX")" +SANDBOX_HOME="$ROOT/home" +NPM_PREFIX="$ROOT/npm-prefix" +REPO="$ROOT/repo" +GLOBAL_CONFIG="$ROOT/global-agent-orchestrator.yaml" +START_LOG="$ROOT/ao-start.log" +mkdir -p "$SANDBOX_HOME" "$NPM_PREFIX" "$REPO" + +START_PID="" + +# --------------------------------------------------------------------------- +# Cleanup: runs on success, failure, and interrupt. Leaves machine as found. +# --------------------------------------------------------------------------- +cleanup() { + local ec=$? + set +e + echo + step "Cleanup" + # 1. Graceful sandbox stop (only sees the sandbox running.json via HOME). + if [ -x "$NPM_PREFIX/bin/ao" ]; then + HOME="$SANDBOX_HOME" PATH="$NPM_PREFIX/bin:$PATH" \ + AO_CALLER_TYPE=agent AO_GLOBAL_CONFIG="$GLOBAL_CONFIG" AO_CONFIG_PATH="$GLOBAL_CONFIG" \ + run_timeout 30 ao stop --all >/dev/null 2>&1 || true + fi + # 2. Kill the recorded daemon pid tree, if still alive. + if [ -n "$START_PID" ] && kill -0 "$START_PID" 2>/dev/null; then + pkill -P "$START_PID" 2>/dev/null || true + kill "$START_PID" 2>/dev/null || true + sleep 1 + kill -9 "$START_PID" 2>/dev/null || true + fi + # 3. Safety net: reap any orphan process whose args reference our unique + # temp root. This path is unique to this run — cannot match real ao. + pkill -f "$ROOT" 2>/dev/null || true + # 4. Remove the single temp root. + if [ "$KEEP" -eq 1 ]; then + warn "--keep set; leaving sandbox at $ROOT" + else + rm -rf "$ROOT" 2>/dev/null || true + ok "Removed sandbox $ROOT" + fi + exit "$ec" +} +trap cleanup EXIT INT TERM + +# --------------------------------------------------------------------------- +# Free-port allocator (asks the kernel for an ephemeral port via node). +# --------------------------------------------------------------------------- +free_port() { + node -e 'const n=require("net");const s=n.createServer();s.listen(0,"127.0.0.1",()=>{const p=s.address().port;s.close(()=>console.log(p));});' +} + +# Snapshot helpers (coexist mode). All read-only — never mutate real state. +snapshot_real() { + local out="$1" + { + echo "## running.json" + if [ -f "$REAL_RUNNING_JSON" ]; then cat "$REAL_RUNNING_JSON"; else echo "(absent)"; fi + echo + echo "## tmux sessions" + tmux ls 2>/dev/null | sort || echo "(no tmux server)" + echo + echo "## listening tcp ports" + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | awk '{print $1, $9}' | sort -u + else + echo "(lsof unavailable)" + fi + } > "$out" +} + +# --------------------------------------------------------------------------- +echo "${BLUE}══════════════════════════════════════════════════════════${NC}" +echo "${BLUE} AO published-package onboarding test${NC}" +echo "${BLUE} package=${PKG_SPEC} mode=${MODE}${NC}" +echo "${BLUE} sandbox=${ROOT}${NC}" +echo "${BLUE}══════════════════════════════════════════════════════════${NC}" + +command -v node >/dev/null 2>&1 || die "node is required" +command -v npm >/dev/null 2>&1 || die "npm is required" +command -v git >/dev/null 2>&1 || die "git is required" +command -v curl >/dev/null 2>&1 || die "curl is required" + +# coexist: snapshot BEFORE. +BEFORE="$ROOT/real-before.txt" +AFTER="$ROOT/real-after.txt" +if [ "$MODE" = "coexist" ]; then + step "coexist: snapshotting real ao state (before)" + snapshot_real "$BEFORE" + if [ -f "$REAL_RUNNING_JSON" ]; then + ok "Real ao appears to be running (running.json present) — good, we will prove it is untouched" + else + warn "No real running.json found — coexist still verifies nothing changes" + fi +fi + +# --------------------------------------------------------------------------- +# Allocate ports (dashboard + 2 terminal WS). Never 3001 / 14800 / 14801. +# --------------------------------------------------------------------------- +step "Allocating free ports" +if [ -n "$FIXED_PORT" ]; then + PORT="$FIXED_PORT" +else + PORT="$(free_port)" +fi +TERM_PORT="$(free_port)" +DIRECT_TERM_PORT="$(free_port)" +for p in "$PORT" "$TERM_PORT" "$DIRECT_TERM_PORT"; do + case "$p" in + 3001|14800|14801) die "Refusing reserved port $p" ;; + esac +done +ok "dashboard=$PORT terminal=$TERM_PORT directTerminal=$DIRECT_TERM_PORT" + +# --------------------------------------------------------------------------- +# Install published package into the temp prefix. +# --------------------------------------------------------------------------- +step "Installing $PKG_SPEC into temp prefix" +npm install -g --prefix "$NPM_PREFIX" "$PKG_SPEC" >"$ROOT/npm-install.log" 2>&1 \ + || { cat "$ROOT/npm-install.log" >&2; die "npm install failed"; } +[ -x "$NPM_PREFIX/bin/ao" ] || die "ao binary not found in temp prefix after install" +ok "Installed" + +# Sandbox env used for ALL ao invocations from here on. +export HOME="$SANDBOX_HOME" +export PATH="$NPM_PREFIX/bin:$PATH" +export AO_CALLER_TYPE=agent +export AO_GLOBAL_CONFIG="$GLOBAL_CONFIG" +export AO_CONFIG_PATH="$GLOBAL_CONFIG" +export PORT="$PORT" +export TERMINAL_PORT="$TERM_PORT" +export DIRECT_TERMINAL_PORT="$DIRECT_TERM_PORT" + +step "Verifying ao binary resolves to the sandbox" +RESOLVED="$(command -v ao)" +case "$RESOLVED" in + "$NPM_PREFIX"/*) ok "ao -> $RESOLVED" ;; + *) die "ao resolved to $RESOLVED (expected under $NPM_PREFIX)" ;; +esac +ao --version || die "ao --version failed" + +# --------------------------------------------------------------------------- +# Throwaway git repo + sandbox global config. +# --------------------------------------------------------------------------- +step "Creating throwaway git repo" +git -C "$REPO" init -q +git -C "$REPO" config user.email "onboarding@example.com" +git -C "$REPO" config user.name "Onboarding Test" +git -C "$REPO" config commit.gpgsign false +echo "# onboarding test repo" > "$REPO/README.md" +git -C "$REPO" add . +git -C "$REPO" commit -qm "initial commit" +ok "Repo at $REPO" + +step "Writing sandbox global config" +cat > "$GLOBAL_CONFIG" <"$START_LOG" 2>&1 ) & +START_PID=$! +ok "ao start launched (wrapper pid $START_PID)" + +# Wait for running.json to appear in the SANDBOX home. +SANDBOX_RUNNING="$SANDBOX_HOME/.agent-orchestrator/running.json" +step "Waiting for sandbox running.json" +RUNNING_PID="" +for _ in $(seq 1 100); do + if [ -f "$SANDBOX_RUNNING" ]; then + RUNNING_PID="$(node -e 'const j=require(process.argv[1]);process.stdout.write(String(j.pid||""))' "$SANDBOX_RUNNING" 2>/dev/null || true)" + [ -n "$RUNNING_PID" ] && break + fi + kill -0 "$START_PID" 2>/dev/null || { cat "$START_LOG" >&2; die "ao start exited early"; } + sleep 0.3 +done +[ -n "$RUNNING_PID" ] || { cat "$START_LOG" >&2; die "running.json never appeared"; } +ok "running.json registered pid $RUNNING_PID on port $PORT" + +# Wait for the dashboard to serve. +step "Waiting for dashboard on http://127.0.0.1:$PORT" +DASH_OK=0 +for _ in $(seq 1 100); do + if curl -sf "http://127.0.0.1:$PORT/api/sessions" >/dev/null 2>&1; then + DASH_OK=1; break + fi + kill -0 "$START_PID" 2>/dev/null || { cat "$START_LOG" >&2; die "ao start exited before dashboard came up"; } + sleep 0.5 +done +[ "$DASH_OK" -eq 1 ] || { tail -n 40 "$START_LOG" >&2; die "dashboard did not respond on port $PORT"; } +ok "Dashboard responding (/api/sessions 200)" + +# Confirm the served port is the sandbox port (and NOT 3001). +curl -sf "http://127.0.0.1:3001/api/sessions" >/dev/null 2>&1 \ + && warn "port 3001 also responds (the REAL ao) — expected in coexist mode" \ + || true + +# --------------------------------------------------------------------------- +# Stop the sandbox daemon — must operate ONLY on the sandbox. +# --------------------------------------------------------------------------- +step "Stopping sandbox ao (ao stop --all)" +( cd "$REPO" && run_timeout 40 ao stop --all ) || die "ao stop --all failed/timed out" + +step "Verifying clean shutdown" +for _ in $(seq 1 40); do + kill -0 "$RUNNING_PID" 2>/dev/null || break + sleep 0.5 +done +kill -0 "$RUNNING_PID" 2>/dev/null && die "daemon pid $RUNNING_PID still alive after stop" +[ -f "$SANDBOX_RUNNING" ] && die "sandbox running.json still present after stop" +curl -sf "http://127.0.0.1:$PORT/api/sessions" >/dev/null 2>&1 \ + && die "dashboard still serving on $PORT after stop" || true +ok "Sandbox daemon stopped, running.json cleared, port released" + +# --------------------------------------------------------------------------- +# coexist: prove the real ao is untouched. +# --------------------------------------------------------------------------- +if [ "$MODE" = "coexist" ]; then + step "coexist: snapshotting real ao state (after) and diffing" + snapshot_real "$AFTER" + if diff -u "$BEFORE" "$AFTER" > "$ROOT/real-diff.txt"; then + ok "Real ao state IDENTICAL before/after (running.json, tmux, ports)" + else + echo "${RED}Real ao state CHANGED — isolation breach:${NC}" >&2 + cat "$ROOT/real-diff.txt" >&2 + die "coexist invariant violated" + fi +fi + +echo +echo "${GREEN}══════════════════════════════════════════════════════════${NC}" +echo "${GREEN} PASS — onboarding works for ${PKG_SPEC} (mode=${MODE})${NC}" +echo "${GREEN}══════════════════════════════════════════════════════════${NC}" +exit 0 diff --git a/skills/README.md b/skills/README.md index e70ce2c7a6..5900a7a960 100644 --- a/skills/README.md +++ b/skills/README.md @@ -10,6 +10,7 @@ Reusable skill documents for AI coding agents working on this repository. Each s | [`agent-orchestrator/`](agent-orchestrator/SKILL.md) | Architecture and conventions for working on the agent-orchestrator codebase | | [`release-notes/`](release-notes/ao-weekly-release/SKILL.md) | Generate weekly release notes from git history | | [`social-media/`](social-media/SKILL.md) | Social media post generation | +| [`onboarding-test/`](onboarding-test/SKILL.md) | Verify a published `@aoagents/ao` version onboards cleanly (install → start → dashboard → stop), fully isolated from any real ao on the machine | ## How to Use diff --git a/skills/onboarding-test/SKILL.md b/skills/onboarding-test/SKILL.md new file mode 100644 index 0000000000..33dc59d079 --- /dev/null +++ b/skills/onboarding-test/SKILL.md @@ -0,0 +1,116 @@ +--- +name: onboarding-test +description: Verify a published @aoagents/ao npm version onboards cleanly (install → ao start → dashboard up → ao stop), fully isolated from any real ao install/runtime on the machine. +trigger: You want to confirm your PR/release does not break first-run onboarding, or to smoke-test a published version before/after a release. +--- + +# Onboarding Test Skill + +Prove that a **published** `@aoagents/ao` version onboards end-to-end on a fresh +machine — `npm install -g` → `ao start` → dashboard serves → `ao stop` — without +touching any real ao install or running daemon on the box. + +The whole flow is one script: [`scripts/test-onboarding.sh`](../../scripts/test-onboarding.sh). + +## When to use + +- Before cutting a release: `--version nightly` (tracks main) should be green. +- After publishing: `--version latest` confirms the released tarball onboards. +- On a machine that already runs ao (your dev box): `--mode coexist` proves the + test leaves your real daemon, sessions, tmux, and ports untouched. + +## Run it + +```bash +# Canonical clean onboarding (clean CI / new machine): +scripts/test-onboarding.sh --version latest --mode fresh + +# On a machine already running ao — proves ZERO change to the real install: +scripts/test-onboarding.sh --version latest --mode coexist + +# A specific published version, or the nightly tag: +scripts/test-onboarding.sh --version 0.9.1 --mode fresh +scripts/test-onboarding.sh --version nightly --mode fresh +``` + +Exit code `0` = onboarding works. Non-zero = it does not — read the printed step +that failed. Use `--keep` to leave the sandbox in place for debugging, and +`--port N` to pin the dashboard port (otherwise a free one is auto-allocated). + +Requirements on the runner: `node`, `npm`, `git`, `curl` (all present on CI and +dev machines). `tmux` is **not** required — the test uses `runtime: process`. + +## Two modes + +| Mode | Assumes | What it adds | +|------|---------|--------------| +| `fresh` | No existing ao (clean CI / new machine) | Canonical clean onboarding path. | +| `coexist` | Machine already runs ao | Snapshots the **real** `~/.agent-orchestrator/running.json`, `tmux ls`, and listening TCP ports BEFORE and AFTER, then asserts they are byte-for-byte identical. Fails loudly if the test perturbed anything. | + +## Isolation guarantees (why it is safe to run next to a live daemon) + +The script enforces all of these — it never mutates real state: + +- **Temp npm prefix.** `npm install -g --prefix `; that `bin/` goes first on + `PATH`. Real global `node_modules` are never touched. +- **Sandbox `HOME`.** A temp `HOME` means `~/.agent-orchestrator` (running.json, + last-stop.json, sessions, locks) is a separate tree. The sandbox `ao stop` + reads only the sandbox `running.json`, so it can only kill the sandbox daemon. +- **Auto-allocated free ports** for the dashboard and both terminal WS servers; + the script refuses the reserved `3001` / `14800` / `14801`. +- **`runtime: process`**, not tmux — the sandbox shares no tmux server with the + real ao, so it cannot see or kill real tmux sessions. (This is the + "dedicated tmux socket *or equivalent*" requirement: process runtime sidesteps + the shared default tmux socket entirely.) +- **Throwaway git repo** (`git init` + one commit) as the onboarded project. +- **Non-interactive:** `AO_CALLER_TYPE=agent` plus `ao start --no-orchestrator` + (no LLM key needed — scoped to install + CLI + dashboard-up + clean stop). +- **One temp root**, removed on `trap EXIT` (success, failure, or interrupt): + graceful sandbox `ao stop --all`, then kill the recorded daemon pid tree, then + a final `pkill -f ` safety net (the path is unique to the run, so it + can never match the real ao), then `rm -rf` the root. + +The env knobs the script sets (verified against `packages/core/src/paths.ts`, +`config.ts`, `global-config.ts`, and `packages/cli/src/lib/running-state.ts`): +`HOME` (controls `~/.agent-orchestrator`), `AO_CONFIG_PATH` + `AO_GLOBAL_CONFIG` +(config location), `AO_CALLER_TYPE=agent` (non-interactive), `PORT` / +`TERMINAL_PORT` / `DIRECT_TERMINAL_PORT` (server ports). + +## What it verifies + +1. `npm install -g` of the published package succeeds and `ao` resolves to the + sandbox prefix. +2. `ao --version` runs. +3. `ao start --no-orchestrator` registers `running.json` in the sandbox HOME. +4. The dashboard serves (`GET /api/sessions` → 200) on the sandbox port. +5. `ao stop --all` exits cleanly, the daemon pid dies, `running.json` is removed, + and the port is released. +6. In `coexist` mode, the real ao snapshot is unchanged. + +## CI + +This harness is a **local / on-demand** test — run it by hand (or wire it into +your own pipeline). It is intentionally not gated per-PR: a PR's own code is not +yet published, so testing the *published* package belongs after a build is +published (nightly tracks main; a release publishes `latest`), not on the PR. + +For per-PR, source-build onboarding coverage see the existing "Test Fresh +Onboarding" job in `.github/workflows/onboarding-test.yml`. + +To run it in CI later, a single step suffices on a Node 20 runner with `git` and +`curl` available: + +```yaml +- run: bash scripts/test-onboarding.sh --version nightly --mode fresh +``` + +## Onboarding-test your own PR/version + +1. Land your change and let it publish (a `nightly` build of main is published + automatically; a release publishes `latest`). +2. Run `scripts/test-onboarding.sh --version nightly --mode fresh` (or the exact + `0.x.y-nightly-` once you know it from `npm view @aoagents/ao dist-tags`). +3. If you are on a machine with a live ao daemon, prefer `--mode coexist` so you + also prove your test run did not disturb your working daemon. +4. Green = onboarding intact. Red = read the failing step; `--keep` then inspect + `/ao-start.log`. From 916045f173f0c6fe9418f8b69e890bff1922fb35 Mon Sep 17 00:00:00 2001 From: i-trytoohard <193449657+i-trytoohard@users.noreply.github.com> Date: Sun, 24 May 2026 00:12:54 +0530 Subject: [PATCH 2/2] test: support testing a branch via --registry + document nightly-sha route Adds --registry so an unmerged branch published to a throwaway local registry (verdaccio) can be onboarding-tested with full packaging fidelity, keeping every isolation guarantee. Documents that merged commits are already covered by the auto-published per-commit nightly (--version 0.x.y-nightly-), and why a source/npm-link build cannot reproduce packaging bugs. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/test-onboarding.sh | 31 ++++++++++++++++++++++++++++--- skills/onboarding-test/SKILL.md | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/scripts/test-onboarding.sh b/scripts/test-onboarding.sh index 4587696a8f..164ebe24ea 100755 --- a/scripts/test-onboarding.sh +++ b/scripts/test-onboarding.sh @@ -25,9 +25,23 @@ # # Usage: # scripts/test-onboarding.sh [--version ] \ +# [--registry ] \ # [--mode ] \ # [--port ] [--keep] [-h|--help] # +# Testing a BRANCH instead of a release: +# The grok-style packaging bugs this harness catches only exist in the +# PUBLISHED tarball, so a branch must be *published* somewhere first — a +# source build does not reproduce them. Two faithful routes: +# 1. Merged to main: a per-commit nightly is auto-published. Run +# --version 0.x.y-nightly- (see: npm view @aoagents/ao versions) +# to test that exact commit's real packaged artifact — no setup. +# 2. Unmerged PR branch: publish it to a throwaway local registry +# (verdaccio) and point the harness there: +# npx verdaccio & # http://localhost:4873 +# pnpm -r publish --registry http://localhost:4873 --no-git-checks +# scripts/test-onboarding.sh --registry http://localhost:4873 --version +# # Exit code 0 = pass, non-zero = fail. set -euo pipefail @@ -39,6 +53,7 @@ VERSION="latest" MODE="fresh" FIXED_PORT="" KEEP=0 +REGISTRY="" usage() { sed -n '2,/^set -euo/p' "$0" | sed 's/^#\{0,1\} \{0,1\}//; /^set -euo/d' @@ -53,6 +68,8 @@ while [ $# -gt 0 ]; do --mode=*) MODE="${1#*=}"; shift ;; --port) FIXED_PORT="${2:?--port needs a value}"; shift 2 ;; --port=*) FIXED_PORT="${1#*=}"; shift ;; + --registry) REGISTRY="${2:?--registry needs a value}"; shift 2 ;; + --registry=*) REGISTRY="${1#*=}"; shift ;; --keep) KEEP=1; shift ;; -h|--help) usage 0 ;; *) echo "Unknown argument: $1" >&2; usage 1 ;; @@ -215,10 +232,18 @@ done ok "dashboard=$PORT terminal=$TERM_PORT directTerminal=$DIRECT_TERM_PORT" # --------------------------------------------------------------------------- -# Install published package into the temp prefix. +# Install the package into the temp prefix. +# Default registry = npmjs. --registry points at a throwaway local registry +# (verdaccio) so an unmerged branch can be tested with full packaging fidelity. # --------------------------------------------------------------------------- -step "Installing $PKG_SPEC into temp prefix" -npm install -g --prefix "$NPM_PREFIX" "$PKG_SPEC" >"$ROOT/npm-install.log" 2>&1 \ +REGISTRY_ARGS=() +if [ -n "$REGISTRY" ]; then + REGISTRY_ARGS=(--registry "$REGISTRY") + step "Installing $PKG_SPEC from $REGISTRY into temp prefix" +else + step "Installing $PKG_SPEC into temp prefix" +fi +npm install -g --prefix "$NPM_PREFIX" ${REGISTRY_ARGS[@]+"${REGISTRY_ARGS[@]}"} "$PKG_SPEC" >"$ROOT/npm-install.log" 2>&1 \ || { cat "$ROOT/npm-install.log" >&2; die "npm install failed"; } [ -x "$NPM_PREFIX/bin/ao" ] || die "ao binary not found in temp prefix after install" ok "Installed" diff --git a/skills/onboarding-test/SKILL.md b/skills/onboarding-test/SKILL.md index 33dc59d079..50d021e2bf 100644 --- a/skills/onboarding-test/SKILL.md +++ b/skills/onboarding-test/SKILL.md @@ -37,6 +37,38 @@ Exit code `0` = onboarding works. Non-zero = it does not — read the printed st that failed. Use `--keep` to leave the sandbox in place for debugging, and `--port N` to pin the dashboard port (otherwise a free one is auto-allocated). +## Testing a branch instead of a release + +This harness deliberately tests the **published tarball**, because the bugs it +catches (e.g. a plugin's `dist` requiring a `package.json` that the published +`files` list omits) only exist in the packed artifact — a plain source/branch +build resolves those paths fine and would give a false green. So a branch must +be **published somewhere** first. Two faithful routes: + +1. **Merged to `main` — zero setup.** Every commit on main publishes a nightly: + ```bash + npm view @aoagents/ao versions # find 0.x.y-nightly- for your commit + scripts/test-onboarding.sh --version 0.x.y-nightly- --mode fresh + ``` + That installs and exercises the **real packaged artifact** of that exact + commit. This is how you confirm a merged fix (e.g. a release retarget) before + it reaches `latest`. + +2. **Unmerged PR branch — throwaway local registry.** Publish the branch's + packages to [verdaccio](https://verdaccio.org/) and point the harness at it + with `--registry`: + ```bash + npx verdaccio & # http://localhost:4873 + pnpm -r publish --registry http://localhost:4873 --no-git-checks + scripts/test-onboarding.sh --registry http://localhost:4873 \ + --version --mode fresh + ``` + `--registry` only changes where `npm install` pulls from — every isolation + guarantee below still holds. + +Do **not** try to onboarding-test a branch by `npm link`/source build: it cannot +reproduce packaging bugs, which is the whole point of this harness. + Requirements on the runner: `node`, `npm`, `git`, `curl` (all present on CI and dev machines). `tmux` is **not** required — the test uses `runtime: process`.