From 3929c4f6037fa5335679012c33392b767478104a Mon Sep 17 00:00:00 2001 From: ree2raz Date: Thu, 4 Jun 2026 19:35:13 +0530 Subject: [PATCH 1/2] docs: pivot to single v1.0 SPEC; remove v0.1/v0.2 docs Replace the v0.1 spec set (PROJECT_SPEC, SCHEMA_V0.1, CLI_V01_SPEC, CORE_CHECKS_SPEC, DETECTOR_AUTHENTICATION_SPEC, ARCHITECTURE) and the v0.2 roadmap (SCOPE.md) with a single authoritative docs/SPEC.md for v1.0. Add a minimal CLAUDE.md pointing to it and stating the non-goals (no LLM in the verification path, no semantic detectors). --- CLAUDE.md | 17 + SCOPE.md | 95 ----- docs/ARCHITECTURE.md | 49 --- docs/CLI_V01_SPEC.md | 325 ----------------- docs/CORE_CHECKS_SPEC.md | 259 -------------- docs/DETECTOR_AUTHENTICATION_SPEC.md | 335 ------------------ docs/PROJECT_SPEC.md | 485 ------------------------- docs/SCHEMA_V0.1.md | 410 --------------------- docs/SPEC.md | 511 +++++++++++++++++++++++++++ 9 files changed, 528 insertions(+), 1958 deletions(-) create mode 100644 CLAUDE.md delete mode 100644 SCOPE.md delete mode 100644 docs/ARCHITECTURE.md delete mode 100644 docs/CLI_V01_SPEC.md delete mode 100644 docs/CORE_CHECKS_SPEC.md delete mode 100644 docs/DETECTOR_AUTHENTICATION_SPEC.md delete mode 100644 docs/PROJECT_SPEC.md delete mode 100644 docs/SCHEMA_V0.1.md create mode 100644 docs/SPEC.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6aafb8a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,17 @@ +# CLAUDE.md + +`docs/SPEC.md` is the authoritative spec. Read it before making design decisions; +when code conflicts with it, the code is wrong until the spec is deliberately changed. + +**attest verifies structure and outcomes, never behavior or semantics.** Hard rules +(from SPEC §2): + +- No LLM anywhere in the verification path. Determinism is the product. +- No semantic/behavioral verification. Behavioral claims return `unverifiable` with + an LLM-review pointer — never a heuristic. Do **not** (re)introduce semantic + detectors into the verification path. +- `@attest/detectors-ts` is a demoted, opt-in, best-effort plugin. It must never + affect the exit code or gate CI. + +The fixture corpus (SPEC §10) is the regression oracle: a change that breaks an +oracle case is wrong by definition. diff --git a/SCOPE.md b/SCOPE.md deleted file mode 100644 index 1c5e468..0000000 --- a/SCOPE.md +++ /dev/null @@ -1,95 +0,0 @@ -# attest — Roadmap & Scope - -This file defines the scope of each release after v0.1. v0.1 is shipped and frozen -(see `docs/PROJECT_SPEC.md` §4 for what it covers). The governing principles below -are inherited from the v0.1 spec and are **non-negotiable across all versions**: - -- Deterministic checks only — no LLM inference anywhere in the verification path. -- The schema is the product; schema stability outranks feature count. -- Stateless by default; no hosted service, no accounts. -- Every rejection returns a structured, actionable reason code. - ---- - -## v0.2 — Breadth of behavior, plus agent integration - -**Thesis**: v0.1 proved the core loop on one behavior (`authentication`) in one language -(TypeScript). v0.2 proves the loop generalizes — across more behavioral properties and -into the agent's own workflow via MCP — without changing the schema shape. - -### IN (must ship) - -1. **Remaining TypeScript behavioral detectors.** Implement the nine `BehavioralProperty` - values not covered in v0.1, each as a dedicated detector with its own fixture suite - (≥10 fixtures each, same `*.expected.json` convention as `authentication`): - - `input_validation` - - `error_handling` - - `null_check` - - `authorization` - - `rate_limiting` - - `logging` - - `sanitization` - - `timeout` - - `retry_logic` - - Suggested sequencing (highest reviewer value first): `input_validation` → - `error_handling` → `authorization` → `null_check` → `sanitization` → - `rate_limiting` → `logging` → `timeout` → `retry_logic`. Each ships as its own PR. - -2. **`@attest/mcp` — Model Context Protocol server.** Exposes `attest verify` as an MCP - tool so an agent can self-verify its claims inside the editing session, before handing - off to a human. Stateless: manifest + diff + repo-root in, `VerdictReport` out. Reuses - `@attest/core` verbatim — no verification logic lives in the MCP layer. - -3. **Pre-declaration / TDD-style flow.** Allow a manifest to be emitted _before_ the diff - exists (claims as intent), then verified against the diff once changes land. Requires a - manifest `mode` discriminator (`declared` vs `asserted`) — this is the one permitted - schema addition in v0.2 and triggers `schema_version` `0.2`. - -4. **Published npm packages.** Release `@attest/schema`, `@attest/core`, `@attest/cli` - (and `@attest/mcp`) to npm under the `@attest/*` scope via changesets. v0.1 was - repo-local only; v0.2 is the first installable release. - -### OUT (deferred) - -- GitHub App / PR-comment renderer → **v0.3** -- Python-language detectors → **v0.3** (port the detector framework to a Python AST - backend once the TS behavioral catalog is complete and stable) -- Multi-session provenance chaining → **v0.3+** -- Test _execution_ or coverage measurement — **never** (we check test _presence_, never - _behavior_; defer to c8/istanbul/Codecov) -- Any form of LLM inference in the verifier — **never** - -### Definition of "v0.2 ready" - -- [ ] All ten TS behavioral detectors implemented; each fixture matches its expected verdict exactly -- [ ] Each detector ≥85% line coverage (same gate as `authentication` in v0.1) -- [ ] `@attest/mcp` server starts, registers the `verify` tool, returns a `VerdictReport` for the golden-path inputs -- [ ] Schema `0.2` adds only the `mode` discriminator; `0.1` manifests still validate (or are explicitly rejected with a version reason code — decide and document) -- [ ] `pnpm lint && pnpm test && pnpm build` green from a clean checkout -- [ ] Packages publishable: `changeset version` + dry-run `pnpm publish --dry-run` succeed -- [ ] README documents MCP setup and the pre-declaration flow -- [ ] SCOPE.md and PROJECT_SPEC.md updated to reflect shipped scope - -### What would invalidate this plan - -- If adding a second behavioral detector forces a schema change beyond `mode`, the - "schema is stable" assumption is wrong — stop and revise the schema spec before - building the remaining eight. -- If MCP adoption among target agents stalls, demote `@attest/mcp` below the detector - work; the detectors are the load-bearing deliverable. - ---- - -## v0.3 — Surface and second language (sketch, not yet committed) - -- GitHub App / PR-comment renderer: post the `VerdictReport` as a PR review comment, - with `reviewer_focus` as the comment body. Reuses `@attest/core`; adds only a renderer - and a webhook handler. -- `@attest/detectors-py`: Python-language detectors, beginning with `authentication`, - reusing the v0.2 detector interface against a Python AST backend. -- Multi-session provenance chaining: link manifests across sessions to verify a sequence - of agent handoffs. - -This section is a direction, not a contract. It will be promoted to a full IN/OUT scope -when v0.2 is shipped. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md deleted file mode 100644 index c884443..0000000 --- a/docs/ARCHITECTURE.md +++ /dev/null @@ -1,49 +0,0 @@ -# Architecture — attest v0.1 - -## Overview - -``` -manifest.json + changes.diff - │ - ▼ - @attest/cli (attest verify) - │ - ├── validates manifest via @attest/schema - ├── parses diff via @attest/core (parse-diff) - └── calls core.verify() - │ - ├── undeclared.ts — computes (diff_paths ∪ files_touched) − declared_files - ├── checks/ — symbol_exists, removed, test_covers, signature_matches - │ └── locate-route.ts — shared route finder (used by checks + detectors) - └── detector registry - └── @attest/detectors-ts - └── authentication/ - ├── express.ts - ├── fastify.ts - ├── nestjs.ts - ├── koa.ts - └── raw-node.ts -``` - -## Package dependency graph - -``` -@attest/schema → (no internal deps) -@attest/core → @attest/schema -@attest/detectors-ts → @attest/core, @attest/schema -@attest/cli → @attest/schema, @attest/core, @attest/detectors-ts -``` - -No circular dependencies. Each package imports only the public `index.ts` of its dependencies. - -## Key design decisions - -**Deterministic only.** The verifier never calls an LLM. Every verdict is reproducible from the same inputs. - -**Read-from-disk, not diff-apply.** Post-diff content comes from reading the file at `repoRoot/path`. The diff is used only to enumerate changed files. This avoids a patch-apply dependency and works naturally with any checkout-based workflow. - -**`locateRoute()` is shared.** Both `symbol_exists` checks on endpoint targets and the authentication detector use the same route-location logic, exported from `@attest/core`. No duplication. - -**Evidence is pedagogical.** The reviewer should learn *why* the verdict is what it is, not just *what* it is. Every chain member classification is reported with the layer that matched it. - -**`reviewer_focus` is the primary human output.** The full claim list is for completeness; the focus section tells the reviewer exactly where to look. It is omitted only when every claim is `verified` and there are zero undeclared findings. diff --git a/docs/CLI_V01_SPEC.md b/docs/CLI_V01_SPEC.md deleted file mode 100644 index f59df12..0000000 --- a/docs/CLI_V01_SPEC.md +++ /dev/null @@ -1,325 +0,0 @@ -# CLI Specification: `attest verify` (v0.1) - -Status: **Ready for implementation.** -Package: `@attest/cli` -Companions: `PROJECT_SPEC.md`, `SCHEMA_V0.1.md`, `DETECTOR_AUTHENTICATION_SPEC.md` - ---- - -## 1. Purpose - -Prove the core loop works end-to-end in a single process: manifest in, diff in, verdict out. This CLI is the "hello world" acceptance artifact for v0.1. Every other future surface (MCP server, GitHub App) is a different driver over the same core. - ---- - -## 2. Binary and command structure - -**Binary**: `attest` (installed via `npm i -g @attest/cli` in future; repo-local via `pnpm link` for v0.1) - -**Commands in v0.1**: exactly one — `verify` - -``` -attest verify --manifest --diff [options] -``` - -No other commands ship in v0.1. Calling `attest` with no args prints help and exits 0. Calling `attest ` prints help and exits 64 (EX_USAGE). - ---- - -## 3. Arguments and flags - -| Flag | Short | Required | Default | Type | Description | -|-------------------|-------|----------|---------|-----------|------------------------------------------------------------------------------| -| `--manifest` | `-m` | yes | — | path | Path to the manifest JSON conforming to `SCHEMA_V0.1.md` | -| `--diff` | `-d` | yes | — | path or `-` | Path to a unified-diff file, or `-` to read from stdin | -| `--repo-root` | `-r` | no | `process.cwd()` | path | Repository root for resolving relative paths in the manifest | -| `--format` | `-f` | no | `human` | enum | Output format: `human` or `json` | -| `--no-color` | | no | `false` | boolean | Disable ANSI color in `human` format. Also respected: `NO_COLOR` env var | -| `--verbose` | `-v` | no | `false` | boolean | Emit detector-level diagnostics to stderr | -| `--help` | `-h` | no | — | boolean | Print help to stdout, exit 0 | -| `--version` | `-V` | no | — | boolean | Print version to stdout, exit 0 | - -**Argument parsing**: `clipanion`. Unknown flags → exit 64 with a clear error on stderr. Missing required flag → exit 64. - ---- - -## 4. Input contracts - -### 4.1 Manifest - -- Must be valid JSON -- Must validate against `@attest/schema` -- Path resolution: absolute paths used as-is; relative paths resolved against `process.cwd()` -- File not found → exit 66 (EX_NOINPUT) -- JSON parse error → exit 65 (EX_DATAERR) -- Schema validation failure → exit 2, with every validation error printed to stderr (one per line, `path: code: message` format) - -### 4.2 Diff - -- Must be a unified diff (output of `git diff`, `git format-patch`, or equivalent) -- When `--diff -` is specified, read from stdin until EOF -- Paths inside the diff are interpreted relative to `--repo-root` -- Binary files in the diff → logged to stderr as `warn: binary file skipped: `, continue processing -- Empty diff (no changes) → exit 65 with stderr `error: diff contains no changes` - -### 4.3 Repo root - -- Must be a directory that exists -- Missing → exit 66 -- Used to resolve paths in `session.files_touched` and `claim.target.path` when those are relative - ---- - -## 5. Output contracts - -### 5.1 `human` format (stdout) - -Matches the mockup in `SCHEMA_V0.1.md` §11 exactly. Structure: - -``` -🤖 Agent: () · tool calls · files touched -📝 Task: - -📋 Declared changes (): - - - ... - -⚠️ Undeclared modifications (): - • — symbol `` modified but not in any claim - • — file modified but not in any claim - ... - -🔍 Reviewer focus: - 1. - 2. - ... -``` - -**Icon mapping**: -- `verified` → `✅` -- `unverified` → `❌` -- `partial` → `⚠️` -- `unverifiable` → `ⓘ` - -**Rules**: -- The "Undeclared modifications" section is omitted if empty -- The "Reviewer focus" section is omitted only when every claim is `verified` AND there are zero undeclared findings. Otherwise it always appears -- One-line evidence summary: the first evidence entry's `note`, or a fallback constructed from `claim.target` and `reason_code` if no evidence — agents implementing this should see §5.3 -- Color: verdict icons are colorized when TTY and `--no-color` not set. `verified` → green, `unverified` → red, `partial` → yellow, `unverifiable` → cyan - -**`reviewer_focus` ordering** (applies to both human and JSON output; arrays are order-sensitive in the golden-path test): -1. `unverified` and `partial` ClaimResult entries, in manifest claim-id order -2. UndeclaredFinding entries, sorted lexicographically by `path` then `symbol` (nulls last) -`unverifiable` claims do not appear in `reviewer_focus`. - -**`reviewer_focus[].reason` string templates** (used verbatim; these are separate from the per-claim one-line summary in §5.3): - -| Condition | Template | -|---|---| -| `behavior_present` claim, `unverified` or `partial` | `" failed — not detected"` | -| Other check kind, `unverified`, has `reason_code` | `" — "` | -| Other check kind, `unverified`, no `reason_code` | `" — failed"` | -| Undeclared symbol finding | `"undeclared change to \`\`"` | -| Undeclared file finding | `"undeclared file "` | - -Humanization: replace underscores with spaces. Example: `no_auth_in_chain` → `"no auth in chain"`, `rate_limiting` → `"rate limiting"`. - -### 5.2 `json` format (stdout) - -Single JSON document matching `VerdictReport` from `PROJECT_SPEC.md` §8.2. Pretty-printed with 2-space indentation. Final newline. - -Example (for the golden-path test in §7): - -```json -{ - "manifest_hash": "sha256:", - "summary": { - "total_claims": 4, - "verified": 3, - "unverified": 1, - "partial": 0, - "unverifiable": 0, - "undeclared_files": 0, - "undeclared_symbols": 1 - }, - "claims": [ - { - "claim_id": "c1", - "verdict": "verified", - "evidence": [ - { "kind": "symbol", "path": "src/auth/email.ts", "symbol": "EmailVerificationService", "note": "class declaration found" } - ] - }, - { - "claim_id": "c2", - "verdict": "verified", - "evidence": [ - { "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /verify", "note": "express route registered" } - ] - }, - { - "claim_id": "c3", - "verdict": "unverified", - "reason_code": "no_auth_in_chain", - "evidence": [ - { "kind": "middleware-chain", "path": "src/routes/auth.ts", "symbol": "POST /verify", "note": "no rate_limiting middleware found in chain" } - ] - }, - { - "claim_id": "c4", - "verdict": "verified", - "evidence": [ - { "kind": "test", "path": "src/auth/email.test.ts", "symbol": "EmailVerificationService", "note": "3 tests reference subject symbol" } - ] - } - ], - "undeclared": [ - { "type": "symbol", "path": "src/auth/email.ts", "symbol": "generateSecret" } - ], - "reviewer_focus": [ - { "claim_id": "c3", "reason": "claim unverified: rate_limiting not detected" }, - { "undeclared": { "type": "symbol", "path": "src/auth/email.ts", "symbol": "generateSecret" }, "reason": "undeclared symbol modification" } - ] -} -``` - -Note: v0.1 ships with the authentication detector only. The c3 rate_limiting example above is drawn from `SCHEMA_V0.1.md` §11 for consistency, but in v0.1 the rate_limiting check will return `unverifiable` with reason `detector_not_implemented`. See §7 for the revised golden-path that v0.1 actually verifies. - -### 5.3 Evidence summarization rules (human format) - -For each claim, the one-line summary is produced by: - -1. Find the first evidence entry with a non-empty `note` field (scanning all entries, not just index 0). If found → use that note verbatim (truncated to 120 chars). -2. Else if `reason_code` is present → ` at :` -3. Else → ` in ` - -`reason_code_humanized` converts `snake_case` to human phrase, e.g., `no_auth_in_chain` → "no auth in chain". - -Note: the authentication detector places a no-note route summary entry at `evidence[0]` (see `DETECTOR_AUTHENTICATION_SPEC.md §6`), so rule 1 finds the first *meaningful* note in `evidence[1+]` for verified claims, and falls through to rule 2 for unverified/no-auth claims where no entry has a note. - -### 5.4 Stderr - -- `--verbose` enables per-detector diagnostics (e.g., "running authentication detector on claim c3") -- Warnings (binary files, etc.) always go to stderr regardless of `--verbose` -- Schema validation errors always go to stderr -- No progress bars, no spinners — pipe-friendly - ---- - -## 6. Exit codes - -| Code | Meaning | -|------|---------------------------------------------------------------------------| -| 0 | All claims verified; zero undeclared findings | -| 1 | At least one claim is `unverified` or `partial`, or ≥1 undeclared finding | -| 2 | Manifest failed schema validation | -| 64 | Usage error (unknown flag, missing required flag, unknown command) | -| 65 | Input data error (malformed JSON, empty diff, etc.) | -| 66 | Input file not found (manifest, diff, or repo-root missing) | -| 70 | Internal error (parse error in verifier, unexpected exception) | - -`unverifiable` alone does **not** cause exit 1 — it is a neutral state ("manual review needed"). Only `unverified`, `partial`, or undeclared findings cause exit 1. Rationale: the CLI should not block CI on claims we cannot check (`cannot_express`, `refactor`). Those are always for humans. - ---- - -## 7. Golden-path end-to-end test (v0.1 version) - -The CLI is acceptance-complete when this test passes exactly. - -### 7.1 Setup - -Create a minimal fixture repo at `packages/cli/test/fixtures/golden-path/`: - -``` -golden-path/ -├── src/ -│ └── routes/ -│ └── auth.ts # post-diff state -├── manifest.json -├── input.diff # unified diff -├── expected-human.txt # expected stdout when --format=human --no-color -└── expected.json # expected stdout when --format=json -``` - -### 7.2 Fixture contents (describe; agent writes) - -**`manifest.json`** — a minimal manifest with exactly two claims: - -1. `c1`: `add_symbol`, target `{ kind: "endpoint", path: "src/routes/auth.ts", symbol: "POST /login" }`, check `symbol_exists` -2. `c2`: `modify_behavior`, target same endpoint, check `behavior_present` with `{ property: "authentication" }` - -**`src/routes/auth.ts`** — post-diff TypeScript source containing: - -- An Express import -- `app.post("/login", handler)` — NO auth middleware -- The handler function (any body) - -**`input.diff`** — the unified diff that: - -- Adds `src/routes/auth.ts` (new file) -- Adds a symbol `unlistedHelper` not claimed in the manifest - -**Expected verdicts**: - -- `c1` → `verified` (endpoint exists) -- `c2` → `unverified`, reason `no_auth_in_chain` (the authentication detector finds no auth) -- Undeclared finding: `unlistedHelper` in `src/routes/auth.ts` -- Exit code: `1` - -### 7.3 Expected `human` output (with `--no-color`) - -``` -🤖 Agent: claude-code (claude-opus-4-7) · 5 tool calls · 1 files touched -📝 Task: Add login endpoint - -📋 Declared changes (2): - ✅ c1 endpoint POST /login in src/routes/auth.ts - ❌ c2 no auth in chain at src/routes/auth.ts:POST /login - -⚠️ Undeclared modifications (1): - • src/routes/auth.ts — symbol `unlistedHelper` modified but not in any claim - -🔍 Reviewer focus: - 1. c2 failed — authentication not detected - 2. undeclared change to `unlistedHelper` -``` - -### 7.4 Test invocation - -``` -attest verify \ - --manifest packages/cli/test/fixtures/golden-path/manifest.json \ - --diff packages/cli/test/fixtures/golden-path/input.diff \ - --repo-root packages/cli/test/fixtures/golden-path \ - --format human \ - --no-color -``` - -The test asserts: -1. Exit code equals `1` -2. Stdout matches `expected-human.txt` byte-for-byte -3. With `--format json`, stdout is JSON-equal to `expected.json` (order-independent on object keys, order-sensitive on arrays) - ---- - -## 8. Non-goals for v0.1 CLI - -- No config file (`.attestrc`). All behavior is flag-driven. -- No plugin system. Detectors are compiled in. -- No interactive mode. -- No GitHub integration. The App is v0.3. -- No auto-fix / suggestion output. The CLI reports; humans act. -- No parallelism tuning. Detectors run sequentially in v0.1. - ---- - -## 9. Implementation notes (hints, not specifications) - -- `clipanion` supports strict typing of flags; use it fully -- Read the manifest with `fs.readFile` + `JSON.parse` — no streaming needed at this scale -- Stdin reading: consume the entire stream via `node:stream/consumers` `text()` when `--diff -` -- Computing `manifest_hash`: SHA-256 of the raw manifest file bytes (not a re-serialization). Prefix `sha256:` to match schema convention -- Color: use `picocolors` only if a color library is strictly necessary. v0.1 default: no extra dep — write ANSI codes inline behind a `useColor` flag - -These are hints to guide the agent's implementation choices. The spec does not require any specific implementation; it only requires the contract. diff --git a/docs/CORE_CHECKS_SPEC.md b/docs/CORE_CHECKS_SPEC.md deleted file mode 100644 index 7a7a5cc..0000000 --- a/docs/CORE_CHECKS_SPEC.md +++ /dev/null @@ -1,259 +0,0 @@ -# Core Verification Checks — v0.1 - -Status: **Ready for implementation.** -Package: `@attest/core` -Module: `src/checks/` -Companion: `PROJECT_SPEC.md`, `SCHEMA_V0.1.md`, `DETECTOR_AUTHENTICATION_SPEC.md` - ---- - -## 1. Purpose - -`@attest/core` implements all verification checks **except** `behavior_present` (which is delegated to registered detectors). This document specifies: - -- Core-level reason codes -- The five non-`behavior_present` check algorithms -- The `locateRoute()` shared utility -- The undeclared-changes detection algorithm (file-level and symbol-level) -- Post-diff content resolution rules -- `VerifyInput.manifestRawBytes` and hash computation - -Anything not covered here defers to `DETECTOR_AUTHENTICATION_SPEC.md` for the authentication detector, and to `PROJECT_SPEC.md §8` for types. - ---- - -## 2. Core-level reason codes - -These are emitted by the verifier's routing/orchestration layer, not by detectors. They extend `@attest/core`'s public exports alongside `Verdict`. - -```ts -export type CoreReasonCode = - | "detector_not_implemented" // behavior_present claim; no detector registered for that property - | "unsupported_check"; // check kind not yet implemented in v0.1 -``` - -**`detector_not_implemented`** — when a `behavior_present` claim arrives for a property with no registered detector (e.g., `rate_limiting` in v0.1), the verifier returns `unverifiable` with this code. This is not an error; it is the designed-in answer for v0.1-unimplemented properties. - -**`unsupported_check`** — used when a check kind runs against an incompatible target kind (e.g., `signature_matches` on a `file` target), or when a parse error prevents the check from running. - -These codes are **not** valid in detector output. The separation is enforced by package boundary: detectors return `DetectorVerdict` (reason codes enumerated in `DETECTOR_AUTHENTICATION_SPEC.md §10`); core produces `ClaimResult` (reason codes from either set, depending on routing path). - ---- - -## 3. Check implementations - -All checks in this section live in `@attest/core/src/checks/`. They are internal to core and are not exported from `@attest/core/src/index.ts`. - -### 3.1 `cannot_verify` - -No-op. Always returns `{ verdict: "unverifiable", evidence: [] }`. No reason code. Used for claims of type `refactor` and for `verification_contract.check: "cannot_verify"`. - -### 3.2 `symbol_exists` - -Parses the post-diff content of `target.path` using ts-morph (syntactic parse, no TypeChecker). Resolves presence by `target.kind`: - -| `target.kind` | Located by | -|---|---| -| `function` | Top-level function declaration OR `const`/`let` variable declaration where the initializer is an arrow function or function expression, named `target.symbol` | -| `class` | Top-level class declaration named `target.symbol` | -| `type` | Top-level type alias (`type X = ...`) or interface (`interface X { ... }`) named `target.symbol` | -| `endpoint` | Route registration matching `target.symbol` via `locateRoute()` (§5) | -| `file` | File present in diff OR on disk at `repoRoot/target.path` | -| `module` | Same as `file` | -| `package` | Package key present in `dependencies` or `devDependencies` of `target.path` (must be a `package.json`) | -| `config_key` | Key path `target.symbol` (dot-separated) present in the target config file, parsed as JSON or treated as line-by-line key scan for non-JSON files | - -**Evidence on pass**: `{ kind: "symbol", path, symbol, note: " declaration found" }` - -**Evidence on fail**: `{ kind: "symbol", path, symbol, note: " declaration not found in post-diff content" }` - -**Verdicts**: -- Symbol found → `verified` -- Symbol not found → `unverified` (no reason code; the check ran and failed cleanly) -- Parse error or incompatible target kind → `unverifiable` with `reason_code: "unsupported_check"` and a descriptive note - -### 3.3 `removed` - -Mirror of `symbol_exists`. Uses the same per-kind lookup logic. - -- Symbol/file **absent** → `verified` -- Symbol/file **present** → `unverified` (no reason code) -- Parse error → `unverifiable` / `unsupported_check` - -### 3.4 `test_covers` - -`verification_contract.params.subject_symbol` is required; if absent, return `unverifiable` / `unsupported_check`. - -Parses `target.path` syntactically and returns `verified` if **any** of: - -1. An `import` declaration names `subject_symbol` as a named import, default import, or namespace import. -2. A string literal inside a `describe`, `it`, `test`, or `suite` call contains `subject_symbol` as a substring. -3. A `new` expression or call expression directly references an identifier named `subject_symbol`. - -Returns `unverified` if none match. - -**Evidence on pass**: `{ kind: "test", path, symbol: subject_symbol, note: " reference(s) found" }` - -**Evidence on fail**: `{ kind: "test", path, symbol: subject_symbol, note: "no references to subject_symbol found" }` - -### 3.5 `signature_matches` - -`verification_contract.params.expected` is required (string). If absent, return `unverifiable` / `unsupported_check`. - -Only valid for `target.kind` of `function`, `class`, or `type`. Any other kind → `unverifiable` / `unsupported_check`. - -Algorithm: - -1. Locate the declaration using the same lookup as `symbol_exists`. -2. For `function`: extract the parameter list and return type annotation as a string by printing the relevant AST nodes. -3. Normalize both the extracted string and `params.expected`: collapse all whitespace runs to a single space, trim leading/trailing whitespace. -4. Compare normalized strings. - -**Evidence on pass**: `{ kind: "symbol", path, symbol, note: "signature matches" }` - -**Evidence on fail**: `{ kind: "symbol", path, symbol, note: "expected: — found: " }` - ---- - -## 4. Undeclared-changes detection - -### 4.1 File-level undeclared - -``` -declared_files = { claim.target.path | claim ∈ manifest.claims } -diff_paths = { change.path | change ∈ diffSet.changes } -touched_files = { path | path ∈ manifest.session.files_touched } - -undeclared_files = (diff_paths ∪ touched_files) − declared_files -``` - -Using the **union** of `diff_paths` and `touched_files` closes the omission vector: a file that appears in the diff but is absent from `files_touched` is still flagged. A file in `files_touched` but absent from the diff generates no finding (agent listed a file it ultimately did not change — not a concern). - -Each undeclared file produces an `UndeclaredFinding` with `type: "file"`. - -### 4.2 Symbol-level undeclared - -For each file in `(declared_files ∩ diff_paths)` — files that are both declared in some claim AND present in the diff: - -1. Parse post-diff content with ts-morph (syntactic, no TypeChecker). -2. Extract **top-level declaration names**: function declarations, class declarations, `const`/`let`/`var` declarations at module scope (top-level, not inside blocks), type aliases, interfaces, exported enums. Do not descend into function bodies. -3. Build the **covered symbol set** for this file: union of `claim.target.symbol` across all claims whose `claim.target.path` equals this file. For claims with `target.kind: "endpoint"`, also attempt a best-effort lookup of the handler function name (the last identifier argument in the route registration call) and add it to the covered set — a failed lookup is silently ignored. -4. `undeclared_symbols = top_level_names − covered_symbol_set` - -Each undeclared symbol produces an `UndeclaredFinding` with `type: "symbol"`. - -**Non-TS files**: skip symbol-level detection entirely. A file-level finding is still emitted if the file is in `undeclared_files`. - -### 4.3 Reviewer focus construction - -After assembling `ClaimResult[]` and `UndeclaredFinding[]`: - -1. Add one `reviewer_focus` entry for each `ClaimResult` whose `verdict` is `unverified` or `partial`, in manifest claim-id order. -2. Add one `reviewer_focus` entry for each `UndeclaredFinding`, sorted lexicographically by `path` then `symbol` (nulls last). -3. `unverifiable` claims do **not** produce reviewer focus entries — they are noise-free by design. - -**Reason string templates** (used verbatim in both human and JSON output): - -| Condition | `reason` template | -|---|---| -| `behavior_present` claim, `unverified` or `partial` | `" failed — not detected"` | -| Other check kind, `unverified` | `" — "` | -| Other check kind, `unverified`, no reason code | `" — failed"` | -| Undeclared symbol | `"undeclared change to \`\`"` | -| Undeclared file | `"undeclared file "` | - -**Humanization rule**: replace underscores with spaces. Examples: `authentication` → `"authentication"`, `rate_limiting` → `"rate limiting"`, `no_auth_in_chain` → `"no auth in chain"`. - ---- - -## 5. `locateRoute()` — shared route-location utility - -Exported from `@attest/core` for use by both core's `symbol_exists` check (endpoint kind) and the `@attest/detectors-ts` authentication detector. - -```ts -import type { SourceFile, Node } from "ts-morph"; - -export type KnownFramework = "express" | "fastify" | "nestjs" | "koa" | "raw-node"; - -export interface RouteLocation { - framework: KnownFramework; - registrationNode: Node; // the CallExpression or MethodDeclaration anchoring the route -} - -export function detectFramework(sourceFile: SourceFile): KnownFramework | null; - -export function locateRoute( - sourceFile: SourceFile, - symbol: string // "METHOD /path" or "ClassName.methodName" -): RouteLocation | null; -``` - -**Framework detection** follows `DETECTOR_AUTHENTICATION_SPEC.md §4.1` exactly (import-scan, first-match wins). `detectFramework` returns `null` if no recognized import is found. - -**Target registration** follows `DETECTOR_AUTHENTICATION_SPEC.md §4.2` exactly, per framework. Returns `null` if the symbol is not found. - -The authentication detector's framework modules (`express.ts`, `fastify.ts`, etc.) import `locateRoute` and `detectFramework` from `@attest/core` rather than re-implementing them. This is the only approved cross-package dependency beyond what `PROJECT_SPEC.md §7` already lists. - ---- - -## 6. Post-diff content resolution - -**Invariant**: at the time `verify()` runs, the working tree at `repoRoot` reflects post-diff state. The CLI is designed to be run after `git apply` or from a checkout that is already in the post-merge state. - -**Resolution rules** (in order): - -1. **File present on disk** at `repoRoot/path` → read from disk. This is the post-diff content. -2. **File absent from disk** (deleted in the diff) → `postDiffFile()` returns `null`. Checks requiring content return `unverifiable` with `reason_code: "unsupported_check"` and `note: "file deleted; cannot inspect post-diff content"`. -3. **Binary file** (flagged by `parse-diff` as binary) → skip all content-based checks; return `unverifiable` with a note. Emit a stderr warning: `warn: binary file skipped: `. - -The diff is used **only** for: -- Enumerating changed files (for undeclared detection) -- Determining change kind: `added | modified | deleted` - -The diff is **not** applied programmatically. `parse-diff` is used for structure only. No patch-apply utility is needed. - ---- - -## 7. `VerifyInput` — manifest hash - -```ts -export interface VerifyInput { - manifest: Manifest; - manifestRawBytes: Uint8Array; // raw bytes of manifest file as read from disk by CLI - diff: DiffSet; - repoRoot: string; -} -``` - -Core computes `manifest_hash` internally: - -```ts -import { createHash } from "node:crypto"; - -const hash = createHash("sha256").update(manifestRawBytes).digest("hex"); -const manifestHash = `sha256:${hash}`; -``` - -The CLI reads the manifest file once with `fs.readFile` (returns a `Buffer`, assignable to `Uint8Array`), passes both the parsed `Manifest` and the raw buffer in `VerifyInput`. Core never touches the filesystem for the manifest — the CLI owns I/O; core owns computation. - ---- - -## 8. Module layout within `@attest/core` - -``` -packages/core/src/ -├── index.ts # re-exports: verify, locateRoute, detectFramework, CoreReasonCode, all types -├── verifier.ts # verify() orchestration — routes claims to checks or detectors -├── checks/ -│ ├── symbol-exists.ts -│ ├── removed.ts -│ ├── test-covers.ts -│ ├── signature-matches.ts -│ └── cannot-verify.ts -├── undeclared.ts # file-level + symbol-level undeclared detection -├── diff.ts # unified diff → DiffSet (parse-diff wrapper) -├── locate-route.ts # locateRoute() + detectFramework() — re-exported from index.ts -└── verdict.ts # VerdictReport constructor, reviewer_focus builder -``` - -The `PROJECT_SPEC.md §6` monorepo layout is updated to add `checks/` and `locate-route.ts` to the `core/src/` tree. diff --git a/docs/DETECTOR_AUTHENTICATION_SPEC.md b/docs/DETECTOR_AUTHENTICATION_SPEC.md deleted file mode 100644 index b28571b..0000000 --- a/docs/DETECTOR_AUTHENTICATION_SPEC.md +++ /dev/null @@ -1,335 +0,0 @@ -# Detector Specification: `authentication` - -Status: **Ready for implementation.** -Package: `@attest/detectors-ts` -Module: `src/authentication/` -Companion: `PROJECT_SPEC.md`, `SCHEMA_V0.1.md` - ---- - -## 1. Purpose - -Given a TypeScript source file and a target symbol representing an HTTP route or endpoint handler, determine whether the target is protected by an authentication check before its business logic executes. - -This detector implements the `behavior_present` check when `verification_contract.params.property === "authentication"`. - -The detector does **not** decide whether authorization is correct, whether the auth mechanism is secure, or whether the token format is right. It decides one thing: is there *any* auth check gating the handler. - ---- - -## 2. Interface - -```ts -import type { Claim } from "@attest/schema"; -import type { DetectorContext, DetectorVerdict } from "@attest/detectors-ts"; - -export async function detectAuthentication( - claim: Claim, - ctx: DetectorContext -): Promise; -``` - -**Accepts** a claim where: -- `type` is `"modify_behavior"` or `"add_symbol"` -- `target.kind` is `"endpoint"` (required for auth — only endpoints can be "protected") -- `target.path` is a TypeScript file in the repo -- `target.symbol` is an endpoint identifier. Format: `"METHOD /path"` for Express/Fastify/Koa/raw Node; `"ClassName.methodName"` for NestJS -- `verification_contract.check` is `"behavior_present"` -- `verification_contract.params.property` is `"authentication"` - -**Rejects** (returns `unverifiable` with reason `invalid_claim_shape`) any claim that does not meet all of the above. - ---- - -## 3. Supported frameworks (v0.1) - -| Framework | Versions | How endpoints are registered | -|---------------|----------|----------------------------------------------------------------------------| -| Express | 4.x, 5.x | `app.METHOD(path, ...middleware, handler)` or `router.METHOD(...)` | -| Fastify | 4.x | `fastify.METHOD(path, { preHandler, handler })` or `fastify.route({...})` | -| NestJS | 10.x | Controller class + method decorators (`@Get`, `@Post`, etc.) | -| Koa | 2.x | `app.use(...)` chain + `@koa/router` `router.METHOD(...)` | -| Raw Node HTTP | N/A | `http.createServer((req, res) => {...})` with in-handler routing | - -Framework detection is automatic based on imports and API shape. An explicit `framework` hint is not supported in v0.1. - ---- - -## 4. Algorithm (language-agnostic steps) - -1. **Parse** `target.path` with `ts-morph`. If parse fails → return `unverifiable` with reason `parse_error`. -2. **Detect framework** by scanning top-of-file imports (see §4.1). If no recognized framework → return `unverifiable` with reason `framework_unsupported`. -3. **Locate target registration** corresponding to `target.symbol` (see §4.2). If not found → return `unverified` with reason `no_route_found`. -4. **Collect the middleware/guard/hook chain** that runs before the handler body (see §4.3, per-framework). -5. **Classify each chain entry** as `auth`, `not-auth`, or `unknown` using the three-layer heuristic in §5. -6. **Compute verdict** per §6. - -### 4.1 Framework detection — import signatures - -Scan all `import` declarations in the file. The *first* match (in the order below) wins: - -| Import source | Framework | -|-----------------------------------------------------------|---------------| -| `express` | Express | -| `fastify` | Fastify | -| `@nestjs/common`, `@nestjs/core` | NestJS | -| `koa`, `@koa/router` | Koa | -| `http`, `https`, `node:http`, `node:https` | Raw Node | - -If the file contains no such imports, the detector walks *up* one level: if a default or named export from this file is consumed by a sibling file that imports `express` etc., that framework applies. In v0.1, this upward resolution is **not implemented** — if the file itself does not import the framework, return `framework_unsupported`. - -### 4.2 Target registration — per-framework location - -**Express / Koa with `@koa/router`**: `target.symbol` is `"METHOD /path"`. Match by finding a CallExpression where: -- The callee is `.` where method matches `get|post|put|delete|patch|options|head|all|use` -- The first argument is a string literal equal to `/path` -- The method (lowercased) equals the METHOD token - -**Fastify**: match either: -- `fastify.("/path", ...)` — same pattern as Express -- `fastify.route({ method: "METHOD", url: "/path", ... })` — an ObjectLiteralExpression where `method` and `url` properties match - -**NestJS**: `target.symbol` is `"ClassName.methodName"`. Match by finding: -- A ClassDeclaration named `ClassName` decorated with `@Controller(...)` -- A MethodDeclaration within it named `methodName` decorated with one of `@Get`, `@Post`, `@Put`, `@Delete`, `@Patch`, `@Options`, `@Head`, `@All` - -**Raw Node**: match the handler function inside `http.createServer(HANDLER)`. If `target.symbol` is `"METHOD /path"`, the detector looks inside the handler body for an `if` branch whose condition matches `req.method === "METHOD"` and `req.url === "/path"` (or a matching regex). The middleware chain is the set of statements executed before entering that branch. - -### 4.3 Middleware/guard/hook chain collection — per-framework - -**Express**: -1. Inline chain: all arguments between the path and the final argument in `app.METHOD(path, ...middleware, handler)` are inline middleware -2. Router-level: if the route lives on a `Router` instance, collect all `router.use(...)` calls *above* the route registration in source order -3. App-level: collect all `app.use(pathOrMiddleware, ...)` calls above the `app.METHOD` call. If `app.use` has a path prefix, it applies only if the route path starts with that prefix -4. Concatenate in order: app-level → router-level → inline → handler - -**Fastify**: -1. Route-options: if the route is registered via `{ preHandler, onRequest, preValidation }`, these are the inline hooks (in the order `onRequest` → `preValidation` → `preHandler`) -2. Global hooks: `fastify.addHook("onRequest"|"preValidation"|"preHandler", fn)` calls *above* the route registration -3. Concatenate: global hooks → route-options hooks → handler - -**NestJS**: -1. Method-level guards: `@UseGuards(...)` on the method -2. Class-level guards: `@UseGuards(...)` on the controller class -3. Global guards: `APP_GUARD` provider in the module OR `app.useGlobalGuards(...)` — **v0.1 does not resolve these** (cross-file). If no method/class guards exist, emit `unknown_middleware_only` with a note that global guards may apply. -4. Concatenate: global (not resolved in v0.1) → class → method → handler - -**Koa**: -1. Router-level middleware: `router.METHOD(path, ...middleware, handler)` — same inline pattern as Express -2. App-level: `app.use(middleware)` calls above the router mount -3. Concatenate: app → router → inline → handler - -**Raw Node**: -1. Statements in the top-level handler body that execute before the matched route branch (conditional returns, header reads, token parses) -2. Treat each such statement as a "chain entry" for classification - ---- - -## 5. The three-layer classification heuristic - -For each chain entry, classify as `auth`, `not-auth`, or `unknown` by applying layers in order. First definitive match wins. - -### Layer 1 — Name match - -The identifier (function name, decorator argument class name, hook callback name) matches at least one of the following patterns (case-insensitive): - -**Positive patterns** (→ `auth`): -- Contains `auth` but **not** `author` (exclude `authorContext`, `authorOnly`, etc.) -- Equals or contains any of: `authenticate`, `authentication`, `authenticated`, `isAuthenticated`, `requireAuth`, `requiresAuth`, `needsAuth`, `ensureAuth`, `withAuth`, `protectRoute`, `protected`, `protect`, `private`, `guarded`, `guard` -- Starts with `require`, `ensure`, `check`, `verify`, `validate` and is immediately followed by any of: `auth`, `user`, `login`, `session`, `token`, `jwt`, `credential` -- NestJS-specific: class name ends with `Guard` (e.g., `AuthGuard`, `JwtAuthGuard`, `RolesGuard` — the detector treats `RolesGuard` as `auth`; authorization is a different detector in future versions) - -**Negative patterns** (→ `not-auth` — do not upgrade to `unknown`): -- Matches exactly: `bodyParser`, `cors`, `compression`, `cookieParser`, `morgan`, `helmet`, `rateLimit`, `logger`, `errorHandler`, `notFound`, `staticFiles`, `json`, `urlencoded`, `multer`, `upload` - -If no positive or negative pattern matches → fall through to Layer 2. - -### Layer 2 — Import origin - -Inspect the import source of the identifier. If imported from any package below → classify as `auth`: - -``` -passport -passport-* -express-jwt -@auth0/* -@clerk/* -@nestjs/passport -@nestjs/jwt -@clerk/clerk-sdk-node -next-auth -@auth/* -lucia -lucia-auth -@supabase/auth-helpers-* -better-auth -firebase-admin/auth -jose (JWT library; strong signal but not definitive) -jsonwebtoken (JWT library; strong signal but not definitive) -``` - -For `jose` and `jsonwebtoken` specifically — Layer 2 match only if the function body *calls* the library's verification API (`jwt.verify`, `jwtVerify`, `jwtDecrypt`). Mere import is not enough. - -If the identifier is imported from a local path (`./`, `../`) — Layer 2 does not apply (we do not resolve cross-file in v0.1; body pattern must carry the decision). - -### Layer 3 — Body pattern - -If the function body is available in the same file, inspect its statements. Classify as `auth` if **any** of the following appear in the body, before the function returns or calls `next()`: - -1. A call matching `jwt.verify(...)`, `jwtVerify(...)`, `jsonwebtoken.verify(...)`, `jose.jwtVerify(...)` -2. A call matching `bcrypt.compare(...)` followed within three statements by a conditional that throws or returns -3. A property read like `req.session.user`, `ctx.state.user`, `request.user` followed within five statements by a conditional that throws or returns or calls `res.status(401|403)` / `throw new UnauthorizedException()` / `throw new HttpException(..., 401)` / `ctx.throw(401)` -4. A call to `res.status(401)`, `res.status(403)`, `res.sendStatus(401)`, `res.sendStatus(403)` -5. `throw` of any of: `UnauthorizedException`, `UnauthorizedError`, `AuthenticationError`, `HttpException` with a 401/403 literal - -If the body is not available in the same file and Layers 1–2 did not match → classify as `unknown`. - ---- - -## 6. Verdict logic - -After classifying every entry in the chain: - -| Chain state | Verdict | Reason code | -|-----------------------------------------------------------|---------------|-------------------------------| -| ≥1 entry is `auth` | `verified` | (none) | -| Chain is empty | `unverified` | `no_middleware_chain` | -| All entries are `not-auth` | `unverified` | `no_auth_in_chain` | -| Mix of `not-auth` and `unknown`, no `auth` | `partial` | `unknown_middleware_only` | -| Framework was not determinable | `unverifiable`| `framework_unsupported` | -| Target not found in file | `unverified` | `no_route_found` | -| Parse error | `unverifiable`| `parse_error` | -| Invalid claim shape (e.g., wrong `target.kind`) | `unverifiable`| `invalid_claim_shape` | - -**Evidence**: in every case, the detector emits evidence entries naming the chain members it found and how each was classified. This gives the reviewer a paper trail. - -**Evidence ordering rule (critical for CLI rendering)**: the first evidence entry must be a **route summary entry** — `{ kind: "route", path, symbol }` with **no `note` field**. Chain-member entries follow. This ensures the CLI human renderer falls through to the `reason_code` humanization rule (§5.3 rule 2 of `CLI_V01_SPEC.md`) rather than printing a chain-member classification note as the one-line summary. - -Example evidence payloads: -```json -[ - { "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /login" }, - { "kind": "middleware", "symbol": "authMiddleware", "note": "classified auth via Layer 1 name match" }, - { "kind": "middleware", "symbol": "bodyParser.json", "note": "classified not-auth via negative list" }, - { "kind": "middleware", "symbol": "customCheck", "note": "classified unknown — no name/import/body signal" } -] -``` - -For `verified` verdicts, the route summary entry is still first, followed by the `auth`-classified entry: `{ kind: "middleware", symbol: "authMiddleware", note: "classified auth via Layer 1 name match" }`. The CLI renders `evidence[1].note` as the summary via rule 1 (first entry with a non-empty note wins). - ---- - -## 7. Fixture catalog (v0.1 requirements) - -Every fixture lives at `packages/detectors-ts/fixtures/authentication/.ts` with a companion `.expected.json`. - -Minimum catalog — **seventeen fixtures**, all must pass before v0.1 ships: - -### Express (4 fixtures) -1. `express-route-level-valid.ts` — `app.post("/x", authMiddleware, handler)` → `verified` -2. `express-app-level-valid.ts` — `app.use(authMiddleware)` above `app.post("/x", handler)` → `verified` -3. `express-no-auth.ts` — `app.post("/x", handler)`, no auth anywhere → `unverified` / `no_auth_in_chain` -4. `express-passport-import.ts` — `passport.authenticate("jwt")` as middleware, imported from `passport` → `verified` (Layer 2) - -### Fastify (3 fixtures) -5. `fastify-preHandler-valid.ts` — `fastify.post("/x", { preHandler: authHook, handler })` → `verified` -6. `fastify-addHook-valid.ts` — `fastify.addHook("onRequest", authHook)` above route → `verified` -7. `fastify-no-auth.ts` — plain `fastify.post("/x", handler)` → `unverified` / `no_auth_in_chain` - -### NestJS (3 fixtures) -8. `nestjs-method-guard-valid.ts` — `@UseGuards(AuthGuard)` on the method → `verified` -9. `nestjs-class-guard-valid.ts` — `@UseGuards(JwtAuthGuard)` on the class → `verified` -10. `nestjs-no-guard.ts` — controller method with no guard → `partial` / `unknown_middleware_only` (because global guards might apply — v0.1 cannot resolve them) - -### Koa (3 fixtures) -11. `koa-app-use-valid.ts` — `app.use(authMiddleware)` before the router → `verified` -12. `koa-router-level-valid.ts` — `router.post("/x", requireAuth, handler)` → `verified` -13. `koa-no-auth.ts` — plain router → `unverified` / `no_auth_in_chain` - -### Raw Node (2 fixtures) -14. `raw-node-token-check-valid.ts` — handler checks `req.headers.authorization` and responds 401 when absent, before routing → `verified` -15. `raw-node-no-check.ts` — handler routes directly with no auth → `unverified` / `no_auth_in_chain` - -### Cross-cutting edge cases (2 fixtures) -16. `ambiguous-custom-middleware.ts` — Express with `app.post("/x", customThing, handler)` where `customThing` is locally defined, has no auth-name, and body is too opaque → `partial` / `unknown_middleware_only` -17. `target-not-found.ts` — claim targets `"POST /missing"` but file has no such route → `unverified` / `no_route_found` - -**Expected JSON format per fixture**: -```json -{ - "verdict": "verified", - "reason_code": null, - "evidence_contains": ["authMiddleware", "Layer 1"] -} -``` - ---- - -## 8. Performance requirements - -| Metric | Requirement | -|---------------------------------------|---------------------------------| -| Single-file detection, <500 LOC | ≤50 ms on modern laptop | -| Single-file detection, <5000 LOC | ≤200 ms | -| Network calls | **Zero** | -| Filesystem reads beyond `target.path` | **Zero** (in v0.1) | -| Memory | <200 MB for full fixture run | - -The detector must not use TypeChecker (`ts-morph`'s `getTypeAtLocation`, etc.) — syntactic AST only. This is explicit: TypeChecker is 100× slower and we do not need type inference to detect the patterns above. - ---- - -## 9. Out of scope (explicit) - -- **Cross-file resolution.** If `authMiddleware` is defined in another file, we classify by name (Layer 1) and import (Layer 2). We do not walk to its definition in v0.1. -- **Conditional auth.** Auth applied only to some HTTP methods on the same path, or only under certain runtime conditions. Treated as "whatever the detector sees in the chain." -- **NestJS global guards** (`APP_GUARD`, `useGlobalGuards`). Surfaced as `partial` / `unknown_middleware_only`. -- **Custom framework wrappers.** If the project wraps Express in a house-built abstraction, the detector will emit `framework_unsupported`. This is the correct behavior — the reviewer must manually confirm. -- **Runtime behavior.** We do not execute anything. -- **Authorization.** A separate detector in a future version. `RolesGuard` and similar RBAC constructs are classified as `auth` in v0.1 because they imply an authenticated caller. A dedicated `authorization` detector will refine this later. - ---- - -## 10. Reason code enumeration (final) - -All reason codes the detector may emit: - -``` -parse_error -framework_unsupported -invalid_claim_shape -no_route_found -no_middleware_chain -no_auth_in_chain -unknown_middleware_only -``` - -No other reason codes are permitted. If the detector encounters a condition not covered by these codes, it must return `unverifiable` with `reason_code: "parse_error"` and include a descriptive `note` in evidence. Add a new reason code only via spec revision. - -These codes apply exclusively to the authentication detector. Core-level reason codes (e.g., `detector_not_implemented`, `unsupported_check`) are defined in `PROJECT_SPEC.md §8.2` and may appear in `ClaimResult.reason_code` when the routing layer — not this detector — produces the verdict. - ---- - -## 11. Acceptance test definition - -The authentication detector is acceptance-complete when: - -1. All seventeen fixtures in §7 pass (verdict + reason_code + evidence_contains) -2. Zero fixtures take longer than the budgets in §8 -3. The detector exports only `detectAuthentication` from `src/authentication/index.ts`; framework modules are internal -4. The detector adds zero dependencies beyond those in `PROJECT_SPEC.md` §5 -5. `pnpm --filter @attest/detectors-ts test` exits 0 -6. Code coverage on `src/authentication/` is ≥85% line coverage - ---- - -## 12. Known limitations to document in README - -When shipping v0.1, the README must list these honestly so users don't hit surprises: - -- Cross-file middleware definitions are classified by name/import only -- NestJS global guards are flagged as `partial`, not `verified` -- Custom framework abstractions will fall through to `framework_unsupported` -- Syntactic analysis only — no type inference, no runtime execution - -Honest limitations build trust. Hiding them burns it. diff --git a/docs/PROJECT_SPEC.md b/docs/PROJECT_SPEC.md deleted file mode 100644 index 927d0e4..0000000 --- a/docs/PROJECT_SPEC.md +++ /dev/null @@ -1,485 +0,0 @@ -# attest — Project Specification v0.1 - -Status: **Ready for implementation.** -Audience: Coding agents (Claude Code, OpenCode, Hermes, or equivalents) building v0.1. -Companion specs: `SCHEMA_V0.1.md`, `DETECTOR_AUTHENTICATION_SPEC.md`, `CLI_V01_SPEC.md`. - ---- - -## 1. Name and namespace - -**Name**: `attest` - -**Rationale**: The agent attests to its changes via structured claims; `attest` verifies those attestations against the actual diff. Short, one syllable, semantic precision, and rides the existing "attestation" idiom in software supply-chain security (SLSA, in-toto, sigstore) without conflicting with it — different layer of the stack. - -**Namespace availability check (MANDATORY before first commit)**: - -- npm: `attest` as an org scope (`@attest/*`) -- GitHub: `github.com/attest` -- Fallback if taken: `attestly` (org scope `@attestly/*`, `github.com/attestly`) - -If the primary name is unavailable, apply the fallback globally. Do not mix. - ---- - -## 2. Mission (one paragraph) - -`attest` closes the gap between what an AI coding agent claims it did and what it actually did. The agent declares structured claims; `attest` verifies each claim deterministically against the diff; a human reviewer reads one report that tells them exactly where to focus. No LLM judgment in the verification path. No SaaS dependency. Open source, Apache-2.0, locally runnable. - ---- - -## 3. Philosophy (non-negotiable) - -1. **Deterministic checks only.** The verifier does not call an LLM to decide whether a claim matches a diff. Every verdict is reproducible from the same inputs. -2. **Silence-by-omission is the adversary.** Undeclared modifications are surfaced as loudly as failed claims. Zero tolerance. -3. **The schema is the product.** Agents learn to emit this vocabulary; reviewers learn to read in it. Schema stability is more important than feature count. -4. **Fail loud, fail specific.** Every rejection returns a structured reason code the agent can act on. -5. **No hidden state.** No databases in v0.1. Input in, verdict out, process exits. Stateless CLI. - ---- - -## 4. v0.1 Scope - -### IN (must ship) - -1. `@attest/schema` — JSON Schema + TypeScript types for the manifest defined in `SCHEMA_V0.1.md` -2. `@attest/core` — verifier orchestration, verdict engine, undeclared-changes detector, diff parser -3. `@attest/detectors-ts` — TypeScript-language detectors (authentication only in v0.1) -4. `@attest/cli` — command-line tool: manifest + diff in, verdict out -5. Fixture suite for the authentication detector (≥17 fixtures, specified in `DETECTOR_AUTHENTICATION_SPEC.md` §7) -6. README, CONTRIBUTING, LICENSE (Apache-2.0), CI workflow - -### OUT (deferred, do not build) - -- MCP (Model Context Protocol) server (v0.2) -- GitHub App / PR comment renderer (v0.3) -- Nine remaining behavioral detectors (`input_validation`, `error_handling`, etc.) — v0.2+ -- Python-language detectors — v0.2+ -- Test execution (we check test _presence_, never _behavior_) -- Pre-declaration / TDD-style flow — v0.2+ -- Multi-session provenance chaining — v0.2+ -- Published npm packages — v0.2+ (v0.1 is repo-local usable via `pnpm build && pnpm link`) -- Any form of LLM inference in the verifier - -**Rationale for narrow scope**: v0.1 must prove the core loop — structured claim in, deterministic verdict out — works end to end on one behavior. The detector is the hardest and most novel piece. If authentication detection is crisp and the CLI produces the mockup from `SCHEMA_V0.1.md` §11, the thesis is proven and everything else is adapters + repetition. - ---- - -## 5. Technology stack (locked) - -These decisions are final for v0.1. Do not introduce alternatives without a spec revision. - -| Concern | Choice | Rationale | -| ------------------ | ------------------------------- | --------------------------------------------------------- | -| Language | TypeScript 5.4+ | Target language, native | -| Module system | ESM only | Modern, forward-compatible | -| Runtime | Node.js 20 LTS or newer | Maintained LTS, native ESM, fetch, test runner available | -| Package manager | pnpm 9+ | Fast, first-class workspaces, strict dependency isolation | -| AST library | `ts-morph` | Object-oriented API over TS compiler; agent-friendly | -| JSON Schema | `ajv` + `ajv-formats` | Fastest, most widely adopted | -| Diff parsing | `parse-diff` | Battle-tested unified-diff parser | -| Test framework | `vitest` | Fast, TS-native, ESM-native | -| Linter | `eslint` + `@typescript-eslint` | Standard | -| Formatter | `prettier` | Standard | -| CLI framework | `clipanion` | Typed, class-based, validates args at compile time | -| Build (libraries) | `tsc` | Canonical | -| Build (CLI bin) | `tsup` | Single-file bundle for `#!/usr/bin/env node` entrypoint | -| Release management | `changesets` | Semver discipline from day one | -| CI | GitHub Actions | Free, standard | -| Coverage | `@vitest/coverage-v8` | vitest coverage provider; required for ≥85% gate | - -No other dependencies may be added without explicit spec revision. If the agent believes a dependency is required, it must stop and ask — not silently introduce. - -TypeScript compiler settings: `strict: true`, `noImplicitAny: true`, `noUncheckedIndexedAccess: true`, `exactOptionalPropertyTypes: true`, `moduleResolution: "bundler"`, `target: "ES2022"`. - ---- - -## 6. Monorepo layout - -``` -attest/ -├── packages/ -│ ├── schema/ -│ │ ├── src/ -│ │ │ ├── index.ts # re-exports -│ │ │ ├── manifest.schema.json -│ │ │ ├── types.ts # TS types mirroring the schema -│ │ │ └── validator.ts # ajv-based validator factory -│ │ ├── test/ -│ │ ├── package.json -│ │ └── tsconfig.json -│ ├── core/ -│ │ ├── src/ -│ │ │ ├── index.ts -│ │ │ ├── verifier.ts # orchestration -│ │ │ ├── undeclared.ts # undeclared-changes detector -│ │ │ ├── diff.ts # unified diff → structured change set -│ │ │ ├── verdict.ts # verdict types and constructors -│ │ │ ├── locate-route.ts # locateRoute() + detectFramework() — shared util -│ │ │ └── checks/ -│ │ │ ├── symbol-exists.ts -│ │ │ ├── removed.ts -│ │ │ ├── test-covers.ts -│ │ │ ├── signature-matches.ts -│ │ │ └── cannot-verify.ts -│ │ ├── test/ -│ │ ├── package.json -│ │ └── tsconfig.json -│ ├── detectors-ts/ -│ │ ├── src/ -│ │ │ ├── index.ts -│ │ │ ├── detector.ts # Detector interface + registry -│ │ │ └── authentication/ -│ │ │ ├── index.ts -│ │ │ ├── express.ts # framework-specific modules -│ │ │ ├── fastify.ts -│ │ │ ├── nestjs.ts -│ │ │ ├── koa.ts -│ │ │ ├── raw-node.ts -│ │ │ ├── heuristics.ts # name/import/body classifier -│ │ │ └── types.ts -│ │ ├── test/ -│ │ ├── fixtures/ -│ │ │ └── authentication/ -│ │ │ ├── express-route-level-valid.ts -│ │ │ ├── express-route-level-valid.expected.json -│ │ │ ├── express-no-auth.ts -│ │ │ ├── express-no-auth.expected.json -│ │ │ └── ... # full catalog per §7 of detector spec -│ │ ├── package.json -│ │ └── tsconfig.json -│ └── cli/ -│ ├── src/ -│ │ ├── index.ts # bin entrypoint -│ │ ├── commands/ -│ │ │ └── verify.ts -│ │ ├── render/ -│ │ │ ├── human.ts -│ │ │ └── json.ts -│ │ └── exit-codes.ts -│ ├── test/ -│ ├── package.json -│ └── tsconfig.json -├── docs/ -│ ├── SCHEMA_V0.1.md # already exists -│ ├── PROJECT_SPEC.md # this file -│ ├── DETECTOR_AUTHENTICATION_SPEC.md -│ ├── CLI_V01_SPEC.md -│ └── ARCHITECTURE.md # derived from this spec, for humans -├── .github/ -│ └── workflows/ -│ └── ci.yml -├── .changeset/ -├── package.json # root workspace manifest -├── pnpm-workspace.yaml -├── tsconfig.base.json -├── .eslintrc.cjs -├── .prettierrc -├── README.md -├── CONTRIBUTING.md -└── LICENSE # Apache-2.0 -``` - ---- - -## 7. Package dependency graph - -``` -schema → (no internal deps) -core → schema -detectors-ts → core, schema -cli → schema, core, detectors-ts -``` - -No circular dependencies. No package imports a sibling's internal module — only its public `index.ts` exports. - ---- - -## 8. Public interface contracts - -These TypeScript types define the boundaries between packages. Specification only — agent writes the declarations and implementations. - -### 8.1 `@attest/schema` - -```ts -// The manifest shape — mirrors the JSON Schema in §3 of SCHEMA_V0.1.md -export type SchemaVersion = "0.1"; -export type AgentId = "claude-code" | "codex" | "cursor" | "opencode" | "other"; -export type TaskSource = "user_prompt" | "issue_reference" | "continuation"; -export type ClaimType = - | "add_symbol" - | "remove_symbol" - | "modify_signature" - | "modify_behavior" - | "add_test" - | "refactor" - | "add_dependency" - | "remove_dependency" - | "config_change"; -export type CheckKind = - | "symbol_exists" - | "behavior_present" - | "test_covers" - | "signature_matches" - | "removed" - | "cannot_verify"; -export type BehavioralProperty = - | "null_check" - | "input_validation" - | "error_handling" - | "authentication" - | "authorization" - | "rate_limiting" - | "logging" - | "sanitization" - | "timeout" - | "retry_logic" - | "cannot_express"; -export type TargetKind = - | "function" - | "class" - | "type" - | "endpoint" - | "file" - | "module" - | "config_key" - | "package"; - -export interface Target { - kind: TargetKind; - path: string; - symbol?: string; -} -export interface VerificationContract { - check: CheckKind; - params?: Record; -} -export interface Claim { - id: string; - type: ClaimType; - target: Target; - description: string; - verification_contract: VerificationContract; -} -export interface Session { - /* mirror SCHEMA_V0.1.md §2 */ -} -export interface Task { - summary: string; - source: TaskSource; -} -export interface Manifest { - schema_version: SchemaVersion; - session: Session; - task: Task; - claims: Claim[]; -} - -// Validator -export interface ValidationError { - path: string; - code: string; - message: string; -} -export interface Validator { - validate( - input: unknown, - ): { ok: true; manifest: Manifest } | { ok: false; errors: ValidationError[] }; -} -export function createValidator(): Validator; -``` - -### 8.2 `@attest/core` - -```ts -import type { Manifest, Claim } from "@attest/schema"; - -export type Verdict = "verified" | "unverified" | "partial" | "unverifiable"; - -// Core-level reason codes — emitted by verifier routing, not by detectors. -// Detector-level reason codes are enumerated in DETECTOR_AUTHENTICATION_SPEC.md §10. -export type CoreReasonCode = - | "detector_not_implemented" // behavior_present claim; no detector registered for that property - | "unsupported_check"; // check kind not yet implemented or incompatible with target kind - -export interface Evidence { - kind: string; - path?: string; - symbol?: string; - note?: string; -} - -export interface ClaimResult { - claim_id: string; - verdict: Verdict; - reason_code?: string; // CoreReasonCode or a detector reason code - evidence: Evidence[]; -} - -export interface UndeclaredFinding { - type: "file" | "symbol"; - path: string; - symbol?: string; -} - -export interface VerdictReport { - manifest_hash: string; - summary: { - total_claims: number; - verified: number; - unverified: number; - partial: number; - unverifiable: number; - undeclared_files: number; - undeclared_symbols: number; - }; - claims: ClaimResult[]; - undeclared: UndeclaredFinding[]; - reviewer_focus: Array<{ claim_id?: string; undeclared?: UndeclaredFinding; reason: string }>; -} - -export interface DiffChange { - path: string; - kind: "added" | "modified" | "deleted"; - post_content?: string; - hunks: unknown[]; -} -export interface DiffSet { - changes: DiffChange[]; -} - -export interface VerifyInput { - manifest: Manifest; - manifestRawBytes: Uint8Array; // raw bytes of the manifest file; core computes manifest_hash from this - diff: DiffSet; - repoRoot: string; -} - -export function verify(input: VerifyInput): Promise; -``` - -### 8.3 `@attest/detectors-ts` - -```ts -import type { Claim, Target } from "@attest/schema"; -import type { Evidence, Verdict } from "@attest/core"; - -export interface DetectorContext { - repoRoot: string; - postDiffFile: (path: string) => Promise; // post-diff content or null if deleted -} - -export interface DetectorVerdict { - verdict: Verdict; - reason_code?: string; - evidence: Evidence[]; -} - -export interface Detector { - id: string; // e.g. "ts.behavior.authentication" - canHandle(claim: Claim): boolean; // routing gate - run(claim: Claim, ctx: DetectorContext): Promise; -} - -export function registerDetectors(): Detector[]; -``` - -### 8.4 `@attest/cli` - -```ts -// clipanion command definition; no runtime types exported to consumers. -// The CLI is a binary, not a library. -``` - ---- - -## 9. Diff format expected by v0.1 - -The CLI accepts **unified diff** format (the output of `git diff` or `git format-patch` single-commit). v0.1 does not read git history directly; the diff is passed as a file or stdin. - -**Expected invariants**: - -- Paths in the diff are relative to `--repo-root`. -- Only textual files are handled. Binary files in the diff are ignored with a warning. -- Post-diff content is resolved by **reading the file from disk at `repoRoot/path`** — the working tree is assumed to be in post-diff state when `attest verify` runs. The diff is used only to enumerate changed files and determine change kind (`added | modified | deleted`); it is never applied programmatically to reconstruct content. A deleted file (absent from disk) causes content-dependent checks to return `unverifiable`. See `CORE_CHECKS_SPEC.md §6` for the full resolution rules. - -**Out of scope**: reading `.git/` directly, multi-commit ranges, renames (for v0.1, rename + modification is treated as delete+add). - ---- - -## 10. Quality gates — definition of "v0.1 ready" - -Every item below must be green before v0.1 is considered shipped. - -- [ ] Repository structure matches §6 exactly -- [ ] All package dependencies match §7; no extras -- [ ] All stack choices in §5 are respected; no extras -- [ ] `@attest/schema` validates the example manifest from `SCHEMA_V0.1.md` §10 (positive test) -- [ ] `@attest/schema` rejects hard-fail rules 1, 3, and 4 from `SCHEMA_V0.1.md` §9 (three negative tests in `packages/schema/test/validator.negative.test.ts`) -- [ ] `@attest/core` rejects hard-fail rules 2 and 5 from `SCHEMA_V0.1.md` §9 (two negative tests in `packages/core/test/verifier.test.ts`; these require diff + repo context not available to the schema package) -- [ ] Every fixture in `DETECTOR_AUTHENTICATION_SPEC.md` §7 passes (matches expected verdict exactly) -- [ ] CLI golden-path test (`CLI_V01_SPEC.md` §7) passes: given the specified manifest + diff, produces the specified output, exits with code 1 -- [ ] `pnpm lint` exits 0 -- [ ] `pnpm test` exits 0 with all tests green -- [ ] `pnpm build` produces dist/ artifacts in every package -- [ ] README.md covers: what it is, install, basic `attest verify` example, link to SCHEMA_V0.1.md -- [ ] CONTRIBUTING.md covers: how to add a detector, how to add a fixture, commit conventions -- [ ] LICENSE is Apache-2.0 -- [ ] CI workflow runs `pnpm lint && pnpm test && pnpm build` on every push and PR - ---- - -## 11. Testing strategy - -1. **Schema tests** (`packages/schema/test/`): - - `validator.positive.test.ts` — valid manifest accepted - - `validator.negative.test.ts` — three rejection cases: hard-fail rules 1, 3, and 4 (JSON-Schema-enforceable; rules 2 and 5 require diff + repo context and are tested in core) -2. **Core tests** (`packages/core/test/`): - - `undeclared.test.ts` — set-subtraction behavior on files and symbols - - `verifier.test.ts` — orchestration: correct detector routing, correct aggregation -3. **Detector tests** (`packages/detectors-ts/test/`): - - `authentication.fixtures.test.ts` — drives the fixture catalog. For each `*.ts` fixture in `fixtures/authentication/`, runs the detector and asserts the verdict matches the companion `*.expected.json` -4. **CLI tests** (`packages/cli/test/`): - - `verify.e2e.test.ts` — the golden-path test - -**Fixture convention**: every fixture file `X.ts` has a companion `X.expected.json` containing `{ verdict, reason_code?, evidence_contains?: string[] }`. The test asserts verdict equality and that every string in `evidence_contains` appears in at least one evidence entry's `note`. - ---- - -## 12. Repository hygiene - -- Commit discipline: one logical change per commit. No "WIP" merged to main. -- Semver: breaking change to `@attest/schema` = major version bump. All downstream packages follow. -- Changesets required for any user-facing change. -- Every new detector or behavior ships as a dedicated PR with its own fixture suite. -- `main` protected: CI must pass; at least one review required (once the repo has collaborators). - ---- - -## 13. Non-goals (hard boundaries) - -- Not a general-purpose PR reviewer. Do one thing. -- Not a security scanner. Defer to CodeQL, Semgrep, Snyk, Dependabot. -- Not a coverage tool. Defer to c8, istanbul, Codecov. -- Not a style enforcer. Defer to eslint, prettier. -- Not a hosted service. No backend, no auth, no accounts. -- Not a replacement for human review. The output is a _focus directive_, not an approval signal. - ---- - -## 14. Versioning and schema evolution - -`schema_version` is the sole gate for manifest compatibility. The verifier rejects manifests with unrecognized versions. - -v0.1 freezes the schema shape defined in `SCHEMA_V0.1.md`. Any field addition or removal triggers a new schema version. The `@attest/schema` package exports the schema version as a constant. - ---- - -## 15. What the agent should do when ambiguity arises - -1. Re-read the relevant spec section. -2. If still ambiguous, STOP. Do not guess. -3. Surface the ambiguity to the user under "Open Questions" before proceeding. -4. Choose the most conservative interpretation (the one that rejects more inputs, constrains more behavior, narrows scope) and proceed, flagging the choice to the user. - -The single worst failure mode is the agent silently making design decisions. The second worst is stalling. Conservative-and-documented beats both. diff --git a/docs/SCHEMA_V0.1.md b/docs/SCHEMA_V0.1.md deleted file mode 100644 index 6fcc8bd..0000000 --- a/docs/SCHEMA_V0.1.md +++ /dev/null @@ -1,410 +0,0 @@ -# Claim Verification Schema — v0.1 - -Status: **Draft — not yet implemented** -Scope: TypeScript first. Python in v0.2. -Surface: MCP (Model Context Protocol) server exposes a single tool, `declare_changes`. Agent must call it before finishing. Output is rendered by the GitHub App as a single PR comment. - ---- - -## 1. Design principles (non-negotiable) - -1. Every claim must be verifiable in principle by deterministic means — no free-text performance or quality claims. -2. `target.symbol` must be AST-locatable in the post-diff tree. Hard-fail if not. -3. The verifier trusts nothing the agent says about outcomes — it re-runs the check independently. -4. `unverifiable` is a valid verdict. Failing gracefully beats false certainty. -5. Undeclared modifications are surfaced as loudly as failed claims. Silence-by-omission is the default attack surface; close it. - ---- - -## 2. Top-level manifest - -```jsonc -{ - "schema_version": "0.1", - "session": { - "agent": "claude-code" | "codex" | "cursor" | "opencode" | "other", - "model": "string", // e.g. "claude-opus-4-7" - "session_id": "uuid", - "started_at": "ISO8601", - "completed_at": "ISO8601", - "prompt_hash": "sha256:...", // hash only — never raw prompt - "tool_calls_count": 0, // integer - "files_touched": ["path1", "path2"] - }, - "task": { - "summary": "string", // one sentence, <=120 chars, human-readable - "source": "user_prompt" | "issue_reference" | "continuation" - }, - "claims": [ /* see §4 */ ] -} -``` - -**Flat claim list in v0.1.** No `parent_id`, no DAG. Grouping deferred until the core verification loop is proven. - ---- - -## 3. JSON Schema (strict validation) - -Reject manifests that do not validate. No soft-fails. No defaults. - -```json -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://example.com/claim-verification/v0.1", - "type": "object", - "required": ["schema_version", "session", "task", "claims"], - "additionalProperties": false, - "properties": { - "schema_version": { "const": "0.1" }, - "session": { - "type": "object", - "required": ["agent", "model", "session_id", "started_at", "completed_at", - "prompt_hash", "tool_calls_count", "files_touched"], - "additionalProperties": false, - "properties": { - "agent": { "enum": ["claude-code", "codex", "cursor", "opencode", "other"] }, - "model": { "type": "string", "minLength": 1 }, - "session_id": { "type": "string", "format": "uuid" }, - "started_at": { "type": "string", "format": "date-time" }, - "completed_at": { "type": "string", "format": "date-time" }, - "prompt_hash": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, - "tool_calls_count": { "type": "integer", "minimum": 0 }, - "files_touched": { "type": "array", "items": { "type": "string" } } - } - }, - "task": { - "type": "object", - "required": ["summary", "source"], - "additionalProperties": false, - "properties": { - "summary": { "type": "string", "maxLength": 120 }, - "source": { "enum": ["user_prompt", "issue_reference", "continuation"] } - } - }, - "claims": { - "type": "array", - "minItems": 1, - "items": { "$ref": "#/$defs/claim" } - } - }, - "$defs": { - "claim": { - "type": "object", - "required": ["id", "type", "target", "description", "verification_contract"], - "additionalProperties": false, - "properties": { - "id": { "type": "string", "pattern": "^c[0-9]+$" }, - "type": { "enum": [ - "add_symbol", "remove_symbol", "modify_signature", - "modify_behavior", "add_test", "refactor", - "add_dependency", "remove_dependency", "config_change" - ]}, - "target": { - "type": "object", - "required": ["kind", "path"], - "additionalProperties": false, - "properties": { - "kind": { "enum": ["function", "class", "type", "endpoint", - "file", "module", "config_key", "package"] }, - "path": { "type": "string" }, - "symbol": { "type": "string" } - } - }, - "description": { "type": "string", "maxLength": 280 }, - "verification_contract": { - "type": "object", - "required": ["check"], - "additionalProperties": false, - "properties": { - "check": { "enum": [ - "symbol_exists", "behavior_present", "test_covers", - "signature_matches", "removed", "cannot_verify" - ]}, - "params": { "type": "object" } - } - } - } - } - } -} -``` - ---- - -## 4. Claim `type` — what the claim asserts - -| `type` | What the agent is declaring | Default `check` | -|----------------------|---------------------------------------------------------|------------------------------| -| `add_symbol` | A new function/class/type/endpoint was introduced | `symbol_exists` | -| `remove_symbol` | An existing symbol was deleted | `removed` | -| `modify_signature` | Params/return type of an existing symbol changed | `signature_matches` | -| `modify_behavior` | Logic of an existing symbol changed in a named way | `behavior_present` | -| `add_test` | A new test targeting a specific symbol was added | `test_covers` | -| `refactor` | Structural change, behavior preserved (unverifiable) | `cannot_verify` | -| `add_dependency` | A package was added to manifest | `symbol_exists` (on manifest)| -| `remove_dependency` | A package was removed from manifest | `removed` | -| `config_change` | A config key was changed to a specific value | `signature_matches` | - -`refactor` is deliberately routed to `cannot_verify`. This is not a loophole — it flags the claim for mandatory human review. Agents that try to launder behavior changes through `refactor` will get caught when the `files_touched` set is larger than the `refactor` claim's target. - ---- - -## 5. `verification_contract.check` — how the verifier checks - -| `check` | Params | What the verifier runs | -|----------------------|-------------------------------------------|-------------------------------------------------------------| -| `symbol_exists` | none | Parse post-diff file; assert `target.symbol` resolves. | -| `behavior_present` | `{ property: }` | Run behavior detector for `property` on target symbol. | -| `test_covers` | `{ subject_symbol: string }` | Assert new test imports or names `subject_symbol`. | -| `signature_matches` | `{ expected: string }` | Compare AST signature of `target.symbol` to `expected`. | -| `removed` | none | Assert `target.symbol` is absent in post-diff tree. | -| `cannot_verify` | none | No-op; verdict = `unverifiable`. | - ---- - -## 6. `behavioral_property` enum — v0.1 ships with ten - -Each property is (a) a short string identifier, (b) a plain-English definition, and (c) a **detector contract** — the set of AST patterns the verifier must recognize as satisfying the claim. The detector is language-specific; TypeScript patterns below. - -### 6.1 `null_check` -- **Definition**: the target symbol guards against null/undefined input before dereference. -- **Detector must find at least one of**: - - Optional chaining (`?.`) applied to a parameter of `target.symbol` - - Nullish coalescing (`??`) applied to a parameter - - Explicit guard: `if (x == null)`, `if (!x)`, `if (x === undefined)` - - Validation library call whose schema marks the field `.nullable()` or `.optional()` — and the call appears before the first dereference -- **Fails if**: no guard appears on any parameter path leading to a dereference. - -### 6.2 `input_validation` -- **Definition**: inputs are validated against a schema or explicit constraints before use. -- **Detector must find**: - - A call to `zod.parse`, `zod.safeParse`, `yup.validate`, `joi.validate`, `class-validator` decorators, or `ajv.compile` - - OR explicit type/range/format checks on each input parameter -- **Fails if**: inputs reach a side-effecting call (DB write, network, FS) without passing through a validator. - -### 6.3 `error_handling` -- **Definition**: the target symbol handles expected failure modes rather than letting them throw uncaught. -- **Detector must find**: - - `try/catch` block wrapping the primary operation - - OR `.catch()` on a promise chain - - OR a `Result` / `Either` return type with explicit error branch -- **Fails if**: the primary operation is a call that can throw and no catch exists on any path. - -### 6.4 `authentication` -- **Definition**: a route/handler requires an authenticated caller before executing. -- **Detector must find**: - - Auth middleware applied to the route (`app.use(authMiddleware)`, decorator, route guard) - - OR an explicit token/session check as the first statement of the handler -- **Fails if**: the handler performs protected work before any auth check. - -### 6.5 `authorization` -- **Definition**: the handler checks that the authenticated caller has permission for the specific resource/action. -- **Detector must find**: - - A call to a permission/role check (`hasPermission`, `can`, `isAdmin`, `authorize`, CASL ability check) that precedes resource access -- **Fails if**: auth exists but no permission check is performed before the resource is read/written. - -### 6.6 `rate_limiting` -- **Definition**: the route is protected against abuse by request-frequency limits. -- **Detector must find**: - - Middleware from `express-rate-limit`, `@fastify/rate-limit`, `rate-limiter-flexible`, or equivalent - - OR an explicit token-bucket / sliding-window check using a datastore -- **Fails if**: the route has no such middleware and no inline check. - -### 6.7 `logging` -- **Definition**: a logger call is added at the relevant path (success, error, or audit event). -- **Detector must find**: - - A call to a known logger (`pino`, `winston`, `bunyan`, `console.error` in a catch block, platform logger) inside `target.symbol` -- **Fails if**: no such call exists inside the symbol body. - -### 6.8 `sanitization` -- **Definition**: user-provided input is sanitized before being used in HTML, SQL, shell, or filesystem context. -- **Detector must find**: - - A call to `DOMPurify`, `validator.escape`, `sqlstring.escape`, `path.resolve` + boundary check, or a parameterized query builder - - OR the input is passed to an ORM method that parameterizes by default (Prisma, TypeORM, Drizzle query builder — not raw SQL) -- **Fails if**: user input flows into `res.send`, `exec`, raw SQL, or unbounded FS path. - -### 6.9 `timeout` -- **Definition**: a bounded wait is enforced on a network or long-running operation. -- **Detector must find**: - - `AbortController` + `signal` passed to fetch/axios - - OR `axios({ timeout })`, `fetch(..., { signal: AbortSignal.timeout(n) })` - - OR `Promise.race` against `setTimeout` -- **Fails if**: the call has no upper bound on completion time. - -### 6.10 `retry_logic` -- **Definition**: a failing operation is retried according to a defined policy. -- **Detector must find**: - - A call to a retry library (`p-retry`, `async-retry`, `cockatiel`, `@aws-sdk/middleware-retry`) - - OR a loop with backoff (`for`/`while` + `setTimeout`/`sleep` + exponential factor) -- **Fails if**: the operation can fail transiently and no retry is present. - -**Escape hatch**: `cannot_express` is an eleventh valid value for `behavioral_property`. Claims using it are routed to `unverifiable` and surfaced as ⚠️ in the PR comment. Do not let this become the default — instrument its usage and if any single agent emits it on more than 20% of claims, that is itself a signal worth surfacing. - ---- - -## 7. Verifier outcomes - -Every claim resolves to exactly one verdict: - -| Verdict | Icon | Meaning | -|----------------|------|-------------------------------------------------------------------------| -| `verified` | ✅ | Deterministic check passed. | -| `unverified` | ❌ | Deterministic check failed — the claim does not match the diff. | -| `partial` | ⚠️ | Target symbol exists and was modified, but behavior check inconclusive. | -| `unverifiable` | ⓘ | Claim used `cannot_verify` or `cannot_express` — manual review needed. | - -The reviewer's attention should flow: ❌ → ⚠️ → ⓘ → ✅. - ---- - -## 8. Undeclared-changes detection (mandatory) - -This is the countermeasure against silence-by-omission. The verifier computes: - -``` -declared_files = union of claim.target.path across all claims -diff_paths = union of changed file paths across all diff entries -undeclared_files = (diff_paths ∪ session.files_touched) − declared_files -``` - -Using the union of `diff_paths` and `files_touched` closes the omission vector: a file present in the diff but absent from `files_touched` is still flagged. See `CORE_CHECKS_SPEC.md §4.1` for the full algorithm. - -For each file in `undeclared_files`, the verifier emits a synthetic `undeclared_modification` entry with verdict `unverified` and a mandatory reviewer focus marker. These are surfaced prominently in the report. - -Further: for each declared file, the verifier walks the diff and identifies modified symbols not named in any claim targeting that file. These become `undeclared_symbol_change` entries with the same verdict. - -Threshold: **zero tolerance**. Any undeclared change, at file or symbol level, is a reviewer focus point. - ---- - -## 9. Hard-fail validation rules - -The MCP server must reject the `declare_changes` call with a structured error if any of: - -1. The manifest fails JSON Schema validation. -2. `target.symbol` is specified but cannot be resolved by AST parse of the post-diff tree for `target.path`. -3. `verification_contract.check = "behavior_present"` but `params.property` is not in the `behavioral_property` enum. -4. `claims` is empty. -5. `files_touched` contains a path outside the repo root. - -Rejection returns an error payload the agent can read and retry against. No partial acceptance. - ---- - -## 10. Example — a valid manifest - -```jsonc -{ - "schema_version": "0.1", - "session": { - "agent": "claude-code", - "model": "claude-opus-4-7", - "session_id": "b3a1c0e2-9e2f-4e6a-8d13-1f2a3b4c5d6e", - "started_at": "2026-04-19T12:34:56Z", - "completed_at": "2026-04-19T12:41:22Z", - "prompt_hash": "sha256:a3f1c2e4b5d6f7a8c9e0b1d2f3a4c5e6b7d8f9a0c1e2b3d4f5a6c7e8b9d0f1a2", - "tool_calls_count": 23, - "files_touched": [ - "src/auth/email.ts", - "src/routes/auth.ts", - "src/auth/email.test.ts", - "package.json" - ] - }, - "task": { - "summary": "Add email verification flow with rate-limited /verify endpoint", - "source": "user_prompt" - }, - "claims": [ - { - "id": "c1", - "type": "add_symbol", - "target": { - "kind": "class", - "path": "src/auth/email.ts", - "symbol": "EmailVerificationService" - }, - "description": "Service for generating and validating email tokens (15-min TTL)", - "verification_contract": { "check": "symbol_exists" } - }, - { - "id": "c2", - "type": "add_symbol", - "target": { - "kind": "endpoint", - "path": "src/routes/auth.ts", - "symbol": "POST /verify" - }, - "description": "Route that validates a token and activates the user", - "verification_contract": { "check": "symbol_exists" } - }, - { - "id": "c3", - "type": "modify_behavior", - "target": { - "kind": "endpoint", - "path": "src/routes/auth.ts", - "symbol": "POST /verify" - }, - "description": "Applied rate limiting to prevent brute-force token guessing", - "verification_contract": { - "check": "behavior_present", - "params": { "property": "rate_limiting" } - } - }, - { - "id": "c4", - "type": "add_test", - "target": { - "kind": "file", - "path": "src/auth/email.test.ts" - }, - "description": "Unit tests for EmailVerificationService token generation and expiry", - "verification_contract": { - "check": "test_covers", - "params": { "subject_symbol": "EmailVerificationService" } - } - } - ] -} -``` - ---- - -## 11. Example — verifier output (rendered as PR comment) - -``` -🤖 Agent: claude-code (claude-opus-4-7) · 23 tool calls · 4 files touched -📝 Task: Add email verification flow with rate-limited /verify endpoint - -📋 Declared changes (4): - ✅ c1 EmailVerificationService exists in src/auth/email.ts - ✅ c2 POST /verify registered in src/routes/auth.ts - ❌ c3 "rate_limiting" NOT detected on POST /verify — no middleware or inline limiter - ✅ c4 src/auth/email.test.ts imports EmailVerificationService (3 tests) - -⚠️ Undeclared modifications (1): - • src/auth/email.ts — symbol `generateSecret` modified but not in any claim - -🔍 Reviewer focus: - 1. c3 failed — rate-limiting claim unverified - 2. undeclared change to `generateSecret` -``` - ---- - -## 12. Out of scope for v0.1 - -- Test execution (we check test presence and subject, not actual branch coverage) -- Concurrency/thread-safety claims -- Performance claims -- Cross-file semantic claims that can't be reduced to AST patterns -- Pre-declaration flow (agent declares intent before writing code) -- Multi-session provenance chaining -- Non-TypeScript languages (Python in v0.2) - ---- - -## 13. Versioning - -Schema versions are semver-major on any breaking field change. The `schema_version` constant in the manifest is the sole gate. Verifier rejects manifests with unrecognized versions. diff --git a/docs/SPEC.md b/docs/SPEC.md new file mode 100644 index 0000000..f9070db --- /dev/null +++ b/docs/SPEC.md @@ -0,0 +1,511 @@ +# attest — Engineering Specification + +**Target:** v1.0 (public release) +**Status:** Design locked for Phase 1. Phases 2–4 specified at decreasing fidelity. +**Audience:** the maintainer, interactive Claude Code sessions, and autonomous routines. + +> This file is the single source of truth. `CLAUDE.md` references it so every +> session and routine inherits it as ground truth. When a decision here conflicts +> with code, the code is wrong until this file is deliberately changed. + +--- + +## 0. How to read this + +Sections 1–5 are stable contracts (product, scope, data model, architecture). +Section 6 is the detailed, build-now spec. Sections 7–9 are forward specs that +will tighten as Phase 1 lands. Section 10 is the test strategy that gates every +phase. Section 11 is the implementation path using Claude Code. + +The single most important rule in this document: **attest verifies structure and +outcomes, never behavior or semantics.** Every design choice descends from that. + +--- + +## 1. Product definition + +**One-liner:** attest is the deterministic verification layer for spec-driven and +agentic development. It confirms that a code change matches what the agent declared +it would do, catches changes the agent did not declare, and produces a +compliance-grade record of the result. No LLM in the verification path. No SaaS. + +**Thesis.** AI coding agents report success in prose ("Done — added the login +endpoint and tests"). That report is not evidence. attest forces the agent to emit +a structured _manifest_ of what it changed, then independently checks every claim +against the actual diff and against real build/test outcomes, and flags anything +the agent changed but did not declare (scope drift). The output is a structured +verdict and an exportable provenance record. + +**Positioning.** attest rides two proven waves rather than standing in an empty +quadrant: + +1. **Outcome-based verification** (the SWE-bench / CI-gate pole): deterministic, + language-agnostic, run-it-and-check-the-result. +2. **Spec-driven development** (spec-kit, Kiro, GSD): developers are voluntarily + adopting "declare intent first." attest is the missing verifier at the end of + that workflow — _did the diff match the spec?_ + +**Beachhead ICP.** Regulated teams shipping high-risk AI systems (EU AI Act +Annex III domains: finance, hiring, healthcare, critical infra) who need +design-level provenance for AI-generated code before the **August 2, 2026** +high-risk enforcement date. Broad-developer adoption is a follow-on, not the +launch target. Calibrate "success" to high-intent users and design partners, not +star counts. + +**License:** Apache-2.0. **Distribution:** local CLI + CI integration. No cloud +dependency, ever. + +--- + +## 2. Scope: goals and non-goals + +### In scope (what attest does) + +- Parse an agent-emitted manifest of declared changes. +- Verify each declared change against the actual diff (file-level and symbol-level, + structurally). +- Detect undeclared changes (files and symbols changed but not declared). +- Verify declared outcomes (build passes, tests pass, lint passes) by execution. +- Emit a structured verdict and a provenance/audit record. +- Work across languages via tree-sitter (Phase 1: TypeScript/TSX, Python, Go). + +### Explicitly NOT in scope (the pivot, made permanent) + +- **No semantic correctness judgment.** attest does not decide whether the code is + _correct_ or _good_. That is the job of LLM review tools (CodeRabbit, Greptile). +- **No behavioral/security property verification.** Claims like "authentication is + enforced on every path," "input is validated," "no SQL injection" are + **semantic** and return `unverifiable`, with a message pointing the user to + semantic/LLM review. attest never tries to answer these with heuristics. (This is + the Camp-3 trap that earlier killed the syntactic-detector approach — do not + reintroduce it.) +- **No per-framework detectors as a core feature.** The legacy `detectors-ts` + authentication detector is demoted to an optional, clearly-labeled best-effort + plugin, out of the core thesis and out of CI gating. +- **No LLM anywhere in the verification path.** Determinism is the product. +- **No SaaS, no telemetry, no required network calls.** + +If a proposed feature requires understanding _what the code means_, it is out of +scope by definition. Route it to the human or to an LLM tool; do not build it into +attest. + +--- + +## 3. The verification model + +attest runs exactly three verifier families. All three are deterministic and +language-agnostic. + +1. **Declared-change verification** — for each claim in the manifest, confirm the + claimed change is structurally present in the diff/post-change tree. +2. **Undeclared-change detection** — compute the set of things actually changed, + subtract the set declared; anything left over is scope drift. (The "union trick.") +3. **Outcome verification** — execute the declared build/test/lint commands in an + isolated checkout and confirm exit codes match the claims. + +Verdict = the union of all three, plus an overall pass/fail and exit code. + +--- + +## 4. Data model + +All three artifacts live in `@attest/schema` as JSON Schema + generated TypeScript +types + an ajv validator. Schemas are versioned via `attest_version`. + +### 4.1 Manifest (input — emitted by the agent) + +```jsonc +{ + "attest_version": "1.0", + "task": { "id": "T-142", "description": "Add login endpoint" }, + "agent": { "id": "claude-code", "model": "claude-opus-4-8", "tool_calls": 5 }, + "generated_at": "2026-05-31T19:04:00Z", + + // Everything the agent claims it touched. Drives undeclared-change detection. + "declared_scope": { + "files": ["src/routes/auth.ts", "tests/auth.test.ts"], + }, + + // Individual verifiable claims. + "claims": [ + { "id": "c1", "kind": "file_change", "op": "modify", "path": "src/routes/auth.ts" }, + { + "id": "c2", + "kind": "symbol_added", + "path": "src/routes/auth.ts", + "symbol": "login", + "symbol_kind": "function", + }, + { "id": "c3", "kind": "test_added", "path": "tests/auth.test.ts", "covers": "login" }, + { "id": "c4", "kind": "outcome", "check": "tests_pass" }, + { "id": "c5", "kind": "outcome", "check": "build_passes" }, + ], +} +``` + +**Claim taxonomy (all structural — this list is closed for v1.0):** + +| `kind` | Fields | Verified by | +| ----------------- | --------------------------------------------- | ------------------ | +| `file_change` | `op` (create/modify/delete), `path` | diff parser | +| `symbol_added` | `path`, `symbol`, `symbol_kind` | tree-sitter | +| `symbol_removed` | `path`, `symbol`, `symbol_kind` | tree-sitter | +| `symbol_modified` | `path`, `symbol`, `symbol_kind` | tree-sitter | +| `test_added` | `path`, `covers?` | diff + tree-sitter | +| `test_modified` | `path`, `covers?` | diff + tree-sitter | +| `outcome` | `check` (build_passes/tests_pass/lint_passes) | runner | + +Any claim whose `kind` is not in this table → `unverifiable` with reason +`unsupported_claim_kind`. Any _semantic_ assertion smuggled into a `description` +field is ignored by the verifier (it is not a claim). + +### 4.2 Verdict (output) + +```jsonc +{ + "attest_version": "1.0", + "task_id": "T-142", + "result": "fail", // "pass" | "fail" + "exit_code": 1, // 0 = all verified + zero undeclared; else 1 + "claims": [ + { "id": "c1", "status": "verified", "evidence": { "op": "modify", "hunks": 2 } }, + { + "id": "c2", + "status": "verified", + "evidence": { "node_kind": "function_declaration", "line": 42 }, + }, + { "id": "c3", "status": "failed", "reason": "no change detected in tests/auth.test.ts" }, + { + "id": "c4", + "status": "verified", + "evidence": { "cmd": "npm test", "exit_code": 0, "duration_ms": 8123 }, + }, + { "id": "c5", "status": "verified", "evidence": { "cmd": "npm run build", "exit_code": 0 } }, + ], + "undeclared_changes": [ + { "path": "src/config/db.ts", "op": "modify", "granularity": "file", "severity": "flag" }, + ], + "summary": { "claims_total": 5, "verified": 4, "failed": 1, "unverifiable": 0, "undeclared": 1 }, +} +``` + +`status` ∈ `verified | failed | unverifiable`. The CLI renders this as the human +view (the v0.1 emoji output is fine); the JSON is the source of truth and the +input to the audit record. + +### 4.3 Audit record (Phase 3 — provenance) + +Append-only JSONL, one record per verification, designed to map onto EU AI Act +Article 12 logging fields. Minimum field set (confirm against primary Article +12/19 text before any pitch copy claims compliance): + +```jsonc +{ + "record_id": "uuid", + "timestamp": "2026-05-31T19:04:10Z", + "invoking_user": "rituraj", // who ran the agent/verifier + "governing_spec": { "source": "spec-kit", "ref": "tasks.md@a1b2c3" }, + "agent": { "id": "claude-code", "model": "claude-opus-4-8" }, + "input_context_hash": "sha256:...", // manifest hash, not raw content + "output_artifact_hash": "sha256:...", // diff hash + "verdict_digest": "sha256:...", // hash of §4.2 verdict + "human_reviewer": null, // filled on review + "disposition": "pending", // pending|accepted|rejected +} +``` + +No raw PII or source in the audit record — hashes only, matching the auditguard +pattern. Retention target ≥ 6 months (Article 19). + +--- + +## 5. Architecture + +Monorepo (pnpm), TypeScript. Reorganize from v0.1 as follows. + +``` +packages/ + schema/ @attest/schema JSON Schema, TS types, ajv validator (manifest, verdict, audit) + diff/ @attest/diff unified diff -> structured file/hunk model + symbols/ @attest/symbols tree-sitter symbol extraction, language-agnostic [NEW] + core/ @attest/core orchestration: load manifest, run 3 verifiers, assemble verdict + runner/ @attest/runner build/test/lint execution + worktree isolation [NEW] + audit/ @attest/audit provenance record emission [Phase 3] + cli/ @attest/cli `attest verify`, `attest derive`, `attest audit` + detectors-ts/ @attest/detectors-ts DEMOTED: optional best-effort behavioral plugin, off by default +``` + +### 5.1 The tree-sitter decision (architectural linchpin) + +Symbol verification uses **tree-sitter** parse trees, not per-language/per-framework +hand-written logic. `@attest/symbols` exposes one operation: + +> Given a file's post-change source and a `symbol` + `symbol_kind`, return whether a +> declaration node of that kind with that name exists, and where. + +Per language we vendor a grammar (TS/TSX, Python, Go in Phase 1) and a small +**node-kind map**: `symbol_kind` → the grammar's declaration node kinds +(e.g. `function` → `function_declaration | method_definition | arrow_function` +assigned to a name binding). Adding a language = vendor a grammar + write a +node-kind map. **No detector logic, ever.** This is what makes "language-agnostic +structural verification" real and what keeps attest out of the Camp-3 trap: it +answers _does this symbol exist and is it of this kind_, never _does this symbol +behave correctly_. + +--- + +## 6. Phase 1 — Core verification engine (BUILD NOW, ~3 weeks of evenings) + +**Goal / definition of done:** a stranger clones attest, points it at a real repo +in TypeScript, Python, _or_ Go, runs `attest verify`, and gets a correct verdict +that includes (a) per-claim structural verification, (b) undeclared-change +detection, and (c) real build/test outcomes — with no framework-specific code in +the path. + +### 6.1 Inputs + +`attest verify --manifest --diff --repo-root ` +plus a repo config file (`attest.toml` or `attest.config.json`) declaring runner +commands (see 6.4). `--diff` may be omitted to default to `git diff` of the working +tree against `HEAD`. + +### 6.2 Declared-change verification + +For each claim: + +- **`file_change`** — parse the diff (`@attest/diff`). Confirm a hunk exists for + `path` with the claimed `op`. `create` = file absent in base, present after; + `delete` = inverse; `modify` = present both sides with hunks. Pure text/diff + operation, language-agnostic. Status `verified` / `failed`. + +- **`symbol_added|removed|modified`** — use `@attest/symbols`. Parse the relevant + pre- and post-change file states (reconstruct from base + diff, or read from the + worktree). Compute the symbol delta for that file at the requested `symbol_kind`. + Confirm the named symbol appears in the added/removed/modified set as claimed. + Status with evidence `{ node_kind, line }`. + +- **`test_added|modified`** — structural only: (1) a diff hunk exists for a path + that the repo's test-glob classifies as a test file, and (2) if `covers` is + given, a tree-sitter test-symbol (e.g. `it`/`test`/`describe` call, `def test_*`, + `func Test*`) referencing or adjacent to `covers` was added/changed. `covers` is + a _structural reference check, not a coverage proof_ — do not claim it verifies + the test actually exercises the symbol. If you cannot confirm structurally, + return `unverifiable`, never a guess. + +- **Behavioral / unknown kinds** — `unverifiable`, reason set, with a message + pointing to LLM review. Never fall through to a heuristic. + +### 6.3 Undeclared-change detection (the moat) + +``` +actual_files = { paths with hunks in the diff } +declared_files = declared_scope.files ∪ { path of every claim } +undeclared_files = actual_files \ declared_files (minus the allowlist) +``` + +Plus **intra-file symbol drift**: for each declared file, compute actual +added/modified symbols (tree-sitter) and subtract those named in claims; leftovers +are undeclared symbol changes inside an otherwise-declared file. This is the +scope-drift case Gergely Orosz named ("the agent fixed something nearby and the +diff no longer corresponds to the intention"). + +**Allowlist** (config, with sane defaults) suppresses noise: lockfiles +(`package-lock.json`, `pnpm-lock.yaml`, `go.sum`, `poetry.lock`), generated dirs, +formatting-only hunks. Without this the detector is too noisy to trust — treat the +allowlist as a Phase-1 requirement, not a polish item. Each undeclared change gets +a `severity` (`flag` default; allowlisted = suppressed). + +### 6.4 Outcome verification (runner) + +`@attest/runner` executes declared `outcome` checks and compares exit codes. + +- **Config** (`attest.toml`): `build_cmd`, `test_cmd`, `lint_cmd`. Auto-detect when + absent (npm/pnpm scripts, `Makefile`, `pyproject`/`pytest`, `go build`/`go test`). +- **Isolation (Phase 1):** create a clean `git worktree` at the post-change state, + run commands there, capture exit code + truncated (head/tail) logs + duration. + **Do not run in the live working tree** — isolation is a correctness requirement. +- **Security note:** Phase 1 assumes the user runs attest on _their own_ change + (locally or in their own CI), where they would run these tests anyway. Running + _untrusted_ agent code under attest requires container isolation — deferred to + Phase 3 (§8). Do not let this gap get silently closed by a "simpler" worktree-less + implementation. + +### 6.5 detectors-ts demotion + +Move the v0.1 authentication detector and the `chain.ts`/`classify.ts` heuristics +into `@attest/detectors-ts` as an **opt-in plugin, disabled by default**, clearly +labeled "best-effort, non-deterministic across frameworks, not part of the core +verdict." It must not affect exit code or gate CI. Keep it (sunk work, occasional +signal); do not centre it. Do not invest further in per-framework coverage. + +### 6.6 CLI surface (Phase 1) + +- `attest verify --manifest --diff? --repo-root --format json|human` + Exit 0 iff all claims `verified`/`unverifiable-but-allowed` AND zero undeclared + (after allowlist). Else exit 1. +- `attest schema [manifest|verdict]` — print the JSON Schema. + +### 6.7 Phase 1 acceptance criteria (the "done" gate — do not skip) + +Phase 1 ships only when **all** hold: + +1. `attest verify` produces a correct verdict on a real repo in **each** of TS, + Python, Go (one fixture repo per language, in the test corpus §10). +2. Undeclared-change detection catches a planted scope-drift change (a file and an + intra-file symbol the manifest did not declare) and suppresses an allowlisted one. +3. Outcome verification runs real build+test in worktree isolation and reports the + true exit code (verified against a deliberately failing-test fixture). +4. A behavioral claim returns `unverifiable` with the LLM-review pointer — never a + heuristic verdict. +5. The fixture corpus (honest / lying / partial / undeclared manifests) passes as + the regression oracle; CI runs it on every commit. +6. detectors-ts is off by default and does not influence exit code. +7. `README` shows a 20-minute zero-to-first-verdict path on a real repo. + +--- + +## 7. Phase 2 — Workflow integration + SDD derivation (~weeks 3–5) + +**Goal:** attest drops into existing workflows with near-zero manual manifest +authoring. + +- **GitHub Action** (highest leverage): fails the check when claims don't verify or + undeclared changes exist. Enters teams' CI without workflow change. +- **Pre-commit hook** and **Claude Code hook**: run attest automatically when the + agent reports completion. +- **`attest derive`** — the friction-killer. Where a spec-driven-development + artifact already exists (spec-kit `tasks.md`, Kiro requirements/design/tasks, + EARS acceptance criteria), derive the manifest from it instead of asking for a + second declaration. This is the core Phase 2 differentiator: attest becomes the + verification half of a movement already in motion. Start with spec-kit `tasks.md` + (largest install base), then Kiro. +- **Acceptance:** a contributor adds attest to a repo's CI in one file; on an + SDD repo, `attest derive` produces a usable manifest with no hand-authoring; the + Action posts a clear pass/fail with the undeclared list. + +--- + +## 8. Phase 3 — Audit trail / provenance (~weeks 5–6) + +**Goal:** emit a regulator-presentable provenance record per verification. + +- `@attest/audit` writes the §4.3 JSONL record, append-only, hashes-not-content. +- Field mapping to EU AI Act Article 12 (logging) and Article 19 (≥6-month + retention). **Verify against primary statute text before any compliance claim in + docs or pitch.** +- `attest audit export` — produce a signed, time-ordered bundle for a date range. +- **Container isolation** for the runner lands here (reproducible, sandboxed + execution; SWE-bench-style pinned image), closing the §6.4 security gap for the + untrusted-code case the regulated buyer cares about. +- **Positioning:** lead with verification, deliver it _in_ the audit format. "An + agent touched this" is satisfiable by a git `Co-Authored-By` footer; your + defensible position is "the agent claimed X, the verifier independently confirmed + Y, here is the signed divergence record." +- **Acceptance:** every `verify` emits a valid audit record; `export` yields a + bundle a non-technical reviewer can read; container-isolated runs are + reproducible across machines. + +--- + +## 9. Phase 4 — Launch (~weeks 6–8) + +Not engineering; positioning and distribution. + +- **Thesis essay** on rituraj.info: "Coding agents lie by omission — make them + prove their work," with attest as the proof-of-concept and the SDD-verifier framing. +- **60-second demo:** an agent claims "done," attest catches an undeclared change + and a failed test claim live. +- **Channels (not HN):** r/LocalLLaMA, r/ExperiencedDevs, the Claude Code / Aider / + spec-kit communities, a build-in-public X thread. One known voice in the + AI-coding-tools space trying it beats any volume of posting. +- **Beachhead message** for the regulated ICP: design-level provenance for + AI-generated code ahead of the Aug 2 2026 high-risk deadline, audit trail inside + your perimeter (no vendor-hosted logs). +- **Success = high-intent users / design partners, not raw stars.** + +--- + +## 10. Cross-cutting: test strategy (the oracle) + +The fixture corpus is the backbone and should be built first (it's also ideal +overnight-routine work). For each supported language, a small real repo plus a set +of (manifest, diff) pairs spanning: + +- **Honest** — every claim true, nothing undeclared → expect `pass`. +- **Lying** — a claim asserts a symbol/file/test that isn't in the diff → `failed`. +- **Partial** — some claims true, some false → mixed verdict, exit 1. +- **Undeclared** — diff changes a file/symbol the manifest didn't declare → flagged. +- **Allowlisted** — only a lockfile changed beyond scope → suppressed, `pass`. +- **Outcome-fail** — claims `tests_pass` but a test fails → `failed` via runner. +- **Behavioral** — a semantic claim → `unverifiable` + pointer, never a guess. + +Every phase regresses against this corpus in CI. A change that breaks an oracle +case is wrong by definition. + +--- + +## 11. Implementation path with Claude Code + +You have a full-time job (in this domain), ~2 evening hours, and Claude Code with +routines (autonomous, cloud, output to a `claude/`-prefixed branch for review; +no mid-run human step). The method below splits work by _what needs your judgment_ +vs _what is mechanical against a clear spec_. + +### 11.1 Set up the context once + +- This file lives at `docs/SPEC.md` (done). +- `CLAUDE.md` at repo root points to `docs/SPEC.md` as authoritative, states the + non-goals (§2) explicitly (so neither a session nor a routine reintroduces + semantic detectors), and names the fixture corpus as the regression oracle. +- Add a few slash commands / saved prompts for the repeated loops (e.g. + "implement package X against its sub-spec and the corpus, open a PR"). + +### 11.2 The division of labor (the core rule) + +- **Routines (overnight, mechanical, well-bounded):** scaffolding, tree-sitter + grammar wiring + node-kind maps, the fixture corpus, first-draft implementations + of a _fully specified_ package against existing tests, doc generation, dependency + bumps. Output is a branch/PR you review at breakfast. +- **Interactive sessions (your 2 evening hours, judgment-heavy):** every design + decision, the symbol-delta and undeclared-detection logic, the runner isolation, + the audit format, and **reviewing/merging every routine PR**. Routines draft; + you decide. Never merge a routine PR unread — the worktree-isolation and + no-semantic-fallthrough invariants are exactly what an autonomous agent will + "helpfully" weaken. + +### 11.3 Build order (bottom-up; each step gated by the corpus) + +1. **`@attest/schema`** — lock manifest/verdict schemas (§4.1–4.2). _Interactive_ + (these are contracts). Routine can generate types + validator after you lock them. +2. **Fixture corpus** (§10) — _routine_ generates first drafts overnight; you curate. + Build this early; it's the oracle for everything after. +3. **`@attest/diff`** — unified-diff parser. _Routine_ drafts against corpus diffs; + you review edge cases (renames, binary, mode changes). +4. **`@attest/symbols`** — tree-sitter wiring + node-kind maps for TS/Python/Go. + Grammar wiring is _routine_; the node-kind maps and the "exists/kind only, never + behavior" boundary are _interactive_. +5. **`@attest/core`** — orchestration + the three verifiers + undeclared detection. + This is the heart and the most judgment-heavy. _Interactive_, with routines + filling in well-specified sub-functions. +6. **`@attest/runner`** — worktree isolation + command execution. _Interactive_ + (isolation correctness matters); routine can draft the auto-detect table. +7. **`@attest/cli`** — wire it together, human + JSON output. Mostly _routine_, + you review UX. +8. Run the full corpus; hit the §6.7 acceptance gate; only then ship Phase 1. + +### 11.4 Routine cadence given run caps + +Run caps are per-day and tier-based. Spend them on the highest-value overnight +unit, not many small triggers. A good nightly pattern: one routine that picks up +the next "implement package X against its sub-spec + corpus, open a PR" task; you +review and merge in the morning, queue the next. Keep a short `docs/BUILD_LOG.md` +the routine appends to, so each night's run has continuity. + +### 11.5 Discipline guardrails (your known failure mode) + +- **Do not start Phase 2 until §6.7 passes.** Finishing the gate is the point. +- **Do not let any routine reintroduce semantic detectors** — `CLAUDE.md` must + forbid it and you must catch it in review. +- **One tool to done before the eval harness.** attest reaches public v1.0 first; + the eval harness is the second instance of the same primitive, reusing + `@attest/audit`. Resist building both at once. From 9bc15eb8c587a42f2b999bab9413a55df163be39 Mon Sep 17 00:00:00 2001 From: ree2raz Date: Thu, 4 Jun 2026 19:38:58 +0530 Subject: [PATCH 2/2] style: apply prettier formatting across the repo Repo-wide prettier sweep (line-wrapping, multi-line arrays, JSON pretty-printing, double quotes). No logic or behavior changes; `git diff -w` shows only reformatting. --- CONTRIBUTING.md | 1 + packages/cli/package.json | 4 +- packages/cli/src/commands/verify.ts | 23 ++++--- packages/cli/src/index.ts | 4 +- packages/cli/src/render/human.ts | 6 +- .../test/fixtures/golden-path/expected.json | 17 +++-- packages/cli/test/golden.test.ts | 46 +++++++++----- packages/core/package.json | 4 +- packages/core/src/checks/removed.ts | 27 +++++++- packages/core/src/checks/signature-matches.ts | 7 ++- packages/core/src/checks/symbol-exists.ts | 4 +- packages/core/src/checks/test-covers.ts | 28 +++++++-- packages/core/src/locate-route.ts | 19 +++++- packages/core/src/verifier.ts | 22 ++++--- .../test/fixtures/basic/manifest-stub.json | 24 ++++++- packages/core/test/undeclared.test.ts | 10 +-- packages/core/test/verifier.test.ts | 4 +- .../fastify-preHandler-valid.ts | 9 ++- .../detectors-ts/src/authentication/chain.ts | 50 +++++++++++---- .../src/authentication/classify.ts | 58 ++++++++++++++--- .../detectors-ts/src/authentication/index.ts | 9 +-- packages/schema/package.json | 4 +- packages/schema/src/manifest.schema.json | 63 +++++++++++++------ packages/schema/src/validator.ts | 6 +- pnpm-workspace.yaml | 2 +- 25 files changed, 324 insertions(+), 127 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c74998b..43292d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,7 @@ Follow conventional commits: `feat(scope): message`, `fix(scope): message`, `tes Scope is the package short name: `schema`, `core`, `detectors-ts`, `cli`. Examples: + - `feat(schema): add ajv validator` - `test(detectors-ts): add express fixtures` - `fix(core): handle deleted files in undeclared detection` diff --git a/packages/cli/package.json b/packages/cli/package.json index 5bad816..c11e56f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -5,7 +5,9 @@ "bin": { "attest": "./dist/index.js" }, - "files": ["dist"], + "files": [ + "dist" + ], "scripts": { "build": "tsup", "test": "vitest run", diff --git a/packages/cli/src/commands/verify.ts b/packages/cli/src/commands/verify.ts index e8bfd20..a3a9e29 100644 --- a/packages/cli/src/commands/verify.ts +++ b/packages/cli/src/commands/verify.ts @@ -25,9 +25,18 @@ export class VerifyCommand extends Command { ], }); - manifest = Option.String("--manifest,-m", { required: true, description: "Path to manifest JSON" }); - diff = Option.String("--diff,-d", { required: true, description: "Path to unified diff file, or - for stdin" }); - repoRoot = Option.String("--repo-root,-r", { required: false, description: "Repository root (default: cwd)" }); + manifest = Option.String("--manifest,-m", { + required: true, + description: "Path to manifest JSON", + }); + diff = Option.String("--diff,-d", { + required: true, + description: "Path to unified diff file, or - for stdin", + }); + repoRoot = Option.String("--repo-root,-r", { + required: false, + description: "Repository root (default: cwd)", + }); format = Option.String("--format,-f", "human", { description: "Output format: human or json" }); noColor = Option.Boolean("--no-color", false, { description: "Disable ANSI color" }); verbose = Option.Boolean("--verbose,-v", false, { description: "Verbose stderr output" }); @@ -36,9 +45,7 @@ export class VerifyCommand extends Command { const { stderr: out } = this.context; // ── Resolve repo root ──────────────────────────────────────────────── - const repoRoot = this.repoRoot - ? resolve(this.repoRoot) - : process.cwd(); + const repoRoot = this.repoRoot ? resolve(this.repoRoot) : process.cwd(); try { await access(repoRoot, constants.R_OK); @@ -48,9 +55,7 @@ export class VerifyCommand extends Command { } // ── Read manifest ──────────────────────────────────────────────────── - const manifestPath = isAbsolute(this.manifest) - ? this.manifest - : resolve(this.manifest); + const manifestPath = isAbsolute(this.manifest) ? this.manifest : resolve(this.manifest); let manifestRawBytes: Buffer; try { diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 3fce442..cb6ca04 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,7 +9,9 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); // Read version from package.json let version = "0.1.0"; try { - const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8")) as { version?: string }; + const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8")) as { + version?: string; + }; version = pkg.version ?? "0.1.0"; } catch { // fallback diff --git a/packages/cli/src/render/human.ts b/packages/cli/src/render/human.ts index 2e528a9..7a1d8f1 100644 --- a/packages/cli/src/render/human.ts +++ b/packages/cli/src/render/human.ts @@ -72,11 +72,7 @@ function evidenceSummary(claim: ClaimResult, manifest: Manifest): string { // ─── Main renderer ───────────────────────────────────────────────────────── -export function renderHuman( - report: VerdictReport, - manifest: Manifest, - useColor: boolean, -): string { +export function renderHuman(report: VerdictReport, manifest: Manifest, useColor: boolean): string { const lines: string[] = []; const { session, task } = manifest; diff --git a/packages/cli/test/fixtures/golden-path/expected.json b/packages/cli/test/fixtures/golden-path/expected.json index f0e57ac..a38d124 100644 --- a/packages/cli/test/fixtures/golden-path/expected.json +++ b/packages/cli/test/fixtures/golden-path/expected.json @@ -13,24 +13,21 @@ { "claim_id": "c1", "verdict": "verified", - "evidence": [ - { "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /login" } - ] + "evidence": [{ "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /login" }] }, { "claim_id": "c2", "verdict": "unverified", "reason_code": "no_auth_in_chain", - "evidence": [ - { "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /login" } - ] + "evidence": [{ "kind": "route", "path": "src/routes/auth.ts", "symbol": "POST /login" }] } ], - "undeclared": [ - { "type": "symbol", "path": "src/routes/auth.ts", "symbol": "unlistedHelper" } - ], + "undeclared": [{ "type": "symbol", "path": "src/routes/auth.ts", "symbol": "unlistedHelper" }], "reviewer_focus": [ { "claim_id": "c2", "reason": "c2 failed — authentication not detected" }, - { "undeclared": { "type": "symbol", "path": "src/routes/auth.ts", "symbol": "unlistedHelper" }, "reason": "undeclared change to `unlistedHelper`" } + { + "undeclared": { "type": "symbol", "path": "src/routes/auth.ts", "symbol": "unlistedHelper" }, + "reason": "undeclared change to `unlistedHelper`" + } ] } diff --git a/packages/cli/test/golden.test.ts b/packages/cli/test/golden.test.ts index adbab1c..33dc752 100644 --- a/packages/cli/test/golden.test.ts +++ b/packages/cli/test/golden.test.ts @@ -11,9 +11,7 @@ const FIXTURES = join(__dirname, "fixtures", "golden-path"); const REPO_ROOT = join(__dirname, "..", "..", "..", ".."); // monorepo root /** Run the CLI via Node directly (ts-node / tsx won't be available; use built dist) */ -async function runCli( - args: string[], -): Promise<{ stdout: string; stderr: string; code: number }> { +async function runCli(args: string[]): Promise<{ stdout: string; stderr: string; code: number }> { try { const { stdout, stderr } = await execFileAsync( process.execPath, @@ -35,10 +33,14 @@ describe("attest verify — golden path", () => { it("exits 1 and produces correct human output", async () => { const { stdout, code } = await runCli([ "verify", - "--manifest", MANIFEST, - "--diff", DIFF, - "--repo-root", REPO, - "--format", "human", + "--manifest", + MANIFEST, + "--diff", + DIFF, + "--repo-root", + REPO, + "--format", + "human", "--no-color", ]); @@ -50,10 +52,14 @@ describe("attest verify — golden path", () => { it("exits 1 and produces correct JSON output", async () => { const { stdout, code } = await runCli([ "verify", - "--manifest", MANIFEST, - "--diff", DIFF, - "--repo-root", REPO, - "--format", "json", + "--manifest", + MANIFEST, + "--diff", + DIFF, + "--repo-root", + REPO, + "--format", + "json", ]); const expected = JSON.parse(readFileSync(join(FIXTURES, "expected.json"), "utf-8")); @@ -67,9 +73,12 @@ describe("attest verify — exit codes", () => { it("exits 66 when manifest file not found", async () => { const { code, stderr } = await runCli([ "verify", - "--manifest", "/nonexistent/manifest.json", - "--diff", DIFF, - "--repo-root", REPO, + "--manifest", + "/nonexistent/manifest.json", + "--diff", + DIFF, + "--repo-root", + REPO, ]); expect(code).toBe(66); expect(stderr).toMatch(/not found|ENOENT/i); @@ -78,9 +87,12 @@ describe("attest verify — exit codes", () => { it("exits 65 when manifest is invalid JSON", async () => { const { code, stderr } = await runCli([ "verify", - "--manifest", join(FIXTURES, "input.diff"), // diff is not JSON - "--diff", DIFF, - "--repo-root", REPO, + "--manifest", + join(FIXTURES, "input.diff"), // diff is not JSON + "--diff", + DIFF, + "--repo-root", + REPO, ]); expect(code).toBe(65); expect(stderr).toMatch(/JSON|parse/i); diff --git a/packages/core/package.json b/packages/core/package.json index a078de5..0887c7a 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -10,7 +10,9 @@ "types": "./dist/index.d.ts" } }, - "files": ["dist"], + "files": [ + "dist" + ], "scripts": { "build": "tsc -p tsconfig.build.json", "test": "vitest run", diff --git a/packages/core/src/checks/removed.ts b/packages/core/src/checks/removed.ts index 6fc0d93..7c6f90b 100644 --- a/packages/core/src/checks/removed.ts +++ b/packages/core/src/checks/removed.ts @@ -35,7 +35,14 @@ export function checkRemoved( return { claim_id, verdict: "verified", - evidence: [{ kind: "symbol", path, ...(symbol ? { symbol } : {}), note: "file deleted; symbol implicitly removed" }], + evidence: [ + { + kind: "symbol", + path, + ...(symbol ? { symbol } : {}), + note: "file deleted; symbol implicitly removed", + }, + ], }; } @@ -46,7 +53,14 @@ export function checkRemoved( return { claim_id, verdict: "unverified", - evidence: [{ kind: "symbol", path, ...(symbol ? { symbol } : {}), note: `${target.kind} still present in post-diff content` }], + evidence: [ + { + kind: "symbol", + path, + ...(symbol ? { symbol } : {}), + note: `${target.kind} still present in post-diff content`, + }, + ], }; } @@ -54,7 +68,14 @@ export function checkRemoved( return { claim_id, verdict: "verified", - evidence: [{ kind: "symbol", path, ...(symbol ? { symbol } : {}), note: `${target.kind} absent from post-diff content` }], + evidence: [ + { + kind: "symbol", + path, + ...(symbol ? { symbol } : {}), + note: `${target.kind} absent from post-diff content`, + }, + ], }; } diff --git a/packages/core/src/checks/signature-matches.ts b/packages/core/src/checks/signature-matches.ts index bb92c28..7636dcc 100644 --- a/packages/core/src/checks/signature-matches.ts +++ b/packages/core/src/checks/signature-matches.ts @@ -63,7 +63,12 @@ export function checkSignatureMatches( verdict: "unverifiable", reason_code: "unsupported_check", evidence: [ - { kind: "symbol", path, symbol, note: `signature_matches not supported for kind "${kind}"` }, + { + kind: "symbol", + path, + symbol, + note: `signature_matches not supported for kind "${kind}"`, + }, ], }; } diff --git a/packages/core/src/checks/symbol-exists.ts b/packages/core/src/checks/symbol-exists.ts index d04403a..7a2e441 100644 --- a/packages/core/src/checks/symbol-exists.ts +++ b/packages/core/src/checks/symbol-exists.ts @@ -85,6 +85,8 @@ export function checkSymbolExists( return { claim_id, verdict: "unverified", - evidence: [{ kind: "symbol", path, symbol, note: `${kind} declaration not found in post-diff content` }], + evidence: [ + { kind: "symbol", path, symbol, note: `${kind} declaration not found in post-diff content` }, + ], }; } diff --git a/packages/core/src/checks/test-covers.ts b/packages/core/src/checks/test-covers.ts index 62e1e7d..7676b1f 100644 --- a/packages/core/src/checks/test-covers.ts +++ b/packages/core/src/checks/test-covers.ts @@ -32,13 +32,22 @@ export function checkTestCovers( // 1. Import declarations for (const importDecl of sourceFile.getImportDeclarations()) { const defaultImport = importDecl.getDefaultImport(); - if (defaultImport?.getText() === subjectSymbol) { refCount++; break; } + if (defaultImport?.getText() === subjectSymbol) { + refCount++; + break; + } const namespaceImport = importDecl.getNamespaceImport(); - if (namespaceImport?.getText() === subjectSymbol) { refCount++; break; } + if (namespaceImport?.getText() === subjectSymbol) { + refCount++; + break; + } const named = importDecl.getNamedImports().find((n) => n.getName() === subjectSymbol); - if (named) { refCount++; break; } + if (named) { + refCount++; + break; + } } // 2. String literals inside test-runner calls @@ -49,7 +58,10 @@ export function checkTestCovers( for (const arg of callExpr.getArguments()) { if (arg.getKind() === SyntaxKind.StringLiteral) { const text = arg.asKindOrThrow(SyntaxKind.StringLiteral).getLiteralValue(); - if (text.includes(subjectSymbol)) { refCount++; break; } + if (text.includes(subjectSymbol)) { + refCount++; + break; + } } } } @@ -71,13 +83,17 @@ export function checkTestCovers( return { claim_id, verdict: "verified", - evidence: [{ kind: "test", path, symbol: subjectSymbol, note: `${refCount} reference(s) found` }], + evidence: [ + { kind: "test", path, symbol: subjectSymbol, note: `${refCount} reference(s) found` }, + ], }; } return { claim_id, verdict: "unverified", - evidence: [{ kind: "test", path, symbol: subjectSymbol, note: "no references to subject_symbol found" }], + evidence: [ + { kind: "test", path, symbol: subjectSymbol, note: "no references to subject_symbol found" }, + ], }; } diff --git a/packages/core/src/locate-route.ts b/packages/core/src/locate-route.ts index de64894..fb2b09f 100644 --- a/packages/core/src/locate-route.ts +++ b/packages/core/src/locate-route.ts @@ -12,11 +12,26 @@ import { SyntaxKind, type SourceFile, type Node } from "ts-morph"; export type KnownFramework = "express" | "fastify" | "nestjs" | "koa" | "raw-node"; const EXPRESS_METHODS = new Set([ - "get", "post", "put", "delete", "patch", "options", "head", "all", "use", + "get", + "post", + "put", + "delete", + "patch", + "options", + "head", + "all", + "use", ]); const NESTJS_HTTP_DECORATORS = new Set([ - "Get", "Post", "Put", "Delete", "Patch", "Options", "Head", "All", + "Get", + "Post", + "Put", + "Delete", + "Patch", + "Options", + "Head", + "All", ]); export interface RouteLocation { diff --git a/packages/core/src/verifier.ts b/packages/core/src/verifier.ts index 8b2bf4c..55aa382 100644 --- a/packages/core/src/verifier.ts +++ b/packages/core/src/verifier.ts @@ -4,7 +4,11 @@ import { Project } from "ts-morph"; import type { Claim } from "@attest/schema"; import type { VerifyInput, VerdictReport, ClaimResult, UndeclaredFinding } from "./types.js"; import { computeManifestHash, buildVerdictReport } from "./verdict.js"; -import { computeUndeclaredFiles, computeUndeclaredSymbols, buildCoveredSymbolSet } from "./undeclared.js"; +import { + computeUndeclaredFiles, + computeUndeclaredSymbols, + buildCoveredSymbolSet, +} from "./undeclared.js"; import { checkCannotVerify } from "./checks/cannot-verify.js"; import { checkSymbolExists } from "./checks/symbol-exists.js"; import { checkRemoved } from "./checks/removed.js"; @@ -19,9 +23,7 @@ function validateFilesTouched(repoRoot: string, filesTouched: readonly string[]) for (const filePath of filesTouched) { const resolved = resolve(normalizedRoot, filePath); if (!resolved.startsWith(normalizedRoot + "/") && resolved !== normalizedRoot) { - throw new Error( - `files_touched path "${filePath}" is outside repo root "${normalizedRoot}"`, - ); + throw new Error(`files_touched path "${filePath}" is outside repo root "${normalizedRoot}"`); } } } @@ -114,7 +116,9 @@ async function dispatchClaim( return { claim_id, verdict: "unverified", - evidence: [{ kind: "test", path: target.path, note: "file not found in post-diff state" }], + evidence: [ + { kind: "test", path: target.path, note: "file not found in post-diff state" }, + ], }; } return checkTestCovers(claim_id, target.path, vc, sourceFile); @@ -199,11 +203,9 @@ export async function verify(input: VerifyInput): Promise { const content = await postDiffFile(repoRoot, filePath); if (!content) continue; - const sourceFile = project.createSourceFile( - `__virtual_undeclared__/${filePath}`, - content, - { overwrite: true }, - ); + const sourceFile = project.createSourceFile(`__virtual_undeclared__/${filePath}`, content, { + overwrite: true, + }); const coveredSymbols = buildCoveredSymbolSet(manifest, filePath); const symbolFindings = computeUndeclaredSymbols(sourceFile, filePath, coveredSymbols); diff --git a/packages/core/test/fixtures/basic/manifest-stub.json b/packages/core/test/fixtures/basic/manifest-stub.json index e6970b8..37fa3ab 100644 --- a/packages/core/test/fixtures/basic/manifest-stub.json +++ b/packages/core/test/fixtures/basic/manifest-stub.json @@ -1 +1,23 @@ -{"schema_version":"0.1","session":{"agent":"claude-code","model":"test","session_id":"b3a1c0e2-9e2f-4e6a-8d13-1f2a3b4c5d6e","started_at":"2026-01-01T00:00:00Z","completed_at":"2026-01-01T00:01:00Z","prompt_hash":"sha256:a3f1c2e4b5d6f7a8c9e0b1d2f3a4c5e6b7d8f9a0c1e2b3d4f5a6c7e8b9d0f1a2","tool_calls_count":1,"files_touched":["src/main.ts"]},"task":{"summary":"test","source":"user_prompt"},"claims":[{"id":"c1","type":"refactor","target":{"kind":"file","path":"src/main.ts"},"description":"refactored","verification_contract":{"check":"cannot_verify"}}]} +{ + "schema_version": "0.1", + "session": { + "agent": "claude-code", + "model": "test", + "session_id": "b3a1c0e2-9e2f-4e6a-8d13-1f2a3b4c5d6e", + "started_at": "2026-01-01T00:00:00Z", + "completed_at": "2026-01-01T00:01:00Z", + "prompt_hash": "sha256:a3f1c2e4b5d6f7a8c9e0b1d2f3a4c5e6b7d8f9a0c1e2b3d4f5a6c7e8b9d0f1a2", + "tool_calls_count": 1, + "files_touched": ["src/main.ts"] + }, + "task": { "summary": "test", "source": "user_prompt" }, + "claims": [ + { + "id": "c1", + "type": "refactor", + "target": { "kind": "file", "path": "src/main.ts" }, + "description": "refactored", + "verification_contract": { "check": "cannot_verify" } + } + ] +} diff --git a/packages/core/test/undeclared.test.ts b/packages/core/test/undeclared.test.ts index f76f39c..86a8d52 100644 --- a/packages/core/test/undeclared.test.ts +++ b/packages/core/test/undeclared.test.ts @@ -23,9 +23,9 @@ describe("computeUndeclaredFiles", () => { it("uses union of diff_paths and files_touched", () => { const result = computeUndeclaredFiles( - new Set(["src/a.ts"]), // in diff only - ["src/b.ts"], // in touched only - new Set([]), // nothing declared + new Set(["src/a.ts"]), // in diff only + ["src/b.ts"], // in touched only + new Set([]), // nothing declared ); expect(result.sort()).toEqual(["src/a.ts", "src/b.ts"]); }); @@ -33,8 +33,8 @@ describe("computeUndeclaredFiles", () => { it("does not flag files in touched but absent from diff (no concern)", () => { // files_touched but absent from diff AND declared → not flagged const result = computeUndeclaredFiles( - new Set([]), // diff is empty - ["src/declared.ts"], // only touched + new Set([]), // diff is empty + ["src/declared.ts"], // only touched new Set(["src/declared.ts"]), // declared ); expect(result).toEqual([]); diff --git a/packages/core/test/verifier.test.ts b/packages/core/test/verifier.test.ts index 66ab982..424c86e 100644 --- a/packages/core/test/verifier.test.ts +++ b/packages/core/test/verifier.test.ts @@ -157,7 +157,9 @@ describe("verify — undeclared detection", () => { }, ], }; - const diffWithExtra = BASIC_DIFF + `\ + const diffWithExtra = + BASIC_DIFF + + `\ diff --git a/src/other.ts b/src/other.ts new file mode 100644 --- /dev/null diff --git a/packages/detectors-ts/fixtures/authentication/fastify-preHandler-valid.ts b/packages/detectors-ts/fixtures/authentication/fastify-preHandler-valid.ts index 3656820..f0720f5 100644 --- a/packages/detectors-ts/fixtures/authentication/fastify-preHandler-valid.ts +++ b/packages/detectors-ts/fixtures/authentication/fastify-preHandler-valid.ts @@ -8,6 +8,9 @@ async function authHook(request: any, reply: any) { } } -app.post("/x", { preHandler: authHook, handler: async (request, reply) => { - reply.send({ ok: true }); -}}); +app.post("/x", { + preHandler: authHook, + handler: async (request, reply) => { + reply.send({ ok: true }); + }, +}); diff --git a/packages/detectors-ts/src/authentication/chain.ts b/packages/detectors-ts/src/authentication/chain.ts index e8b1d5e..1213985 100644 --- a/packages/detectors-ts/src/authentication/chain.ts +++ b/packages/detectors-ts/src/authentication/chain.ts @@ -42,7 +42,11 @@ function lineOf(node: Node): number { } /** Collect app.use(middleware) calls ABOVE the given line in source */ -function collectAppUseBefore(sourceFile: SourceFile, beforeLine: number, routePath: string): string[] { +function collectAppUseBefore( + sourceFile: SourceFile, + beforeLine: number, + routePath: string, +): string[] { const result: string[] = []; for (const callExpr of sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression)) { if (lineOf(callExpr) >= beforeLine) continue; @@ -100,7 +104,8 @@ function findFastifyRoute( const firstArg = args[0]; if (firstArg?.getKind() === SyntaxKind.ObjectLiteralExpression) { const obj = firstArg.asKindOrThrow(SyntaxKind.ObjectLiteralExpression); - let methodMatch = false, urlMatch = false; + let methodMatch = false, + urlMatch = false; for (const p of obj.getProperties()) { if (p.getKind() !== SyntaxKind.PropertyAssignment) continue; const pa = p.asKindOrThrow(SyntaxKind.PropertyAssignment); @@ -111,7 +116,10 @@ function findFastifyRoute( const v = init.asKindOrThrow(SyntaxKind.StringLiteral).getLiteralValue(); if (v.toUpperCase() === method.toUpperCase()) methodMatch = true; } - if ((pname === "url" || pname === "path") && init.getKind() === SyntaxKind.StringLiteral) { + if ( + (pname === "url" || pname === "path") && + init.getKind() === SyntaxKind.StringLiteral + ) { const v = init.asKindOrThrow(SyntaxKind.StringLiteral).getLiteralValue(); if (v === path) urlMatch = true; } @@ -161,7 +169,10 @@ function fastifyGlobalHooks(sourceFile: SourceFile, beforeLine: number): string[ const firstArg = args[0]; const secondArg = args[1]; if (!firstArg || firstArg.getKind() !== SyntaxKind.StringLiteral) continue; - const hookName = firstArg.asKindOrThrow(SyntaxKind.StringLiteral).getLiteralValue().toLowerCase(); + const hookName = firstArg + .asKindOrThrow(SyntaxKind.StringLiteral) + .getLiteralValue() + .toLowerCase(); if (!hookNames.has(hookName)) continue; if (secondArg) result.push(secondArg.getText()); } @@ -187,7 +198,16 @@ function nestjsChain( if (!method) return null; // Verify HTTP decorator on method - const HTTP_DECORATORS = new Set(["Get","Post","Put","Delete","Patch","Options","Head","All"]); + const HTTP_DECORATORS = new Set([ + "Get", + "Post", + "Put", + "Delete", + "Patch", + "Options", + "Head", + "All", + ]); const hasHttpDecorator = method.getDecorators().some((d) => HTTP_DECORATORS.has(d.getName())); if (!hasHttpDecorator) return null; @@ -218,11 +238,7 @@ function nestjsChain( // ─── Raw Node helpers ────────────────────────────────────────────────────── -function rawNodeChain( - sourceFile: SourceFile, - method: string, - path: string, -): ChainEntry[] | null { +function rawNodeChain(sourceFile: SourceFile, method: string, path: string): ChainEntry[] | null { // Find http.createServer((req, res) => { ... }) for (const callExpr of sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression)) { const expr = callExpr.getExpression(); @@ -260,7 +276,11 @@ function rawNodeChain( /res\.(status|sendStatus)\s*\(\s*(401|403)\s*\)/.test(preRouteText); if (hasAuthHeader || hasStatus401) { - entries.push({ name: "pre-route auth check", classification: "auth", layer: "Layer 3 body pattern" }); + entries.push({ + name: "pre-route auth check", + classification: "auth", + layer: "Layer 3 body pattern", + }); } else if (preRouteText.trim()) { entries.push({ name: "pre-route statements", classification: "unknown", layer: "no signal" }); } @@ -284,7 +304,13 @@ export function collectChain( if (entries === null) return "not_found"; // NestJS: if no guards found, emit unknown (global guards may apply) if (entries.length === 0) { - return [{ name: "global guards unresolved", classification: "unknown", layer: "NestJS global guards not resolved in v0.1" }]; + return [ + { + name: "global guards unresolved", + classification: "unknown", + layer: "NestJS global guards not resolved in v0.1", + }, + ]; } return entries; } diff --git a/packages/detectors-ts/src/authentication/classify.ts b/packages/detectors-ts/src/authentication/classify.ts index 12d7b9a..8808092 100644 --- a/packages/detectors-ts/src/authentication/classify.ts +++ b/packages/detectors-ts/src/authentication/classify.ts @@ -4,15 +4,39 @@ import type { Classification } from "./types.js"; // Layer 1 — Name match const NEGATIVE_NAMES = new Set([ - "bodyparser", "cors", "compression", "cookieparser", "morgan", "helmet", - "ratelimit", "logger", "errorhandler", "notfound", "staticfiles", "json", - "urlencoded", "multer", "upload", + "bodyparser", + "cors", + "compression", + "cookieparser", + "morgan", + "helmet", + "ratelimit", + "logger", + "errorhandler", + "notfound", + "staticfiles", + "json", + "urlencoded", + "multer", + "upload", ]); const POSITIVE_EXACT = new Set([ - "authenticate", "authentication", "authenticated", "isauthenticated", - "requireauth", "requiresauth", "needsauth", "ensureauth", "withauth", - "protectroute", "protected", "protect", "private", "guarded", "guard", + "authenticate", + "authentication", + "authenticated", + "isauthenticated", + "requireauth", + "requiresauth", + "needsauth", + "ensureauth", + "withauth", + "protectroute", + "protected", + "protect", + "private", + "guarded", + "guard", ]); const AUTH_REQUIRE_PREFIXES = ["require", "ensure", "check", "verify", "validate"]; @@ -20,12 +44,25 @@ const AUTH_REQUIRE_SUFFIXES = ["auth", "user", "login", "session", "token", "jwt // Layer 2 — Auth packages const AUTH_PACKAGES = new Set([ - "passport", "express-jwt", "lucia", "lucia-auth", "next-auth", - "@nestjs/passport", "@nestjs/jwt", "@clerk/clerk-sdk-node", "better-auth", + "passport", + "express-jwt", + "lucia", + "lucia-auth", + "next-auth", + "@nestjs/passport", + "@nestjs/jwt", + "@clerk/clerk-sdk-node", + "better-auth", "firebase-admin/auth", ]); -const AUTH_PACKAGE_PREFIXES = ["passport-", "@auth0/", "@clerk/", "@auth/", "@supabase/auth-helpers-"]; +const AUTH_PACKAGE_PREFIXES = [ + "passport-", + "@auth0/", + "@clerk/", + "@auth/", + "@supabase/auth-helpers-", +]; // jose/jsonwebtoken need body verification check const JWT_VERIFY_PACKAGES = new Set(["jose", "jsonwebtoken"]); @@ -210,7 +247,8 @@ export function classifyEntry( // For passport specifically: passport.authenticate is Layer 2 if (rootName !== name) { const pkg = resolveImportPackage(rootName, sourceFile); - if (pkg && isAuthPackage(pkg)) return { classification: "auth", layer: "Layer 2 import origin" }; + if (pkg && isAuthPackage(pkg)) + return { classification: "auth", layer: "Layer 2 import origin" }; } // JWT libs: check body diff --git a/packages/detectors-ts/src/authentication/index.ts b/packages/detectors-ts/src/authentication/index.ts index ac50593..b05bace 100644 --- a/packages/detectors-ts/src/authentication/index.ts +++ b/packages/detectors-ts/src/authentication/index.ts @@ -22,9 +22,8 @@ const FRAMEWORK_IMPORTS: Array<{ pattern: string | RegExp; framework: KnownFrame function detectFramework(content: string): KnownFramework | null { // Quick scan using regex to avoid full parse for this step for (const { pattern, framework } of FRAMEWORK_IMPORTS) { - const escaped = typeof pattern === "string" - ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - : pattern.source; + const escaped = + typeof pattern === "string" ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern.source; const re = new RegExp(`from\\s+["']${escaped}["']`); if (re.test(content)) return framework; } @@ -120,9 +119,7 @@ export async function detectAuthentication( return { verdict: "unverifiable", reason_code: "invalid_claim_shape", - evidence: [ - { kind: "symbol", path: target.path, note: "target.symbol is required" }, - ], + evidence: [{ kind: "symbol", path: target.path, note: "target.symbol is required" }], }; } diff --git a/packages/schema/package.json b/packages/schema/package.json index a3371e5..c59a09f 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -10,7 +10,9 @@ "types": "./dist/index.d.ts" } }, - "files": ["dist"], + "files": [ + "dist" + ], "scripts": { "build": "tsc -p tsconfig.build.json && cp src/manifest.schema.json dist/manifest.schema.json", "test": "vitest run", diff --git a/packages/schema/src/manifest.schema.json b/packages/schema/src/manifest.schema.json index 5ed381e..879a11b 100644 --- a/packages/schema/src/manifest.schema.json +++ b/packages/schema/src/manifest.schema.json @@ -9,19 +9,25 @@ "session": { "type": "object", "required": [ - "agent", "model", "session_id", "started_at", "completed_at", - "prompt_hash", "tool_calls_count", "files_touched" + "agent", + "model", + "session_id", + "started_at", + "completed_at", + "prompt_hash", + "tool_calls_count", + "files_touched" ], "additionalProperties": false, "properties": { - "agent": { "enum": ["claude-code", "codex", "cursor", "opencode", "other"] }, - "model": { "type": "string", "minLength": 1 }, - "session_id": { "type": "string", "format": "uuid" }, - "started_at": { "type": "string", "format": "date-time" }, - "completed_at": { "type": "string", "format": "date-time" }, - "prompt_hash": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "agent": { "enum": ["claude-code", "codex", "cursor", "opencode", "other"] }, + "model": { "type": "string", "minLength": 1 }, + "session_id": { "type": "string", "format": "uuid" }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" }, + "prompt_hash": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, "tool_calls_count": { "type": "integer", "minimum": 0 }, - "files_touched": { "type": "array", "items": { "type": "string" } } + "files_touched": { "type": "array", "items": { "type": "string" } } } }, "task": { @@ -30,7 +36,7 @@ "additionalProperties": false, "properties": { "summary": { "type": "string", "maxLength": 120 }, - "source": { "enum": ["user_prompt", "issue_reference", "continuation"] } + "source": { "enum": ["user_prompt", "issue_reference", "continuation"] } } }, "claims": { @@ -45,12 +51,18 @@ "required": ["id", "type", "target", "description", "verification_contract"], "additionalProperties": false, "properties": { - "id": { "type": "string", "pattern": "^c[0-9]+$" }, + "id": { "type": "string", "pattern": "^c[0-9]+$" }, "type": { "enum": [ - "add_symbol", "remove_symbol", "modify_signature", - "modify_behavior", "add_test", "refactor", - "add_dependency", "remove_dependency", "config_change" + "add_symbol", + "remove_symbol", + "modify_signature", + "modify_behavior", + "add_test", + "refactor", + "add_dependency", + "remove_dependency", + "config_change" ] }, "target": { @@ -58,8 +70,19 @@ "required": ["kind", "path"], "additionalProperties": false, "properties": { - "kind": { "enum": ["function", "class", "type", "endpoint", "file", "module", "config_key", "package"] }, - "path": { "type": "string" }, + "kind": { + "enum": [ + "function", + "class", + "type", + "endpoint", + "file", + "module", + "config_key", + "package" + ] + }, + "path": { "type": "string" }, "symbol": { "type": "string" } } }, @@ -71,8 +94,12 @@ "properties": { "check": { "enum": [ - "symbol_exists", "behavior_present", "test_covers", - "signature_matches", "removed", "cannot_verify" + "symbol_exists", + "behavior_present", + "test_covers", + "signature_matches", + "removed", + "cannot_verify" ] }, "params": { "type": "object" } diff --git a/packages/schema/src/validator.ts b/packages/schema/src/validator.ts index 7f97421..ff521c8 100644 --- a/packages/schema/src/validator.ts +++ b/packages/schema/src/validator.ts @@ -35,7 +35,9 @@ export interface ValidationError { } export interface Validator { - validate(input: unknown): { ok: true; manifest: Manifest } | { ok: false; errors: ValidationError[] }; + validate( + input: unknown, + ): { ok: true; manifest: Manifest } | { ok: false; errors: ValidationError[] }; } function ajvErrorToValidationError(err: ErrorObject): ValidationError { @@ -84,7 +86,7 @@ function validateBehaviorPresentParams(input: unknown): ValidationError[] { errors.push({ path: `/claims/${i}/verification_contract/params/property`, code: "required", - message: 'behavior_present check requires params.property', + message: "behavior_present check requires params.property", }); } else if ( typeof property !== "string" || diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 18ec407..dee51e9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,2 @@ packages: - - 'packages/*' + - "packages/*"