From 2688ee2a0c7b227d7420800c826d4f3910018957 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 5 Jul 2026 17:37:12 -0400 Subject: [PATCH] =?UTF-8?q?spec:=20add=20ReasoningAssay=20+=20AssayStandar?= =?UTF-8?q?d=20=E2=80=94=20epistemic=20verdicts=20as=20projections?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claim trustworthiness was being reduced to a flat ok/sad/bad scalar, but confidence, authenticity, and verification-vs-generation are orthogonal axes. ReasoningAssay stores five axes (method, binding, verifier, agreement, authority) and makes ok/sad/bad a render-time projection that re-derives when a verifier's reliability improves. AssayStandard records that reliability (confusion matrix, sample size, calibrated flag) as versioned canon, so a verdict must point at measured — not assumed — verifier reliability to project above 'sad'. validate-reasoning-examples enforces schema conformance AND projection soundness (recomputes assay() from the axes, fails on drift), and brings the previously-unvalidated reasoning family under make validate. --- CHANGELOG.md | 1 + Makefile | 8 +- docs/adr/ADR-epistemic-assay-verdict-v0-1.md | 82 ++++++++++++++ examples/assay_standard.json | 25 +++++ examples/reasoning_assay.json | 25 +++++ examples/reasoning_assay.refuted.json | 25 +++++ examples/reasoning_assay.unassayed.json | 26 +++++ schemas/AssayStandard.json | 73 +++++++++++++ schemas/ReasoningAssay.json | 82 ++++++++++++++ schemas/reasoning-contracts.README.md | 26 +++++ tools/validate_reasoning_examples.py | 109 +++++++++++++++++++ 11 files changed, 480 insertions(+), 2 deletions(-) create mode 100644 docs/adr/ADR-epistemic-assay-verdict-v0-1.md create mode 100644 examples/assay_standard.json create mode 100644 examples/reasoning_assay.json create mode 100644 examples/reasoning_assay.refuted.json create mode 100644 examples/reasoning_assay.unassayed.json create mode 100644 schemas/AssayStandard.json create mode 100644 schemas/ReasoningAssay.json create mode 100644 tools/validate_reasoning_examples.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a0439c0..c5a4a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Thi ## [Unreleased] ### Added +- Epistemic Assay contracts: `ReasoningAssay` (a typed verdict on a claim over five orthogonal axes — method, binding, verifier, agreement, authority — whose `ok`/`sad`/`bad` `projectedState` is a render-time projection, not a stored scalar) and `AssayStandard` (a verifier's measured, versioned reliability — the calibration reference every assay must point at). Includes canonical `ok`/`sad`/`bad` examples, ADR (`docs/adr/ADR-epistemic-assay-verdict-v0-1.md`), and a `validate-reasoning-examples` target that also enforces projection soundness (recomputes `assay()` from the stored axes and fails on drift) and retroactively brings the existing reasoning family under `make validate`. - Onboarding control-plane contract family: `WorkspaceScope`, `TrustMode`, `CapabilityPack`, `ConnectorActionScope`, `AutomationTemplate`, and `OnboardingReceipt`, with canonical first-run examples, semantic vocabulary seed, ADR, and `validate-onboarding-examples` validation target. - `ReasoningEvent.controlFlow` (optional) + a reserved control-flow `eventType` vocabulary (`reasoning.tool.called` / `reasoning.decision.branched` / `reasoning.subrun.spawned` / `reasoning.subrun.joined` / `reasoning.run.completed`) so emitters can carry a run's operational control flow for downstream narration-fidelity verification (SP-TRACE-CFR). Backward-compatible; operational structure only (no raw reasoning). See `docs/reasoning-control-flow-events.md` and `examples/reasoning_event_control_flow.json`. - SourceOS interaction substrate top-level index and README discovery links for `SourceOSInteractionEvent`, generated TypeScript/Python artifacts, and the Noetica → Superconscious → AgentPlane → AgentTerm reference flow. diff --git a/Makefile b/Makefile index f8bfb11..ca53060 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,12 @@ -.PHONY: validate validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples +.PHONY: validate validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples -validate: validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples +validate: validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples @echo "OK: validate" +validate-reasoning-examples: + python3 -m pip install --user jsonschema >/dev/null + python3 tools/validate_reasoning_examples.py + validate-agentic-os-examples: python3 -m pip install --user jsonschema >/dev/null python3 tools/validate_agentic_os_examples.py diff --git a/docs/adr/ADR-epistemic-assay-verdict-v0-1.md b/docs/adr/ADR-epistemic-assay-verdict-v0-1.md new file mode 100644 index 0000000..58ca04e --- /dev/null +++ b/docs/adr/ADR-epistemic-assay-verdict-v0-1.md @@ -0,0 +1,82 @@ +# ADR: The Epistemic Assay — Verdicts as Projections, Not Scalars (v0.1) + +## Status + +Proposed. + +## Context + +The operational health of a subsystem is well modelled by a tri-state readout — `ok` / `sad` / `bad` +(came up / degraded-but-survived / hard failure). These are mutually exclusive states of a single +thing (the outcome of an operation), and the middle state is deliberately first-class. + +That same three-state readout was being reached for to describe the trustworthiness of a **claim**. +This conflates concerns that are not points on one severity scale. Recent evaluation work surfaced at +least four independent facts hiding behind a single "confidence" pixel: + +- **method** — was the claim computed, retrieved, or generated? These behave as decorrelated *classes*, + not degrees; the verified-compute arm is decorrelated from grounding arms precisely because it is a + different method. +- **binding time** — was the supporting evidence produced *inline* with the computation, or attached + *post-hoc*? Generative paths are post-hoc bound; only deterministic paths are inline. A post-hoc + "verified" is a different animal from an inline one, and nothing recorded which was obtained. +- **verifier authority** — who judged, and how reliable is that judge, *measured*? A deployed verifier + grading at low F1 with only "slight" inter-rater agreement does not issue a weaker verdict — it + issues an **unanchored** one, because its own reliability is unrecorded. +- **agreement** — arm concurrence overstates confidence when arms are correlated (they fail together), + so raw "N arms agree" is closer to one vote than N. + +Collapsing orthogonal axes into one stored scalar is the design flaw. It also makes history +un-re-judgeable: when the verifier improves, every past verdict is stuck at the reliability it was +stamped with. + +## Decision + +### 1. Keep the tri-state readout for operational health; do not store it as a claim's verdict + +`ok` / `sad` / `bad` remains the canonical readout for operational outcomes. For **claims**, the verdict +is no longer a stored scalar. + +### 2. Introduce `ReasoningAssay` — a typed verdict over five orthogonal axes + +`schemas/ReasoningAssay.json` records `method`, `binding`, `verifier`, `agreement`, and `authority`. +The `ok` / `sad` / `bad` value (`projectedState`) is a **render-time projection** of these axes, cached +at `assayedAt` and re-derivable — never the source of truth. This is the "operation ⊥ verdict" principle +(separate the fact from the judgement so history stays re-judgeable) applied to epistemics. + +The projection is defined so that: + +- `ok` requires inline binding **and** an attestable method (computed/retrieved) **and** a calibrated verifier; +- `sad` ("unassayed") covers everything real-but-unresolved — post-hoc binding, generation-with-citations, + an uncalibrated verifier, or correlated-arm agreement. The amber band is deliberately wide and *earned*; +- `bad` covers refuted / failed / authority-broken. + +### 3. Introduce `AssayStandard` — the verifier's measured reliability as canon + +`schemas/AssayStandard.json` records a verifier's confusion matrix, sample size, optional derived metrics +and inter-rater agreement, and a `calibrated` flag, versioned per measurement. A `ReasoningAssay`'s +`verifier.calibrationRef` **must** point at one; a verdict without it cannot project above `sad`. This +gives "who verifies the verifier" a mechanical answer: the verifier's own measured receipts. Re-measuring +publishes a new version, and referencing assays re-project against it — the same discipline as keeping +every benchmark arm rather than only its current winner. + +### 4. The projection is executable and CI-enforced + +`tools/validate_reasoning_examples.py` recomputes `assay()` from each example's stored axes and fails the +build if the recorded `projectedState` does not follow from them. The projection cannot silently drift from +the axes it claims to summarise. + +## Consequences + +- On the current stack, essentially only inline-bound verified-compute paths project to `ok`. This is + honest, and renders the verified-compute moat directly as UX. +- Authenticity is a property of the actor/channel (mirroring `EventEnvelope.actor`/`integrity`), so a + confident-sounding answer cannot launder its own authority. +- The single highest-leverage follow-on is populating real `AssayStandard` records from verifier calibration + runs, so production verdicts reference measured — not assumed — reliability. + +## Non-goals + +- This ADR does not change operational-health telemetry. +- It does not mandate a runtime; it defines the canonical shapes that emitters (e.g. superconscious, + agentplane evidence sealing) conform to. diff --git a/examples/assay_standard.json b/examples/assay_standard.json new file mode 100644 index 0000000..fff5069 --- /dev/null +++ b/examples/assay_standard.json @@ -0,0 +1,25 @@ +{ + "id": "urn:srcos:assay-standard:narration-fidelity-verifier:0.2.0", + "type": "AssayStandard", + "specVersion": "2.0.0", + "verifierId": "urn:srcos:verifier:narration-fidelity", + "version": "0.2.0", + "confusionMatrix": { + "truePositive": 41, + "falsePositive": 12, + "trueNegative": 38, + "falseNegative": 9 + }, + "metrics": { + "f1": 0.8, + "precision": 0.77, + "recall": 0.82 + }, + "interRaterAgreement": { + "kappa": 0.68, + "interpretation": "substantial" + }, + "sampleSize": 100, + "calibrated": true, + "measuredAt": "2026-07-05T00:00:00Z" +} diff --git a/examples/reasoning_assay.json b/examples/reasoning_assay.json new file mode 100644 index 0000000..bb3b07b --- /dev/null +++ b/examples/reasoning_assay.json @@ -0,0 +1,25 @@ +{ + "id": "urn:srcos:reasoning-assay:superconscious-demo-ok", + "type": "ReasoningAssay", + "specVersion": "2.0.0", + "runRef": "urn:srcos:reasoning-run:superconscious-demo", + "claimRef": "sha256:claim-computed-answer", + "method": "computed", + "binding": "inline", + "verifier": { + "verifierId": "urn:srcos:verifier:narration-fidelity", + "calibrationRef": "urn:srcos:assay-standard:narration-fidelity-verifier:0.2.0" + }, + "agreement": { + "arms": 3, + "correlation": 0.18, + "effectiveVotes": 2.6 + }, + "authority": { + "actorRef": "urn:srcos:agent:superconscious", + "channel": "tool", + "integrityVerified": true + }, + "projectedState": "ok", + "assayedAt": "2026-07-05T00:00:01Z" +} diff --git a/examples/reasoning_assay.refuted.json b/examples/reasoning_assay.refuted.json new file mode 100644 index 0000000..2cc31a0 --- /dev/null +++ b/examples/reasoning_assay.refuted.json @@ -0,0 +1,25 @@ +{ + "id": "urn:srcos:reasoning-assay:superconscious-demo-refuted", + "type": "ReasoningAssay", + "specVersion": "2.0.0", + "runRef": "urn:srcos:reasoning-run:superconscious-demo", + "claimRef": "sha256:claim-authority-broken", + "method": "generated", + "binding": "post-hoc", + "verifier": { + "verifierId": "urn:srcos:verifier:narration-fidelity", + "calibrationRef": "urn:srcos:assay-standard:narration-fidelity-verifier:0.2.0" + }, + "agreement": { + "arms": 3, + "correlation": 0.22, + "effectiveVotes": 2.4 + }, + "authority": { + "actorRef": "urn:srcos:agent:superconscious", + "channel": "sdk", + "integrityVerified": false + }, + "projectedState": "bad", + "assayedAt": "2026-07-05T00:00:01Z" +} diff --git a/examples/reasoning_assay.unassayed.json b/examples/reasoning_assay.unassayed.json new file mode 100644 index 0000000..8b938e7 --- /dev/null +++ b/examples/reasoning_assay.unassayed.json @@ -0,0 +1,26 @@ +{ + "id": "urn:srcos:reasoning-assay:superconscious-demo-unassayed", + "type": "ReasoningAssay", + "specVersion": "2.0.0", + "runRef": "urn:srcos:reasoning-run:superconscious-demo", + "claimRef": "sha256:claim-generated-with-citations", + "method": "generated", + "binding": "post-hoc", + "verifier": { + "verifierId": "urn:srcos:verifier:deployed-nli", + "calibrationRef": "urn:srcos:assay-standard:deployed-nli:0.1.0" + }, + "agreement": { + "arms": 3, + "correlation": 0.66, + "effectiveVotes": 1.2 + }, + "authority": { + "actorRef": "urn:srcos:agent:superconscious", + "channel": "sdk", + "integrityVerified": true + }, + "projectedState": "sad", + "unassayedReason": "post-hoc-binding+correlated-arms+uncalibrated-verifier", + "assayedAt": "2026-07-05T00:00:01Z" +} diff --git a/schemas/AssayStandard.json b/schemas/AssayStandard.json new file mode 100644 index 0000000..893433e --- /dev/null +++ b/schemas/AssayStandard.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/AssayStandard.json", + "title": "AssayStandard", + "description": "The measured, versioned reliability of a verifier ('judge') — the calibrated reference every ReasoningAssay is checked against. Answers 'who verifies the verifier' mechanically: a verdict may only claim 'ok' when it points at a standard whose reliability was actually measured. Re-measuring publishes a new version; historical assays that reference an older version can be re-projected against the newer one.", + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "specVersion", "verifierId", "version", "confusionMatrix", "sampleSize", "measuredAt"], + "properties": { + "id": { + "type": "string", + "pattern": "^urn:srcos:assay-standard:", + "description": "Stable URN for this measured standard. Pattern: urn:srcos:assay-standard::." + }, + "type": { "const": "AssayStandard" }, + "specVersion": { "type": "string", "description": "Spec version, e.g. \"2.0.0\"." }, + "verifierId": { + "type": "string", + "description": "URN or stable identifier of the verifier/judge whose reliability this standard records." + }, + "version": { + "type": "string", + "description": "Version of the verifier at the time of measurement. A new measurement of the same verifier yields a new AssayStandard with a bumped version." + }, + "confusionMatrix": { + "type": "object", + "additionalProperties": false, + "required": ["truePositive", "falsePositive", "trueNegative", "falseNegative"], + "description": "Raw counts from the calibration run. Facts, not interpretation — metrics below are derived from these and may be recomputed.", + "properties": { + "truePositive": { "type": "integer", "minimum": 0 }, + "falsePositive": { "type": "integer", "minimum": 0 }, + "trueNegative": { "type": "integer", "minimum": 0 }, + "falseNegative": { "type": "integer", "minimum": 0 } + } + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "description": "Optional derived scores (recomputable from confusionMatrix). Present as a convenience; the matrix is authoritative.", + "properties": { + "f1": { "type": "number", "minimum": 0, "maximum": 1 }, + "precision": { "type": "number", "minimum": 0, "maximum": 1 }, + "recall": { "type": "number", "minimum": 0, "maximum": 1 } + } + }, + "interRaterAgreement": { + "type": "object", + "additionalProperties": false, + "description": "Optional inter-rater reliability when the standard was set against multiple human/reference judges.", + "properties": { + "kappa": { "type": "number", "minimum": -1, "maximum": 1 }, + "interpretation": { + "enum": ["poor", "slight", "fair", "moderate", "substantial", "almost-perfect"] + } + } + }, + "sampleSize": { + "type": "integer", + "minimum": 1, + "description": "Number of labelled items the standard was measured over. A standard measured on too few items is itself weak evidence." + }, + "calibrated": { + "type": "boolean", + "description": "Whether this standard clears the project's minimum bar to let a referencing assay project to 'ok'. Uncalibrated standards can still be recorded — a referencing assay then projects no better than 'sad'." + }, + "measuredAt": { + "type": "string", + "format": "date-time", + "description": "When the measurement was taken. Standards are perishable; consumers may treat an old measurement as stale." + } + } +} diff --git a/schemas/ReasoningAssay.json b/schemas/ReasoningAssay.json new file mode 100644 index 0000000..e7585f5 --- /dev/null +++ b/schemas/ReasoningAssay.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.srcos.ai/v2/ReasoningAssay.json", + "title": "ReasoningAssay", + "description": "A typed epistemic verdict on a claim produced during a reasoning run. Replaces a flat confidence scalar: confidence, authenticity, and verification-vs-generation are ORTHOGONAL axes, so they are recorded separately and the ok/sad/bad readout is a render-time PROJECTION of the tuple, not a stored severity. Because the axes are stored, a verdict can be re-projected when the verifier's AssayStandard improves — the same discipline as keeping every benchmark arm rather than only its current winner.", + "type": "object", + "additionalProperties": false, + "required": ["id", "type", "specVersion", "runRef", "method", "binding", "verifier", "assayedAt"], + "properties": { + "id": { + "type": "string", + "pattern": "^urn:srcos:reasoning-assay:", + "description": "Stable URN for this assay. Pattern: urn:srcos:reasoning-assay:." + }, + "type": { "const": "ReasoningAssay" }, + "specVersion": { "type": "string", "description": "Spec version, e.g. \"2.0.0\"." }, + "runRef": { + "type": "string", + "pattern": "^urn:srcos:reasoning-run:", + "description": "The reasoning run this claim was produced in." + }, + "claimRef": { + "type": "string", + "description": "Optional URN or content hash of the specific claim being assayed. Absent when the assay covers the run's overall answer." + }, + "method": { + "enum": ["computed", "retrieved", "generated"], + "description": "AXIS 1 — how the claim was produced. Categorical, not a severity: computed (deterministic/executed), retrieved (grounded in a source), or generated (model-authored). These behave as decorrelated classes, not points on one scale." + }, + "binding": { + "enum": ["inline", "post-hoc"], + "description": "AXIS 2 — when the supporting evidence was produced. inline: emitted as part of the computation. post-hoc: attached after the fact. A post-hoc 'verified' is a weaker animal than an inline one; the difference must be recorded, not collapsed." + }, + "verifier": { + "type": "object", + "additionalProperties": false, + "required": ["calibrationRef"], + "description": "AXIS 3 — who judged, and how reliable that judge is measured to be. A verdict is unanchored unless it points at the verifier's own measured reliability.", + "properties": { + "verifierId": { "type": "string", "description": "Identifier of the judge that issued this verdict." }, + "calibrationRef": { + "type": "string", + "pattern": "^urn:srcos:assay-standard:", + "description": "The AssayStandard recording this verifier's measured reliability. Required — a verdict without a calibration reference cannot project above 'sad'." + } + } + }, + "agreement": { + "type": "object", + "additionalProperties": false, + "description": "AXIS 4 — decorrelation-weighted concurrence across arms. Raw arm count overstates confidence when arms are correlated (they fail together); effectiveVotes discounts for that.", + "properties": { + "arms": { "type": "integer", "minimum": 1, "description": "Number of arms/methods that weighed in." }, + "correlation": { "type": "number", "minimum": 0, "maximum": 1, "description": "Measured pairwise correlation among the arms." }, + "effectiveVotes": { "type": "number", "minimum": 0, "description": "Decorrelation-adjusted vote count; approaches 1 as arms become correlated." } + } + }, + "authority": { + "type": "object", + "additionalProperties": false, + "description": "AXIS 5 — authenticity as a property of the actor and channel, never of the content. A confident-sounding answer cannot launder its own authority. Corroborated across tiers (telemetry / substrate / peripheral) rather than asserted.", + "properties": { + "actorRef": { "type": "string", "description": "URN of the actor that produced the claim (mirrors EventEnvelope.actor.subjectId)." }, + "channel": { "type": "string", "description": "The channel the claim arrived over (e.g. sdk, typed, tool)." }, + "integrityVerified": { "type": "boolean", "description": "Whether the envelope integrity (hash/signature) checked out for this claim." } + } + }, + "projectedState": { + "enum": ["ok", "sad", "bad"], + "description": "The render-time projection assay() produced from the axes AT assayedAt. A cache, not the source of truth: 'ok' = inline-bound + computed/attested + calibrated verifier; 'sad' = unassayed/unresolved (post-hoc, generated-with-citations, uncalibrated verifier, or correlated-arm agreement); 'bad' = refuted, failed, or authority-broken. Re-derivable from the stored axes at any time." + }, + "unassayedReason": { + "type": "string", + "description": "When projectedState is 'sad', the specific unresolved condition (e.g. 'uncalibrated-verifier', 'post-hoc-binding', 'correlated-arms'). Frames amber as honest pending-work, not soft failure." + }, + "assayedAt": { + "type": "string", + "format": "date-time", + "description": "When this assay was produced. Re-projection against a newer AssayStandard yields a later assay rather than mutating this record." + } + } +} diff --git a/schemas/reasoning-contracts.README.md b/schemas/reasoning-contracts.README.md index 3e334ee..b876f41 100644 --- a/schemas/reasoning-contracts.README.md +++ b/schemas/reasoning-contracts.README.md @@ -13,6 +13,8 @@ These contracts were introduced to move the Superconscious reference loop out of | `ReasoningReceipt.json` | `urn:srcos:receipt:reasoning:` | Final receipt for a reasoning run. | | `ReasoningReplayPlan.json` | `urn:srcos:reasoning-replay-plan:` | Replay classification and replay input/constraint record. | | `ReasoningBenchmark.json` | `urn:srcos:reasoning-benchmark:` | Benchmark result for a reasoning run. | +| `ReasoningAssay.json` | `urn:srcos:reasoning-assay:` | Typed epistemic verdict on a claim; ok/sad/bad is a render-time projection of five orthogonal axes. | +| `AssayStandard.json` | `urn:srcos:assay-standard:` | Measured, versioned reliability of a verifier — the calibration reference every assay is checked against. | ## Examples @@ -23,11 +25,35 @@ These contracts were introduced to move the Superconscious reference loop out of | `examples/reasoning_receipt.json` | `ReasoningReceipt` | | `examples/reasoning_replay_plan.json` | `ReasoningReplayPlan` | | `examples/reasoning_benchmark.json` | `ReasoningBenchmark` | +| `examples/assay_standard.json` | `AssayStandard` | +| `examples/reasoning_assay.json` | `ReasoningAssay` (projects `ok`) | +| `examples/reasoning_assay.unassayed.json` | `ReasoningAssay` (projects `sad`) | +| `examples/reasoning_assay.refuted.json` | `ReasoningAssay` (projects `bad`) | ## Design intent Reasoning contracts expose safe operational traces and evidence-backed coordination records. They do not depend on raw private reasoning content. +### The Assay: verdicts are projections, not scalars + +`ReasoningAssay` records an epistemic verdict on a claim. Confidence, authenticity, and +verification-vs-generation are **orthogonal axes**, not points on one severity scale, so they are +stored separately and the `ok`/`sad`/`bad` readout is a render-time **projection** (`projectedState`), +never a stored opinion. The five axes: + +1. **method** — `computed` / `retrieved` / `generated` (categorical; these behave as decorrelated classes). +2. **binding** — `inline` / `post-hoc` (was evidence produced with the computation, or attached after?). +3. **verifier** — points at an `AssayStandard` (the judge's *measured* reliability). A verdict without a + calibration reference cannot project above `sad`. +4. **agreement** — decorrelation-weighted; correlated arms fail together, so `effectiveVotes` discounts raw arm count. +5. **authority** — authenticity as a property of the actor/channel (mirrors `EventEnvelope.actor`/`integrity`), + never of the content. + +Because the axes are stored, a verdict is **re-projectable**: when a verifier's `AssayStandard` improves, +historical assays re-project without mutating their records — the same discipline as keeping every benchmark +arm rather than only its current winner. `tools/validate_reasoning_examples.py` enforces this: it recomputes +`assay()` from the stored axes and fails CI if the recorded `projectedState` does not follow from them. + Expected consumers: - `SocioProphet/superconscious` as the reference governed cognition loop; diff --git a/tools/validate_reasoning_examples.py b/tools/validate_reasoning_examples.py new file mode 100644 index 0000000..765c8a8 --- /dev/null +++ b/tools/validate_reasoning_examples.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Validate the reasoning-run contract family, including the epistemic Assay. + +Two checks, not one: + 1. schema conformance — every example validates against its schema; + 2. projection soundness — for each ReasoningAssay, the reference assay() + projection recomputed from the stored axes matches the recorded + projectedState. This keeps the ok/sad/bad readout an *executable* + projection of the tuple rather than a stored opinion that can drift. +""" +from __future__ import annotations + +import json +from pathlib import Path + +import jsonschema + +ROOT = Path(__file__).resolve().parents[1] + +PAIRS = [ + (ROOT / "schemas" / "ReasoningRun.json", ROOT / "examples" / "reasoning_run.json"), + (ROOT / "schemas" / "ReasoningEvent.json", ROOT / "examples" / "reasoning_event.json"), + (ROOT / "schemas" / "ReasoningEvent.json", ROOT / "examples" / "reasoning_event_control_flow.json"), + (ROOT / "schemas" / "ReasoningReceipt.json", ROOT / "examples" / "reasoning_receipt.json"), + (ROOT / "schemas" / "ReasoningReplayPlan.json", ROOT / "examples" / "reasoning_replay_plan.json"), + (ROOT / "schemas" / "ReasoningBenchmark.json", ROOT / "examples" / "reasoning_benchmark.json"), + (ROOT / "schemas" / "AssayStandard.json", ROOT / "examples" / "assay_standard.json"), + (ROOT / "schemas" / "ReasoningAssay.json", ROOT / "examples" / "reasoning_assay.json"), + (ROOT / "schemas" / "ReasoningAssay.json", ROOT / "examples" / "reasoning_assay.unassayed.json"), + (ROOT / "schemas" / "ReasoningAssay.json", ROOT / "examples" / "reasoning_assay.refuted.json"), +] + +ASSAY_EXAMPLES = [ + ROOT / "examples" / "reasoning_assay.json", + ROOT / "examples" / "reasoning_assay.unassayed.json", + ROOT / "examples" / "reasoning_assay.refuted.json", +] + + +def load_standards() -> dict[str, dict]: + """Index every AssayStandard fixture by its URN so assay() can look up + whether a verdict's verifier is actually calibrated.""" + standards: dict[str, dict] = {} + for path in ROOT.glob("examples/*.json"): + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + continue + if isinstance(doc, dict) and doc.get("type") == "AssayStandard": + standards[doc["id"]] = doc + return standards + + +def assay(record: dict, standards: dict[str, dict]) -> str: + """Reference projection: recompute the ok/sad/bad readout from the axes. + + bad — authority is broken (envelope integrity did not verify). + ok — inline-bound AND method is attestable (computed/retrieved) + AND the verifier points at a calibrated AssayStandard. + sad — everything else: unassayed / unresolved but not refuted. + """ + authority = record.get("authority", {}) + if authority.get("integrityVerified") is False: + return "bad" + + calibration_ref = record["verifier"]["calibrationRef"] + standard = standards.get(calibration_ref) + calibrated = bool(standard and standard.get("calibrated")) + + inline = record["binding"] == "inline" + attestable = record["method"] in ("computed", "retrieved") + + if inline and attestable and calibrated: + return "ok" + return "sad" + + +def validate_pair(schema_path: Path, example_path: Path) -> None: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + jsonschema.validators.validator_for(schema).check_schema(schema) + example = json.loads(example_path.read_text(encoding="utf-8")) + jsonschema.validate(example, schema) + + +def main() -> int: + checks: dict[str, bool] = {} + + for schema_path, example_path in PAIRS: + validate_pair(schema_path, example_path) + checks[example_path.name] = True + + standards = load_standards() + for example_path in ASSAY_EXAMPLES: + record = json.loads(example_path.read_text(encoding="utf-8")) + recorded = record.get("projectedState") + recomputed = assay(record, standards) + if recorded != recomputed: + raise SystemExit( + f"projection mismatch in {example_path.name}: " + f"recorded={recorded!r} but assay()={recomputed!r}" + ) + checks[f"projection:{example_path.name}"] = True + + print(json.dumps({"ok": all(checks.values()), "checks": checks}, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())