From 2d0ac72f47b53fe2d68a63d6164c9620fe063892 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 10:10:45 -0400 Subject: [PATCH 01/13] Add Holmes surface schema --- schemas/holmes-surface.schema.json | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 schemas/holmes-surface.schema.json diff --git a/schemas/holmes-surface.schema.json b/schemas/holmes-surface.schema.json new file mode 100644 index 0000000..d35777b --- /dev/null +++ b/schemas/holmes-surface.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.socioprophet.dev/holmes/holmes-surface.v1.schema.json", + "title": "HolmesSurface", + "type": "object", + "required": ["apiVersion", "kind", "metadata", "spec"], + "properties": { + "apiVersion": { + "const": "holmes.socioprophet.dev/v1" + }, + "kind": { + "const": "HolmesSurface" + }, + "metadata": { + "type": "object", + "required": ["name", "version"], + "properties": { + "name": {"type": "string"}, + "version": {"type": "string"} + }, + "additionalProperties": true + }, + "spec": { + "type": "object", + "required": [ + "product", + "components", + "componentFamilies", + "nlpTasks", + "methodFamilies", + "requiredPromotionEvidence", + "integrations" + ], + "properties": { + "product": {"const": "Holmes"}, + "tagline": {"type": "string"}, + "components": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "componentFamilies": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "nlpTasks": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "methodFamilies": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "requiredPromotionEvidence": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "integrations": {"type": "object", "additionalProperties": {"type": "string"}} + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} From fff25fdafd74a96f3b842a5b9b54195531fe4526 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 10:11:33 -0400 Subject: [PATCH 02/13] Add Holmes proof-claim contract schema --- .../holmes-proof-claim-contract.schema.json | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 schemas/holmes-proof-claim-contract.schema.json diff --git a/schemas/holmes-proof-claim-contract.schema.json b/schemas/holmes-proof-claim-contract.schema.json new file mode 100644 index 0000000..8fcc57f --- /dev/null +++ b/schemas/holmes-proof-claim-contract.schema.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.socioprophet.dev/holmes/holmes-proof-claim-contract.v1.schema.json", + "title": "HolmesReasoningContract", + "type": "object", + "required": ["apiVersion", "kind", "metadata", "spec"], + "properties": { + "apiVersion": { + "const": "holmes.socioprophet.dev/v1" + }, + "kind": { + "const": "HolmesReasoningContract" + }, + "metadata": { + "type": "object", + "required": ["name", "version"], + "properties": { + "name": {"type": "string"}, + "version": {"type": "string"} + }, + "additionalProperties": true + }, + "spec": { + "type": "object", + "required": [ + "canonicalLoop", + "holmesOwnedSegment", + "admissionInvariant", + "candidateOnlyStatus", + "policyDecisions", + "contractMappings", + "workedExamples" + ], + "properties": { + "canonicalLoop": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "holmesOwnedSegment": {"type": "array", "items": {"type": "string"}, "minItems": 3}, + "admissionInvariant": {"type": "string"}, + "candidateOnlyStatus": {"const": "candidate_only"}, + "policyDecisions": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "contractMappings": {"type": "object", "additionalProperties": {"type": "string"}}, + "workedExamples": {"type": "object"} + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} From 12cfe4d6f8a74aea921d6f5c05b4b4bb8de64e3a Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:01:42 -0400 Subject: [PATCH 03/13] Add invalid Holmes surface fixture for wrong kind --- .../invalid/holmes-surface-wrong-kind.json | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 fixtures/invalid/holmes-surface-wrong-kind.json diff --git a/fixtures/invalid/holmes-surface-wrong-kind.json b/fixtures/invalid/holmes-surface-wrong-kind.json new file mode 100644 index 0000000..3642c75 --- /dev/null +++ b/fixtures/invalid/holmes-surface-wrong-kind.json @@ -0,0 +1,24 @@ +{ + "apiVersion": "holmes.socioprophet.dev/v1", + "kind": "NotHolmesSurface", + "metadata": { + "name": "bad-surface-wrong-kind", + "version": "0.1.0" + }, + "spec": { + "product": "Holmes", + "components": ["sherlock-search"], + "componentFamilies": ["basic-primitives"], + "nlpTasks": ["language-identification"], + "methodFamilies": ["language.topic.v1/Propose", "language.topic.v1/Train"], + "requiredPromotionEvidence": ["corpusRef", "pipelineOrModelRef", "algorithmFamily", "taskContract", "evalRecord", "latencyFootprintRecord", "slashTopicsTrainingRef", "guardrailPolicy", "evidenceReceipt", "promotionRecord", "rollbackRef"], + "integrations": { + "standards": "SocioProphet/functional-model-surfaces", + "platform": "SocioProphet/prophet-platform", + "search": "SocioProphet/sherlock-search", + "slashTopics": "SocioProphet/slash-topics", + "lab": "SociOS-Linux/nlplab", + "sourceosCarry": "SourceOS-Linux/sourceos-model-carry" + } + } +} From 7712137f965715269eab16918f54e0b7b38a266c Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:03:38 -0400 Subject: [PATCH 04/13] Add invalid Holmes surface fixture for missing Slash Topics integration --- .../holmes-surface-missing-slash-topics.json | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 fixtures/invalid/holmes-surface-missing-slash-topics.json diff --git a/fixtures/invalid/holmes-surface-missing-slash-topics.json b/fixtures/invalid/holmes-surface-missing-slash-topics.json new file mode 100644 index 0000000..5cdde8e --- /dev/null +++ b/fixtures/invalid/holmes-surface-missing-slash-topics.json @@ -0,0 +1,23 @@ +{ + "apiVersion": "holmes.socioprophet.dev/v1", + "kind": "HolmesSurface", + "metadata": { + "name": "bad-surface-missing-slash-topics", + "version": "0.1.0" + }, + "spec": { + "product": "Holmes", + "components": ["sherlock-search", "221b", "mycroft-router", "moriarty-bench", "irene-shield", "the-canon", "deduction-engine"], + "componentFamilies": ["basic-primitives", "advanced-primitives", "rule-techniques", "classical-ml", "neural-nlp", "transformers", "foundation-language-services", "retrieval-and-knowledge", "guardrails-and-governance", "agent-and-tool-orchestration"], + "nlpTasks": ["language-identification", "sentence-segmentation", "tokenization", "lemmatization", "part-of-speech-tagging", "morphological-features", "dependency-parsing", "semantic-role-labeling", "entity-extraction", "numeric-entity-extraction", "pii-extraction", "coreference-resolution", "relation-extraction", "text-classification", "zero-shot-classification", "sentiment-classification", "target-sentiment-extraction", "keyword-extraction", "category-classification", "concept-linking", "topic-modeling", "topic-model-training", "topic-taxonomy-induction", "topic-pack-generation", "topical-clustering", "text-similarity", "table-header-identification", "claim-extraction", "contradiction-detection", "semantic-graph-conversion", "evidence-governance"], + "methodFamilies": ["language.topic.v1/Propose", "language.topic.v1/Train"], + "requiredPromotionEvidence": ["corpusRef", "pipelineOrModelRef", "algorithmFamily", "taskContract", "evalRecord", "latencyFootprintRecord", "slashTopicsTrainingRef", "guardrailPolicy", "evidenceReceipt", "promotionRecord", "rollbackRef"], + "integrations": { + "standards": "SocioProphet/functional-model-surfaces", + "platform": "SocioProphet/prophet-platform", + "search": "SocioProphet/sherlock-search", + "lab": "SociOS-Linux/nlplab", + "sourceosCarry": "SourceOS-Linux/sourceos-model-carry" + } + } +} From 08fd849dea3098d8dcd838c17219b0bba4e10963 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:24:03 -0400 Subject: [PATCH 05/13] Add invalid proof-claim fixture for wrong Holmes segment --- .../proof-claim-wrong-owned-segment.json | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 fixtures/invalid/proof-claim-wrong-owned-segment.json diff --git a/fixtures/invalid/proof-claim-wrong-owned-segment.json b/fixtures/invalid/proof-claim-wrong-owned-segment.json new file mode 100644 index 0000000..d2e9d35 --- /dev/null +++ b/fixtures/invalid/proof-claim-wrong-owned-segment.json @@ -0,0 +1,40 @@ +{ + "apiVersion": "holmes.socioprophet.dev/v1", + "kind": "HolmesReasoningContract", + "metadata": { + "name": "bad-proof-claim-wrong-segment", + "version": "0.1.0" + }, + "spec": { + "canonicalLoop": ["Observe", "Anchor", "Normalize", "Propose", "Explain", "Verify", "Govern", "Act", "Receipt", "Learn"], + "holmesOwnedSegment": ["Propose", "Verify", "Explain"], + "admissionInvariant": "VectorCandidate | ModelOutput | GraphCandidate -> ProposedClaim -> ExplanationTrace + ProofCertificate? + ContradictionReport", + "candidateOnlyStatus": "candidate_only", + "policyDecisions": ["allow", "deny", "require_review", "provisional"], + "contractMappings": { + "Claim": "ontogenesis.socioprophet.dev/v1/Claim", + "ProofCertificate": "ontogenesis.socioprophet.dev/v1/ProofCertificate", + "ExplanationTrace": "ontogenesis.socioprophet.dev/v1/ExplanationTrace", + "ContradictionReport": "ontogenesis.socioprophet.dev/v1/ContradictionReport", + "TruthBounds": "ontogenesis.socioprophet.dev/v1/TruthBounds" + }, + "workedExamples": { + "documentSpanToPolicyReadyClaim": { + "reasoningTrace": [ + { + "ruleName": "rule.bad", + "premises": ["p"], + "conclusion": "c", + "evidenceRefs": ["evidence://bad"], + "confidence": 0.5, + "truthBounds": {"lower": 0.1, "upper": 0.9, "method": "fixture"} + } + ] + }, + "vectorCandidateVerificationPath": { + "candidateClaim": {"status": "candidate_only"}, + "verificationPath": {"result": "rejected_before_policy"} + } + } + } +} From c40bf5345de447c116eabbae42194aad1006f09c Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:28:17 -0400 Subject: [PATCH 06/13] Add invalid proof-claim fixture for incomplete reasoning trace --- .../proof-claim-missing-reasoning-field.json | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 fixtures/invalid/proof-claim-missing-reasoning-field.json diff --git a/fixtures/invalid/proof-claim-missing-reasoning-field.json b/fixtures/invalid/proof-claim-missing-reasoning-field.json new file mode 100644 index 0000000..601e6d8 --- /dev/null +++ b/fixtures/invalid/proof-claim-missing-reasoning-field.json @@ -0,0 +1,39 @@ +{ + "apiVersion": "holmes.socioprophet.dev/v1", + "kind": "HolmesReasoningContract", + "metadata": { + "name": "bad-proof-claim-missing-reasoning-field", + "version": "0.1.0" + }, + "spec": { + "canonicalLoop": ["Observe", "Anchor", "Normalize", "Propose", "Explain", "Verify", "Govern", "Act", "Receipt", "Learn"], + "holmesOwnedSegment": ["Propose", "Explain", "Verify"], + "admissionInvariant": "VectorCandidate | ModelOutput | GraphCandidate -> ProposedClaim -> ExplanationTrace + ProofCertificate? + ContradictionReport", + "candidateOnlyStatus": "candidate_only", + "policyDecisions": ["allow", "deny", "require_review", "provisional"], + "contractMappings": { + "Claim": "ontogenesis.socioprophet.dev/v1/Claim", + "ProofCertificate": "ontogenesis.socioprophet.dev/v1/ProofCertificate", + "ExplanationTrace": "ontogenesis.socioprophet.dev/v1/ExplanationTrace", + "ContradictionReport": "ontogenesis.socioprophet.dev/v1/ContradictionReport", + "TruthBounds": "ontogenesis.socioprophet.dev/v1/TruthBounds" + }, + "workedExamples": { + "documentSpanToPolicyReadyClaim": { + "reasoningTrace": [ + { + "ruleName": "rule.bad.missing_truth_bounds", + "premises": ["p"], + "conclusion": "c", + "evidenceRefs": ["evidence://bad"], + "confidence": 0.5 + } + ] + }, + "vectorCandidateVerificationPath": { + "candidateClaim": {"status": "candidate_only"}, + "verificationPath": {"result": "rejected_before_policy"} + } + } + } +} From 3b399460ccc9ab77c074e865ee80c3d789cfc36c Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:42:31 -0400 Subject: [PATCH 07/13] Refactor Holmes validator for negative fixtures --- tools/validate_holmes.py | 98 ++++++++++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/tools/validate_holmes.py b/tools/validate_holmes.py index 0d87a11..372bf04 100644 --- a/tools/validate_holmes.py +++ b/tools/validate_holmes.py @@ -6,8 +6,13 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +SCHEMA_DIR = ROOT / "schemas" EXAMPLE = ROOT / "examples" / "holmes-surface.json" REASONING_EXAMPLE = ROOT / "examples" / "holmes-proof-claim-contract.json" +SCHEMA_FILES = [ + SCHEMA_DIR / "holmes-surface.schema.json", + SCHEMA_DIR / "holmes-proof-claim-contract.schema.json", +] REQUIRED_COMPONENTS = { "sherlock-search", "221b", @@ -79,6 +84,14 @@ "promotionRecord", "rollbackRef", } +REQUIRED_INTEGRATIONS = { + "standards", + "platform", + "search", + "slashTopics", + "lab", + "sourceosCarry", +} REQUIRED_MAPPINGS = { "Claim", "ProofCertificate", @@ -103,6 +116,10 @@ def fail(message: str) -> int: return 1 +def load_json(path: Path) -> dict: + return json.loads(path.read_text()) + + def require_set(spec: dict, field: str, required: set[str]) -> int | None: observed = set(spec.get(field, [])) missing = required - observed @@ -111,15 +128,28 @@ def require_set(spec: dict, field: str, required: set[str]) -> int | None: return None -def validate_surface() -> int | None: - if not EXAMPLE.exists(): - return fail("missing examples/holmes-surface.json") - data = json.loads(EXAMPLE.read_text()) +def validate_schema_files() -> int | None: + for schema_path in SCHEMA_FILES: + if not schema_path.exists(): + return fail(f"missing schema: {schema_path.relative_to(ROOT)}") + try: + load_json(schema_path) + except json.JSONDecodeError as exc: + return fail(f"invalid schema JSON {schema_path.relative_to(ROOT)}: {exc}") + return None + + +def validate_surface_data(data: dict, source: str = "HolmesSurface") -> int | None: if data.get("apiVersion") != "holmes.socioprophet.dev/v1": - return fail("wrong apiVersion") + return fail(f"{source}: wrong apiVersion") if data.get("kind") != "HolmesSurface": - return fail("wrong kind") + return fail(f"{source}: wrong kind") + metadata = data.get("metadata", {}) + if not metadata.get("name") or not metadata.get("version"): + return fail(f"{source}: missing metadata.name or metadata.version") spec = data.get("spec", {}) + if spec.get("product") != "Holmes": + return fail(f"{source}: spec.product must be Holmes") for field, required in [ ("components", REQUIRED_COMPONENTS), ("componentFamilies", REQUIRED_COMPONENT_FAMILIES), @@ -131,57 +161,77 @@ def validate_surface() -> int | None: if result is not None: return result integrations = spec.get("integrations", {}) - for key in ["standards", "platform", "search", "slashTopics", "lab", "sourceosCarry"]: - if key not in integrations: - return fail(f"missing integration: {key}") + missing_integrations = sorted(REQUIRED_INTEGRATIONS - set(integrations.keys())) + if missing_integrations: + return fail(f"{source}: missing integrations: {missing_integrations}") return None -def validate_reasoning_contract() -> int | None: - if not REASONING_EXAMPLE.exists(): - return fail("missing examples/holmes-proof-claim-contract.json") - reasoning = json.loads(REASONING_EXAMPLE.read_text()) +def validate_surface(path: Path = EXAMPLE) -> int | None: + if not path.exists(): + return fail(f"missing {path.relative_to(ROOT)}") + try: + data = load_json(path) + except json.JSONDecodeError as exc: + return fail(f"invalid JSON {path.relative_to(ROOT)}: {exc}") + return validate_surface_data(data, str(path.relative_to(ROOT))) + + +def validate_reasoning_contract_data(reasoning: dict, source: str = "HolmesReasoningContract") -> int | None: if reasoning.get("apiVersion") != "holmes.socioprophet.dev/v1": - return fail("wrong reasoning contract apiVersion") + return fail(f"{source}: wrong reasoning contract apiVersion") if reasoning.get("kind") != "HolmesReasoningContract": - return fail("wrong reasoning contract kind") + return fail(f"{source}: wrong reasoning contract kind") + metadata = reasoning.get("metadata", {}) + if not metadata.get("name") or not metadata.get("version"): + return fail(f"{source}: missing metadata.name or metadata.version") reasoning_spec = reasoning.get("spec", {}) if reasoning_spec.get("candidateOnlyStatus") != "candidate_only": - return fail("candidateOnlyStatus must be candidate_only") + return fail(f"{source}: candidateOnlyStatus must be candidate_only") actual_segment = reasoning_spec.get("holmesOwnedSegment", []) if actual_segment != REQUIRED_HOLMES_SEGMENT: return fail( - "holmesOwnedSegment must be ordered exactly as [Propose, Explain, Verify]; " + f"{source}: holmesOwnedSegment must be ordered exactly as [Propose, Explain, Verify]; " f"got {actual_segment}" ) mappings = set(reasoning_spec.get("contractMappings", {}).keys()) missing_mappings = REQUIRED_MAPPINGS - mappings if missing_mappings: - return fail(f"missing contract mappings: {sorted(missing_mappings)}") + return fail(f"{source}: missing contract mappings: {sorted(missing_mappings)}") worked_examples = reasoning_spec.get("workedExamples", {}) for key in ["documentSpanToPolicyReadyClaim", "vectorCandidateVerificationPath"]: if key not in worked_examples: - return fail(f"missing worked example: {key}") + return fail(f"{source}: missing worked example: {key}") doc_example = worked_examples["documentSpanToPolicyReadyClaim"] reasoning_trace = doc_example.get("reasoningTrace", []) if not reasoning_trace: - return fail("documentSpanToPolicyReadyClaim must include reasoningTrace") + return fail(f"{source}: documentSpanToPolicyReadyClaim must include reasoningTrace") for index, entry in enumerate(reasoning_trace): missing_reasoning_fields = REQUIRED_REASONING_TRACE - set(entry.keys()) if missing_reasoning_fields: return fail( - f"missing reasoningTrace fields at index {index}: {sorted(missing_reasoning_fields)}" + f"{source}: missing reasoningTrace fields at index {index}: {sorted(missing_reasoning_fields)}" ) vector_example = worked_examples["vectorCandidateVerificationPath"] if vector_example.get("candidateClaim", {}).get("status") != "candidate_only": - return fail("vector candidateClaim status must be candidate_only") + return fail(f"{source}: vector candidateClaim status must be candidate_only") if vector_example.get("verificationPath", {}).get("result") != REJECTED_BEFORE_POLICY: - return fail("vector verification path result must be rejected_before_policy") + return fail(f"{source}: vector verification path result must be rejected_before_policy") return None +def validate_reasoning_contract(path: Path = REASONING_EXAMPLE) -> int | None: + if not path.exists(): + return fail(f"missing {path.relative_to(ROOT)}") + try: + reasoning = load_json(path) + except json.JSONDecodeError as exc: + return fail(f"invalid JSON {path.relative_to(ROOT)}: {exc}") + return validate_reasoning_contract_data(reasoning, str(path.relative_to(ROOT))) + + def main() -> int: - for validator in [validate_surface, validate_reasoning_contract]: + for validator in [validate_schema_files, validate_surface, validate_reasoning_contract]: result = validator() if result is not None: return result From 85b2ca52bccb61f9477c93b4712a7690f187639d Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:43:46 -0400 Subject: [PATCH 08/13] Add negative fixture runner --- tools/run_negative_fixtures.py | 67 ++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tools/run_negative_fixtures.py diff --git a/tools/run_negative_fixtures.py b/tools/run_negative_fixtures.py new file mode 100644 index 0000000..8c1beb5 --- /dev/null +++ b/tools/run_negative_fixtures.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import contextlib +import io +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +INVALID_DIR = ROOT / "fixtures" / "invalid" +sys.path.insert(0, str(ROOT / "tools")) + +import validate_holmes # noqa: E402 + + +def fail(message: str) -> int: + print(f"ERROR: {message}", file=sys.stderr) + return 1 + + +def validate_invalid_fixture(path: Path) -> tuple[bool, str]: + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + return True, f"invalid JSON rejected: {exc}" + + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + if path.name.startswith("holmes-surface-"): + result = validate_holmes.validate_surface_data(data, str(path.relative_to(ROOT))) + elif path.name.startswith("proof-claim-"): + result = validate_holmes.validate_reasoning_contract_data(data, str(path.relative_to(ROOT))) + else: + return False, "unknown fixture family; expected holmes-surface-* or proof-claim-*" + + if result is None: + return False, "fixture unexpectedly passed validation" + return True, stderr.getvalue().strip() or "fixture rejected" + + +def main() -> int: + if not INVALID_DIR.exists(): + return fail("missing fixtures/invalid") + fixtures = sorted(INVALID_DIR.glob("*.json")) + if len(fixtures) < 4: + return fail(f"expected at least 4 invalid fixtures; got {len(fixtures)}") + + failures: list[str] = [] + for fixture in fixtures: + rejected, reason = validate_invalid_fixture(fixture) + if not rejected: + failures.append(f"{fixture.relative_to(ROOT)}: {reason}") + else: + print(f"OK: rejected {fixture.relative_to(ROOT)} — {reason}") + + if failures: + for failure in failures: + print(f"ERROR: {failure}", file=sys.stderr) + return 1 + + print("OK: Holmes negative fixtures rejected") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d65c44da0c269b880512e06fd18104dbd0beac7b Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:45:12 -0400 Subject: [PATCH 09/13] Run negative fixtures in Holmes validation --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6071ca5..51793fa 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build test validate validate-maturity dist release-dry-run clean +.PHONY: build test validate validate-contracts validate-maturity validate-negative-fixtures dist release-dry-run clean BIN := holmes DIST_DIR := dist @@ -20,8 +20,13 @@ test: validate-maturity: python3 tools/validate_maturity.py -validate: build validate-maturity +validate-contracts: python3 tools/validate_holmes.py + +validate-negative-fixtures: + python3 tools/run_negative_fixtures.py + +validate: build validate-maturity validate-contracts validate-negative-fixtures bin/$(BIN) --version bin/$(BIN) doctor bin/$(BIN) self-test From 34d09ee87f2ad1f02b7dc5ff9464006a5bad865a Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:46:24 -0400 Subject: [PATCH 10/13] Promote Holmes maturity manifest to M3 evidence set --- repo.maturity.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/repo.maturity.yaml b/repo.maturity.yaml index e31c8ca..ae9bba5 100644 --- a/repo.maturity.yaml +++ b/repo.maturity.yaml @@ -6,17 +6,22 @@ canonicality: canonical owners: - SocioProphet maturity: - level: M2 - targetLevel: M3 + level: M3 + targetLevel: M4 evidence: - README.md defines Holmes purpose, product family, Slash Topics training role, proof-claim reasoning role, repo boundaries, and integration path. - docs/ARCHITECTURE.md defines Holmes product components, service boundaries, TriTRPC method families, and promotion rules. - docs/NLP_COMPONENT_ALIGNMENT.md defines governed NLP component coverage and Slash Topics topic-model training alignment. - docs/PROOF_CLAIM_CONTRACT.md defines the Propose -> Explain -> Verify proof-claim contract and policy-admission boundary. - - examples/holmes-surface.json provides the machine-readable Holmes product surface fixture. - - examples/holmes-proof-claim-contract.json provides deterministic proof-claim reasoning fixture coverage. - - tools/validate_holmes.py validates Holmes surface and proof-claim contract fixtures. - - Makefile provides build, test, validate, dist, and release-dry-run targets. + - schemas/holmes-surface.schema.json defines the Holmes surface contract schema. + - schemas/holmes-proof-claim-contract.schema.json defines the proof-claim reasoning contract schema. + - examples/holmes-surface.json provides the positive Holmes product surface fixture. + - examples/holmes-proof-claim-contract.json provides deterministic positive proof-claim reasoning fixture coverage. + - fixtures/invalid/ contains 4 negative fixtures covering Holmes surface and proof-claim rejection paths. + - tools/validate_holmes.py validates schema JSON, Holmes surface fixture semantics, and proof-claim contract fixture semantics. + - tools/run_negative_fixtures.py confirms invalid fixtures are rejected by the same validation functions used for positive fixtures. + - tools/validate_maturity.py validates this maturity manifest and required M3 evidence references. + - Makefile provides build, test, validate, validate-contracts, validate-negative-fixtures, validate-maturity, dist, and release-dry-run targets. - .github/workflows/validate.yml runs make validate and make release-dry-run on pull requests and main pushes. validation: commands: @@ -55,8 +60,7 @@ integrations: relationship: workspace governance and maturity ingestion required: false nextActions: - - Add negative fixtures and a negative-fixture runner so Holmes can honestly claim M3. + - Confirm CI status and update validation.lastKnownStatus to passing after the M3 validation workflow is green. - Replace deterministic not-yet-wired CLI stubs with service-backed Sherlock, graph, and governance adapters. - - Add schema files for Holmes surface, proof-claim contract, and Slash Topics training references. - - Add release notes, checksums, and provenance attestations for M4 readiness. + - Add release notes, checksums, SBOM, and provenance attestations for M4 readiness. - Register Holmes in workspace-inventory or Sociosphere scorecard ingestion. From a221f7d9bf5d014ae3252fa4b01375800aee048d Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 12:53:51 -0400 Subject: [PATCH 11/13] Validate Holmes M3 maturity evidence --- tools/validate_maturity.py | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/tools/validate_maturity.py b/tools/validate_maturity.py index 13b4dfa..b41121a 100644 --- a/tools/validate_maturity.py +++ b/tools/validate_maturity.py @@ -19,8 +19,8 @@ "owners:", " - SocioProphet", "maturity:", - " level: M2", - " targetLevel: M3", + " level: M3", + " targetLevel: M4", "validation:", " commands:", " - make validate", @@ -28,6 +28,21 @@ "integrations:", "nextActions:", ] +REQUIRED_PATHS = [ + "README.md", + "docs/ARCHITECTURE.md", + "docs/NLP_COMPONENT_ALIGNMENT.md", + "docs/PROOF_CLAIM_CONTRACT.md", + "schemas/holmes-surface.schema.json", + "schemas/holmes-proof-claim-contract.schema.json", + "examples/holmes-surface.json", + "examples/holmes-proof-claim-contract.json", + "tools/validate_holmes.py", + "tools/run_negative_fixtures.py", + "tools/validate_maturity.py", + "Makefile", + ".github/workflows/validate.yml", +] REQUIRED_INTEGRATIONS = { "SocioProphet/functional-model-surfaces", "SocioProphet/prophet-platform", @@ -77,6 +92,12 @@ def main() -> int: if required not in text: return fail(f"missing required maturity text: {required}") + for relative_path in REQUIRED_PATHS: + if relative_path not in text: + return fail(f"maturity evidence must name {relative_path}") + if not (ROOT / relative_path).exists(): + return fail(f"maturity evidence path does not exist: {relative_path}") + status = scalar(text, "lastKnownStatus") if status not in ALLOWED_VALIDATION_STATUS: return fail(f"validation.lastKnownStatus must be one of {sorted(ALLOWED_VALIDATION_STATUS)}; got {status!r}") @@ -85,9 +106,15 @@ def main() -> int: if missing_integrations: return fail(f"missing integrations: {missing_integrations}") + invalid_fixtures = sorted((ROOT / "fixtures" / "invalid").glob("*.json")) + if len(invalid_fixtures) < 4: + return fail(f"fixtures/invalid must contain at least 4 negative fixtures; got {len(invalid_fixtures)}") + if "fixtures/invalid/ contains 4 negative fixtures" not in text: + return fail("maturity evidence must explicitly cite fixtures/invalid/ negative fixtures") + evidence_count = count_list_items_in_section(text, " evidence") - if evidence_count < 6: - return fail(f"maturity.evidence should contain at least 6 concrete evidence items; got {evidence_count}") + if evidence_count < 10: + return fail(f"maturity.evidence should contain at least 10 concrete evidence items at M3; got {evidence_count}") next_actions = count_list_items_in_section(text, "nextActions") if next_actions < 3: From be074d38ab51e97c555d26b996d0acc05208fb61 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 13:02:47 -0400 Subject: [PATCH 12/13] Allow nested maturity scalar fields --- tools/validate_maturity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/validate_maturity.py b/tools/validate_maturity.py index b41121a..bd58d7c 100644 --- a/tools/validate_maturity.py +++ b/tools/validate_maturity.py @@ -64,7 +64,7 @@ def fail(message: str) -> int: def scalar(text: str, key: str) -> str | None: - match = re.search(rf"^{re.escape(key)}:\s*(.+?)\s*$", text, flags=re.MULTILINE) + match = re.search(rf"^\s*{re.escape(key)}:\s*(.+?)\s*$", text, flags=re.MULTILINE) if not match: return None return match.group(1).strip().strip('"') From 2014db210341900e87092a765c0a0d2a5e5b0d05 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Wed, 13 May 2026 13:06:53 -0400 Subject: [PATCH 13/13] Mark Holmes M3 validation passing --- repo.maturity.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repo.maturity.yaml b/repo.maturity.yaml index ae9bba5..b238998 100644 --- a/repo.maturity.yaml +++ b/repo.maturity.yaml @@ -27,7 +27,7 @@ validation: commands: - make validate ciRequired: true - lastKnownStatus: pending + lastKnownStatus: passing integrations: - repository: SocioProphet/functional-model-surfaces relationship: normative cross-surface standards and maturity model reference @@ -60,7 +60,6 @@ integrations: relationship: workspace governance and maturity ingestion required: false nextActions: - - Confirm CI status and update validation.lastKnownStatus to passing after the M3 validation workflow is green. - Replace deterministic not-yet-wired CLI stubs with service-backed Sherlock, graph, and governance adapters. - Add release notes, checksums, SBOM, and provenance attestations for M4 readiness. - Register Holmes in workspace-inventory or Sociosphere scorecard ingestion.