From 0d1cec6ca6b50e6cf58226531103857116e33903 Mon Sep 17 00:00:00 2001 From: mdheller <21163552+mdheller@users.noreply.github.com> Date: Sat, 4 Jul 2026 20:02:03 -0400 Subject: [PATCH] harden: negative fixtures + M3 (schemas must reject malformed manifests) Add fixtures/invalid/ (bad function enum, mutable-carry safety violation, missing schemaVersion, invalid maturity level, empty owners) + run_negative_fixtures.py, wired into CI. A malformed manifest can no longer pass validation silently. Bump maturity M2->M3. --- .github/workflows/validate.yml | 2 + Makefile | 5 +- .../functional-service.bad-function.json | 54 +++++++++++++++++++ ...tional-service.missing-schema-version.json | 53 ++++++++++++++++++ .../functional-service.mutable-carry.json | 54 +++++++++++++++++++ fixtures/invalid/maturity.empty-owners.yaml | 47 ++++++++++++++++ fixtures/invalid/maturity.invalid-level.yaml | 48 +++++++++++++++++ repo.maturity.yaml | 8 +-- tools/run_negative_fixtures.py | 48 +++++++++++++++++ 9 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 fixtures/invalid/functional-service.bad-function.json create mode 100644 fixtures/invalid/functional-service.missing-schema-version.json create mode 100644 fixtures/invalid/functional-service.mutable-carry.json create mode 100644 fixtures/invalid/maturity.empty-owners.yaml create mode 100644 fixtures/invalid/maturity.invalid-level.yaml create mode 100644 tools/run_negative_fixtures.py diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 36b5cb5..fc536b5 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -21,3 +21,5 @@ jobs: run: make smoke - name: Strict jsonschema validation of maturity record run: python3 tools/validate_maturity.py schemas/repo-maturity.schema.json repo.maturity.yaml + - name: Negative fixtures (must be rejected) + run: python3 tools/run_negative_fixtures.py diff --git a/Makefile b/Makefile index 8fe8aff..ba03dd3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: validate validate-spine smoke carry +.PHONY: validate validate-spine validate-negative smoke carry validate: validate-spine python3 tools/validate.py @@ -11,3 +11,6 @@ smoke: carry: python3 tools/emit_sourceos_carry.py > examples/sourceos-carry.speechlab.json + +validate-negative: + python3 tools/run_negative_fixtures.py diff --git a/fixtures/invalid/functional-service.bad-function.json b/fixtures/invalid/functional-service.bad-function.json new file mode 100644 index 0000000..5ad992a --- /dev/null +++ b/fixtures/invalid/functional-service.bad-function.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "functional-service.v1", + "service": { + "id": "speechlab.holmes.speech-pilot", + "name": "SpeechLab Holmes Speech Pilot", + "ownerRepository": "SociOS-Linux/speechlab", + "status": "experimental", + "description": "Offline deterministic speech functional surface emitted by speechlab for governed promotion." + }, + "function": "not-a-real-function", + "model": { + "modelRef": "urn:socioprophet:functional-model:speechlab:deterministic-speech:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "audio/query", + "audio/corpus" + ], + "outputs": [ + "speech/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": false, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:speechlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/functional-service.missing-schema-version.json b/fixtures/invalid/functional-service.missing-schema-version.json new file mode 100644 index 0000000..feb7545 --- /dev/null +++ b/fixtures/invalid/functional-service.missing-schema-version.json @@ -0,0 +1,53 @@ +{ + "service": { + "id": "speechlab.holmes.speech-pilot", + "name": "SpeechLab Holmes Speech Pilot", + "ownerRepository": "SociOS-Linux/speechlab", + "status": "experimental", + "description": "Offline deterministic speech functional surface emitted by speechlab for governed promotion." + }, + "function": "speech", + "model": { + "modelRef": "urn:socioprophet:functional-model:speechlab:deterministic-speech:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "audio/query", + "audio/corpus" + ], + "outputs": [ + "speech/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": false, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:speechlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/functional-service.mutable-carry.json b/fixtures/invalid/functional-service.mutable-carry.json new file mode 100644 index 0000000..dca588d --- /dev/null +++ b/fixtures/invalid/functional-service.mutable-carry.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "functional-service.v1", + "service": { + "id": "speechlab.holmes.speech-pilot", + "name": "SpeechLab Holmes Speech Pilot", + "ownerRepository": "SociOS-Linux/speechlab", + "status": "experimental", + "description": "Offline deterministic speech functional surface emitted by speechlab for governed promotion." + }, + "function": "speech", + "model": { + "modelRef": "urn:socioprophet:functional-model:speechlab:deterministic-speech:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "audio/query", + "audio/corpus" + ], + "outputs": [ + "speech/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": true, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:speechlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/maturity.empty-owners.yaml b/fixtures/invalid/maturity.empty-owners.yaml new file mode 100644 index 0000000..44f23f6 --- /dev/null +++ b/fixtures/invalid/maturity.empty-owners.yaml @@ -0,0 +1,47 @@ +schemaVersion: repo-maturity.v1 +repository: SociOS-Linux/speechlab +plane: socios-lab +status: experimental +canonicality: reference +owners: [] +maturity: + level: M2 + targetLevel: M3 + evidence: + - README.md states lab purpose, lab-only boundary, and integration points. + - service-manifest/functional-service.v1.json conforms to functional-service.v1. + - lab.manifest.json enumerates 4 governed lab surfaces. + - make validate exits 0 (offline, deterministic). + - CI runs make validate + strict jsonschema validation on every push/PR. +validation: + commands: + - make validate + - make smoke + ciRequired: true + lastKnownStatus: passing +integrations: + - repository: SocioProphet/functional-model-surfaces + relationship: emits functional-service.v1 manifests + required: true + - repository: SocioProphet/model-governance-ledger + relationship: sends promotion evidence + required: true + - repository: SocioProphet/sociosphere + relationship: registers maturity/evidence status + required: true + - repository: SocioProphet/holmes + relationship: provides the speech functional surface to Holmes + required: false + - repository: SocioProphet/model-router + relationship: routable governed surface + required: false + - repository: SocioProphet/guardrail-fabric + relationship: guardrail policy enforcement + required: false + - repository: SourceOS-Linux/sourceos-model-carry + relationship: disabled-by-default SourceOS carry reference + required: false +nextActions: + - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. + - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/fixtures/invalid/maturity.invalid-level.yaml b/fixtures/invalid/maturity.invalid-level.yaml new file mode 100644 index 0000000..57abb5f --- /dev/null +++ b/fixtures/invalid/maturity.invalid-level.yaml @@ -0,0 +1,48 @@ +schemaVersion: repo-maturity.v1 +repository: SociOS-Linux/speechlab +plane: socios-lab +status: experimental +canonicality: reference +owners: + - SociOS-Linux +maturity: + level: M9 + targetLevel: M3 + evidence: + - README.md states lab purpose, lab-only boundary, and integration points. + - service-manifest/functional-service.v1.json conforms to functional-service.v1. + - lab.manifest.json enumerates 4 governed lab surfaces. + - make validate exits 0 (offline, deterministic). + - CI runs make validate + strict jsonschema validation on every push/PR. +validation: + commands: + - make validate + - make smoke + ciRequired: true + lastKnownStatus: passing +integrations: + - repository: SocioProphet/functional-model-surfaces + relationship: emits functional-service.v1 manifests + required: true + - repository: SocioProphet/model-governance-ledger + relationship: sends promotion evidence + required: true + - repository: SocioProphet/sociosphere + relationship: registers maturity/evidence status + required: true + - repository: SocioProphet/holmes + relationship: provides the speech functional surface to Holmes + required: false + - repository: SocioProphet/model-router + relationship: routable governed surface + required: false + - repository: SocioProphet/guardrail-fabric + relationship: guardrail policy enforcement + required: false + - repository: SourceOS-Linux/sourceos-model-carry + relationship: disabled-by-default SourceOS carry reference + required: false +nextActions: + - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. + - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/repo.maturity.yaml b/repo.maturity.yaml index 0dd25a8..549c699 100644 --- a/repo.maturity.yaml +++ b/repo.maturity.yaml @@ -6,14 +6,15 @@ canonicality: reference owners: - SociOS-Linux maturity: - level: M2 - targetLevel: M3 + level: M3 + targetLevel: M4 evidence: - README.md states lab purpose, lab-only boundary, and integration points. - service-manifest/functional-service.v1.json conforms to functional-service.v1. - lab.manifest.json enumerates 4 governed lab surfaces. - make validate exits 0 (offline, deterministic). - CI runs make validate + strict jsonschema validation on every push/PR. + - Negative fixtures under fixtures/invalid/ confirm the schemas reject malformed documents (CI-enforced). validation: commands: - make validate @@ -43,6 +44,5 @@ integrations: relationship: disabled-by-default SourceOS carry reference required: false nextActions: - - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. - - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Produce tagged releases with sha256 checksums / provenance attestation (M4). - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/tools/run_negative_fixtures.py b/tools/run_negative_fixtures.py new file mode 100644 index 0000000..4682ef1 --- /dev/null +++ b/tools/run_negative_fixtures.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""M3 negative-fixture runner: every fixture under fixtures/invalid/ MUST be rejected +by its schema. Exits non-zero if any invalid document unexpectedly validates — that would +mean the schema/validator has regressed and could admit a malformed manifest.""" +import json, os, sys, glob +import jsonschema +try: + import yaml +except Exception: + yaml=None + +FS="schemas/functional-service.schema.json" +RM="schemas/repo-maturity.schema.json" + +def load(p): + if p.endswith((".yaml",".yml")): + return None if yaml is None else yaml.safe_load(open(p)) + return json.load(open(p)) + +def schema_for(p): + b=os.path.basename(p) + if b.startswith("functional-service"): return json.load(open(FS)) + if b.startswith("maturity"): return json.load(open(RM)) + return None + +def main(): + fixtures=sorted(glob.glob("fixtures/invalid/*")) + if not fixtures: + print("FAIL: no negative fixtures found under fixtures/invalid/", file=sys.stderr); return 1 + leaked=0 + for p in fixtures: + s=schema_for(p) + if s is None: + print(f" SKIP (no schema mapping): {p}"); continue + d=load(p) + if d is None: + print(f" SKIP (pyyaml unavailable): {p}"); continue + try: + jsonschema.validate(d, s) + print(f" FAIL: {p} unexpectedly VALIDATED — schema too loose"); leaked+=1 + except jsonschema.ValidationError as e: + print(f" OK rejected: {os.path.basename(p)} :: {e.message[:70]}") + if leaked: + print(f"{leaked} invalid fixture(s) passed validation", file=sys.stderr); return 1 + print("OK: all negative fixtures correctly rejected"); return 0 + +if __name__=="__main__": + raise SystemExit(main())