diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 36b5cb5..fc536b5 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -21,3 +21,5 @@ jobs: run: make smoke - name: Strict jsonschema validation of maturity record run: python3 tools/validate_maturity.py schemas/repo-maturity.schema.json repo.maturity.yaml + - name: Negative fixtures (must be rejected) + run: python3 tools/run_negative_fixtures.py diff --git a/Makefile b/Makefile index 3cd5bc3..d97854e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: validate validate-spine smoke carry +.PHONY: validate validate-spine validate-negative smoke carry validate: validate-spine python3 tools/validate.py @@ -11,3 +11,6 @@ smoke: carry: python3 tools/emit_sourceos_carry.py > examples/sourceos-carry.ocrlab.json + +validate-negative: + python3 tools/run_negative_fixtures.py diff --git a/fixtures/invalid/functional-service.bad-function.json b/fixtures/invalid/functional-service.bad-function.json new file mode 100644 index 0000000..40c9335 --- /dev/null +++ b/fixtures/invalid/functional-service.bad-function.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "functional-service.v1", + "service": { + "id": "ocrlab.holmes.ocr-pilot", + "name": "OCRLab Holmes OCR Pilot", + "ownerRepository": "SociOS-Linux/ocrlab", + "status": "experimental", + "description": "Offline deterministic ocr functional surface emitted by ocrlab for governed promotion." + }, + "function": "not-a-real-function", + "model": { + "modelRef": "urn:socioprophet:functional-model:ocrlab:deterministic-ocr:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "image/document", + "text/corpus" + ], + "outputs": [ + "ocr/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": false, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:ocrlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/functional-service.missing-schema-version.json b/fixtures/invalid/functional-service.missing-schema-version.json new file mode 100644 index 0000000..b7db9cf --- /dev/null +++ b/fixtures/invalid/functional-service.missing-schema-version.json @@ -0,0 +1,53 @@ +{ + "service": { + "id": "ocrlab.holmes.ocr-pilot", + "name": "OCRLab Holmes OCR Pilot", + "ownerRepository": "SociOS-Linux/ocrlab", + "status": "experimental", + "description": "Offline deterministic ocr functional surface emitted by ocrlab for governed promotion." + }, + "function": "ocr", + "model": { + "modelRef": "urn:socioprophet:functional-model:ocrlab:deterministic-ocr:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "image/document", + "text/corpus" + ], + "outputs": [ + "ocr/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": false, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:ocrlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/functional-service.mutable-carry.json b/fixtures/invalid/functional-service.mutable-carry.json new file mode 100644 index 0000000..8880bc1 --- /dev/null +++ b/fixtures/invalid/functional-service.mutable-carry.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "functional-service.v1", + "service": { + "id": "ocrlab.holmes.ocr-pilot", + "name": "OCRLab Holmes OCR Pilot", + "ownerRepository": "SociOS-Linux/ocrlab", + "status": "experimental", + "description": "Offline deterministic ocr functional surface emitted by ocrlab for governed promotion." + }, + "function": "ocr", + "model": { + "modelRef": "urn:socioprophet:functional-model:ocrlab:deterministic-ocr:v0", + "adapterRefs": [], + "runtime": "python-stdlib-offline-smoke", + "mutableStatePolicy": "forbidden-in-sourceos" + }, + "inputs": [ + "image/document", + "text/corpus" + ], + "outputs": [ + "ocr/features", + "ranking/candidates", + "evidence/smoke-receipt" + ], + "evals": { + "required": true, + "references": [ + "examples/smoke-input.json", + "examples/smoke-output.json", + "evidence/smoke-receipt.example.json" + ], + "minimumPromotionGate": "candidate requires signed eval and dataset provenance" + }, + "governance": { + "ledgerRequired": true, + "guardrailRequired": true, + "routingRequired": true, + "policyRefs": [ + "SocioProphet/model-governance-ledger", + "SocioProphet/guardrail-fabric", + "SocioProphet/model-router", + "SocioProphet/holmes" + ] + }, + "sourceosCarry": { + "allowed": true, + "carriesMutableModelState": true, + "clientRefRequired": true, + "launchProfileRefs": [ + "urn:srcos:model-carry:functional-service:ocrlab-holmes-pilot" + ] + } +} diff --git a/fixtures/invalid/maturity.empty-owners.yaml b/fixtures/invalid/maturity.empty-owners.yaml new file mode 100644 index 0000000..caa7434 --- /dev/null +++ b/fixtures/invalid/maturity.empty-owners.yaml @@ -0,0 +1,47 @@ +schemaVersion: repo-maturity.v1 +repository: SociOS-Linux/ocrlab +plane: socios-lab +status: experimental +canonicality: reference +owners: [] +maturity: + level: M2 + targetLevel: M3 + evidence: + - README.md states lab purpose, lab-only boundary, and integration points. + - service-manifest/functional-service.v1.json conforms to functional-service.v1. + - lab.manifest.json enumerates 5 governed lab surfaces. + - make validate exits 0 (offline, deterministic). + - CI runs make validate + strict jsonschema validation on every push/PR. +validation: + commands: + - make validate + - make smoke + ciRequired: true + lastKnownStatus: passing +integrations: + - repository: SocioProphet/functional-model-surfaces + relationship: emits functional-service.v1 manifests + required: true + - repository: SocioProphet/model-governance-ledger + relationship: sends promotion evidence + required: true + - repository: SocioProphet/sociosphere + relationship: registers maturity/evidence status + required: true + - repository: SocioProphet/holmes + relationship: provides the ocr functional surface to Holmes + required: false + - repository: SocioProphet/model-router + relationship: routable governed surface + required: false + - repository: SocioProphet/guardrail-fabric + relationship: guardrail policy enforcement + required: false + - repository: SourceOS-Linux/sourceos-model-carry + relationship: disabled-by-default SourceOS carry reference + required: false +nextActions: + - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. + - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/fixtures/invalid/maturity.invalid-level.yaml b/fixtures/invalid/maturity.invalid-level.yaml new file mode 100644 index 0000000..9a40c9d --- /dev/null +++ b/fixtures/invalid/maturity.invalid-level.yaml @@ -0,0 +1,48 @@ +schemaVersion: repo-maturity.v1 +repository: SociOS-Linux/ocrlab +plane: socios-lab +status: experimental +canonicality: reference +owners: + - SociOS-Linux +maturity: + level: M9 + targetLevel: M3 + evidence: + - README.md states lab purpose, lab-only boundary, and integration points. + - service-manifest/functional-service.v1.json conforms to functional-service.v1. + - lab.manifest.json enumerates 5 governed lab surfaces. + - make validate exits 0 (offline, deterministic). + - CI runs make validate + strict jsonschema validation on every push/PR. +validation: + commands: + - make validate + - make smoke + ciRequired: true + lastKnownStatus: passing +integrations: + - repository: SocioProphet/functional-model-surfaces + relationship: emits functional-service.v1 manifests + required: true + - repository: SocioProphet/model-governance-ledger + relationship: sends promotion evidence + required: true + - repository: SocioProphet/sociosphere + relationship: registers maturity/evidence status + required: true + - repository: SocioProphet/holmes + relationship: provides the ocr functional surface to Holmes + required: false + - repository: SocioProphet/model-router + relationship: routable governed surface + required: false + - repository: SocioProphet/guardrail-fabric + relationship: guardrail policy enforcement + required: false + - repository: SourceOS-Linux/sourceos-model-carry + relationship: disabled-by-default SourceOS carry reference + required: false +nextActions: + - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. + - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/repo.maturity.yaml b/repo.maturity.yaml index fb21ed3..bc70cdd 100644 --- a/repo.maturity.yaml +++ b/repo.maturity.yaml @@ -6,14 +6,15 @@ canonicality: reference owners: - SociOS-Linux maturity: - level: M2 - targetLevel: M3 + level: M3 + targetLevel: M4 evidence: - README.md states lab purpose, lab-only boundary, and integration points. - service-manifest/functional-service.v1.json conforms to functional-service.v1. - lab.manifest.json enumerates 5 governed lab surfaces. - make validate exits 0 (offline, deterministic). - CI runs make validate + strict jsonschema validation on every push/PR. + - Negative fixtures under fixtures/invalid/ confirm the schemas reject malformed documents (CI-enforced). validation: commands: - make validate @@ -43,6 +44,5 @@ integrations: relationship: disabled-by-default SourceOS carry reference required: false nextActions: - - Add negative fixtures under fixtures/invalid/ (>=2 per schema) to reach M3. - - Wire promotion evidence into SocioProphet/model-governance-ledger. + - Produce tagged releases with sha256 checksums / provenance attestation (M4). - Register maturity record with SocioProphet/sociosphere workspace-inventory. diff --git a/tools/run_negative_fixtures.py b/tools/run_negative_fixtures.py new file mode 100644 index 0000000..4682ef1 --- /dev/null +++ b/tools/run_negative_fixtures.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""M3 negative-fixture runner: every fixture under fixtures/invalid/ MUST be rejected +by its schema. Exits non-zero if any invalid document unexpectedly validates — that would +mean the schema/validator has regressed and could admit a malformed manifest.""" +import json, os, sys, glob +import jsonschema +try: + import yaml +except Exception: + yaml=None + +FS="schemas/functional-service.schema.json" +RM="schemas/repo-maturity.schema.json" + +def load(p): + if p.endswith((".yaml",".yml")): + return None if yaml is None else yaml.safe_load(open(p)) + return json.load(open(p)) + +def schema_for(p): + b=os.path.basename(p) + if b.startswith("functional-service"): return json.load(open(FS)) + if b.startswith("maturity"): return json.load(open(RM)) + return None + +def main(): + fixtures=sorted(glob.glob("fixtures/invalid/*")) + if not fixtures: + print("FAIL: no negative fixtures found under fixtures/invalid/", file=sys.stderr); return 1 + leaked=0 + for p in fixtures: + s=schema_for(p) + if s is None: + print(f" SKIP (no schema mapping): {p}"); continue + d=load(p) + if d is None: + print(f" SKIP (pyyaml unavailable): {p}"); continue + try: + jsonschema.validate(d, s) + print(f" FAIL: {p} unexpectedly VALIDATED — schema too loose"); leaked+=1 + except jsonschema.ValidationError as e: + print(f" OK rejected: {os.path.basename(p)} :: {e.message[:70]}") + if leaked: + print(f"{leaked} invalid fixture(s) passed validation", file=sys.stderr); return 1 + print("OK: all negative fixtures correctly rejected"); return 0 + +if __name__=="__main__": + raise SystemExit(main())