diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b30a88a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,55 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + # Postgres as a service container -> the gated integration tests run for real + # (recursive-CTE reachability, ON CONFLICT merge, JSONB, migrations, score history). + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: reachability + POSTGRES_PASSWORD: reachability + POSTGRES_DB: reachability + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U reachability" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + env: + PYTHONPATH: core + REACHABILITY_TEST_DSN: postgresql://reachability:reachability@localhost:5432/reachability + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install package + dev deps + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Install grype + # grype on PATH -> the gated live SCA test scans the SBOM fixture for real + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Lint + run: ruff check core + + - name: Test (Postgres service + live grype) + run: pytest core/tests -q diff --git a/.gitignore b/.gitignore index 5d4a82e..93df2db 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ desktop.ini # --- Logs --- *.log logs/ + +# --- local tool binaries (grype, etc.) --- +.tools/ diff --git a/adapters/zap/adapter.py b/adapters/zap/adapter.py index d0fb54e..ae67ae9 100644 --- a/adapters/zap/adapter.py +++ b/adapters/zap/adapter.py @@ -1,34 +1,149 @@ """ZAP DAST adapter — code adapter (effort-ladder rung 2). Default OSS scanner for the `dast` capability. ZAP runs as a daemon with a REST -API (stateful session, spider + active scan), so it needs code rather than a -declarative YAML. Subclasses CodeAdapter; ``normalize()`` is kept pure and -unit-testable against recorded ZAP alert JSON (see core/tests/test_normalize/). -Contract: schema/03-adapter-interface.md. +API (stateful session: spider + passive/active scan), so it needs code rather than +a declarative YAML. ``normalize()`` is kept pure and unit-testable against recorded +ZAP alert JSON (see core/tests/fixtures/zap_alerts.json). Contract: +schema/01-finding-schema.md + schema/03-adapter-interface.md. """ from __future__ import annotations +import hashlib +import json +import time +import urllib.parse +import urllib.request +import uuid from typing import Iterator from reachability.adapters.code_adapter import CodeAdapter from reachability.adapters.protocol import ScanTarget +_FINDING_NS = uuid.uuid5(uuid.NAMESPACE_URL, "https://reachability.dev/finding") + +# ZAP risk -> canonical level (ZAP has no "critical"). +_RISK_LEVEL = {"Informational": "info", "Info": "info", "Low": "low", + "Medium": "medium", "High": "high"} +# ZAP confidence -> DAST evidence confidence (01-finding-schema.md). +_CONFIDENCE = {"False Positive": "tentative", "Low": "tentative", + "Medium": "firm", "High": "confirmed", "Confirmed": "confirmed"} +# CWE -> our category vocabulary (categories.md). Unmapped -> "unknown" (never +# auto-downgraded, only flagged — we don't guess a category we can't justify). +_CWE_CATEGORY = { + "79": "xss", "89": "sql_injection", "78": "command_injection", "77": "command_injection", + "22": "path_traversal", "98": "path_traversal", "918": "ssrf", + "94": "code_injection", "502": "code_injection", "434": "file_upload", + "200": "info_disclosure", "215": "info_disclosure", "548": "info_disclosure", + "497": "info_disclosure", + "287": "auth_bypass", "288": "auth_bypass", "306": "missing_authentication", + "639": "broken_access_control", "284": "broken_access_control", "798": "secret_exposure", + "16": "security_misconfiguration", "693": "security_misconfiguration", + "1021": "security_misconfiguration", "1004": "security_misconfiguration", + "525": "security_misconfiguration", "614": "security_misconfiguration", +} + class ZapAdapter(CodeAdapter): name = "zap" kind = "dast" + def __init__(self, base_url: str = "http://127.0.0.1:8090", api_key: str | None = None) -> None: + self.base_url = base_url.rstrip("/") + self.api_key = api_key + + # --- lifecycle -------------------------------------------------------- def healthcheck(self) -> bool: - raise NotImplementedError("scaffold: ping ZAP daemon REST API") + try: + # short timeout: discovery must stay fast when no ZAP daemon is running + self.version = self._get("/JSON/core/view/version/", _timeout=3).get("version", "unknown") + return True + except Exception: + return False def run(self, target: ScanTarget) -> Iterator[dict]: - raise NotImplementedError("scaffold: spider + active scan, stream alerts -> normalize") + active = bool(target.options.get("active")) + for url in target.targets: + self._spider(url) + self._wait_passive() + if active: + self._active_scan(url) + for alert in self._get("/JSON/alert/view/alerts/", baseurl=url).get("alerts", []): + yield self.normalize(alert) + # --- mapping (pure, testable) ----------------------------------------- def normalize(self, raw_item: dict) -> dict: - # Map one ZAP alert -> normalized finding. evidence.confidence is REQUIRED - # for dast (01-finding-schema.md): map ZAP confidence -> tentative|firm|confirmed. - raise NotImplementedError("scaffold") + cwe = str(raw_item.get("cweid", "")).strip() + cwes = [f"CWE-{cwe}"] if cwe and cwe not in ("-1", "0") else [] + category = _CWE_CATEGORY.get(cwe, "unknown") + risk = (raw_item.get("risk") or "").split(" ")[0] + url = raw_item.get("url", "") + parts = urllib.parse.urlsplit(url) if url else urllib.parse.SplitResult("", "", "", "", "") + port = parts.port or (443 if parts.scheme == "https" else 80 if parts.scheme else None) + + finding = { + "schema_version": "1.0", + "adapter": {"name": self.name, "kind": self.kind, "version": getattr(self, "version", "unknown")}, + "title": raw_item.get("name") or raw_item.get("alert", ""), + "description": raw_item.get("description", ""), + "category": {"type": category, "cwe": cwes}, + "cve": [], + "base_severity": {"level": _RISK_LEVEL.get(risk, "info"), "source": "scanner"}, + "asset_ref": {"locator": {"url": url, "host": parts.hostname, "port": port}}, + "evidence": { + "confidence": _CONFIDENCE.get(raw_item.get("confidence"), "tentative"), + "request": f"{raw_item.get('method', 'GET')} {url}".strip(), + "response_snippet": raw_item.get("evidence", ""), + }, + "remediation": { + "summary": raw_item.get("solution", ""), + "references": [r for r in [raw_item.get("reference")] if r], + }, + "raw": raw_item, + } + fp = self._fingerprint(finding) + finding["fingerprint"] = fp + finding["finding_id"] = str(uuid.uuid5(_FINDING_NS, f"{self.name}:{fp}")) + return finding + + # --- ZAP REST helpers ------------------------------------------------- + def _get(self, path: str, _timeout: int = 30, **params) -> dict: + if self.api_key: + params["apikey"] = self.api_key + query = urllib.parse.urlencode(params) + url = f"{self.base_url}{path}" + (f"?{query}" if query else "") + with urllib.request.urlopen(url, timeout=_timeout) as resp: + return json.loads(resp.read().decode()) + + def _spider(self, target_url: str, timeout_s: int = 180) -> None: + scan_id = self._get("/JSON/spider/action/scan/", url=target_url).get("scan") + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if int(self._get("/JSON/spider/view/status/", scanId=scan_id).get("status", 0)) >= 100: + return + time.sleep(1) + + def _wait_passive(self, timeout_s: int = 120) -> None: + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if int(self._get("/JSON/pscan/view/recordsToScan/").get("recordsToScan", 0)) == 0: + return + time.sleep(1) + + def _active_scan(self, target_url: str, timeout_s: int = 600) -> None: + scan_id = self._get("/JSON/ascan/action/scan/", url=target_url).get("scan") + deadline = time.monotonic() + timeout_s + while time.monotonic() < deadline: + if int(self._get("/JSON/ascan/view/status/", scanId=scan_id).get("status", 0)) >= 100: + return + time.sleep(2) + + def _fingerprint(self, finding: dict) -> str: + basis = {"category": finding["category"]["type"], + "url": finding["asset_ref"]["locator"].get("url"), + "title": finding["title"]} + digest = hashlib.sha256(json.dumps(basis, sort_keys=True, default=str).encode()).hexdigest() + return f"sha256-{digest}" ADAPTER = ZapAdapter # registry entrypoint diff --git a/core/reachability/adapters/conditions_map.py b/core/reachability/adapters/conditions_map.py index 6f9e498..cd2f10e 100644 --- a/core/reachability/adapters/conditions_map.py +++ b/core/reachability/adapters/conditions_map.py @@ -1,23 +1,197 @@ """Category -> default pre/postconditions + mitigating controls. -Loaded from schema/categories.md (the join table). Adapters inherit a category's -defaults best-effort; the engine refines them with environment facts. +The data here is a faithful transcription of schema/categories.md — the join +table that makes attack-path chaining and control-based downgrading work. Adapters +inherit a category's defaults best-effort; the engine refines them with +environment facts ("defaults are seeds, not verdicts", categories.md §Engine notes). -Two invariants from categories.md that this table MUST honor and the engine -must not flatten away: +Tokens use the shared namespace:value vocabulary. Some carry zone placeholders the +static table can't fill — ```` (the zone the finding's asset sits in) +and ```` (an internal zone the asset can pivot to). ``enrich()`` +resolves them against the context graph; unresolved placeholders are dropped +rather than guessed. + +Two invariants from categories.md this module MUST honor and the engine must not +flatten away: * ``vulnerable_dependency`` is deliberately UNDER-specified — its risk is - dominated by reachability, not category. No flat pre/postconditions. - * ``unknown`` has no defaults and no mitigation — never auto-downgraded, - only auto-flagged for review. + dominated by reachability, not category. No flat pre/postconditions; base + severity stays scanner/NVD-driven (default_base_severity is None). + * ``unknown`` (and any category absent from the table) has no defaults and no + mitigation — never auto-downgraded, only auto-flagged for review. """ from __future__ import annotations +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CategoryDefaults: + type: str + cwe: tuple[str, ...] + preconditions: tuple[str, ...] + postconditions: tuple[str, ...] + mitigated_by: tuple[str, ...] # control_types that genuinely block this class + default_base_severity: str | None # fallback when a scanner omits severity + + +CATEGORIES: dict[str, CategoryDefaults] = {} + + +def _register(names: list[str], **kw) -> None: + for name in names: + CATEGORIES[name] = CategoryDefaults(type=name, **kw) + + +# --- Injection & code execution --- +_register(["sql_injection"], cwe=("CWE-89",), + preconditions=("network:internet_reachable",), + postconditions=("access:db_read", "access:db_write"), + mitigated_by=("parameterized_queries", "input_validation", "waf"), + default_base_severity="critical") +_register(["command_injection"], cwe=("CWE-77", "CWE-78"), + preconditions=("network:internet_reachable",), + postconditions=("exec:code_on_host", "access:read_filesystem", "position:foothold:"), + mitigated_by=("input_validation", "waf"), + default_base_severity="critical") +_register(["code_injection", "deserialization"], cwe=("CWE-94", "CWE-502"), + preconditions=("network:internet_reachable",), + postconditions=("exec:code_on_host", "position:foothold:"), + mitigated_by=("input_validation",), + default_base_severity="critical") +_register(["xss"], cwe=("CWE-79",), + preconditions=("network:internet_reachable", "auth:unauthenticated"), + postconditions=("exec:javascript_in_victim_browser", "access:session_token"), + mitigated_by=("output_encoding", "csp", "waf"), + default_base_severity="high") +_register(["ssti"], cwe=("CWE-1336",), + preconditions=("network:internet_reachable",), + postconditions=("exec:code_on_host",), + mitigated_by=("input_validation",), + default_base_severity="critical") + +# --- Access control & auth --- +_register(["broken_access_control", "idor"], cwe=("CWE-639", "CWE-284"), + preconditions=("auth:authenticated_user",), + postconditions=("access:db_read", "access:db_write"), + mitigated_by=("auth_gate",), + default_base_severity="high") +_register(["auth_bypass"], cwe=("CWE-287", "CWE-288"), + preconditions=("network:internet_reachable", "auth:unauthenticated"), + postconditions=("auth:authenticated_user",), + mitigated_by=("auth_gate", "mfa"), + default_base_severity="critical") +_register(["privilege_escalation"], cwe=("CWE-269",), + preconditions=("auth:authenticated_user",), + postconditions=("auth:authenticated_admin",), + mitigated_by=("auth_gate",), + default_base_severity="high") +_register(["missing_authentication"], cwe=("CWE-306",), + preconditions=("network:internet_reachable",), + postconditions=("access:db_read",), # "depends; commonly db_read" — coarse seed + mitigated_by=("auth_gate",), + default_base_severity="high") + +# --- Server-side request & file --- +_register(["ssrf"], cwe=("CWE-918",), + preconditions=("network:internet_reachable",), + postconditions=("exec:ssrf_outbound", "access:cloud_creds", "position:pivot_to:"), + mitigated_by=("segmentation", "input_validation"), + default_base_severity="high") +_register(["path_traversal", "lfi"], cwe=("CWE-22", "CWE-98"), + preconditions=("network:internet_reachable",), + postconditions=("access:read_filesystem",), + mitigated_by=("input_validation", "waf"), + default_base_severity="high") +_register(["file_upload"], cwe=("CWE-434",), + preconditions=("network:internet_reachable",), + postconditions=("exec:code_on_host",), + mitigated_by=("input_validation", "waf"), + default_base_severity="high") + +# --- Exposure & configuration --- +_register(["info_disclosure"], cwe=("CWE-200",), + preconditions=("network:internet_reachable",), + postconditions=("access:secrets",), # "if creds leak" — engine treats secret-bearing + mitigated_by=("output_encoding",), # disclosures very differently from version banners + default_base_severity="low") +_register(["exposed_service"], cwe=("CWE-668",), + preconditions=("network:internet_reachable",), + postconditions=("position:foothold:",), + mitigated_by=("segmentation", "reverse_proxy", "auth_gate"), + default_base_severity="medium") +_register(["security_misconfiguration"], cwe=("CWE-16",), + preconditions=(), postconditions=(), mitigated_by=(), # "varies" — no firm seed + default_base_severity="medium") +_register(["secret_exposure"], cwe=("CWE-798",), + preconditions=(), # "read_filesystem OR internet_reachable" — no firm seed + postconditions=("access:secrets", "access:cloud_creds", "access:db_read"), + mitigated_by=("secrets_manager",), + default_base_severity="high") -def defaults_for(category_type: str) -> dict: - """Return {preconditions, postconditions, mitigated_by, default_base_severity}. +# --- Dependency / supply chain (deliberately under-specified) --- +_register(["vulnerable_dependency"], cwe=(), + preconditions=(), postconditions=(), mitigated_by=(), + default_base_severity=None) # use scanner/NVD severity; risk is reachability-driven - Scaffold: source of truth is schema/categories.md; this will load a parsed - form of that table (or a generated data file derived from it). + +def defaults_for(category_type: str) -> CategoryDefaults | None: + """Return the default seeds for a category, or None if it has no entry + (unknown/exotic categories are never auto-enriched — flagged, not guessed).""" + return CATEGORIES.get(category_type) + + +def enrich(finding, graph=None): + """Best-effort fill of a finding's empty pre/postconditions from its category + defaults, resolving zone placeholders against the context graph. Never + overwrites conditions an adapter already supplied. Returns the finding. + + vulnerable_dependency and unknown categories are no-ops (empty/absent defaults), + preserving the categories.md invariants. """ - raise NotImplementedError("scaffold: back this with schema/categories.md") + d = defaults_for(finding.category.type) + if d is None: + return finding + if not finding.preconditions: + finding.preconditions = _resolve(d.preconditions, finding, graph) + if not finding.postconditions: + finding.postconditions = _resolve(d.postconditions, finding, graph) + return finding + + +# --- placeholder resolution ---------------------------------------------------- + +def _resolve(tokens: tuple[str, ...], finding, graph) -> list[str]: + asset_zone = _asset_zone_name(finding, graph) + internal_zones = _connected_zone_names(finding, graph) if graph is not None else [] + out: list[str] = [] + for tok in tokens: + if "" in tok or "" in tok: + if asset_zone: + out.append(tok.replace("", asset_zone).replace("", asset_zone)) + # else: cannot resolve -> drop rather than emit a placeholder token + elif "" in tok: + out.extend(tok.replace("", z) for z in internal_zones) + else: + out.append(tok) + return out + + +def _asset_zone_name(finding, graph) -> str | None: + if graph is None or not finding.asset_ref.node_id: + return None + for edge in graph.neighbors(finding.asset_ref.node_id, ["resides_in"], "out"): + return edge.dst.split(":", 1)[-1] # "zone:vlan-20" -> "vlan-20" + return None + + +def _connected_zone_names(finding, graph) -> list[str]: + if graph is None or not finding.asset_ref.node_id: + return [] + zone = None + for edge in graph.neighbors(finding.asset_ref.node_id, ["resides_in"], "out"): + zone = edge.dst + break + if zone is None: + return [] + return [e.dst.split(":", 1)[-1] for e in graph.neighbors(zone, ["connects_to"], "out")] diff --git a/core/reachability/adapters/registry.py b/core/reachability/adapters/registry.py index 3f6686a..d58430b 100644 --- a/core/reachability/adapters/registry.py +++ b/core/reachability/adapters/registry.py @@ -1,30 +1,83 @@ """Adapter auto-discovery + role binding. Contract: schema/03-adapter-interface.md. On startup the registry scans the top-level ``adapters/`` directory: each -``adapter.yaml`` -> a ``YamlAdapter``, each ``adapter.py`` -> the class it -exports. It runs ``healthcheck()`` and registers each under its name and kind. -Config (``config/environment.yaml`` ``scanners:`` block) binds a kind -> a name; -that single block is the entire "swap a scanner" surface. +``adapter.yaml`` -> a ``YamlAdapter``, each ``adapter.py`` -> the class it exports +as ``ADAPTER``. It runs ``healthcheck()`` (recording liveness, never failing +discovery — ``docker compose up`` should report which scanners are live without +external services) and registers each under its name and kind. Config +(``config/environment.yaml`` ``scanners:`` block) binds a kind -> a name; that +single block is the entire "swap a scanner" surface. """ from __future__ import annotations +import importlib.util +from pathlib import Path + +import yaml + from .protocol import Adapter +from .yaml_adapter import YamlAdapter class AdapterRegistry: def __init__(self, adapters_dir: str) -> None: - self._dir = adapters_dir + self._dir = Path(adapters_dir) self._by_name: dict[str, Adapter] = {} self._by_kind: dict[str, list[Adapter]] = {} + self._health: dict[str, bool] = {} def discover(self) -> None: - """Scan adapters_dir; load adapter.yaml/adapter.py; healthcheck; register.""" - raise NotImplementedError("scaffold") + """Scan adapters_dir; load adapter.yaml/adapter.py; healthcheck; register. + Folders starting with '_' (e.g. _template) are skipped.""" + if not self._dir.is_dir(): + return + for sub in sorted(self._dir.iterdir()): + if not sub.is_dir() or sub.name.startswith("_"): + continue + adapter = self._load(sub) + if adapter is not None: + self._register(adapter) def get(self, name: str) -> Adapter: - raise NotImplementedError("scaffold") + if name not in self._by_name: + raise KeyError(f"no adapter registered under name {name!r}") + return self._by_name[name] def resolve_kind(self, kind: str, name: str) -> Adapter: """Bind a capability (e.g. 'dast') to the configured adapter name.""" - raise NotImplementedError("scaffold") + adapter = self._by_name.get(name) + if adapter is None: + raise KeyError(f"scanners.{kind} -> {name!r}, but no such adapter is registered") + if adapter.kind != kind: + raise ValueError(f"adapter {name!r} is kind {adapter.kind!r}, not {kind!r}") + return adapter + + def is_healthy(self, name: str) -> bool: + return self._health.get(name, False) + + # --- internals -------------------------------------------------------- + def _load(self, folder: Path) -> Adapter | None: + if (folder / "adapter.yaml").exists(): + spec = yaml.safe_load((folder / "adapter.yaml").read_text(encoding="utf-8")) + return YamlAdapter(spec) + if (folder / "adapter.py").exists(): + return self._load_py(folder / "adapter.py") + return None + + def _load_py(self, path: Path) -> Adapter | None: + spec = importlib.util.spec_from_file_location(f"_adapter_{path.parent.name}", path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + cls = getattr(module, "ADAPTER", None) + return cls() if cls is not None else None + + def _register(self, adapter: Adapter) -> None: + try: + self._health[adapter.name] = bool(adapter.healthcheck()) + except Exception: + self._health[adapter.name] = False + self._by_name[adapter.name] = adapter + self._by_kind.setdefault(adapter.kind, []).append(adapter) diff --git a/core/reachability/adapters/yaml_adapter.py b/core/reachability/adapters/yaml_adapter.py index 64ed433..5a745c7 100644 --- a/core/reachability/adapters/yaml_adapter.py +++ b/core/reachability/adapters/yaml_adapter.py @@ -1,32 +1,195 @@ """Generic declarative adapter (effort-ladder rung 1). Contract: 03-adapter-interface.md. -Interprets an ``adapter.yaml`` (invoke spec + JSONPath field map) so adding most -CLI scanners is "drop a YAML file in adapters/" — zero code. ``normalize()`` runs -the field map over one native item; ``run()`` invokes the scanner and funnels -each native result through ``normalize()``. +Interprets an ``adapter.yaml`` (invoke spec + field map) so adding most CLI +scanners is "drop a YAML file in adapters/" — zero code. ``normalize()`` runs the +field map over one native item and is kept pure so it is unit-testable against +recorded scanner output with no live scanner. + +Field-map expression language (matches the examples in 03-adapter-interface.md): + * ``$.a.b.c`` — dotted path into the native item + * ``$.list[*]`` / ``[n]`` — array wildcard / index (used by ``iterator``) + * ``$.x | lower`` — pipe a transform (lower|upper|str) + * ``$.a + '@' + $.b`` — concatenate paths and 'single-quoted' literals + * ``@const:value`` — a literal constant +Target keys are dotted (``base_severity.level``) and assigned nested. Fields whose +target is list-typed (cve, category.cwe, ...) coerce a scalar into a 1-element list. """ from __future__ import annotations +import hashlib +import json +import re +import subprocess +import uuid from typing import Iterator from .protocol import ScanTarget +# Normalized fields that are arrays — a scalar mapped here is wrapped into a list. +_LIST_FIELDS = { + "cve", "category.cwe", "category.capec", "category.owasp", + "preconditions", "postconditions", +} +_FINDING_NS = uuid.uuid5(uuid.NAMESPACE_URL, "https://reachability.dev/finding") + + +# --- tiny JSON path resolver (no external dep; covers the contract's grammar) --- + +def _tokens(path: str) -> list[tuple]: + body = path.strip().lstrip("$").lstrip(".") + out: list[tuple] = [] + for seg in filter(None, body.split(".")): + key = re.match(r"^([^\[\]]*)", seg).group(1) + if key: + out.append(("key", key)) + for br in re.findall(r"\[([^\]]*)\]", seg): + out.append(("wild",) if br == "*" else ("index", int(br))) + return out + + +def _json_query(data: object, path: str) -> list: + """Return every value matching ``path`` (always a list).""" + results = [data] + for tok in _tokens(path): + nxt: list = [] + for cur in results: + if tok[0] == "key" and isinstance(cur, dict) and tok[1] in cur: + nxt.append(cur[tok[1]]) + elif tok[0] == "wild" and isinstance(cur, list): + nxt.extend(cur) + elif tok[0] == "index" and isinstance(cur, list) and -len(cur) <= tok[1] < len(cur): + nxt.append(cur[tok[1]]) + results = nxt + return results + + +def _apply_func(value: object, func: str) -> object: + if value is None: + return None + if func == "lower": + return str(value).lower() + if func == "upper": + return str(value).upper() + if func == "str": + return str(value) + raise ValueError(f"unknown transform {func!r} in adapter map") + + +def _extract_version(output: str) -> str: + """Best-effort scanner version from `version_cmd` output: a 'version' field if + the tool prints JSON (e.g. `grype version -o json`), else the first line.""" + if output.startswith("{"): + try: + data = json.loads(output) + except json.JSONDecodeError: + data = {} + for key in ("version", "Version", "applicationVersion"): + if isinstance(data.get(key), str): + return data[key][:64] + return output.splitlines()[0][:64] + + +def _set_nested(target: dict, dotted: str, value: object) -> None: + keys = dotted.split(".") + node = target + for k in keys[:-1]: + node = node.setdefault(k, {}) + node[keys[-1]] = value + class YamlAdapter: def __init__(self, spec: dict) -> None: self.spec = spec self.name = spec["name"] self.kind = spec["kind"] - self.version = "unknown" # resolved via version_cmd at healthcheck + self.version = "unknown" # resolved by healthcheck via version_cmd + self.invoke = spec.get("invoke", {}) + mp = spec.get("map", {}) + self.iterator = mp.get("iterator", "$[*]") + self.fields = mp.get("fields", {}) + self.defaults = mp.get("defaults", {}) + # --- lifecycle -------------------------------------------------------- def healthcheck(self) -> bool: - raise NotImplementedError("scaffold: run version_cmd / probe invoke target") + vc = self.spec.get("version_cmd") + if not vc: + return True + try: + proc = subprocess.run(vc, capture_output=True, text=True, timeout=15) + except (FileNotFoundError, OSError, subprocess.SubprocessError): + return False + if proc.returncode == 0 and proc.stdout.strip(): + self.version = _extract_version(proc.stdout.strip()) + return proc.returncode == 0 def run(self, target: ScanTarget) -> Iterator[dict]: - raise NotImplementedError("scaffold: invoke (subprocess|docker|rest) -> normalize each") + if self.invoke.get("type") != "subprocess": + raise NotImplementedError(f"invoke.type {self.invoke.get('type')!r} not yet supported") + if self.invoke.get("output") != "stdout_json": + raise NotImplementedError("only stdout_json output is supported in this slice") + for tgt in target.targets: + cmd = [a.replace("{target}", tgt) for a in self.invoke["command"]] + proc = subprocess.run(cmd, capture_output=True, text=True) + data = json.loads(proc.stdout) + for item in self.iterate(data): + finding = self.normalize(item) + finding["scan_id"] = "" # stamped by the scan runner + yield finding + + # --- mapping (pure) --------------------------------------------------- + def iterate(self, data: object) -> list: + """Apply the iterator path to raw scanner output -> list of native items.""" + return _json_query(data, self.iterator) def normalize(self, raw_item: dict) -> dict: - raise NotImplementedError( - "scaffold: apply spec['map'] JSONPaths + defaults; preserve raw_item under 'raw'" - ) + finding: dict = {} + for key, val in self.defaults.items(): + self._assign(finding, key, val) + for key, expr in self.fields.items(): + value = self._eval(expr, raw_item) + if value is not None: + self._assign(finding, key, value) + + finding["schema_version"] = "1.0" + finding["adapter"] = {"name": self.name, "kind": self.kind, "version": self.version} + finding.setdefault("category", {}).setdefault("type", "unknown") + finding["raw"] = raw_item + fp = self._fingerprint(finding) + finding["fingerprint"] = fp + # Stable per (scanner, fingerprint) and deterministic (project principle). + finding["finding_id"] = str(uuid.uuid5(_FINDING_NS, f"{self.name}:{fp}")) + return finding + + # --- internals -------------------------------------------------------- + def _assign(self, finding: dict, dotted: str, value: object) -> None: + if dotted in _LIST_FIELDS and not isinstance(value, list): + value = [] if value is None else [value] + _set_nested(finding, dotted, value) + + def _eval(self, expr: str, item: dict): + expr = expr.strip() + if expr.startswith("@const:"): + return expr[len("@const:"):] + parts = [p.strip() for p in expr.split("+")] + if len(parts) == 1: + return self._eval_term(parts[0], item) + return "".join("" if (v := self._eval_term(p, item)) is None else str(v) for p in parts) + + def _eval_term(self, term: str, item: dict): + if len(term) >= 2 and term[0] == "'" and term[-1] == "'": + return term[1:-1] + path, sep, func = term.partition("|") + vals = _json_query(item, path.strip()) + value = vals[0] if vals else None + return _apply_func(value, func.strip()) if sep else value + + def _fingerprint(self, finding: dict) -> str: + basis = { + "category": finding.get("category", {}).get("type"), + "locator": finding.get("asset_ref", {}).get("locator", {}), + "cve": finding.get("cve", []), + "title": finding.get("title"), + } + digest = hashlib.sha256(json.dumps(basis, sort_keys=True, default=str).encode()).hexdigest() + return f"sha256-{digest}" diff --git a/core/reachability/cli.py b/core/reachability/cli.py index 1de1226..0475fdb 100644 --- a/core/reachability/cli.py +++ b/core/reachability/cli.py @@ -1,17 +1,242 @@ """`reachability` CLI — the operator entrypoint. -Subcommands map to the pipeline stages: - ingest sources -> context graph (+ reconciliation report) - scan run configured adapters -> normalized findings (*.findings.jsonl) - reason reach -> chain -> score -> write contextual blocks - report render re-prioritized findings + attack chains +Subcommands map to the pipeline stages and pass file artifacts between each other +(a graph JSON + line-delimited findings), so the whole thing runs self-hosted with +no database required: + + ingest questionnaire -> context graph artifact (graph.json) + scan run a configured adapter -> normalized findings (*.findings.jsonl) + reason graph + findings -> contextual blocks + attack chains + report scored findings (+ chains) -> human-readable or JSON report + +The command functions are importable and side-effect-explicit so they can be +tested without spawning a process. """ from __future__ import annotations +import argparse +import json +import os +from collections import Counter +from dataclasses import asdict +from pathlib import Path + +from .adapters import AdapterRegistry +from .adapters.protocol import ScanTarget +from .config import Config +from .engine.chaining import Chain +from .engine.pipeline import ReasoningPipeline +from .graphstore import InMemoryGraphStore +from .ingestion import diagrams, iac, questionnaire, sbom +from .ingestion import report as recon_report +from .ingestion.pipeline import IngestionPipeline +from .models.finding import Finding +from .models.policy import Policy +from .report.render import render_report + + +def _read_jsonl(path: str): + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if line: + yield json.loads(line) + + +# --- commands (importable, testable) ------------------------------------------ + +def cmd_ingest(questionnaire_path: str | None = None, out_path: str = "graph.json", + *, sbom_paths: list[str] | None = None, terraform_paths: list[str] | None = None, + cloudformation_paths: list[str] | None = None, diagram_paths: list[str] | None = None, + sbom_asset: str | None = None, reconcile_out: str | None = None) -> dict: + """Questionnaire / SBOM / IaC / diagram -> context graph, reconciled across sources. + + Returns a summary including the reconciliation report (conflicts + orphans).""" + sources = [] + if questionnaire_path: + sources.append(questionnaire.load(questionnaire_path)) + for path in sbom_paths or []: + sources.append(sbom.load(path, asset=sbom_asset)) + for path in (terraform_paths or []) + (cloudformation_paths or []): + sources.append(iac.load(path)) # auto-detects terraform vs cloudformation + for path in diagram_paths or []: + sources.append(diagrams.load(path)) + if not sources: + raise SystemExit("ingest needs a source (--questionnaire / --sbom / --terraform " + "/ --cloudformation / --diagram)") + + graph = InMemoryGraphStore() + report = IngestionPipeline(graph).ingest(sources) # reconcile + upsert + Path(out_path).write_text(json.dumps(graph.to_dict(), indent=2), encoding="utf-8") + if reconcile_out: + Path(reconcile_out).write_text(recon_report.render(report), encoding="utf-8") + return {"nodes": len(report.nodes), "edges": len(report.edges), + "by_type": Counter(n.type for n in report.nodes), + "conflicts": len(report.conflicts), "orphans": len(report.orphans), "report": report} + + +def cmd_scan(adapters_dir: str, name: str, targets: list[str], env_id: str, out_path: str) -> int: + """Run one registered adapter -> normalized findings jsonl.""" + registry = AdapterRegistry(adapters_dir) + registry.discover() + adapter = registry.get(name) + if not registry.is_healthy(name): + raise SystemExit(f"adapter {name!r} healthcheck failed (binary/API not available)") + target = ScanTarget(env_id=env_id, targets=list(targets)) + count = 0 + with open(out_path, "w", encoding="utf-8") as fh: + for finding in adapter.run(target): + fh.write(json.dumps(finding) + "\n") + count += 1 + return count + + +def cmd_reason(graph_path: str, findings_path: str, policy_path: str, + out_path: str, chains_path: str | None = None) -> tuple[list[Finding], list[Chain]]: + """Graph + findings -> contextual blocks (+ chains). Order: reach -> chain -> score.""" + graph = InMemoryGraphStore.from_dict(json.loads(Path(graph_path).read_text(encoding="utf-8"))) + findings = [Finding.from_dict(d) for d in _read_jsonl(findings_path)] + policy = Policy.load(policy_path) + pipeline = ReasoningPipeline(graph, policy) + pipeline.run(findings) + with open(out_path, "w", encoding="utf-8") as fh: + for f in findings: + fh.write(json.dumps(f.to_dict()) + "\n") + if chains_path: + Path(chains_path).write_text( + json.dumps([asdict(c) for c in pipeline.chains], indent=2), encoding="utf-8" + ) + return findings, pipeline.chains + + +def cmd_init_db(dsn: str | None, *, store=None) -> list[str]: + """Apply schema migrations to Postgres (idempotent). For ``docker compose up``.""" + if store is None: + from .graphstore.postgres import PostgresGraphStore + store = PostgresGraphStore(dsn, env_id="") + return store.init_schema() + + +def cmd_propose(findings_path: str, out_path: str | None = None, *, provider=None) -> list[dict]: + """Propose refined pre/postconditions for findings (LLM, advisory). The + proposals are for HUMAN REVIEW and are never applied to scoring automatically.""" + from .llm.enrichment import propose_conditions + from .llm.provider import default_provider + provider = provider or default_provider() + findings = [Finding.from_dict(d) for d in _read_jsonl(findings_path)] + proposals = [] + for f in findings: + p = propose_conditions(f, provider=provider) + if p is not None: + proposals.append(asdict(p)) + if out_path: + Path(out_path).write_text(json.dumps(proposals, indent=2), encoding="utf-8") + return proposals + + +def cmd_report(findings_path: str, chains_path: str | None, fmt: str, + show_suppressed: bool = False, narrate: bool = False) -> str: + findings = [Finding.from_dict(d) for d in _read_jsonl(findings_path)] + chains: list[Chain] = [] + if chains_path and Path(chains_path).exists(): + chains = [Chain(**d) for d in json.loads(Path(chains_path).read_text(encoding="utf-8"))] + narratives = None + if narrate and chains: + from .llm.narrative import narrate_all + from .llm.provider import default_provider + narratives = narrate_all(chains, findings, provider=default_provider()) + return render_report(findings, chains, fmt=fmt, show_suppressed=show_suppressed, + narratives=narratives) + + +# --- argparse wiring ---------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="reachability", description=__doc__.splitlines()[0]) + sub = p.add_subparsers(dest="cmd") + + pi = sub.add_parser("ingest", help="questionnaire / SBOM / Terraform -> context graph artifact") + pi.add_argument("--questionnaire", default=None) + pi.add_argument("--sbom", action="append", default=[], help="CycloneDX/SPDX SBOM (repeatable)") + pi.add_argument("--sbom-asset", default=None, help="attach SBOM components to this asset id") + pi.add_argument("--terraform", action="append", default=[], + help="terraform show -json state/plan (repeatable)") + pi.add_argument("--cloudformation", action="append", default=[], + help="CloudFormation template JSON/YAML (repeatable)") + pi.add_argument("--diagram", action="append", default=[], + help="Mermaid diagram .mmd (repeatable; declared confidence)") + pi.add_argument("--reconcile-out", default=None, help="write the reconciliation report here") + pi.add_argument("--out", default="graph.json") + + ps = sub.add_parser("scan", help="run an adapter -> normalized findings jsonl") + ps.add_argument("--adapters-dir", default="adapters") + ps.add_argument("--adapter", required=True, help="registered adapter name (e.g. grype)") + ps.add_argument("--target", action="append", default=[], help="repeatable scan target") + ps.add_argument("--config", default=None, help="environment.yaml (for env_id)") + ps.add_argument("--out", default="findings.jsonl") + + pr = sub.add_parser("reason", help="graph + findings -> contextual scoring + chains") + pr.add_argument("--graph", required=True) + pr.add_argument("--findings", required=True) + pr.add_argument("--policy", default="config/policy.default.yaml") + pr.add_argument("--out", default="scored.findings.jsonl") + pr.add_argument("--chains-out", default="chains.json") + + pd = sub.add_parser("init-db", help="apply Postgres schema migrations (idempotent)") + pd.add_argument("--dsn", default=None, help="defaults to $DATABASE_URL or --config") + pd.add_argument("--config", default=None) + + pc = sub.add_parser("propose", help="LLM-propose refined pre/postconditions (human review)") + pc.add_argument("--findings", required=True) + pc.add_argument("--out", default=None, help="write proposals JSON here") + + pp = sub.add_parser("report", help="scored findings (+ chains) -> report") + pp.add_argument("--findings", required=True) + pp.add_argument("--chains", default=None) + pp.add_argument("--format", default="text", choices=["text", "json"]) + pp.add_argument("--show-suppressed", action="store_true") + pp.add_argument("--narrate", action="store_true", + help="add chain narratives (LLM if ANTHROPIC_API_KEY set, else template prose)") + return p + + +def main(argv: list[str] | None = None) -> None: + args = build_parser().parse_args(argv) -def main() -> None: - raise NotImplementedError("scaffold: click group: ingest | scan | reason | report") + if args.cmd == "ingest": + s = cmd_ingest(args.questionnaire, args.out, sbom_paths=args.sbom, + terraform_paths=args.terraform, cloudformation_paths=args.cloudformation, + diagram_paths=args.diagram, sbom_asset=args.sbom_asset, + reconcile_out=args.reconcile_out) + print(f"Ingested {s['nodes']} nodes, {s['edges']} edges -> {args.out}") + print(f" by type: {dict(s['by_type'])}") + if s["conflicts"] or s["orphans"]: + print(f" reconciliation: {s['conflicts']} conflict(s), {s['orphans']} orphan(s)") + print(recon_report.render(s["report"])) + elif args.cmd == "scan": + env_id = Config.load(args.config).env_id if args.config else "" + n = cmd_scan(args.adapters_dir, args.adapter, args.target, env_id, args.out) + print(f"Wrote {n} findings -> {args.out}") + elif args.cmd == "reason": + findings, chains = cmd_reason(args.graph, args.findings, args.policy, args.out, args.chains_out) + print(f"Scored {len(findings)} findings, found {len(chains)} chains -> {args.out}") + elif args.cmd == "init-db": + dsn = args.dsn or os.environ.get("DATABASE_URL") + if not dsn and args.config: + dsn = Config.load(args.config).database_dsn + applied = cmd_init_db(dsn) + print(f"Applied migrations: {', '.join(applied)}") + elif args.cmd == "propose": + proposals = cmd_propose(args.findings, args.out) + if proposals: + print(f"{len(proposals)} proposal(s) for review" + (f" -> {args.out}" if args.out else "")) + else: + print("No proposals (no LLM provider configured, or nothing to refine).") + elif args.cmd == "report": + print(cmd_report(args.findings, args.chains, args.format, args.show_suppressed, args.narrate)) + else: + build_parser().print_help() if __name__ == "__main__": diff --git a/core/reachability/config.py b/core/reachability/config.py index 0a10548..538afe9 100644 --- a/core/reachability/config.py +++ b/core/reachability/config.py @@ -6,10 +6,11 @@ "swap a scanner" surface. """ -from __future__ import annotations - +import os from dataclasses import dataclass, field +import yaml + @dataclass class Config: @@ -21,4 +22,16 @@ class Config: @classmethod def load(cls, path: str = "config/environment.yaml") -> "Config": - raise NotImplementedError("scaffold: parse YAML; validate; resolve env vars for secrets") + with open(path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + env = data.get("environment", {}) or {} + db = data.get("database", {}) or {} + # DATABASE_URL env var overrides the file (secrets never committed). + dsn = os.environ.get("DATABASE_URL", db.get("dsn", "")) + return cls( + env_id=str(env.get("id", "")), + scanners=data.get("scanners", {}) or {}, + database_dsn=dsn, + policy_path=(data.get("policy", {}) or {}).get("path", "config/policy.default.yaml"), + raw=data, + ) diff --git a/core/reachability/engine/chaining.py b/core/reachability/engine/chaining.py index 446150b..9b2739e 100644 --- a/core/reachability/engine/chaining.py +++ b/core/reachability/engine/chaining.py @@ -4,22 +4,39 @@ constrained at every hop by the CONTEXT graph (what is physically reachable). Two graphs, never confused. Deterministic and reproducible — no LLM here. +A step is applicable when (1) every precondition token is in the attacker's +current state (or satisfiable by a context-graph fact — a network:reachable_from +: holds once the attacker has pivoted into ), AND (2) its target +asset is physically reachable from the attacker's current position. Applying a +step unions its postconditions into state and records the hop. + Termination, three independent guarantees: 1. No finding reused in a path (cycle guard). 2. Monotonic state with growth check (only expand if state actually grew). 3. Hard MAX_DEPTH (default 6) — real reportable chains are short. -Seed: {network:internet_reachable, auth:unauthenticated}. +Seed: {network:internet_reachable, auth:unauthenticated}, position {internet}. """ from __future__ import annotations +import uuid from dataclasses import dataclass, field from ..graphstore.base import GraphStore +from ..models.conditions import Namespace, parse_condition from ..models.finding import Finding +from ..models.graph import Confidence, NodeFilter MAX_DEPTH = 6 +MIN_REPORT_LEN = 2 # "multi-step": a single finding on a sensitive asset isn't a chain + +_PUBLIC_ENTRYPOINTS = NodeFilter(type="entrypoint", attributes={"public": True}) +_CHAIN_NS = uuid.uuid5(uuid.NAMESPACE_URL, "https://reachability.dev/chain") + +# State tokens that make a chain worth reporting even without a sensitive target. +_VALUABLE_TOKENS = {"auth:authenticated_admin", "access:cloud_creds", "access:secrets"} +_SENSITIVITY_WEIGHT = {"crown_jewel": 4, "high": 3} @dataclass @@ -31,7 +48,171 @@ class Chain: score: float = 0.0 -def find_chains(findings: list[Finding], graph: GraphStore) -> list[Chain]: - """Forward search; reportable iff it reaches high/crown_jewel data, admin, - or creds/secrets. Returns ranked, deduped chains. Scaffold.""" - raise NotImplementedError("scaffold: frontier search per 06; then dedupe_and_rank") +def find_chains(findings: list[Finding], graph: GraphStore, *, max_depth: int = MAX_DEPTH) -> list[Chain]: + """Forward search seeded from the internet; record any path that becomes + reportable, keep exploring, then dedupe and rank. Reportable iff it reaches + high/crown_jewel data, authenticated_admin, or cloud_creds/secrets.""" + seed_state = {"network:internet_reachable", "auth:unauthenticated"} + # worklist of partial paths: (attacker_state, path[Finding], position{zone-ids|"internet"}) + frontier: list[tuple[set, list[Finding], set]] = [(seed_state, [], {"internet"})] + recorded: list[list[Finding]] = [] + + while frontier: + state, path, pos = frontier.pop() + if len(path) >= max_depth: + continue + for step in findings: + if any(s.finding_id == step.finding_id for s in path): # no reuse + continue + if not _preconditions_met(step, state, pos): + continue + if not _reachable(step.asset_ref.node_id, pos, graph): + continue + new_state = state | set(step.postconditions) + new_path = path + [step] + new_pos = pos | _positions_from(step.postconditions) + asset = graph.get_node(step.asset_ref.node_id) + if _reportable(new_path, new_state, asset): + recorded.append(new_path) + if new_state != state: # growth check + frontier.append((new_state, new_path, new_pos)) + + return _dedupe_and_rank(recorded, graph) + + +def chains_by_finding(chains: list[Chain]) -> dict[str, list[Chain]]: + """Back-reference: finding_id -> the chains it participates in (feeds the + scoring policy's in_chain / chain_* conditions).""" + out: dict[str, list[Chain]] = {} + for chain in chains: + for fid in chain.steps: + out.setdefault(fid, []).append(chain) + return out + + +# --- applicability ------------------------------------------------------------- + +def _preconditions_met(step: Finding, state: set, pos: set) -> bool: + for tok in step.preconditions: + if tok in state: + continue + try: + cond = parse_condition(tok) + except ValueError: + return False + # Only network reachability tokens are satisfiable by graph/position facts; + # auth/access/exec/position tokens must have been granted by an earlier hop. + if cond.namespace is Namespace.network: + if cond.value == "internet_reachable" and "network:internet_reachable" in state: + continue + if cond.value == "reachable_from" and cond.param and f"zone:{cond.param}" in pos: + continue + if cond.value in ("internal_only", "same_segment") and any(p != "internet" for p in pos): + continue + return False + return True + + +def _positions_from(postconditions: list[str]) -> set: + out: set = set() + for tok in postconditions: + try: + cond = parse_condition(tok) + except ValueError: + continue + if cond.namespace is Namespace.position and cond.param: + out.add(f"zone:{cond.param}") + return out + + +def _reachable(asset_id: str | None, pos: set, graph: GraphStore) -> bool: + """Is the target asset physically reachable from the attacker's position?""" + if not asset_id: + return False + asset = graph.get_node(asset_id) + if asset is None: + return False + if "internet" in pos: + if asset.attributes.get("internet_reachable"): + return True + if graph.paths_to(asset_id, from_filter=_PUBLIC_ENTRYPOINTS): + return True + zone = _asset_zone(asset_id, graph) + if zone is not None: + if zone in pos: + return True + held = [p for p in pos if p != "internet"] + if held and _zone_reachable(held, zone, graph): + return True + return False + + +def _asset_zone(asset_id: str, graph: GraphStore) -> str | None: + for edge in graph.neighbors(asset_id, ["resides_in"], "out"): + return edge.dst + return None + + +def _zone_reachable(from_zones: list[str], to_zone: str, graph: GraphStore) -> bool: + seen = set(from_zones) + stack = list(from_zones) + while stack: + z = stack.pop() + if z == to_zone: + return True + for edge in graph.neighbors(z, ["connects_to"], "out"): + if edge.dst not in seen: + seen.add(edge.dst) + stack.append(edge.dst) + return to_zone in seen + + +# --- reportability + ranking --------------------------------------------------- + +def _target_sensitivity(asset) -> int: + if asset is None: + return 0 + return _SENSITIVITY_WEIGHT.get(asset.attributes.get("data_sensitivity"), 0) + + +def _reportable(path: list[Finding], state: set, asset) -> bool: + if len(path) < MIN_REPORT_LEN: + return False + return _target_sensitivity(asset) >= 3 or bool(state & _VALUABLE_TOKENS) + + +def _exploit_score(finding: Finding) -> float: + e = finding.exploitability + return (2.0 if e.kev else 0.0) + (1.0 if e.exploit_public else 0.0) + float(e.epss_score or 0.0) + + +def _confidence_weight(path: list[Finding], graph: GraphStore) -> float: + confs = [n.confidence for n in (graph.get_node(s.asset_ref.node_id) for s in path) if n is not None] + return 1.0 if confs and all(c is Confidence.ground_truth for c in confs) else 0.0 + + +def _dedupe_and_rank(paths: list[list[Finding]], graph: GraphStore) -> list[Chain]: + unique: dict[tuple, list[Finding]] = {} + for path in paths: + unique.setdefault(tuple(s.finding_id for s in path), path) + chains = [_build_chain(p, graph) for p in unique.values()] + chains.sort(key=lambda c: (-c.score, c.id)) # worst rises to the top, deterministically + return chains + + +def _build_chain(path: list[Finding], graph: GraphStore) -> Chain: + steps = [s.finding_id for s in path] + final = graph.get_node(path[-1].asset_ref.node_id) + score = ( + _target_sensitivity(final) + + sum(_exploit_score(s) for s in path) + - 0.5 * len(path) # length penalty: shorter is more practical + + _confidence_weight(path, graph) + ) + return Chain( + id=f"chain:{uuid.uuid5(_CHAIN_NS, ','.join(steps))}", + steps=steps, + target=path[-1].asset_ref.node_id, + reaches_crown_jewel=final is not None and final.attributes.get("data_sensitivity") == "crown_jewel", + score=round(score, 4), + ) diff --git a/core/reachability/engine/pipeline.py b/core/reachability/engine/pipeline.py index 9cb84b5..b4e3220 100644 --- a/core/reachability/engine/pipeline.py +++ b/core/reachability/engine/pipeline.py @@ -13,18 +13,46 @@ from __future__ import annotations +from ..adapters import conditions_map from ..graphstore.base import GraphStore +from ..ingestion import resolve from ..models.finding import Finding from ..models.policy import Policy +from . import chaining, reachability +from .policy_eval import Facts, evaluate class ReasoningPipeline: def __init__(self, graph: GraphStore, policy: Policy) -> None: self._graph = graph self._policy = policy + self.chains: list = [] # populated by run(); consumed by reporting - def run(self, findings: list[Finding]) -> list[Finding]: + def run(self, findings: list[Finding], *, scored_at: str | None = None) -> list[Finding]: """Annotate each finding's ``contextual`` block. Order is load-bearing.""" - raise NotImplementedError( - "scaffold: reachability -> chaining -> policy_eval -> clamp; never reorder" - ) + # 0. Resolve + enrich. Resolve a finding's locator to a graph node — an SCA + # component to its owning asset (depends_on), or a DAST host/url to the + # asset the matching entrypoint fronts — so the finding scores against + # that asset's reachability. Then fill empty pre/postconditions from + # category defaults so scanner findings can chain (never overwriting + # adapter-supplied conditions). + for finding in findings: + resolve.resolve_asset_ref(finding, self._graph) + conditions_map.enrich(finding, self._graph) + # 1. Reachability — per-finding internet/zone reachability + path controls. + rfacts = reachability.compute(findings, self._graph) + # 2. Chaining — BEFORE downgrades, so chain hops are known when scoring. + chains = chaining.find_chains(findings, self._graph) + self.chains = chains + by_finding = chaining.chains_by_finding(chains) + # 3. Policy — evaluate with the full fact set; write the contextual block. + for finding in findings: + node_id = finding.asset_ref.node_id + asset = self._graph.get_node(node_id) if node_id else None + facts = Facts.from_reachability( + rfacts[finding.finding_id], + asset=asset, + chains=by_finding.get(finding.finding_id, []), + ) + finding.contextual = evaluate(finding, facts, self._policy, scored_at=scored_at) + return findings diff --git a/core/reachability/engine/policy_eval.py b/core/reachability/engine/policy_eval.py index 7631d07..a9a8f64 100644 --- a/core/reachability/engine/policy_eval.py +++ b/core/reachability/engine/policy_eval.py @@ -5,26 +5,257 @@ further evaluation for that finding. Renders each fired rule's rationale template from the match — the rationale lines ARE the product. No LLM, ever. -Invariants from categories.md the evaluator must honor: - * unknown_category_downgrade=False -> never auto-downgrade type:"unknown". - * vulnerable_dependency risk is reachability-driven, not flat-category. +Invariants enforced regardless of policy text (categories.md): + * type:"unknown" is never auto-downgraded (only flagged) unless settings opt in + — enforced as a final floor at base severity, and suppression is ignored. + * vulnerable_dependency carries no synthetic pre/postconditions; its risk is + reachability-driven, which the policy expresses via reachable_from_internet + conditions rather than the evaluator special-casing it. """ from __future__ import annotations -from ..models.finding import Contextual, Finding -from ..models.policy import Policy +import re +from dataclasses import dataclass, field +from ..models.finding import Contextual, Finding, Severity +from ..models.graph import Confidence, Node +from ..models.policy import Policy, Settings -def evaluate(finding: Finding, facts: dict, policy: Policy) -> Contextual: - """Apply policy with full reachability + chain facts; return the contextual block. +_CONF_ORDER = {"inferred": 0, "declared": 1, "ground_truth": 2} +_PLACEHOLDER = re.compile(r"\{([^}]+)\}") - ``facts`` carries reachable_from_*, in_chain, chain_* etc. so rule conditions - resolve. Scaffold. + +@dataclass +class Facts: + """The reachability + chain + asset facts a finding is scored against. + + Assembled by the engine pipeline from the reachability pass (02), the + chaining pass (06), and the target asset node. This is the seam the chaining + slice fills: pass real chains and in_chain/chain_* follow. """ - raise NotImplementedError("scaffold: match when -> apply effect -> sum deltas -> clamp -> render") + + reachable_from_internet: bool | None = None + reachable_from_zones: list[str] = field(default_factory=list) + controls_on_all_paths: list[Node] = field(default_factory=list) + controls_on_any_path: list[Node] = field(default_factory=list) + asset: Node | None = None + in_chain: bool = False + chain_reaches_crown_jewel: bool = False + chains: list = field(default_factory=list) # chain objects (id, steps, target, ...) + max_chain_length: int = 0 + + @classmethod + def from_reachability(cls, rfacts, asset: Node | None = None, chains: list | None = None) -> "Facts": + chains = chains or [] + return cls( + reachable_from_internet=rfacts.reachable_from_internet, + reachable_from_zones=list(rfacts.reachable_from_zones), + controls_on_all_paths=list(rfacts.controls_on_all_paths), + controls_on_any_path=list(rfacts.controls_on_any_path), + asset=asset, + in_chain=bool(chains), + chain_reaches_crown_jewel=any(getattr(c, "reaches_crown_jewel", False) for c in chains), + chains=chains, + max_chain_length=max((len(getattr(c, "steps", []) or []) for c in chains), default=0), + ) + + +def evaluate(finding: Finding, facts: Facts, policy: Policy, *, scored_at: str | None = None) -> Contextual: + """Apply policy with full reachability + chain facts; return the contextual block.""" + settings = policy.settings + is_unknown = finding.category.type == "unknown" + + total_delta = 0 + override: Severity | None = None + tags: list[str] = [] + rationale: list[str] = [] + suppressed = False + + for rule in policy.rules: + bindings: dict = {} + if not _match_when(rule.when.spec, finding, facts, settings, bindings): + continue + eff = rule.effect + do_suppress = eff.suppress and not (is_unknown and not settings.unknown_category_downgrade) + total_delta += eff.delta + if eff.set_severity: + override = Severity.coerce(eff.set_severity) + if do_suppress: + suppressed = True + tags.extend(eff.tag) + if rule.rationale: + rationale.append(render_rationale(rule.rationale, _render_context(finding, facts, bindings))) + if eff.stop: + break + + # Clamp summed delta against the safety rails (04 §Severity ladder). + if total_delta < 0: + total_delta = max(total_delta, -settings.max_downgrade) + elif total_delta > 0: + total_delta = min(total_delta, settings.max_upgrade) + + base = finding.base_severity.level + if override is not None: + final = override + else: + final = Severity(max(0, min(4, int(base) + total_delta))) + + # Floor: an unknown category is never silently buried (categories.md). + if is_unknown and not settings.unknown_category_downgrade and int(final) < int(base): + final = base + + return Contextual( + severity=final, + delta=int(final) - int(base), + rationale=rationale, + chains=[getattr(c, "id", str(c)) for c in facts.chains], + tags=tags, + suppressed=suppressed, + scored_at=scored_at, + policy_version=policy.policy_version, + ) + + +def render_rationale(template: str, context: dict) -> str: + """Fill {finding.*}/{control.*}/{chain.*}/{asset.*}/{reachable_from_zone}. + Unknown placeholders are left literal so a template typo is visible, not silent.""" + return _PLACEHOLDER.sub(lambda m: str(context.get(m.group(1), m.group(0))), template) + + +# --- condition matching -------------------------------------------------------- + +def _match_when(when: dict, finding: Finding, facts: Facts, settings: Settings, bindings: dict) -> bool: + """All present keys must match (AND). Records bindings used by rationale.""" + for key, val in when.items(): + if key == "category": + if finding.category.type not in val: + return False + elif key == "base_severity_at_least": + if finding.base_severity.level < Severity.coerce(val): + return False + elif key == "base_severity_at_most": + if finding.base_severity.level > Severity.coerce(val): + return False + elif key == "reachable_from_internet": + if bool(facts.reachable_from_internet) != bool(val): + return False + elif key == "reachable_from_zone": + if val not in facts.reachable_from_zones: + return False + bindings["reachable_from_zone"] = val + elif key == "asset_data_sensitivity": + sens = facts.asset.attributes.get("data_sensitivity") if facts.asset else None + if sens not in val: + return False + if facts.asset: + bindings["asset"] = facts.asset + elif key == "asset_attribute": + if facts.asset is None: + return False + if any(facts.asset.attributes.get(k) != v for k, v in val.items()): + return False + bindings["asset"] = facts.asset + elif key == "control_on_all_paths": + ctl = _match_control(val, facts.controls_on_all_paths, finding, settings) + if ctl is None: + return False + bindings["control"] = ctl + elif key == "control_on_any_path": + ctl = _match_control(val, facts.controls_on_any_path, finding, settings) + if ctl is None: + return False + bindings["control"] = ctl + elif key == "in_chain": + if bool(facts.in_chain) != bool(val): + return False + elif key == "chain_reaches_crown_jewel": + if bool(facts.chain_reaches_crown_jewel) != bool(val): + return False + if val: + cj = next((c for c in facts.chains if getattr(c, "reaches_crown_jewel", False)), None) + if cj is not None: + bindings["chain"] = cj + elif key == "chain_min_length": + if facts.max_chain_length < int(val): + return False + if facts.chains: + bindings.setdefault("chain", max(facts.chains, key=lambda c: len(getattr(c, "steps", []) or []))) + elif key == "exploit": + if not _match_exploit(val, finding): + return False + elif key == "evidence_confidence": + if finding.evidence.get("confidence") not in val: + return False + else: + raise ValueError(f"unknown policy condition key: {key!r}") + return True + + +def _resolve_mitigates(value, finding: Finding) -> list[str]: + if value is None: + return [] + if isinstance(value, str): + return [finding.category.type if "{finding.category}" in value else value] + return list(value) + + +def _conf_rank(conf) -> int: + name = conf.value if isinstance(conf, Confidence) else str(conf) + return _CONF_ORDER.get(name, 1) + + +def _match_control(spec: dict, controls: list[Node], finding: Finding, settings: Settings) -> Node | None: + wanted = set(_resolve_mitigates(spec.get("mitigates"), finding)) + req_mode = spec.get("mode") + if settings.require_block_mode and not req_mode: + req_mode = "block" + min_conf = spec.get("min_confidence", settings.min_control_confidence) + for c in controls: + mitigates = set(c.attributes.get("mitigates", []) or []) + if wanted and not (wanted & mitigates): + continue + if req_mode and c.attributes.get("mode") != req_mode: + continue + if _conf_rank(c.confidence) < _conf_rank(min_conf): + continue + return c + return None + + +def _match_exploit(spec: dict, finding: Finding) -> bool: + ex = finding.exploitability + if "kev" in spec and bool(ex.kev) != bool(spec["kev"]): + return False + if "exploit_public" in spec and bool(ex.exploit_public) != bool(spec["exploit_public"]): + return False + if "epss_at_least" in spec: + if ex.epss_score is None or ex.epss_score < float(spec["epss_at_least"]): + return False + return True -def render_rationale(template: str, match_context: dict) -> str: - """Fill {finding.*}/{control.*}/{chain.*}/{asset.*}/{reachable_from_zone}.""" - raise NotImplementedError("scaffold") +def _render_context(finding: Finding, facts: Facts, bindings: dict) -> dict: + ctx: dict = { + "finding.category": finding.category.type, + "finding.title": finding.title, + "finding.id": finding.finding_id, + } + ctl = bindings.get("control") + if ctl is not None: + ctx["control.id"] = ctl.id + ctx["control.control_type"] = ctl.attributes.get("control_type") + ctx["control.vendor"] = ctl.attributes.get("vendor") + ctx["control.mode"] = ctl.attributes.get("mode") + chain = bindings.get("chain") + if chain is not None: + ctx["chain.id"] = getattr(chain, "id", None) + ctx["chain.target"] = getattr(chain, "target", None) + ctx["chain.length"] = len(getattr(chain, "steps", []) or []) + asset = bindings.get("asset") or facts.asset + if asset is not None: + ctx["asset.label"] = asset.label + ctx["asset.id"] = asset.id + if "reachable_from_zone" in bindings: + ctx["reachable_from_zone"] = bindings["reachable_from_zone"] + return {k: ("" if v is None else v) for k, v in ctx.items()} diff --git a/core/reachability/engine/reachability.py b/core/reachability/engine/reachability.py index 55c830b..b551ca4 100644 --- a/core/reachability/engine/reachability.py +++ b/core/reachability/engine/reachability.py @@ -1,26 +1,81 @@ """Reachability pass (the downgrade engine). Contract: schema/02-context-graph-schema.md. For finding F on asset A: traverse INBOUND from public entrypoints to A. For each -path collect ``protected_by`` controls. If a control on EVERY path ``mitigates`` -F's category in ``block`` mode, F is downgradeable and each control becomes a -rationale line. No inbound path at all -> hard internal-only drop. Deterministic. +path collect ``protected_by`` controls. A control is "on a path" if any node in +that path is protected by it. The engine reports: + + * ``reachable_from_internet`` — does any inbound path exist? (no path -> hard + internal-only downgrade by policy) + * ``controls_on_all_paths`` — controls present on EVERY inbound path; only these + can justify a control-based downgrade (the policy still checks that a control + mitigates F's category in ``block`` mode — that gate lives in policy_eval). + * ``controls_on_any_path`` — weaker signal, present on at least one path. + * ``reachable_from_zones`` — zones traversed on inbound paths (position-aware). + +Deterministic: same graph + findings -> same facts. No LLM. """ from __future__ import annotations +from dataclasses import dataclass, field + from ..graphstore.base import GraphStore from ..models.finding import Finding +from ..models.graph import Node, NodeFilter +# Public entrypoints seed the inbound traversal (02: attributes->>'public'='true'). +_PUBLIC_ENTRYPOINTS = NodeFilter(type="entrypoint", attributes={"public": True}) + +@dataclass class ReachabilityFacts: - """Per-finding annotations consumed by policy conditions.""" + """Per-finding annotations consumed by the scoring policy's conditions.""" - reachable_from_internet: bool - reachable_from_zones: list[str] - controls_on_all_paths: list[dict] - controls_on_any_path: list[dict] + reachable_from_internet: bool = False + reachable_from_zones: list[str] = field(default_factory=list) + controls_on_all_paths: list[Node] = field(default_factory=list) + controls_on_any_path: list[Node] = field(default_factory=list) def compute(findings: list[Finding], graph: GraphStore) -> dict[str, ReachabilityFacts]: - """Return reachability facts keyed by finding_id. Scaffold.""" - raise NotImplementedError("scaffold: paths_to(asset, public-entrypoints) + control collection") + """Return reachability facts keyed by finding_id.""" + return {f.finding_id: compute_one(f, graph) for f in findings} + + +def compute_one(finding: Finding, graph: GraphStore) -> ReachabilityFacts: + target = finding.asset_ref.node_id + facts = ReachabilityFacts() + if not target or graph.get_node(target) is None: + # Unresolvable asset -> treat as not internet-reachable; the policy's + # unknown handling decides what to do (never silently suppress). + return facts + + paths = graph.paths_to(target, from_filter=_PUBLIC_ENTRYPOINTS) + facts.reachable_from_internet = len(paths) > 0 + if not paths: + return facts + + zones: set[str] = set() + per_path_controls: list[dict[str, Node]] = [] + for path in paths: + controls: dict[str, Node] = {} + for node_id in path.nodes: + node = graph.get_node(node_id) + if node is not None and node.type == "zone": + zones.add(node_id) + for edge in graph.neighbors(node_id, ["protected_by"], "out"): + control = graph.get_node(edge.dst) + if control is not None: + controls[control.id] = control + per_path_controls.append(controls) + + all_ids: set[str] = set.intersection(*(set(c) for c in per_path_controls)) + any_ids: set[str] = set().union(*(set(c) for c in per_path_controls)) + merged: dict[str, Node] = {} + for c in per_path_controls: + merged.update(c) + + facts.reachable_from_zones = sorted(zones) + facts.controls_on_all_paths = [merged[i] for i in sorted(all_ids)] + facts.controls_on_any_path = [merged[i] for i in sorted(any_ids)] + return facts diff --git a/core/reachability/graphstore/__init__.py b/core/reachability/graphstore/__init__.py index d9c20b3..60cb815 100644 --- a/core/reachability/graphstore/__init__.py +++ b/core/reachability/graphstore/__init__.py @@ -7,6 +7,7 @@ """ from .base import GraphStore +from .memory import InMemoryGraphStore from .postgres import PostgresGraphStore -__all__ = ["GraphStore", "PostgresGraphStore"] +__all__ = ["GraphStore", "InMemoryGraphStore", "PostgresGraphStore"] diff --git a/core/reachability/graphstore/base.py b/core/reachability/graphstore/base.py index ec749a7..f1a6a17 100644 --- a/core/reachability/graphstore/base.py +++ b/core/reachability/graphstore/base.py @@ -14,6 +14,7 @@ class GraphQuery: class GraphStore(Protocol): def upsert_node(self, node: Node) -> None: ... def upsert_edge(self, edge: Edge) -> None: ... + def get_node(self, node_id: str) -> Node | None: ... def neighbors(self, node_id: str, edge_types: list[str], direction: str) -> list[Edge]: ... def paths_to( self, target_id: str, *, from_filter: NodeFilter, max_depth: int = 12 diff --git a/core/reachability/graphstore/memory.py b/core/reachability/graphstore/memory.py new file mode 100644 index 0000000..28a19b3 --- /dev/null +++ b/core/reachability/graphstore/memory.py @@ -0,0 +1,141 @@ +"""In-memory GraphStore — the infra-free default for tests and small environments. + +Implements the same ``GraphStore`` Protocol as ``PostgresGraphStore`` (the +production default), so the reasoning engine is identical against either. This +exists so the graph + reasoning pipeline can be built and unit-tested with zero +infrastructure — directly in the spirit of 05-context-ingestion.md's build order +("prove the graph end to end" before any parser/DB is required). + +``paths_to`` mirrors the inbound-reachability recursive CTE in +schema/02-context-graph-schema.md: it traverses the topology edge set +(``routes_to``, ``connects_to``) forward from matching roots to a target, with a +per-path cycle guard and depth bound. +""" + +from __future__ import annotations + +from ..models.graph import Confidence, Edge, Node, NodeFilter, Path +from .base import GraphQuery, GraphStore + +# The topology edges reachability traversal walks (same set as the CTE). +_REACH_EDGE_TYPES = ("routes_to", "connects_to") + + +def _val_eq(a: object, b: object) -> bool: + """Loose equality so YAML bools/strings ('true' vs True) compare equal.""" + return a == b or str(a).lower() == str(b).lower() + + +class InMemoryGraphStore(GraphStore): + def __init__(self) -> None: + self._nodes: dict[str, Node] = {} + self._edges: list[Edge] = [] + + # --- writes ----------------------------------------------------------- + def upsert_node(self, node: Node) -> None: + existing = self._nodes.get(node.id) + if existing is None: + self._nodes[node.id] = node + return + # Merge attributes; a higher-confidence write wins on conflict but the + # full reconciliation policy lives in ingestion/reconcile.py. + existing.label = node.label or existing.label + existing.attributes = {**existing.attributes, **node.attributes} + + def upsert_edge(self, edge: Edge) -> None: + for e in self._edges: + if e.type == edge.type and e.src == edge.src and e.dst == edge.dst: + e.attributes = {**e.attributes, **edge.attributes} + return + self._edges.append(edge) + + # --- reads ------------------------------------------------------------ + def get_node(self, node_id: str) -> Node | None: + return self._nodes.get(node_id) + + def neighbors(self, node_id: str, edge_types: list[str], direction: str) -> list[Edge]: + types = set(edge_types) + if direction == "out": + return [e for e in self._edges if e.src == node_id and e.type in types] + if direction == "in": + return [e for e in self._edges if e.dst == node_id and e.type in types] + raise ValueError(f"direction must be 'in' or 'out', got {direction!r}") + + def nodes_by_type(self, node_type: str) -> list[Node]: + return [n for n in self._nodes.values() if n.type == node_type] + + def paths_to( + self, target_id: str, *, from_filter: NodeFilter, max_depth: int = 12 + ) -> list[Path]: + """Every simple inbound path from a node matching ``from_filter`` to + ``target_id``, traversing routes_to/connects_to forward. Cycle-guarded + per path (no node revisited) and bounded by ``max_depth``.""" + roots = [n.id for n in self._nodes.values() if self._matches(n, from_filter)] + paths: list[Path] = [] + for root in roots: + self._dfs(root, target_id, [root], [], paths, max_depth) + return paths + + def query(self, spec: GraphQuery) -> list[Path]: + raise NotImplementedError("scaffold: declarative path DSL") + + # --- serialization (file-based graph artifact, no DB required) --------- + def to_dict(self) -> dict: + return { + "nodes": [ + {"id": n.id, "type": n.type, "label": n.label, "attributes": n.attributes, + "confidence": n.confidence.value, "source": n.source} + for n in self._nodes.values() + ], + "edges": [ + {"type": e.type, "src": e.src, "dst": e.dst, "attributes": e.attributes, + "confidence": e.confidence.value, "source": e.source} + for e in self._edges + ], + } + + @classmethod + def from_dict(cls, data: dict) -> "InMemoryGraphStore": + graph = cls() + for n in data.get("nodes", []): + graph.upsert_node(Node( + id=n["id"], type=n["type"], label=n["label"], + attributes=n.get("attributes", {}) or {}, + confidence=Confidence(n.get("confidence", "declared")), source=n.get("source"), + )) + for e in data.get("edges", []): + graph.upsert_edge(Edge( + type=e["type"], src=e["src"], dst=e["dst"], + attributes=e.get("attributes", {}) or {}, + confidence=Confidence(e.get("confidence", "declared")), source=e.get("source"), + )) + return graph + + # --- internals -------------------------------------------------------- + def _matches(self, node: Node, flt: NodeFilter) -> bool: + if flt.type is not None and node.type != flt.type: + return False + return all(_val_eq(node.attributes.get(k), v) for k, v in flt.attributes.items()) + + def _dfs( + self, + current: str, + target: str, + node_path: list[str], + edge_path: list[Edge], + out: list[Path], + max_depth: int, + ) -> None: + if current == target: + out.append(Path(nodes=list(node_path), edges=list(edge_path))) + return + if len(node_path) - 1 >= max_depth: + return + for e in self.neighbors(current, list(_REACH_EDGE_TYPES), "out"): + if e.dst in node_path: # cycle guard + continue + node_path.append(e.dst) + edge_path.append(e) + self._dfs(e.dst, target, node_path, edge_path, out, max_depth) + node_path.pop() + edge_path.pop() diff --git a/core/reachability/graphstore/migrations/0002_edge_unique.sql b/core/reachability/graphstore/migrations/0002_edge_unique.sql new file mode 100644 index 0000000..2e12205 --- /dev/null +++ b/core/reachability/graphstore/migrations/0002_edge_unique.sql @@ -0,0 +1,7 @@ +-- Upsert support for edges. The contract's graph_edge (0001) has a bigserial PK +-- and no natural key; this unique index makes (env_id, type, src, dst) the +-- conflict target so upsert_edge can ON CONFLICT ... DO UPDATE, matching the +-- in-memory store's dedup-by-(type,src,dst) semantics. + +CREATE UNIQUE INDEX IF NOT EXISTS graph_edge_natural_key + ON graph_edge (env_id, type, src, dst); diff --git a/core/reachability/graphstore/migrations/0003_findings.sql b/core/reachability/graphstore/migrations/0003_findings.sql new file mode 100644 index 0000000..8703fa7 --- /dev/null +++ b/core/reachability/graphstore/migrations/0003_findings.sql @@ -0,0 +1,36 @@ +-- Findings + score history. Source design: schema/01-finding-schema.md (the +-- normalized finding + its engine-written contextual block) and +-- schema/04-scoring-policy.md §Trust ("a re-score under a new policy is +-- distinguishable from the old one in history"). +-- +-- `finding` — the normalized finding (full to_dict in JSONB), one row per +-- finding_id (stable per scanner+fingerprint across rescans). +-- `finding_score` — append-only contextual results: one row per scoring run, so +-- the policy_version history is preserved, never overwritten. + +CREATE TABLE IF NOT EXISTS finding ( + finding_id text PRIMARY KEY, + env_id uuid NOT NULL, + scan_id uuid NOT NULL, + category text, + fingerprint text, + data jsonb NOT NULL, -- full normalized finding (incl. latest contextual) + created_at timestamptz DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS finding_score ( + id bigserial PRIMARY KEY, + finding_id text NOT NULL REFERENCES finding(finding_id), + env_id uuid NOT NULL, + policy_version text NOT NULL, + severity text NOT NULL, + delta int NOT NULL, + suppressed boolean NOT NULL DEFAULT false, + contextual jsonb NOT NULL, -- full contextual block for this run + scored_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS finding_env_scan ON finding (env_id, scan_id); +CREATE INDEX IF NOT EXISTS finding_env_category ON finding (env_id, category); +CREATE INDEX IF NOT EXISTS finding_score_finding ON finding_score (finding_id); +CREATE INDEX IF NOT EXISTS finding_score_env_policy ON finding_score (env_id, policy_version); diff --git a/core/reachability/graphstore/migrations/__init__.py b/core/reachability/graphstore/migrations/__init__.py new file mode 100644 index 0000000..90a692b --- /dev/null +++ b/core/reachability/graphstore/migrations/__init__.py @@ -0,0 +1,36 @@ +"""Schema migrations applied in filename order. Plain .sql, idempotent (IF NOT +EXISTS), so ``apply_all`` is safe to run on every startup (docker compose up).""" + +from __future__ import annotations + +from pathlib import Path + +_DIR = Path(__file__).resolve().parent + + +def migration_files() -> list[Path]: + return sorted(_DIR.glob("*.sql")) + + +def split_statements(sql: str) -> list[str]: + """Split a .sql file into individual statements. + + psycopg3's extended protocol rejects multiple commands in one execute(), so + each statement is run separately. Our migrations are plain DDL with no ';' + inside strings or function bodies, so a naive split on ';' (ignoring '--' + comment lines) is safe.""" + cleaned = "\n".join( + line for line in sql.splitlines() if not line.strip().startswith("--") + ) + return [s.strip() for s in cleaned.split(";") if s.strip()] + + +def apply_all(conn) -> list[str]: + """Execute every migration on a psycopg connection; return the names applied.""" + applied = [] + for path in migration_files(): + for statement in split_statements(path.read_text(encoding="utf-8")): + conn.execute(statement) + applied.append(path.name) + conn.commit() + return applied diff --git a/core/reachability/graphstore/postgres.py b/core/reachability/graphstore/postgres.py index 3cba34e..f68e469 100644 --- a/core/reachability/graphstore/postgres.py +++ b/core/reachability/graphstore/postgres.py @@ -3,35 +3,145 @@ No heavyweight graph DB required to self-host. Inbound-reachability paths come from the recursive CTE in schema/02-context-graph-schema.md; control collection over returned paths is done in the engine, not in SQL. + +psycopg is imported lazily so the package (and the in-memory path) works without +it installed — you only need the driver to actually talk to Postgres. """ from __future__ import annotations -from ..models.graph import Edge, Node, NodeFilter, Path +import uuid as _uuid + +from ..models.graph import Confidence, Edge, Node, NodeFilter, Path +from . import migrations from .base import GraphQuery, GraphStore +from .queries import INBOUND_PATHS_TO_ASSET + +try: # optional driver + import psycopg + from psycopg.rows import dict_row + from psycopg.types.json import Jsonb + _HAS_PSYCOPG = True +except ImportError: # pragma: no cover - exercised by absence + _HAS_PSYCOPG = False + + +def _require_psycopg() -> None: + if not _HAS_PSYCOPG: + raise RuntimeError("psycopg is required for PostgresGraphStore: pip install 'psycopg[binary]'") + + +# --- pure row mappers (unit-testable without a connection) --------------------- + +def _node_from_row(row: dict) -> Node: + return Node( + id=row["id"], type=row["type"], label=row["label"], + attributes=row.get("attributes") or {}, + confidence=Confidence(row.get("confidence", "declared")), + source=row.get("source"), + ) + + +def _edge_from_row(row: dict) -> Edge: + return Edge( + type=row["type"], src=row["src"], dst=row["dst"], + attributes=row.get("attributes") or {}, + confidence=Confidence(row.get("confidence", "declared")), + source=row.get("source"), id=row.get("id"), + ) class PostgresGraphStore(GraphStore): - def __init__(self, dsn: str, env_id: str) -> None: - self._dsn = dsn - self._env_id = env_id # multi-environment / multi-tenant scope + def __init__(self, dsn: str | None = None, env_id: str = "", *, conn=None) -> None: + self._env_id = _uuid.UUID(env_id) if env_id else None # multi-tenant scope + if conn is not None: + self._conn = conn + else: + _require_psycopg() + # autocommit: single-statement ops, and reads must not leave the + # connection idle-in-transaction holding locks. + self._conn = psycopg.connect(dsn, row_factory=dict_row, autocommit=True) + def init_schema(self) -> list[str]: + """Apply all migrations (idempotent). Safe on every startup.""" + return migrations.apply_all(self._conn) + + # --- writes ----------------------------------------------------------- def upsert_node(self, node: Node) -> None: - raise NotImplementedError("scaffold") + self._conn.execute( + """ + INSERT INTO graph_node (id, env_id, type, label, attributes, confidence, source) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (id) DO UPDATE SET + label = EXCLUDED.label, + attributes = graph_node.attributes || EXCLUDED.attributes, + confidence = EXCLUDED.confidence, + source = EXCLUDED.source + """, + (node.id, self._env_id, node.type, node.label, + self._json(node.attributes), node.confidence.value, node.source), + ) + self._conn.commit() def upsert_edge(self, edge: Edge) -> None: - raise NotImplementedError("scaffold") + self._conn.execute( + """ + INSERT INTO graph_edge (env_id, type, src, dst, attributes, confidence, source) + VALUES (%s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (env_id, type, src, dst) DO UPDATE SET + attributes = graph_edge.attributes || EXCLUDED.attributes, + confidence = EXCLUDED.confidence, + source = EXCLUDED.source + """, + (self._env_id, edge.type, edge.src, edge.dst, + self._json(edge.attributes), edge.confidence.value, edge.source), + ) + self._conn.commit() - def neighbors(self, node_id: str, edge_types: list[str], direction: str) -> list[Edge]: - raise NotImplementedError("scaffold") + # --- reads ------------------------------------------------------------ + def get_node(self, node_id: str) -> Node | None: + row = self._conn.execute( + "SELECT id, type, label, attributes, confidence, source " + "FROM graph_node WHERE id = %s AND env_id = %s", + (node_id, self._env_id), + ).fetchone() + return _node_from_row(row) if row else None - def paths_to( - self, target_id: str, *, from_filter: NodeFilter, max_depth: int = 12 - ) -> list[Path]: - raise NotImplementedError("scaffold: see graphstore/queries.py inbound-reachability CTE") + def neighbors(self, node_id: str, edge_types: list[str], direction: str) -> list[Edge]: + if direction == "out": + col = "src" + elif direction == "in": + col = "dst" + else: + raise ValueError(f"direction must be 'in' or 'out', got {direction!r}") + rows = self._conn.execute( + f"SELECT id, type, src, dst, attributes, confidence, source " + f"FROM graph_edge WHERE env_id = %s AND {col} = %s AND type = ANY(%s)", + (self._env_id, node_id, list(edge_types)), + ).fetchall() + return [_edge_from_row(r) for r in rows] def nodes_by_type(self, node_type: str) -> list[Node]: - raise NotImplementedError("scaffold") + rows = self._conn.execute( + "SELECT id, type, label, attributes, confidence, source " + "FROM graph_node WHERE env_id = %s AND type = %s", + (self._env_id, node_type), + ).fetchall() + return [_node_from_row(r) for r in rows] + + def paths_to(self, target_id: str, *, from_filter: NodeFilter, max_depth: int = 12) -> list[Path]: + # The shipped CTE seeds from public entrypoints (the reachability root); + # from_filter is honored for that canonical case (entrypoint + public). + rows = self._conn.execute( + INBOUND_PATHS_TO_ASSET, + {"env_id": self._env_id, "target": target_id, "max_depth": max_depth}, + ).fetchall() + return [Path(nodes=list(r["path"]), edges=[]) for r in rows] def query(self, spec: GraphQuery) -> list[Path]: - raise NotImplementedError("scaffold") + raise NotImplementedError("scaffold: declarative path DSL") + + # --- helpers ---------------------------------------------------------- + @staticmethod + def _json(value): + return Jsonb(value) if _HAS_PSYCOPG else value diff --git a/core/reachability/ingestion/diagrams.py b/core/reachability/ingestion/diagrams.py index ee88ed6..a102691 100644 --- a/core/reachability/ingestion/diagrams.py +++ b/core/reachability/ingestion/diagrams.py @@ -1,15 +1,162 @@ -"""Diagrams-as-code (Mermaid / Structurizr C4), lowest confidence, optional. +"""Diagrams-as-code (Mermaid), lowest confidence, optional. Contract: 05 §4. -Contract: 05-context-ingestion.md §4. Parsed at ``declared`` confidence, always +Parses a Mermaid flowchart into nodes/edges at ``declared`` confidence — always reconciled against and subordinate to machine sources. A hand-drawn diagram may RAISE suspicion (add a node to investigate) but never, by itself, justify burying a critical. Primarily a bootstrapping aid for teams without IaC. + +Mapping (heuristic, declared): + id[label] / id(label) / id{label} -> asset + id[(label)] -> datastore (cylinder) + id((label)) -> entrypoint{public} if the label looks like + internet/user/..., else asset + subgraph NAME ... end -> zone; nodes inside resides_in it + A --> B (directed) -> routes_to A B (connects_to between zones) + A --- B (undirected) -> routes_to in written order """ from __future__ import annotations -from ..models.graph import Edge, Node +import re + +from ..models.graph import Confidence, Edge, Node + +_DECL = Confidence.declared +_SOURCE = "diagram" +_ENTRYPOINT_WORDS = {"internet", "user", "users", "attacker", "client", "clients", + "public", "browser", "www", "external"} +_OP = re.compile(r"(-\.->|==>|-->|---|===|-\.-)") +_DIRECTED = {"-->", "==>", "-.->"} + + +def _slug(value: str) -> str: + return re.sub(r"[^a-z0-9_.-]+", "-", str(value).strip().lower()).strip("-") + + +def _clean_label(text: str | None) -> str | None: + if text is None: + return None + return text.strip().strip('"').strip("'").strip() or None + + +def _parse_node(token: str): + """Return (raw_id, kind, label) for a Mermaid node token, or None.""" + token = token.strip() + m = re.match(r"^([A-Za-z0-9_.-]+)\s*(.*)$", token) + if not m: + return None + rid, rest = m.group(1), m.group(2).strip() + pairs = [("[(", "datastore", 2), ("((", "circle", 2), ("[[", "asset", 2), + ("([", "asset", 2), ("{{", "asset", 2), + ("[", "asset", 1), ("(", "asset", 1), ("{", "asset", 1)] + for prefix, kind, n in pairs: + if rest.startswith(prefix): + return rid, kind, _clean_label(rest[n:-n]) + return rid, "asset", None # bare reference def load(diagram_path: str) -> tuple[list[Node], list[Edge]]: - raise NotImplementedError("scaffold") + with open(diagram_path, encoding="utf-8") as fh: + return build(fh.read()) + + +def build(text: str) -> tuple[list[Node], list[Edge]]: + statements = _statements(text) + subgraphs = {_slug(name) for name in _subgraph_names(statements)} + + nodes: dict[str, Node] = {} + edges: dict[tuple, Edge] = {} + node_zone: dict[str, str] = {} + zone_stack: list[str] = [] + + for kind, payload in statements: + if kind == "subgraph": + zone_stack.append(_slug(payload)) + _ensure_zone(nodes, zone_stack[-1], payload) + elif kind == "end": + if zone_stack: + zone_stack.pop() + elif kind == "edge": + _process_edge(payload, subgraphs, zone_stack, nodes, edges, node_zone) + + return list(nodes.values()), list(edges.values()) + + +def _add_edge(edges: dict, type_: str, src: str, dst: str) -> None: + key = (type_, src, dst) + if key not in edges: + edges[key] = Edge(type=type_, src=src, dst=dst, confidence=_DECL, source=_SOURCE) + + +def _statements(text: str) -> list[tuple[str, str]]: + out: list[tuple[str, str]] = [] + for raw in text.splitlines(): + line = raw.split("%%", 1)[0].strip() # strip comments + if not line: + continue + low = line.lower() + if low.startswith(("graph ", "flowchart ", "graph\t", "flowchart\t")) or low in ("graph", "flowchart"): + continue + if low.startswith("subgraph"): + out.append(("subgraph", line[len("subgraph"):].strip().strip('"'))) + elif low == "end": + out.append(("end", "")) + elif _OP.search(line): + out.append(("edge", line)) + # standalone node decls without an edge are ignored (no topology value) + return out + + +def _subgraph_names(statements: list[tuple[str, str]]) -> list[str]: + return [payload for kind, payload in statements if kind == "subgraph"] + + +def _ensure_zone(nodes: dict, zid: str, label: str) -> str: + node_id = f"zone:{zid}" + internet_facing = any(w in label.lower() for w in ("dmz", "public", "edge", "internet")) + if node_id not in nodes: + nodes[node_id] = Node(id=node_id, type="zone", label=label, + attributes={"internet_facing": internet_facing}, + confidence=_DECL, source=_SOURCE) + return node_id + + +def _resolve(token: str, subgraphs: set, zone_stack: list, nodes: dict, edges: dict, + node_zone: dict) -> str | None: + parsed = _parse_node(token) + if parsed is None: + return None + rid, kind, label = parsed + slug = _slug(rid) + if slug in subgraphs: # an edge endpoint that is a subgraph + return f"zone:{slug}" + if kind == "circle": + kind = "entrypoint" if (label or rid).lower() in _ENTRYPOINT_WORDS else "asset" + prefix = {"datastore": "datastore", "entrypoint": "entrypoint"}.get(kind, "asset") + node_id = f"{prefix}:{slug}" + attrs = {"public": True} if prefix == "entrypoint" else {} + if node_id not in nodes: + nodes[node_id] = Node(id=node_id, type=prefix, label=label or rid, + attributes=attrs, confidence=_DECL, source=_SOURCE) + elif label: + nodes[node_id].label = label + # subgraph membership: first zone the node appears inside (Mermaid semantics) + if prefix in ("asset", "datastore") and zone_stack and node_id not in node_zone: + node_zone[node_id] = zone_stack[-1] + _add_edge(edges, "resides_in", node_id, f"zone:{zone_stack[-1]}") + return node_id + + +def _process_edge(line: str, subgraphs: set, zone_stack: list, nodes: dict, + edges: dict, node_zone: dict) -> None: + cleaned = re.sub(r"\|[^|]*\|", "", line) # drop |edge labels| + parts = _OP.split(cleaned) + tokens, ops = parts[0::2], parts[1::2] + resolved = [_resolve(t, subgraphs, zone_stack, nodes, edges, node_zone) for t in tokens] + for i in range(len(ops)): + src, dst = resolved[i], resolved[i + 1] + if not src or not dst: + continue + both_zones = src.startswith("zone:") and dst.startswith("zone:") + edge_type = "connects_to" if both_zones else "routes_to" + _add_edge(edges, edge_type, src, dst) diff --git a/core/reachability/ingestion/iac/__init__.py b/core/reachability/ingestion/iac/__init__.py index 8440c34..98f295b 100644 --- a/core/reachability/ingestion/iac/__init__.py +++ b/core/reachability/ingestion/iac/__init__.py @@ -4,8 +4,28 @@ ``ground_truth`` nodes/edges. One provider plugin per IaC dialect, each emitting the same generic node/edge set, so core never hard-codes a cloud — the same swappability pattern as scanners and the graph backend. + +``load`` auto-detects the dialect from the resource type prefixes and dispatches. """ +from __future__ import annotations + +from ...models.graph import Edge, Node from .base import IacProvider +from .cloudformation import CloudFormationProvider, load_template, looks_like_cloudformation +from .common import load_resources +from .terraform_aws import TerraformAwsProvider +from .terraform_azure import TerraformAzureProvider + +__all__ = ["IacProvider", "TerraformAwsProvider", "TerraformAzureProvider", + "CloudFormationProvider", "load"] + -__all__ = ["IacProvider"] +def load(source_path: str) -> tuple[list[Node], list[Edge]]: + """Detect the IaC dialect and parse it into nodes/edges.""" + if looks_like_cloudformation(load_template(source_path)): + return CloudFormationProvider().parse(source_path) + types = {r.get("type", "") for r in load_resources(source_path)} + if any(t.startswith("azurerm_") for t in types): + return TerraformAzureProvider().parse(source_path) + return TerraformAwsProvider().parse(source_path) # default: AWS terraform diff --git a/core/reachability/ingestion/iac/cloudformation.py b/core/reachability/ingestion/iac/cloudformation.py new file mode 100644 index 0000000..3cafdf6 --- /dev/null +++ b/core/reachability/ingestion/iac/cloudformation.py @@ -0,0 +1,186 @@ +"""CloudFormation provider plugin. Contract: 05-context-ingestion.md §1. + +Third IaC dialect (after terraform-aws/azure), emitting the same generic node/edge +set. A CFN template references resources by LOGICAL ID via ``Ref`` / ``Fn::GetAtt``, +so wiring resolves logical ids to graph nodes (cleaner than matching resolved +ARNs/IPs as in Terraform state). + +Resource -> graph mapping (AWS::Service::Resource): + AWS::EC2::Subnet -> zone (internet_facing from MapPublicIpOnLaunch) + AWS::ElasticLoadBalancingV2::LoadBalancer -> asset + control{lb}; internet-facing + Scheme -> entrypoint; Subnets -> resides_in + AWS::WAFv2::WebACL -> control{waf, block} + AWS::WAFv2::WebACLAssociation -> protected_by (ResourceArn -> WebACLArn) + AWS::ElasticLoadBalancingV2::{TargetGroup,Listener} -> routes_to LB -> backend instances + AWS::EC2::Instance -> asset; SubnetId -> resides_in + AWS::RDS::DBInstance / DBCluster -> datastore; DBSubnetGroupName -> resides_in zones + AWS::RDS::DBSubnetGroup -> (indexed) SubnetIds + AWS::S3::Bucket -> datastore + +Input is the template (the source of truth), JSON or YAML; the YAML short-form +intrinsics (!Ref, !GetAtt, ...) are parsed via a CloudFormation-aware loader. +""" + +from __future__ import annotations + +import json + +import yaml + +from ...models.graph import Confidence, Edge, Node +from .base import IacProvider +from .common import DEFAULT_WAF_MITIGATES, Emitter, Index, slug + + +class _CfnLoader(yaml.SafeLoader): + """SafeLoader that understands CloudFormation short-form intrinsics.""" + + +def _cfn_multi(loader, tag_suffix, node): + if isinstance(node, yaml.ScalarNode): + value = loader.construct_scalar(node) + elif isinstance(node, yaml.SequenceNode): + value = loader.construct_sequence(node, deep=True) + else: + value = loader.construct_mapping(node, deep=True) + if tag_suffix == "Ref": + return {"Ref": value} + if tag_suffix == "GetAtt": + return {"Fn::GetAtt": value.split(".") if isinstance(value, str) else value} + return {f"Fn::{tag_suffix}": value} + + +_CfnLoader.add_multi_constructor("!", _cfn_multi) + + +def load_template(source_path: str) -> dict: + text = open(source_path, encoding="utf-8").read() + try: + return json.loads(text) + except json.JSONDecodeError: + return yaml.load(text, Loader=_CfnLoader) or {} + + +def looks_like_cloudformation(data: dict) -> bool: + res = data.get("Resources") if isinstance(data, dict) else None + return isinstance(res, dict) and any( + isinstance(r, dict) and "Type" in r for r in res.values()) + + +def _ref(value): + """The logical id a Ref/GetAtt points to, or a literal string, else None.""" + if isinstance(value, str): + return value + if isinstance(value, dict): + if "Ref" in value: + return value["Ref"] + if "Fn::GetAtt" in value: + att = value["Fn::GetAtt"] + return att[0] if isinstance(att, list) and att else None + return None + + +def _tag(props: dict, key: str): + for t in props.get("Tags", []) or []: + if t.get("Key") == key: + return t.get("Value") + return None + + +class CloudFormationProvider(IacProvider): + dialect = "cloudformation" + + def parse(self, source_path: str) -> tuple[list[Node], list[Edge]]: + resources = (load_template(source_path).get("Resources") or {}) + em = Emitter("cloudformation", Confidence.ground_truth) + idx = Index() # subnet logical id -> zone node id + db_subnet_groups: dict[str, list] = {} # group logical id -> [subnet logical ids] + tg_targets: dict[str, list[str]] = {} # target group logical id -> [backend node ids] + lb_tgs: dict[str, set] = {} # lb node id -> {target group logical id} + + for lid, r in resources.items(): # pass 1: primary nodes + t, p = r.get("Type"), r.get("Properties", {}) or {} + if t == "AWS::EC2::Subnet": + zid = em.node(f"zone:{slug(lid)}", "zone", lid, + internet_facing=bool(p.get("MapPublicIpOnLaunch")), + cidr=p.get("CidrBlock")) + idx.by_id[lid] = zid + elif t in ("AWS::ElasticLoadBalancingV2::LoadBalancer", + "AWS::ElasticLoadBalancing::LoadBalancer"): + _lb(lid, p, em) + elif t == "AWS::EC2::Instance": + _instance(lid, p, em) + elif t == "AWS::WAFv2::WebACL": + em.node(f"control:{slug(lid)}", "control", p.get("Name") or lid, + control_type="waf", mode="block", vendor="aws", + mitigates=list(DEFAULT_WAF_MITIGATES)) + elif t in ("AWS::RDS::DBInstance", "AWS::RDS::DBCluster"): + _db(lid, p, em) + elif t == "AWS::RDS::DBSubnetGroup": + db_subnet_groups[lid] = [_ref(s) for s in p.get("SubnetIds", []) or []] + elif t == "AWS::S3::Bucket": + em.node(f"datastore:{slug(p.get('BucketName') or lid)}", "datastore", + p.get("BucketName") or lid, engine="s3", + sensitivity=_tag(p, "sensitivity"), data_sensitivity=_tag(p, "sensitivity")) + + for lid, r in resources.items(): # pass 2: relationships + t, p = r.get("Type"), r.get("Properties", {}) or {} + if t == "AWS::WAFv2::WebACLAssociation": + res = _ref(p.get("ResourceArn")) + acl = _ref(p.get("WebACLArn")) + if res and acl: + em.edge("protected_by", f"asset:{slug(res)}", f"control:{slug(acl)}") + elif t == "AWS::ElasticLoadBalancingV2::TargetGroup": + for tgt in p.get("Targets", []) or []: + ref = _ref(tgt.get("Id") if isinstance(tgt, dict) else tgt) + if ref: + tg_targets.setdefault(lid, []).append(f"asset:{slug(ref)}") + elif t == "AWS::ElasticLoadBalancingV2::Listener": + lb = _ref(p.get("LoadBalancerArn")) + for action in p.get("DefaultActions", []) or []: + tg = _ref(action.get("TargetGroupArn")) + if lb and tg: + lb_tgs.setdefault(f"asset:{slug(lb)}", set()).add(tg) + elif t in ("AWS::RDS::DBInstance", "AWS::RDS::DBCluster"): + did = f"datastore:{slug(p.get('DBInstanceIdentifier') or lid)}" + for sid in db_subnet_groups.get(_ref(p.get("DBSubnetGroupName")), []): + zone = idx.by_id.get(sid) + if zone: + em.edge("resides_in", did, zone) + + for lb, tgs in lb_tgs.items(): # LB routes_to each backend behind it + for tg in tgs: + for target in tg_targets.get(tg, []): + em.edge("routes_to", lb, target) + + return em.nodes(), em.edges() + + +def _lb(lid: str, p: dict, em: Emitter) -> None: + internet_facing = p.get("Scheme") == "internet-facing" + aid = em.node(f"asset:{slug(lid)}", "asset", lid, internet_reachable=internet_facing) + em.edge("protected_by", aid, em.node(f"control:{slug(lid)}-lb", "control", + f"{lid} lb", control_type="lb", vendor="aws")) + subnets = [_ref(s) for s in p.get("Subnets", []) or []] + if subnets and subnets[0]: + em.edge("resides_in", aid, f"zone:{slug(subnets[0])}") + if internet_facing: + eid = em.node(f"entrypoint:{slug(lid)}", "entrypoint", f"{lid} ingress", + public=True, protocol="https") + em.edge("routes_to", eid, aid) + + +def _instance(lid: str, p: dict, em: Emitter) -> None: + public = any(nic.get("AssociatePublicIpAddress") for nic in p.get("NetworkInterfaces", []) or []) + aid = em.node(f"asset:{slug(lid)}", "asset", lid, internet_reachable=public) + subnet = _ref(p.get("SubnetId")) + if subnet: + em.edge("resides_in", aid, f"zone:{slug(subnet)}") + + +def _db(lid: str, p: dict, em: Emitter) -> None: + ident = p.get("DBInstanceIdentifier") or lid + sens = _tag(p, "sensitivity") + em.node(f"datastore:{slug(ident)}", "datastore", ident, + sensitivity=sens, data_sensitivity=sens, + engine=p.get("Engine"), encrypted=bool(p.get("StorageEncrypted"))) diff --git a/core/reachability/ingestion/iac/common.py b/core/reachability/ingestion/iac/common.py new file mode 100644 index 0000000..4a18124 --- /dev/null +++ b/core/reachability/ingestion/iac/common.py @@ -0,0 +1,91 @@ +"""Shared helpers for IaC provider plugins. Contract: 05-context-ingestion.md §1. + +Every dialect plugin (terraform-aws, terraform-azure, ...) emits the same generic +node/edge set at ``ground_truth`` confidence, so the core never hard-codes a cloud +— the same swappability pattern as scanners and the graph backend. +""" + +from __future__ import annotations + +import json +import re + +from ...models.graph import Confidence, Edge, Node + +# An IaC-detected WAF's exact rule coverage isn't always in state; assume the +# common managed-ruleset coverage. The questionnaire can refine per deployment. +DEFAULT_WAF_MITIGATES = ["xss", "sql_injection", "path_traversal"] + + +def slug(value: str) -> str: + return re.sub(r"[^a-z0-9-]+", "-", str(value).strip().lower()).strip("-") + + +def walk_resources(data: dict) -> list[dict]: + """Flatten every resource from a `terraform show -json` state or plan.""" + out: list[dict] = [] + + def walk(module: dict) -> None: + out.extend(module.get("resources", []) or []) + for child in module.get("child_modules", []) or []: + walk(child) + + root = (data.get("values", {}) or {}).get("root_module", {}) or {} + walk(root) + if not out: # plan files put it under planned_values + walk((data.get("planned_values", {}) or {}).get("root_module", {}) or {}) + return out + + +def load_resources(source_path: str) -> list[dict]: + with open(source_path, encoding="utf-8") as fh: + return walk_resources(json.load(fh)) + + +class Index: + """Maps cloud identifiers (arn / resource id) to emitted graph node ids, so + relationship resources (associations, attachments) can be wired in a 2nd pass.""" + + def __init__(self) -> None: + self.by_arn: dict[str, str] = {} + self.by_id: dict[str, str] = {} + + +class Emitter: + """Accumulates nodes (deduped by id, attributes merged) and edges (deduped by + type/src/dst) at a fixed source + confidence.""" + + def __init__(self, source: str, confidence: Confidence) -> None: + self._source = source + self._confidence = confidence + self._nodes: dict[str, Node] = {} + self._edges: dict[tuple, Edge] = {} + + def node(self, node_id: str, type_: str, label: str, **attributes) -> str: + attrs = {k: v for k, v in attributes.items() if v is not None} + if node_id in self._nodes: + self._nodes[node_id].attributes.update(attrs) + else: + self._nodes[node_id] = Node(id=node_id, type=type_, label=label, attributes=attrs, + confidence=self._confidence, source=self._source) + return node_id + + def edge(self, type_: str, src: str, dst: str, **attributes) -> None: + key = (type_, src, dst) + if key not in self._edges: + self._edges[key] = Edge(type=type_, src=src, dst=dst, + attributes={k: v for k, v in attributes.items() if v is not None}, + confidence=self._confidence, source=self._source) + + def has(self, node_id: str) -> bool: + return node_id in self._nodes + + def set_attr(self, node_id: str, key: str, value) -> None: + if node_id in self._nodes: + self._nodes[node_id].attributes[key] = value + + def nodes(self) -> list[Node]: + return list(self._nodes.values()) + + def edges(self) -> list[Edge]: + return list(self._edges.values()) diff --git a/core/reachability/ingestion/iac/terraform_aws.py b/core/reachability/ingestion/iac/terraform_aws.py index 505053e..2c55340 100644 --- a/core/reachability/ingestion/iac/terraform_aws.py +++ b/core/reachability/ingestion/iac/terraform_aws.py @@ -1,19 +1,147 @@ """terraform-aws provider plugin (do first). Contract: 05-context-ingestion.md §1. -Resource -> graph mapping (subset): aws_lb -> asset + control{lb}; -aws_wafv2_web_acl assoc -> control{waf, block} + protected_by; SG rules -> zone -boundaries + connects_to; aws_instance -> asset (internet_reachable derived from -subnet + SG); RDS/storage -> datastore; ingress/public IP -> entrypoint{public}. +Resource -> graph mapping (from the table in 05): + aws_subnet -> zone{internet_facing from map_public_ip_on_launch} + aws_lb / aws_alb / aws_elb -> asset (the LB) + control{lb}; if internet-facing, + an entrypoint{public} that routes_to it + aws_wafv2_web_acl -> control{waf, block} + aws_wafv2_web_acl_association -> protected_by edge from the associated asset + aws_lb_listener + target group -> routes_to from the LB asset to its backends + aws_instance -> asset{internet_reachable}; resides_in its subnet + aws_db_instance / aws_rds_cluster-> datastore{sensitivity, engine, encrypted} + aws_s3_bucket -> datastore + +Two passes: primary nodes + identifier index first, then relationship resources +(associations, attachments, listeners) wired against the index. """ from __future__ import annotations -from ...models.graph import Edge, Node +from ...models.graph import Confidence, Edge, Node from .base import IacProvider +from .common import DEFAULT_WAF_MITIGATES, Emitter, Index, load_resources, slug class TerraformAwsProvider(IacProvider): dialect = "terraform-aws" def parse(self, source_path: str) -> tuple[list[Node], list[Edge]]: - raise NotImplementedError("scaffold") + resources = load_resources(source_path) + em = Emitter("terraform", Confidence.ground_truth) + idx = Index() + tg_targets: dict[str, list[str]] = {} # target_group_arn -> [backend node ids] + lb_tgs: dict[str, set] = {} # lb node id -> {target_group_arn} + db_subnet_groups: dict[str, list] = {} # db_subnet_group name -> [subnet ids] + + for r in resources: # pass 1: primary nodes + index + t, v, name = r.get("type"), r.get("values", {}) or {}, r.get("name", "") + if t == "aws_subnet": + _subnet(v, em, idx) + elif t in ("aws_lb", "aws_alb", "aws_elb"): + _lb(name, v, em, idx) + elif t == "aws_instance": + _instance(name, v, em, idx) + elif t == "aws_wafv2_web_acl": + _waf(name, v, em, idx) + elif t in ("aws_db_instance", "aws_rds_cluster"): + _db(name, v, em, idx) + elif t == "aws_db_subnet_group": + if v.get("name"): + db_subnet_groups[v["name"]] = v.get("subnet_ids") or [] + elif t == "aws_s3_bucket": + _s3(name, v, em, idx) + + for r in resources: # pass 2: relationships + t, v, name = r.get("type"), r.get("values", {}) or {}, r.get("name", "") + if t == "aws_wafv2_web_acl_association": + res = idx.by_arn.get(v.get("resource_arn")) + waf = idx.by_arn.get(v.get("web_acl_arn")) + if res and waf: + em.edge("protected_by", res, waf) + elif t == "aws_lb_target_group_attachment": + tgt = idx.by_id.get(v.get("target_id")) + if v.get("target_group_arn") and tgt: + tg_targets.setdefault(v["target_group_arn"], []).append(tgt) + elif t == "aws_lb_listener": + lb = idx.by_arn.get(v.get("load_balancer_arn")) + for action in v.get("default_action", []) or []: + if lb and action.get("target_group_arn"): + lb_tgs.setdefault(lb, set()).add(action["target_group_arn"]) + elif t in ("aws_db_instance", "aws_rds_cluster"): + # place the datastore in its DB subnet group's zone(s) + did = f"datastore:{slug(v.get('identifier') or name)}" + for sid in db_subnet_groups.get(v.get("db_subnet_group_name"), []): + zone = idx.by_id.get(sid) + if zone: + em.edge("resides_in", did, zone) + + for lb, tgs in lb_tgs.items(): # LB routes_to each backend behind it + for tg in tgs: + for target in tg_targets.get(tg, []): + em.edge("routes_to", lb, target) + + return em.nodes(), em.edges() + + +def _subnet(v: dict, em: Emitter, idx: Index) -> None: + sid = v.get("id") + if not sid: + return + zid = em.node(f"zone:{sid}", "zone", (v.get("tags") or {}).get("Name") or sid, + internet_facing=bool(v.get("map_public_ip_on_launch")), cidr=v.get("cidr_block")) + idx.by_id[sid] = zid + + +def _lb(name: str, v: dict, em: Emitter, idx: Index) -> None: + internet_facing = not v.get("internal", True) + aid = em.node(f"asset:{slug(name)}", "asset", v.get("name") or name, + internet_reachable=internet_facing) + cid = em.node(f"control:{slug(name)}-lb", "control", f"{name} lb", + control_type="lb", vendor="aws") + em.edge("protected_by", aid, cid) + if v.get("arn"): + idx.by_arn[v["arn"]] = aid + subnets = v.get("subnets") or [] + if subnets and isinstance(subnets[0], str): + em.edge("resides_in", aid, f"zone:{subnets[0]}") + if internet_facing: + eid = em.node(f"entrypoint:{slug(name)}", "entrypoint", f"{name} ingress", + public=True, protocol="https", host=v.get("dns_name")) + em.edge("routes_to", eid, aid) + + +def _instance(name: str, v: dict, em: Emitter, idx: Index) -> None: + label = (v.get("tags") or {}).get("Name") or name + aid = em.node(f"asset:{slug(label)}", "asset", label, + internet_reachable=bool(v.get("associate_public_ip_address") or v.get("public_ip")), + os=v.get("ami")) + if v.get("id"): + idx.by_id[v["id"]] = aid + if v.get("subnet_id"): + em.edge("resides_in", aid, f"zone:{v['subnet_id']}") + + +def _waf(name: str, v: dict, em: Emitter, idx: Index) -> None: + cid = em.node(f"control:{slug(name)}", "control", v.get("name") or name, + control_type="waf", mode="block", vendor="aws", + mitigates=list(DEFAULT_WAF_MITIGATES)) + if v.get("arn"): + idx.by_arn[v["arn"]] = cid + + +def _db(name: str, v: dict, em: Emitter, idx: Index) -> None: + ident = v.get("identifier") or name + sens = (v.get("tags") or {}).get("sensitivity") + did = em.node(f"datastore:{slug(ident)}", "datastore", ident, + sensitivity=sens, data_sensitivity=sens, # both: policy reads asset/datastore + engine=v.get("engine"), encrypted=bool(v.get("storage_encrypted"))) + if v.get("id"): + idx.by_id[v["id"]] = did + if v.get("arn"): + idx.by_arn[v["arn"]] = did + + +def _s3(name: str, v: dict, em: Emitter, idx: Index) -> None: + sens = (v.get("tags") or {}).get("sensitivity") + em.node(f"datastore:{slug(v.get('bucket') or name)}", "datastore", v.get("bucket") or name, + engine="s3", sensitivity=sens, data_sensitivity=sens) diff --git a/core/reachability/ingestion/iac/terraform_azure.py b/core/reachability/ingestion/iac/terraform_azure.py index d84e26b..29ba27c 100644 --- a/core/reachability/ingestion/iac/terraform_azure.py +++ b/core/reachability/ingestion/iac/terraform_azure.py @@ -1,17 +1,325 @@ -"""terraform-azure provider plugin (after aws). Contract: 05-context-ingestion.md §1. +"""terraform-azure provider plugin. Contract: 05-context-ingestion.md §1. -e.g. azurerm_lb -> asset + control{lb}; azurerm_frontdoor -> control{cdn+waf} + -entrypoint; NSG rules -> zone boundaries + connects_to. +Resource -> graph mapping (same generic node/edge set as terraform-aws, so the +engine never knows the cloud). Parsed in explicit phases: + + 1. primary nodes + indexes (subnets->zones, public IPs->entrypoints, NICs, WAF + policies, LBs, app gateways, web apps, VMs, datastores, NSG rules) + 2. VM placement: resides_in its NIC's subnet; internet_reachable + entrypoint if + the NIC has a public IP; build private-IP -> asset map for backend wiring + 3. backend routing: application_gateway backend pool IPs and LB backend pools + (via NIC associations) -> routes_to the backend asset (parity with AWS LB) + 4. segmentation: NSG association marks a subnet internet_facing (Internet inbound + rule) and derives connects_to zone->zone from inter-subnet allow rules + 5. public-IP frontends -> entrypoint routes_to the LB / app gateway + +Backend routing + connects_to let chains pivot through Azure topology like AWS. """ from __future__ import annotations -from ...models.graph import Edge, Node +from ...models.graph import Confidence, Edge, Node from .base import IacProvider +from .common import DEFAULT_WAF_MITIGATES, Emitter, Index, load_resources, slug + +_INTERNET_SOURCES = {"internet", "*", "0.0.0.0/0", "any"} +_DATASTORE_ENGINES = { + "azurerm_mssql_database": "mssql", + "azurerm_postgresql_flexible_server": "postgresql", + "azurerm_mysql_flexible_server": "mysql", + "azurerm_cosmosdb_account": "cosmosdb", + "azurerm_storage_account": "storage_account", + "azurerm_key_vault": "key_vault", +} +_WEBAPP_TYPES = {"azurerm_linux_web_app", "azurerm_windows_web_app", + "azurerm_app_service", "azurerm_function_app", "azurerm_linux_function_app"} +_VM_TYPES = {"azurerm_linux_virtual_machine", "azurerm_windows_virtual_machine"} class TerraformAzureProvider(IacProvider): dialect = "terraform-azure" def parse(self, source_path: str) -> tuple[list[Node], list[Edge]]: - raise NotImplementedError("scaffold") + resources = load_resources(source_path) + em = Emitter("terraform", Confidence.ground_truth) + idx = Index() # subnet id/name -> zone node id + s = _State() + + for r in resources: # phase 1: primary nodes + indexes + self._phase1(r, em, idx, s) + for r in resources: # phase 2: VM placement + private-IP map + if r.get("type") in _VM_TYPES: + _wire_vm(r.get("name", ""), r.get("values", {}) or {}, em, idx, s) + self._phase3_backends(resources, em, s) # ingress -> backend routes_to + self._phase4_segmentation(resources, em, idx, s) # internet_facing + connects_to + self._place_datastores(em, idx, s) # datastore resides_in its subnet + for asset_id, pip_id in s.frontends.items(): # phase 5: public-IP frontends + entry = s.pip.get(pip_id) + if entry: + em.edge("routes_to", entry, asset_id) + em.set_attr(asset_id, "internet_reachable", True) + + return em.nodes(), em.edges() + + # --- phase 1 ---------------------------------------------------------- + def _phase1(self, r: dict, em: Emitter, idx: Index, s: "_State") -> None: + t, v, name = r.get("type"), r.get("values", {}) or {}, r.get("name", "") + if t == "azurerm_subnet": + _subnet(v, name, em, idx, s) + elif t == "azurerm_network_security_group": + s.nsg_internet[v.get("id") or name] = s.nsg_internet[name] = _nsg_allows_internet(v) + sources = _nsg_internal_sources(v) + s.nsg_sources[v.get("id") or name] = sources + s.nsg_sources[name] = sources + elif t == "azurerm_public_ip": + eid = em.node(f"entrypoint:{slug(name)}", "entrypoint", v.get("name") or name, + public=True, protocol="tcp", host=v.get("fqdn") or v.get("ip_address")) + if v.get("id"): + s.pip[v["id"]] = eid + elif t == "azurerm_network_interface": + if v.get("id"): + s.nics[v["id"]] = _nic_config(v) + elif t == "azurerm_web_application_firewall_policy": + cid = em.node(f"control:{slug(name)}", "control", v.get("name") or name, + control_type="waf", mode="block", vendor="azure", + mitigates=list(DEFAULT_WAF_MITIGATES)) + if v.get("id"): + s.waf[v["id"]] = cid + elif t == "azurerm_lb": + aid = em.node(f"asset:{slug(name)}", "asset", v.get("name") or name) + em.edge("protected_by", aid, em.node(f"control:{slug(name)}-lb", "control", + f"{name} lb", control_type="lb", vendor="azure")) + if v.get("id"): + s.lb_by_id[v["id"]] = aid + fe = _frontend_public_ip(v) + if fe: + s.frontends[aid] = fe + elif t == "azurerm_lb_backend_address_pool": + if v.get("id") and s.lb_by_id.get(v.get("loadbalancer_id")): + s.pool_to_lb[v["id"]] = s.lb_by_id[v["loadbalancer_id"]] + elif t == "azurerm_network_interface_backend_address_pool_association": + if v.get("network_interface_id") and v.get("backend_address_pool_id"): + s.nic_to_pools.setdefault(v["network_interface_id"], []).append( + v["backend_address_pool_id"]) + elif t == "azurerm_application_gateway": + _app_gateway(v, name, em, s) + elif t == "azurerm_frontdoor": # classic Front Door (backend pools inline) + eid = _frontdoor_node(name, v, em) + for pool in v.get("backend_pool", []) or []: + for be in pool.get("backend", []) or []: + if be.get("address"): + s.fd_classic_backends.append((eid, str(be["address"]).lower())) + elif t == "azurerm_cdn_frontdoor_profile": # Standard/Premium (origins are separate) + eid = _frontdoor_node(name, v, em) + if v.get("id"): + s.fd_profile_entrypoint[v["id"]] = eid + elif t == "azurerm_cdn_frontdoor_origin_group": + if v.get("id"): + s.fd_origin_groups[v["id"]] = v.get("cdn_frontdoor_profile_id") + elif t == "azurerm_cdn_frontdoor_origin": + if v.get("host_name") and v.get("cdn_frontdoor_origin_group_id"): + s.fd_origins.append((str(v["host_name"]).lower(), v["cdn_frontdoor_origin_group_id"])) + elif t in _WEBAPP_TYPES: + host = v.get("default_hostname") + aid = em.node(f"asset:{slug(name)}", "asset", v.get("name") or name, + internet_reachable=True, runtime="app_service", host=host) + em.edge("routes_to", em.node(f"entrypoint:{slug(name)}", "entrypoint", + f"{name} ingress", public=True, protocol="https"), aid) + if host: # lets Front Door / App Gateway FQDN backends resolve + s.host_to_asset[str(host).lower()] = aid + elif t in _VM_TYPES: + em.node(f"asset:{slug(name)}", "asset", v.get("name") or name, + internet_reachable=False, os=v.get("source_image_reference")) + elif t == "azurerm_private_endpoint": + for conn in v.get("private_service_connection", []) or []: + rid = conn.get("private_connection_resource_id") + if rid and v.get("subnet_id"): + s.private_endpoints[rid] = v["subnet_id"] + elif t in _DATASTORE_ENGINES: + sens = (v.get("tags") or {}).get("sensitivity") + did = em.node(f"datastore:{slug(v.get('name') or name)}", "datastore", v.get("name") or name, + engine=_DATASTORE_ENGINES[t], sensitivity=sens, data_sensitivity=sens) + if v.get("id"): + s.datastore_by_id[v["id"]] = did + if v.get("delegated_subnet_id"): # VNet-integrated flexible servers + s.datastore_subnet[did] = v["delegated_subnet_id"] + + # --- phase 3: ingress -> backend ------------------------------------- + def _phase3_backends(self, resources: list, em: Emitter, s: "_State") -> None: + for r in resources: + t, v, name = r.get("type"), r.get("values", {}) or {}, r.get("name", "") + if t == "azurerm_application_gateway": + aid = f"asset:{slug(name)}" + for pool in v.get("backend_address_pool", []) or []: + for ip in pool.get("ip_addresses", []) or []: + target = s.private_ip_to_asset.get(ip) + if target: + em.edge("routes_to", aid, target) + for fqdn in pool.get("fqdns", []) or []: + target = s.host_to_asset.get(str(fqdn).lower()) + if target: + em.edge("routes_to", aid, target) + # LB -> backends via NIC backend-pool associations + for nic_id, pools in s.nic_to_pools.items(): + backend = s.nic_to_asset.get(nic_id) + for pool_id in pools: + lb = s.pool_to_lb.get(pool_id) + if lb and backend: + em.edge("routes_to", lb, backend) + # Front Door -> backend by FQDN: classic backend pools, and Std/Premium + # origins (origin -> origin group -> profile entrypoint). + for eid, address in s.fd_classic_backends: + backend = s.host_to_asset.get(address) + if backend: + em.edge("routes_to", eid, backend) + for host, og_id in s.fd_origins: + eid = s.fd_profile_entrypoint.get(s.fd_origin_groups.get(og_id)) + backend = s.host_to_asset.get(host) + if eid and backend: + em.edge("routes_to", eid, backend) + + # --- phase 4: segmentation ------------------------------------------- + def _phase4_segmentation(self, resources: list, em: Emitter, idx: Index, s: "_State") -> None: + for r in resources: + if r.get("type") != "azurerm_subnet_network_security_group_association": + continue + v = r.get("values", {}) or {} + dest_zone = idx.by_id.get(v.get("subnet_id")) + nsg = v.get("network_security_group_id") or v.get("network_security_group_name") + if not dest_zone: + continue + if s.nsg_internet.get(v.get("network_security_group_id")) or s.nsg_internet.get( + v.get("network_security_group_name")): + em.set_attr(dest_zone, "internet_facing", True) + for cidr in s.nsg_sources.get(nsg, []): + src_zone = s.cidr_to_zone.get(cidr) + if src_zone and src_zone != dest_zone: + em.edge("connects_to", src_zone, dest_zone) + + # --- datastore placement --------------------------------------------- + def _place_datastores(self, em: Emitter, idx: Index, s: "_State") -> None: + for did, subnet_id in s.datastore_subnet.items(): # delegated subnet + zone = idx.by_id.get(subnet_id) + if zone: + em.edge("resides_in", did, zone) + for rid, subnet_id in s.private_endpoints.items(): # private endpoint + did, zone = s.datastore_by_id.get(rid), idx.by_id.get(subnet_id) + if did and zone: + em.edge("resides_in", did, zone) + + +class _State: + def __init__(self) -> None: + self.nics: dict[str, dict] = {} + self.nsg_internet: dict[str, bool] = {} + self.nsg_sources: dict[str, list] = {} + self.pip: dict[str, str] = {} + self.waf: dict[str, str] = {} + self.frontends: dict[str, str] = {} + self.lb_by_id: dict[str, str] = {} + self.pool_to_lb: dict[str, str] = {} + self.nic_to_pools: dict[str, list] = {} + self.cidr_to_zone: dict[str, str] = {} + self.private_ip_to_asset: dict[str, str] = {} + self.host_to_asset: dict[str, str] = {} + self.nic_to_asset: dict[str, str] = {} + self.datastore_by_id: dict[str, str] = {} # resource id -> datastore node id + self.datastore_subnet: dict[str, str] = {} # datastore node id -> delegated subnet id + self.private_endpoints: dict[str, str] = {} # connected resource id -> subnet id + self.fd_classic_backends: list = [] # (frontdoor entrypoint id, backend address) + self.fd_profile_entrypoint: dict[str, str] = {} # profile id -> entrypoint node id + self.fd_origin_groups: dict[str, str] = {} # origin-group id -> profile id + self.fd_origins: list = [] # (origin host_name, origin-group id) + + +def _subnet(v: dict, name: str, em: Emitter, idx: Index, s: "_State") -> None: + cidr = (v.get("address_prefixes") or [None])[0] + zid = em.node(f"zone:{slug(v.get('name') or name)}", "zone", v.get("name") or name, + cidr=cidr, internet_facing=False) + for key in (v.get("id"), v.get("name"), name): + if key: + idx.by_id[key] = zid + if cidr: + s.cidr_to_zone[cidr] = zid + + +def _nsg_allows_internet(v: dict) -> bool: + return any(_rule_sources(rule) & _INTERNET_SOURCES for rule in _inbound_allow_rules(v)) + + +def _nsg_internal_sources(v: dict) -> list[str]: + out: list[str] = [] + for rule in _inbound_allow_rules(v): + for src in _rule_sources(rule): + if src not in _INTERNET_SOURCES and src not in out: + out.append(src) + return out + + +def _inbound_allow_rules(v: dict) -> list[dict]: + return [r for r in (v.get("security_rule", []) or []) + if str(r.get("access", "")).lower() == "allow" + and str(r.get("direction", "")).lower() == "inbound"] + + +def _rule_sources(rule: dict) -> set[str]: + out = {str(rule.get("source_address_prefix", "")).lower()} if rule.get("source_address_prefix") else set() + out |= {str(p).lower() for p in (rule.get("source_address_prefixes") or [])} + return out + + +def _nic_config(v: dict) -> dict: + first = (v.get("ip_configuration") or [{}])[0] if v.get("ip_configuration") else {} + return {"subnet": first.get("subnet_id"), "public_ip": first.get("public_ip_address_id"), + "private_ip": first.get("private_ip_address")} + + +def _frontend_public_ip(v: dict) -> str | None: + for fe in v.get("frontend_ip_configuration", []) or []: + if fe.get("public_ip_address_id"): + return fe["public_ip_address_id"] + return None + + +def _frontdoor_node(name: str, v: dict, em: Emitter) -> str: + """control{cdn+waf} + public entrypoint; returns the entrypoint node id.""" + cid = em.node(f"control:{slug(name)}", "control", v.get("name") or name, + control_type="cdn+waf", mode="block", vendor="azure", + mitigates=list(DEFAULT_WAF_MITIGATES)) + eid = em.node(f"entrypoint:{slug(name)}", "entrypoint", f"{name} ingress", + public=True, protocol="https", host=v.get("host_name") or v.get("cname")) + em.edge("protected_by", eid, cid) + return eid + + +def _app_gateway(v: dict, name: str, em: Emitter, s: "_State") -> None: + aid = em.node(f"asset:{slug(name)}", "asset", v.get("name") or name) + em.edge("protected_by", aid, em.node(f"control:{slug(name)}-agw", "control", + f"{name} app gateway", control_type="app_gateway", vendor="azure")) + fe = _frontend_public_ip(v) + if fe: + s.frontends[aid] = fe + policy = v.get("firewall_policy_id") + if policy and policy in s.waf: + em.edge("protected_by", aid, s.waf[policy]) + + +def _wire_vm(name: str, v: dict, em: Emitter, idx: Index, s: "_State") -> None: + aid = f"asset:{slug(name)}" + if not em.has(aid): + return + for nic_id in v.get("network_interface_ids") or []: + nic = s.nics.get(nic_id) + if not nic: + continue + s.nic_to_asset[nic_id] = aid + if nic.get("private_ip"): + s.private_ip_to_asset[nic["private_ip"]] = aid + zone = idx.by_id.get(nic.get("subnet")) + if zone: + em.edge("resides_in", aid, zone) + if nic.get("public_ip"): + em.set_attr(aid, "internet_reachable", True) + entry = s.pip.get(nic["public_ip"]) + if entry: + em.edge("routes_to", entry, aid) diff --git a/core/reachability/ingestion/pipeline.py b/core/reachability/ingestion/pipeline.py index e8aa32e..9998be1 100644 --- a/core/reachability/ingestion/pipeline.py +++ b/core/reachability/ingestion/pipeline.py @@ -7,18 +7,24 @@ from __future__ import annotations from ..graphstore.base import GraphStore +from ..models.graph import Edge, Node +from .reconcile import ReconciliationReport, reconcile class IngestionPipeline: def __init__(self, graph: GraphStore) -> None: self._graph = graph - def ingest(self, sources: list) -> "ReconciliationReport": - """Run each source -> nodes/edges, reconcile, upsert, return the report.""" - raise NotImplementedError("scaffold") - - -class ReconciliationReport: - """What was ingested, from where, at what confidence, and every conflict found. - Itself an onboarding artifact: where documented architecture and real - deployment disagree (05-context-ingestion.md §Output).""" + def ingest(self, sources: list[tuple[list[Node], list[Edge]]]) -> ReconciliationReport: + """Reconcile (nodes, edges) from each source into one graph and upsert it. + + Returns the reconciliation report: what was ingested, from where, at what + confidence, and every conflict/orphan found.""" + all_nodes = [n for nodes, _ in sources for n in nodes] + all_edges = [e for _, edges in sources for e in edges] + report = reconcile(all_nodes, all_edges) + for node in report.nodes: + self._graph.upsert_node(node) + for edge in report.edges: + self._graph.upsert_edge(edge) + return report diff --git a/core/reachability/ingestion/questionnaire.py b/core/reachability/ingestion/questionnaire.py index 72e16ad..9571292 100644 --- a/core/reachability/ingestion/questionnaire.py +++ b/core/reachability/ingestion/questionnaire.py @@ -4,13 +4,135 @@ Produces ``declared``-confidence nodes/edges: exposure, controls not visible in IaC, the crown-jewel map (cannot be inferred), and segmentation. Starting here proves the graph + reasoning pipeline end-to-end before any parser exists. + +Modeling choices (declared confidence throughout): + * Each ``exposure.internet_facing_services`` entry -> an ``asset`` plus a + synthesized public ``entrypoint`` with a ``routes_to`` edge, so the + reachability traversal (which seeds from public entrypoints) can reach it. + * Each ``controls`` entry -> a ``control`` node + a ``protected_by`` edge from + every asset it ``protects``. + * Each ``data.crown_jewels`` entry -> an ``asset`` carrying ``data_sensitivity``. + * Each ``segmentation.zones`` entry -> a ``zone``; ``connections`` -> ``connects_to``. + +Identity resolution is intra-document: a service named in several sections +becomes one node with merged attributes (full cross-source reconciliation lives +in ingestion/reconcile.py). """ from __future__ import annotations -from ..models.graph import Edge, Node +import re + +import yaml + +from ..models.graph import Confidence, Edge, Node + +_SOURCE = "questionnaire" +_CONF = Confidence.declared + + +def _slug(value: str) -> str: + return re.sub(r"[^a-z0-9-]+", "-", str(value).strip().lower()).strip("-") + + +class _Builder: + """Accumulates nodes (deduped by id, attributes merged) and edges.""" + + def __init__(self) -> None: + self.nodes: dict[str, Node] = {} + self.edges: list[Edge] = [] + + def node(self, node_id: str, type_: str, label: str, **attributes) -> Node: + existing = self.nodes.get(node_id) + if existing is None: + existing = Node( + id=node_id, type=type_, label=label, + attributes={k: v for k, v in attributes.items() if v is not None}, + confidence=_CONF, source=_SOURCE, + ) + self.nodes[node_id] = existing + else: + existing.attributes.update({k: v for k, v in attributes.items() if v is not None}) + return existing + + def edge(self, type_: str, src: str, dst: str, **attributes) -> None: + self.edges.append( + Edge(type=type_, src=src, dst=dst, + attributes={k: v for k, v in attributes.items() if v is not None}, + confidence=_CONF, source=_SOURCE) + ) def load(answers_path: str) -> tuple[list[Node], list[Edge]]: - """Parse questionnaire.answers.yaml into declared-confidence nodes/edges.""" - raise NotImplementedError("scaffold") + """Parse questionnaire answers YAML into declared-confidence nodes/edges.""" + with open(answers_path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + return build(data) + + +def build(data: dict) -> tuple[list[Node], list[Edge]]: + """Pure transform: questionnaire dict -> (nodes, edges). Testable without I/O.""" + b = _Builder() + + # --- exposure: internet-facing services -> asset + public entrypoint --- + # An entry may be a bare name or a {name, host, url} mapping; the host lets DAST + # findings (which carry only a URL/host) resolve to this asset (see resolve.py). + for svc in (data.get("exposure", {}) or {}).get("internet_facing_services", []) or []: + if isinstance(svc, dict): + name = svc.get("name") or svc.get("service") + host, url = svc.get("host") or svc.get("hostname"), svc.get("url") + else: + name, host, url = svc, None, None + asset_id = f"asset:{_slug(name)}" + entry_id = f"entrypoint:{_slug(name)}" + b.node(asset_id, "asset", name, internet_reachable=True, host=host) + b.node(entry_id, "entrypoint", f"{name} ingress", public=True, host=host, url=url) + b.edge("routes_to", entry_id, asset_id) + + # --- controls not visible in IaC -> control + protected_by --- + for ctl in data.get("controls", []) or []: + vendor = ctl.get("vendor") or ctl.get("type") + ctl_id = f"control:{_slug(vendor)}" + b.node( + ctl_id, "control", vendor, + control_type=ctl.get("type"), vendor=ctl.get("vendor"), + mode=ctl.get("mode"), mitigates=ctl.get("mitigates") or [], + ) + for asset in ctl.get("protects", []) or []: + asset_id = f"asset:{_slug(asset)}" + b.node(asset_id, "asset", asset) + b.edge("protected_by", asset_id, ctl_id) + + # --- crown-jewel map (cannot be inferred) -> asset data_sensitivity --- + for cj in (data.get("data", {}) or {}).get("crown_jewels", []) or []: + asset = cj.get("asset") + if not asset: + continue + b.node(f"asset:{_slug(asset)}", "asset", asset, + data_sensitivity=cj.get("sensitivity", "crown_jewel")) + + # --- segmentation -> zones + connects_to --- + seg = data.get("segmentation", {}) or {} + for zone in seg.get("zones", []) or []: + zid = zone.get("id") + if not zid: + continue + b.node(f"zone:{_slug(zid)}", "zone", zid, + internet_facing=zone.get("internet_facing")) + for conn in seg.get("connections", []) or []: + src, dst = conn.get("from"), conn.get("to") + if not (src and dst): + continue + b.node(f"zone:{_slug(src)}", "zone", src) + b.node(f"zone:{_slug(dst)}", "zone", dst) + b.edge("connects_to", f"zone:{_slug(src)}", f"zone:{_slug(dst)}") + + return list(b.nodes.values()), b.edges + + +def populate(graph, nodes: list[Node], edges: list[Edge]) -> None: + """Convenience: upsert a node/edge set into any GraphStore.""" + for n in nodes: + graph.upsert_node(n) + for e in edges: + graph.upsert_edge(e) diff --git a/core/reachability/ingestion/reconcile.py b/core/reachability/ingestion/reconcile.py index 734f12f..e60c386 100644 --- a/core/reachability/ingestion/reconcile.py +++ b/core/reachability/ingestion/reconcile.py @@ -1,18 +1,108 @@ """Reconciliation. Contract: schema/05-context-ingestion.md §Reconciliation. - 1. Identity resolution — match nodes across sources by stable keys (ARN/id, - hostname, purl). Same entity = one node, attributes merged. + 1. Identity resolution — match nodes across sources by stable id. Same entity + from IaC + questionnaire = one node, attributes merged. 2. Confidence wins ties — ground_truth overrides declared on conflict, but the conflict is RECORDED, not erased ("you said X, your IaC says Y"). - 3. Orphans flagged — a questionnaire control protecting an asset with no inbound - path in the IaC graph is suspicious; flag for review, don't wire it in blindly. + 3. Orphans flagged — a questionnaire control that protects an asset with no + inbound path in the IaC graph is suspicious; flag for review, don't wire it + in blindly. + +Output is a ReconciliationReport: the merged graph plus the provenance, conflicts, +and orphans — itself a useful onboarding artifact (05 §Output). """ from __future__ import annotations -from ..models.graph import Edge, Node +from collections import Counter +from dataclasses import dataclass, field + +from ..models.graph import Confidence, Edge, Node + +_CONF_RANK = {Confidence.inferred: 0, Confidence.declared: 1, Confidence.ground_truth: 2} + + +@dataclass +class ReconciliationReport: + nodes: list[Node] + edges: list[Edge] + conflicts: list[dict] = field(default_factory=list) + orphans: list[dict] = field(default_factory=list) + sources: dict = field(default_factory=dict) # source name -> nodes contributed + + +def reconcile(nodes: list[Node], edges: list[Edge]) -> ReconciliationReport: + merged: dict[str, Node] = {} + conflicts: list[dict] = [] + sources: Counter = Counter() + + for n in nodes: + sources[n.source or "unknown"] += 1 + if n.id in merged: + _merge_node(merged[n.id], n, conflicts) + else: + merged[n.id] = Node(id=n.id, type=n.type, label=n.label, + attributes=dict(n.attributes), confidence=n.confidence, source=n.source) + + medges: dict[tuple, Edge] = {} + for e in edges: + key = (e.type, e.src, e.dst) + if key in medges: + if _rank(e.confidence) > _rank(medges[key].confidence): + medges[key].confidence = e.confidence + medges[key].attributes.update(e.attributes) + else: + medges[key] = Edge(type=e.type, src=e.src, dst=e.dst, + attributes=dict(e.attributes), confidence=e.confidence, source=e.source) + + node_list, edge_list = list(merged.values()), list(medges.values()) + return ReconciliationReport( + nodes=node_list, edges=edge_list, conflicts=conflicts, + orphans=_find_orphans(merged, edge_list), sources=dict(sources), + ) + + +def _rank(conf: Confidence) -> int: + return _CONF_RANK.get(conf, 1) + + +def _merge_node(existing: Node, incoming: Node, conflicts: list[dict]) -> None: + ex_rank, in_rank = _rank(existing.confidence), _rank(incoming.confidence) + for key, value in incoming.attributes.items(): + old = existing.attributes.get(key) + if key in existing.attributes and old is not None and value is not None and old != value: + incoming_wins = in_rank > ex_rank + conflicts.append({ + "node": existing.id, "attribute": key, + "values": {existing.source or "?": old, incoming.source or "?": value}, + "winner": (incoming.source if incoming_wins else existing.source), + "winning_value": value if incoming_wins else old, + }) + if incoming_wins: + existing.attributes[key] = value + elif old is None: + existing.attributes[key] = value + if in_rank > ex_rank: # the entity is confirmed by a stronger source + existing.confidence = incoming.confidence + existing.source = incoming.source -def reconcile(nodes: list[Node], edges: list[Edge]) -> tuple[list[Node], list[Edge], list[dict]]: - """Return merged nodes, merged edges, and a list of recorded conflicts/orphans.""" - raise NotImplementedError("scaffold") +def _find_orphans(nodes: dict[str, Node], edges: list[Edge]) -> list[dict]: + """A declared control protecting an asset that the (stronger) graph shows no + inbound path to is suspicious — surface it rather than trusting it blindly.""" + has_inbound = {e.dst for e in edges if e.type == "routes_to"} + orphans: list[dict] = [] + for e in edges: + if e.type != "protected_by": + continue + control = nodes.get(e.dst) + asset = nodes.get(e.src) + if control is None or control.confidence is not Confidence.declared: + continue + reachable = e.src in has_inbound or (asset and asset.attributes.get("internet_reachable")) + if not reachable: + orphans.append({ + "control": e.dst, "protects": e.src, + "reason": "declared control protects an asset with no inbound path in the graph", + }) + return orphans diff --git a/core/reachability/ingestion/report.py b/core/reachability/ingestion/report.py index 7fde45d..b682dd2 100644 --- a/core/reachability/ingestion/report.py +++ b/core/reachability/ingestion/report.py @@ -1,7 +1,43 @@ -"""Render the reconciliation report (05-context-ingestion.md §Output).""" +"""Render the reconciliation report (05-context-ingestion.md §Output). + +ASCII-only (printed to terminals incl. Windows consoles). The report tells the +team where their documented architecture and their actual deployment disagree. +""" from __future__ import annotations +from collections import Counter + +from .reconcile import ReconciliationReport + + +def render(report: ReconciliationReport) -> str: + out: list[str] = [ + "Reconciliation report", + "=" * 60, + f"Ingested {len(report.nodes)} nodes, {len(report.edges)} edges.", + ] + if report.sources: + by_src = ", ".join(f"{src}: {count}" for src, count in sorted(report.sources.items())) + out.append(f" sources (nodes contributed): {by_src}") + by_type = Counter(n.type for n in report.nodes) + out.append(f" by type: {dict(by_type)}") + out.append("") + + if report.conflicts: + out.append(f"Conflicts ({len(report.conflicts)}) - you said X, your IaC says Y:") + for c in report.conflicts: + vals = ", ".join(f"{src}={val!r}" for src, val in c["values"].items()) + out.append(f" {c['node']}.{c['attribute']}: {vals} -> kept {c['winner']}" + f"={c['winning_value']!r}") + out.append("") + + if report.orphans: + out.append(f"Orphans / suspicious ({len(report.orphans)}):") + for o in report.orphans: + out.append(f" {o['control']} protects {o['protects']}: {o['reason']}") + out.append("") -def render(report) -> str: - raise NotImplementedError("scaffold") + if not report.conflicts and not report.orphans: + out.append("No conflicts or orphans - sources agree.") + return "\n".join(out) diff --git a/core/reachability/ingestion/resolve.py b/core/reachability/ingestion/resolve.py new file mode 100644 index 0000000..977b34c --- /dev/null +++ b/core/reachability/ingestion/resolve.py @@ -0,0 +1,67 @@ +"""Asset-ref resolution: map a finding's raw locator to a context-graph node. + +The finding schema (01) says ``asset_ref.locator`` is used to resolve ``node_id`` +when it isn't pre-known. Two locator shapes are resolved here: + + * ``component`` (SCA) -> the asset that depends_on that component (via SBOM). + * ``host`` / ``url`` (DAST) -> the entrypoint whose hostname matches, resolved to + the asset it routes to (so the finding scores against that asset's reachability + and controls). This is what lets a ZAP finding on https://app.example.com flow + through reachability/chaining/policy like any other. + +Hostnames come from ingestion: the questionnaire (per-service host) and IaC (an LB +DNS name, a public-IP FQDN, a Front Door host). +""" + +from __future__ import annotations + +from urllib.parse import urlsplit + +from . import sbom + + +def resolve_asset_ref(finding, graph) -> bool: + """Resolve a finding's node_id from its locator. Returns True if it set one.""" + if finding.asset_ref.node_id: + return False + if sbom.resolve_component_asset(finding, graph): + return True + return _resolve_host(finding, graph) + + +def _resolve_host(finding, graph) -> bool: + host = finding.asset_ref.locator.get("host") + if not host: + url = finding.asset_ref.locator.get("url") + host = urlsplit(url).hostname if url else None + if not host: + return False + host = host.lower() + + # Prefer an entrypoint (it carries the public hostname) and resolve to the asset + # it fronts; fall back to an asset that directly advertises the host. + for node in graph.nodes_by_type("entrypoint"): + if _host_matches(node, host): + finding.asset_ref.node_id = _fronted_asset(node, graph) or node.id + return True + for node in graph.nodes_by_type("asset"): + if _host_matches(node, host): + finding.asset_ref.node_id = node.id + return True + return False + + +def _host_matches(node, host: str) -> bool: + candidates = [] + if node.attributes.get("host"): + candidates.append(node.attributes["host"]) + candidates.extend(node.attributes.get("hostnames") or []) + return any(str(c).lower() == host for c in candidates) + + +def _fronted_asset(entrypoint, graph) -> str | None: + for edge in graph.neighbors(entrypoint.id, ["routes_to"], "out"): + target = graph.get_node(edge.dst) + if target is not None and target.type == "asset": + return target.id + return None diff --git a/core/reachability/ingestion/sbom.py b/core/reachability/ingestion/sbom.py index f796d97..703f68c 100644 --- a/core/reachability/ingestion/sbom.py +++ b/core/reachability/ingestion/sbom.py @@ -4,12 +4,148 @@ linked to its asset via ``depends_on``. This is what lets vulnerable_dependency findings resolve to a real place in the graph — enabling the reachability downgrade that kills most dependency-CVE noise. + +Component node ids use the ``component:@`` convention so a scanner +finding's ``asset_ref.locator.component`` (e.g. Grype's "lodash@4.17.20") resolves +to the owning asset with a direct lookup (see ``resolve_component_asset``). """ from __future__ import annotations -from ..models.graph import Edge, Node +import json +import re + +from ..models.graph import Confidence, Edge, Node + +_GT = Confidence.ground_truth +_SOURCE = "sbom" + + +def _slug(value: str) -> str: + return re.sub(r"[^a-z0-9-]+", "-", str(value).strip().lower()).strip("-") + + +def _ecosystem_from_purl(purl: str | None) -> str | None: + # "pkg:npm/lodash@4.17.20" -> "npm"; "pkg:golang/github.com/x/y" -> "golang" + if purl and purl.startswith("pkg:"): + return purl[4:].split("/", 1)[0] + return None + + +def _component_node(name: str, version: str | None, purl: str | None, + ecosystem: str | None, direct: bool | None) -> tuple[str, Node]: + key = f"{name}@{version}" if version else name + cid = f"component:{key}" + return cid, Node( + id=cid, type="component", label=key, + attributes={"purl": purl, "version": version, "ecosystem": ecosystem, "direct": direct}, + confidence=_GT, source=_SOURCE, + ) + + +def load(sbom_path: str, asset: str | None = None) -> tuple[list[Node], list[Edge]]: + """Parse a CycloneDX or SPDX JSON SBOM into component nodes + depends_on edges. + + ``asset`` overrides the owning asset node id; otherwise it is derived from the + SBOM's primary component/document (``asset:``).""" + with open(sbom_path, encoding="utf-8") as fh: + data = json.load(fh) + return build(data, asset) + + +def build(data: dict, asset: str | None = None) -> tuple[list[Node], list[Edge]]: + if data.get("bomFormat") == "CycloneDX" or "components" in data: + return _from_cyclonedx(data, asset) + if str(data.get("spdxVersion", "")).startswith("SPDX") or "packages" in data: + return _from_spdx(data, asset) + raise ValueError("unrecognized SBOM format (expected CycloneDX or SPDX JSON)") + + +def _from_cyclonedx(data: dict, asset: str | None) -> tuple[list[Node], list[Edge]]: + primary = (data.get("metadata", {}) or {}).get("component", {}) or {} + asset_id = asset or (f"asset:{_slug(primary['name'])}" if primary.get("name") else "asset:application") + direct_refs = _cdx_direct_refs(data, primary.get("bom-ref")) + + nodes: list[Node] = [Node(id=asset_id, type="asset", + label=primary.get("name", asset_id), attributes={}, + confidence=_GT, source=_SOURCE)] + edges: list[Edge] = [] + for comp in data.get("components", []) or []: + name = comp.get("name") + if not name: + continue + purl = comp.get("purl") + direct = (comp.get("bom-ref") in direct_refs) if direct_refs is not None else True + cid, node = _component_node(name, comp.get("version"), purl, + _ecosystem_from_purl(purl) or comp.get("type"), direct) + nodes.append(node) + edges.append(Edge(type="depends_on", src=asset_id, dst=cid, confidence=_GT, source=_SOURCE)) + return nodes, edges + + +def _cdx_direct_refs(data: dict, primary_ref: str | None) -> set | None: + """Refs the primary component depends on directly, or None if no dependency + graph is present (then everything is treated as direct).""" + deps = data.get("dependencies") + if not primary_ref or not deps: + return None + for d in deps: + if d.get("ref") == primary_ref: + return set(d.get("dependsOn", []) or []) + return set() + + +def _from_spdx(data: dict, asset: str | None) -> tuple[list[Node], list[Edge]]: + described = set(data.get("documentDescribes", []) or []) + packages = data.get("packages", []) or [] + primary = next((p for p in packages if p.get("SPDXID") in described), None) + asset_id = asset or (f"asset:{_slug(primary['name'])}" if primary and primary.get("name") + else "asset:application") + direct = {r.get("relatedSpdxElement") for r in data.get("relationships", []) or [] + if r.get("relationshipType") == "DEPENDS_ON" and r.get("spdxElementId") in described} + + nodes: list[Node] = [Node(id=asset_id, type="asset", + label=primary.get("name") if primary else asset_id, + attributes={}, confidence=_GT, source=_SOURCE)] + edges: list[Edge] = [] + for pkg in packages: + if pkg.get("SPDXID") in described: # the primary application itself, not a dependency + continue + name = pkg.get("name") + if not name: + continue + purl = _spdx_purl(pkg) + is_direct = (pkg.get("SPDXID") in direct) if direct else True + cid, node = _component_node(name, pkg.get("versionInfo"), purl, + _ecosystem_from_purl(purl), is_direct) + nodes.append(node) + edges.append(Edge(type="depends_on", src=asset_id, dst=cid, confidence=_GT, source=_SOURCE)) + return nodes, edges + + +def _spdx_purl(pkg: dict) -> str | None: + for ref in pkg.get("externalRefs", []) or []: + if ref.get("referenceType") == "purl": + return ref.get("referenceLocator") + return None + +def resolve_component_asset(finding, graph) -> bool: + """Resolve a component finding to its owning asset via the depends_on graph. -def load(sbom_path: str) -> tuple[list[Node], list[Edge]]: - raise NotImplementedError("scaffold: detect CycloneDX vs SPDX; emit component nodes + depends_on") + For a finding with ``asset_ref.locator.component`` but no resolved ``node_id`` + (the typical SCA shape), find the component node and the asset that depends on + it, and set the finding's node_id. This is what makes a vulnerable_dependency + finding score against its asset's reachability (the dependency-noise downgrade).""" + if finding.asset_ref.node_id: + return False + locator = finding.asset_ref.locator.get("component") + if not locator: + return False + cid = f"component:{locator}" + if graph.get_node(cid) is None: + return False + for edge in graph.neighbors(cid, ["depends_on"], "in"): # asset --depends_on--> component + finding.asset_ref.node_id = edge.src + return True + return False diff --git a/core/reachability/llm/enrichment.py b/core/reachability/llm/enrichment.py index 029ce55..c67372f 100644 --- a/core/reachability/llm/enrichment.py +++ b/core/reachability/llm/enrichment.py @@ -1,16 +1,119 @@ -"""Pre/postcondition enrichment PROPOSALS. Contract: 06 §LLM-assisted. +"""Pre/postcondition enrichment PROPOSALS. Contract: 06-attack-path-chaining.md §LLM. When a finding's category defaults are too coarse, the LLM may PROPOSE more -specific pre/postconditions. Proposals go to a human review queue and never -silently alter scoring — they enter the deterministic pipeline only after -confirmation. +specific pre/postconditions. Proposals go to a HUMAN REVIEW queue and never +silently alter scoring — they enter the deterministic pipeline only after a human +confirms (``apply_proposal``). "The LLM proposes; the engine decides." + +Two guardrails keep the model out of the authoritative path: + 1. Without a configured provider there is no proposal — the pipeline uses the + deterministic category defaults (conditions_map). The model is never required. + 2. Every proposed token is validated against the controlled namespace:value + vocabulary; anything the engine can't parse is dropped, so the model cannot + inject arbitrary state into chaining. """ from __future__ import annotations +import json +import re +from dataclasses import dataclass, field + +from ..models.conditions import parse_condition from ..models.finding import Finding +_SYSTEM = ( + "You refine the preconditions and postconditions of a security finding so an " + "attack-path chaining engine can reason about it. Use ONLY tokens of the form " + "namespace:value[:param] with these namespaces: network (internet_reachable, " + "internal_only, same_segment, reachable_from:), auth (unauthenticated, " + "authenticated_user, authenticated_admin), access (session_token, " + "read_filesystem, write_filesystem, db_read, db_write, cloud_creds, secrets), " + "exec (code_on_host, javascript_in_victim_browser, ssrf_outbound, " + "command_injection), position (foothold:, pivot_to:). Preconditions " + "are what the attacker needs; postconditions are what exploitation grants. You " + "are PROPOSING for human review — do not assign severity or scores. Respond with " + 'ONLY JSON: {"preconditions": [...], "postconditions": [...], "rationale": "..."}.' +) + + +@dataclass +class Proposal: + finding_id: str + preconditions: list[str] = field(default_factory=list) + postconditions: list[str] = field(default_factory=list) + rationale: str = "" + status: str = "proposed" # proposed -> (human) -> applied + + +def propose_conditions(finding: Finding, *, provider=None) -> Proposal | None: + """Ask the model to PROPOSE refined pre/postconditions for human review. + + Returns None when no model is configured (the deterministic defaults stand). + Proposed tokens are validated against the vocabulary; invalid ones are dropped. + """ + if provider is None or not provider.available(): + return None + try: + raw = provider.complete(_prompt(finding), system=_SYSTEM) + data = _parse_json(raw) + except Exception: + return None + if data is None: + return None + return Proposal( + finding_id=finding.finding_id, + preconditions=_valid_tokens(data.get("preconditions")), + postconditions=_valid_tokens(data.get("postconditions")), + rationale=str(data.get("rationale", "")), + ) + + +def apply_proposal(finding: Finding, proposal: Proposal) -> Finding: + """Apply a HUMAN-APPROVED proposal to a finding. This is the only path by which + a proposal enters the finding; it is never called automatically by the pipeline.""" + finding.preconditions = list(proposal.preconditions) + finding.postconditions = list(proposal.postconditions) + proposal.status = "applied" + return finding + + +# --- helpers ------------------------------------------------------------------- + +def _prompt(finding: Finding) -> str: + return "\n".join([ + f"category: {finding.category.type}", + f"cwe: {', '.join(finding.category.cwe) or '(none)'}", + f"title: {finding.title}", + f"description: {finding.description}", + f"current preconditions: {finding.preconditions or '(none)'}", + f"current postconditions: {finding.postconditions or '(none)'}", + "Propose refined preconditions and postconditions.", + ]) + + +def _parse_json(text: str) -> dict | None: + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + m = re.search(r"\{.*\}", text, re.DOTALL) # tolerate prose around the JSON + if not m: + return None + try: + return json.loads(m.group(0)) + except json.JSONDecodeError: + return None + -def propose_conditions(finding: Finding) -> dict: - """Return proposed {preconditions, postconditions} for HUMAN REVIEW only.""" - raise NotImplementedError("scaffold") +def _valid_tokens(tokens) -> list[str]: + out: list[str] = [] + for t in tokens or []: + try: + parse_condition(str(t)) # must be a known namespace:value token + except ValueError: + continue + if t not in out: + out.append(t) + return out diff --git a/core/reachability/llm/narrative.py b/core/reachability/llm/narrative.py index caff8d9..a92f68f 100644 --- a/core/reachability/llm/narrative.py +++ b/core/reachability/llm/narrative.py @@ -1,13 +1,142 @@ -"""Chain -> readable prose. Contract: 06 §LLM-assisted (narrative generation). +"""Chain -> readable prose. Contract: 06-attack-path-chaining.md §LLM-assisted. Pure presentation over a FINISHED, deterministic chain. The chain (steps, score, -target) is already computed; the LLM only renders the decided path into English. +target) is already computed; this only renders the decided path into English. Build order: this is the LAST thing added. + +Two narrators behind one entry point: + * a deterministic TEMPLATE narrator — the default, no network, always works + (keeps the product self-hostable with no phone-home); + * an optional LLM narrator that rephrases the SAME decided facts into smoother + prose when a provider is configured. It never changes, adds, or re-ranks any + fact; on any error it falls back to the template. "The LLM explains; the + engine decides." """ from __future__ import annotations +from dataclasses import dataclass + +_GAINS = { + "access:db_read": "database read access", + "access:db_write": "database write access", + "access:cloud_creds": "cloud credentials", + "access:secrets": "secrets", + "access:session_token": "a victim session token", + "access:read_filesystem": "filesystem read access", + "access:write_filesystem": "filesystem write access", + "exec:code_on_host": "code execution on the host", + "exec:ssrf_outbound": "server-side request forgery", + "exec:javascript_in_victim_browser": "script execution in a victim's browser", + "auth:authenticated_user": "an authenticated user session", + "auth:authenticated_admin": "admin privileges", +} + +_SYSTEM = ( + "You turn a security attack chain into 2-3 sentences of plain English for a " + "security report. The chain, its steps, its target, and its severity were " + "already computed deterministically. Do NOT change, add, re-order, or re-rank " + "any fact — only describe the given path. Be concise and concrete." +) + + +@dataclass +class _Step: + category: str + asset: str + post: list[str] + + +@dataclass +class _Facts: + steps: list[_Step] + target: str | None + reaches_crown_jewel: bool + length: int + + +def narrate_chain(chain, findings_by_id: dict, *, provider=None) -> str: + """Render one chain to prose: LLM if a provider is available, else template.""" + facts = _facts(chain, findings_by_id) + if provider is not None and provider.available(): + try: + text = provider.complete(_prompt(facts), system=_SYSTEM) + if text: + return text + except Exception: + pass # advisory only — never let the model break reporting + return _template_narrative(facts) + + +def narrate_all(chains: list, findings: list, *, provider=None) -> dict: + by_id = {f.finding_id: f for f in findings} + return {c.id: narrate_chain(c, by_id, provider=provider) for c in chains} + + +# --- shared structured facts --------------------------------------------------- + +def _facts(chain, findings_by_id: dict) -> _Facts: + steps = [] + for fid in chain.steps: + f = findings_by_id.get(fid) + if f is not None: + steps.append(_Step(category=f.category.type, asset=f.asset_ref.node_id or fid, + post=list(f.postconditions))) + else: + steps.append(_Step(category="step", asset=fid, post=[])) + return _Facts(steps=steps, target=getattr(chain, "target", None), + reaches_crown_jewel=getattr(chain, "reaches_crown_jewel", False), + length=len(chain.steps)) + + +# --- deterministic template narrator (default) --------------------------------- + +def _template_narrative(facts: _Facts) -> str: + clauses = [] + for i, s in enumerate(facts.steps): + lead = "An unauthenticated attacker on the internet exploits" if i == 0 else "they then exploit" + clause = f"{lead} the {s.category} on {s.asset}" + gains = _gains_phrase(s.post) + if gains: + clause += f", {gains}" + clauses.append(clause) + body = "; ".join(clauses) if clauses else "No reportable steps" + n = facts.length + cj = "the crown-jewel " if facts.reaches_crown_jewel else "" + tail = f". This chain reaches {cj}{facts.target} across {n} hop{'s' if n != 1 else ''}." + return body[:1].upper() + body[1:] + tail + + +def _gains_phrase(post: list[str]) -> str: + pivots, gains = [], [] + for t in post: + if t.startswith("position:pivot_to:") or t.startswith("position:foothold:"): + pivots.append(t.split(":")[-1]) + elif t in _GAINS: + gains.append(_GAINS[t]) + phrases = [] + if pivots: + phrases.append("pivoting to " + _join(pivots)) + if gains: + phrases.append("gaining " + _join(gains)) + return " and ".join(phrases) + + +def _join(items: list[str]) -> str: + items = list(dict.fromkeys(items)) # dedupe, keep order + if len(items) <= 1: + return items[0] if items else "" + return ", ".join(items[:-1]) + " and " + items[-1] + + +# --- LLM narrator (optional) --------------------------------------------------- -def render_chain_narrative(chain, findings_by_id: dict, graph) -> str: - """Turn step1 -> step2 -> step3 into prose. Never alters the chain or scores.""" - raise NotImplementedError("scaffold") +def _prompt(facts: _Facts) -> str: + lines = ["Attack chain (already decided — render only):"] + for i, s in enumerate(facts.steps, 1): + post = ", ".join(s.post) if s.post else "(no new capability)" + lines.append(f" {i}. exploit {s.category} on {s.asset} -> grants {post}") + sens = "crown-jewel " if facts.reaches_crown_jewel else "" + lines.append(f"Terminates at {sens}{facts.target} ({facts.length} hops).") + lines.append("Write the narrative.") + return "\n".join(lines) diff --git a/core/reachability/llm/provider.py b/core/reachability/llm/provider.py index 4b6217f..eac3acd 100644 --- a/core/reachability/llm/provider.py +++ b/core/reachability/llm/provider.py @@ -1,21 +1,61 @@ """Pluggable LLM client (Claude default). Optional dependency (`pip install .[llm]`). Kept behind a thin interface so self-hosters can swap or disable the model -entirely — the deterministic pipeline runs fully without it. +entirely — the deterministic pipeline (and the template narrator) run fully +without it. ``available()`` lets callers fall back gracefully when no model is +configured, honoring the "no external SaaS dependency by default" principle. """ from __future__ import annotations +import os from typing import Protocol class LlmProvider(Protocol): - def complete(self, prompt: str, **kwargs) -> str: ... + def available(self) -> bool: ... + def complete(self, prompt: str, *, system: str | None = None, max_tokens: int = 512) -> str: ... + + +class NullProvider(LlmProvider): + """The default when no model is configured: never available, never called.""" + + def available(self) -> bool: + return False + + def complete(self, prompt: str, *, system: str | None = None, max_tokens: int = 512) -> str: + raise RuntimeError("no LLM provider configured") class ClaudeProvider(LlmProvider): - def __init__(self, model: str = "claude-opus-4-8") -> None: + def __init__(self, model: str = "claude-opus-4-8", api_key: str | None = None) -> None: self._model = model - - def complete(self, prompt: str, **kwargs) -> str: - raise NotImplementedError("scaffold: anthropic SDK; advisory use only") + self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY") + + def available(self) -> bool: + if not self._api_key: + return False + try: + import anthropic # noqa: F401 + except ImportError: + return False + return True + + def complete(self, prompt: str, *, system: str | None = None, max_tokens: int = 512) -> str: + import anthropic + + client = anthropic.Anthropic(api_key=self._api_key) + kwargs: dict = {"model": self._model, "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}]} + if system: + # Cache the (static) system instruction across the many chains in a run. + kwargs["system"] = [{"type": "text", "text": system, + "cache_control": {"type": "ephemeral"}}] + msg = client.messages.create(**kwargs) + return "".join(b.text for b in msg.content if getattr(b, "type", None) == "text").strip() + + +def default_provider() -> LlmProvider: + """A ClaudeProvider if ANTHROPIC_API_KEY + the SDK are present, else NullProvider.""" + provider = ClaudeProvider() + return provider if provider.available() else NullProvider() diff --git a/core/reachability/models/conditions.py b/core/reachability/models/conditions.py index 5a6c0ea..57dc807 100644 --- a/core/reachability/models/conditions.py +++ b/core/reachability/models/conditions.py @@ -39,5 +39,20 @@ def __str__(self) -> str: def parse_condition(token: str) -> Condition: - """Parse 'network:reachable_from:vlan-20' into a Condition. Scaffold.""" - raise NotImplementedError("scaffold: split on ':' into namespace / value / optional param") + """Parse a 'namespace:value[:param]' token into a Condition. + + Examples: + network:internet_reachable -> (network, internet_reachable, None) + network:reachable_from:vlan-20 -> (network, reachable_from, "vlan-20") + position:foothold:vlan-20 -> (position, foothold, "vlan-20") + """ + parts = token.split(":") + if len(parts) < 2: + raise ValueError(f"not a namespace:value token: {token!r}") + namespace_raw, value, *rest = parts + try: + namespace = Namespace(namespace_raw) + except ValueError as exc: + raise ValueError(f"unknown namespace {namespace_raw!r} in {token!r}") from exc + param = ":".join(rest) if rest else None + return Condition(namespace=namespace, value=value, param=param) diff --git a/core/reachability/models/finding.py b/core/reachability/models/finding.py index 9d7aa2a..9c1678b 100644 --- a/core/reachability/models/finding.py +++ b/core/reachability/models/finding.py @@ -12,6 +12,15 @@ from enum import IntEnum +# Scanner-native severity words mapped onto the canonical ladder. Adapters emit +# lowercased levels; this absorbs vocabulary differences (Grype's +# "Negligible"/"Unknown", Trivy's "Unknown", CVSS qualitative labels). +_SEVERITY_ALIASES = { + "negligible": "info", "none": "info", "informational": "info", "unknown": "info", + "moderate": "medium", "important": "high", "warning": "medium", "error": "high", +} + + class Severity(IntEnum): """Ordinal ladder (04-scoring-policy.md). Deltas move up/down; clamp at ends.""" @@ -21,6 +30,16 @@ class Severity(IntEnum): high = 3 critical = 4 + @classmethod + def coerce(cls, value: "str | int | Severity") -> "Severity": + """Map a scanner-native level (str/int) onto the canonical ladder.""" + if isinstance(value, Severity): + return value + if isinstance(value, int): + return cls(value) + key = str(value).strip().lower() + return cls[_SEVERITY_ALIASES.get(key, key)] + @dataclass class Category: @@ -63,6 +82,7 @@ class Contextual: delta: int # signed levels moved from base (negative = downgraded) rationale: list[str] = field(default_factory=list) # one line per rule that fired chains: list[str] = field(default_factory=list) + tags: list[str] = field(default_factory=list) # effect.tag labels for filter/report (04) suppressed: bool = False scored_at: str | None = None policy_version: str | None = None @@ -92,8 +112,113 @@ class Finding: @classmethod def from_dict(cls, data: dict) -> "Finding": - """Build from a normalized finding dict (e.g. adapter output, jsonl line).""" - raise NotImplementedError("scaffold: validate against schema/json/finding.schema.json") + """Build from a normalized finding dict (e.g. adapter output, jsonl line). + + Tolerant by design: adapters fill what they can and the engine enriches + the rest, so missing optional blocks default rather than raise. (Strict + structural validation belongs at the trust boundary against + schema/json/finding.schema.json, separately from this typed view.) + """ + cat = data.get("category", {}) or {} + bs = data.get("base_severity", {}) or {} + ar = data.get("asset_ref", {}) or {} + ex = data.get("exploitability", {}) or {} + ctx = data.get("contextual") + return cls( + finding_id=data.get("finding_id", ""), + scan_id=data.get("scan_id", ""), + adapter=data.get("adapter", {}) or {}, + category=Category( + type=cat.get("type", "unknown"), + cwe=cat.get("cwe", []) or [], + capec=cat.get("capec", []) or [], + owasp=cat.get("owasp", []) or [], + ), + base_severity=BaseSeverity( + level=Severity.coerce(bs.get("level", "info")), + cvss_vector=bs.get("cvss_vector"), + cvss_score=bs.get("cvss_score"), + source=bs.get("source", "scanner"), + ), + asset_ref=AssetRef(node_id=ar.get("node_id"), locator=ar.get("locator", {}) or {}), + title=data.get("title", ""), + description=data.get("description", ""), + cve=data.get("cve", []) or [], + exploitability=Exploitability( + epss_score=ex.get("epss_score"), + kev=bool(ex.get("kev", False)), + exploit_public=bool(ex.get("exploit_public", False)), + ), + preconditions=data.get("preconditions", []) or [], + postconditions=data.get("postconditions", []) or [], + evidence=data.get("evidence", {}) or {}, + remediation=data.get("remediation", {}) or {}, + discovered_at=data.get("discovered_at"), + fingerprint=data.get("fingerprint"), + raw=data.get("raw", {}) or {}, + schema_version=data.get("schema_version", "1.0"), + contextual=cls._contextual_from_dict(ctx) if ctx else None, + ) + + @staticmethod + def _contextual_from_dict(ctx: dict) -> Contextual: + return Contextual( + severity=Severity.coerce(ctx.get("severity", "info")), + delta=int(ctx.get("delta", 0)), + rationale=ctx.get("rationale", []) or [], + chains=ctx.get("chains", []) or [], + tags=ctx.get("tags", []) or [], + suppressed=bool(ctx.get("suppressed", False)), + scored_at=ctx.get("scored_at"), + policy_version=ctx.get("policy_version"), + ) def to_dict(self) -> dict: - raise NotImplementedError("scaffold") + """Serialize to the normalized finding shape (severity as level name).""" + out: dict = { + "schema_version": self.schema_version, + "finding_id": self.finding_id, + "scan_id": self.scan_id, + "adapter": self.adapter, + "discovered_at": self.discovered_at, + "title": self.title, + "description": self.description, + "category": { + "type": self.category.type, + "cwe": self.category.cwe, + "capec": self.category.capec, + "owasp": self.category.owasp, + }, + "cve": self.cve, + "base_severity": { + "level": self.base_severity.level.name, + "cvss_vector": self.base_severity.cvss_vector, + "cvss_score": self.base_severity.cvss_score, + "source": self.base_severity.source, + }, + "exploitability": { + "epss_score": self.exploitability.epss_score, + "kev": self.exploitability.kev, + "exploit_public": self.exploitability.exploit_public, + }, + "asset_ref": {"node_id": self.asset_ref.node_id, "locator": self.asset_ref.locator}, + "preconditions": self.preconditions, + "postconditions": self.postconditions, + "evidence": self.evidence, + "remediation": self.remediation, + "fingerprint": self.fingerprint, + "raw": self.raw, + } + if self.contextual is not None: + c = self.contextual + out["contextual"] = { + "severity": c.severity.name, + "delta": c.delta, + "rationale": c.rationale, + "chains": c.chains, + "tags": c.tags, + "suppressed": c.suppressed, + "scored_at": c.scored_at, + "policy_version": c.policy_version, + } + return out diff --git a/core/reachability/models/policy.py b/core/reachability/models/policy.py index c843097..a79e2b7 100644 --- a/core/reachability/models/policy.py +++ b/core/reachability/models/policy.py @@ -9,6 +9,8 @@ from dataclasses import dataclass, field +import yaml + @dataclass class Settings: @@ -55,5 +57,35 @@ class Policy: @classmethod def load(cls, path: str) -> "Policy": - """Load + validate a policy YAML against schema/json/scoring-policy.schema.json.""" - raise NotImplementedError("scaffold") + """Load a policy YAML the user owns and version-controls.""" + with open(path, encoding="utf-8") as fh: + return cls.from_dict(yaml.safe_load(fh) or {}) + + @classmethod + def from_dict(cls, data: dict) -> "Policy": + s = data.get("settings", {}) or {} + settings = Settings( + max_downgrade=s.get("max_downgrade", 3), + max_upgrade=s.get("max_upgrade", 4), + require_block_mode=s.get("require_block_mode", True), + unknown_category_downgrade=s.get("unknown_category_downgrade", False), + min_control_confidence=s.get("min_control_confidence", "declared"), + ) + rules: list[Rule] = [] + for r in data.get("rules", []) or []: + eff = r.get("effect", {}) or {} + rules.append( + Rule( + id=r["id"], + when=When(spec=r.get("when", {}) or {}), + effect=Effect( + delta=int(eff.get("delta", 0)), + set_severity=eff.get("set_severity"), + suppress=bool(eff.get("suppress", False)), + tag=eff.get("tag", []) or [], + stop=bool(eff.get("stop", False)), + ), + rationale=r.get("rationale", ""), + ) + ) + return cls(policy_version=str(data.get("policy_version", "0")), settings=settings, rules=rules) diff --git a/core/reachability/report/render.py b/core/reachability/report/render.py index f7d7371..06f5a65 100644 --- a/core/reachability/report/render.py +++ b/core/reachability/report/render.py @@ -1,14 +1,105 @@ """Render the re-prioritized report. Contract: README §How it works (4. Report). A re-prioritized list and a set of attack chains, each expandable to WHY — the -rationale lines from contextual.rationale plus chain narratives. The explanation -is the product. +rationale lines from contextual.rationale plus the chain hops. The explanation is +the product, so the rationale is front and centre. Suppressed findings are hidden +from the default view (a filter, never a deletion — 04 §Effect) but counted. """ from __future__ import annotations -from ..models.finding import Finding +import json +from dataclasses import asdict, is_dataclass +from ..models.finding import Finding, Severity -def render_report(findings: list[Finding], chains: list, fmt: str = "text") -> str: - raise NotImplementedError("scaffold: text | json | html") + +def render_report(findings: list[Finding], chains: list | None = None, fmt: str = "text", + *, show_suppressed: bool = False, narratives: dict | None = None) -> str: + chains = chains or [] + narratives = narratives or {} + if fmt == "json": + return _json(findings, chains, narratives) + if fmt == "text": + return _text(findings, chains, show_suppressed, narratives) + raise ValueError(f"unknown report format {fmt!r} (use text|json)") + + +def _sev(f: Finding) -> Severity: + return f.contextual.severity if f.contextual else f.base_severity.level + + +def _text(findings: list[Finding], chains: list, show_suppressed: bool, narratives: dict) -> str: + visible = [f for f in findings + if show_suppressed or not (f.contextual and f.contextual.suppressed)] + visible.sort(key=lambda f: (-int(_sev(f)), -int(f.base_severity.level))) + + pv = next((f.contextual.policy_version for f in findings if f.contextual), None) or "—" + out: list[str] = [ + f"Reachability report - {len(findings)} findings, policy {pv}", + "=" * 60, + "", + "Re-prioritized (by contextual severity):", + "", + ] + for f in visible: + c = f.contextual + sev = _sev(f).name.upper() + base = f.base_severity.level.name + asset = f.asset_ref.node_id or f.asset_ref.locator.get("component") or "—" + flags = "" + if c and c.chains: + flags += " (chain hop)" + if c and c.suppressed: + flags += " (suppressed)" + # ASCII-only output: the report is printed to terminals (incl. Windows + # cp1252 consoles) and piped to files; no non-encodable glyphs. + out.append(f" {sev:<8} <- {base:<8} {f.category.type:<24} {asset}{flags}") + if c: + sign = f"{c.delta:+d}" + if c.rationale: + for line in c.rationale: + out.append(f" [{sign}] {line}") + else: + out.append(f" [{sign}] (no rule fired)") + out.append("") + + if chains: + out.append(f"Attack chains ({len(chains)}):") + by_id = {f.finding_id: f for f in findings} + for ch in chains: + cj = " crown-jewel" if getattr(ch, "reaches_crown_jewel", False) else "" + out.append( + f" [{ch.id}] score {getattr(ch, 'score', 0)} - " + f"reaches{cj} {ch.target} ({len(ch.steps)} hops)" + ) + for i, sid in enumerate(ch.steps, 1): + sf = by_id.get(sid) + desc = f"{sf.category.type} on {sf.asset_ref.node_id}" if sf else sid + out.append(f" {i}. {desc}") + if narratives.get(ch.id): + out.append(f" narrative: {narratives[ch.id]}") + out.append("") + + up = sum(1 for f in findings if f.contextual and f.contextual.delta > 0) + down = sum(1 for f in findings if f.contextual and f.contextual.delta < 0) + sup = sum(1 for f in findings if f.contextual and f.contextual.suppressed) + out.append(f"Summary: {up} upgraded, {down} downgraded, {sup} suppressed.") + return "\n".join(out) + + +def _json(findings: list[Finding], chains: list, narratives: dict) -> str: + def _chain(c): + d = asdict(c) if is_dataclass(c) else dict(c) + if narratives.get(getattr(c, "id", None)): + d["narrative"] = narratives[c.id] + return d + + return json.dumps( + { + "findings": [f.to_dict() for f in findings], + "chains": [_chain(c) for c in chains], + }, + indent=2, + default=str, + ) diff --git a/core/reachability/store/findings_store.py b/core/reachability/store/findings_store.py index f1e7a20..1329f54 100644 --- a/core/reachability/store/findings_store.py +++ b/core/reachability/store/findings_store.py @@ -1,23 +1,117 @@ -"""Findings + scan + contextual-result storage. +"""Findings + scan + contextual-result storage (Postgres). -Findings stream in as line-delimited JSON (*.findings.jsonl, 01-finding-schema.md); -the contextual block (and policy_version) is stamped on every result so a re-score -under a new policy is distinguishable from the old one in history (04 §Trust). -Suppressed findings are kept queryable — suppression is a view filter, never a delete. +Findings stream in as line-delimited JSON (*.findings.jsonl, 01-finding-schema.md). +Each finding's full normalized form is stored in ``finding``; every scoring run +appends to ``finding_score``, so the contextual block (and its policy_version) is +stamped on every result and a re-score under a new policy is distinguishable from +the old one in history (04 §Trust). Suppressed findings are kept queryable — +suppression is a view filter, never a delete. + +psycopg is imported lazily so importing this module never requires the driver. """ from __future__ import annotations +import uuid as _uuid + from ..models.finding import Finding +try: + import psycopg + from psycopg.rows import dict_row + from psycopg.types.json import Jsonb + _HAS_PSYCOPG = True +except ImportError: # pragma: no cover + _HAS_PSYCOPG = False + class FindingsStore: - def __init__(self, dsn: str, env_id: str) -> None: - self._dsn = dsn - self._env_id = env_id + def __init__(self, dsn: str | None = None, env_id: str = "", *, conn=None) -> None: + self._env_id = _uuid.UUID(env_id) if env_id else None + if conn is not None: + self._conn = conn + elif _HAS_PSYCOPG: + # autocommit: reads must not leave the connection idle-in-transaction. + self._conn = psycopg.connect(dsn, row_factory=dict_row, autocommit=True) + else: + raise RuntimeError("psycopg is required for FindingsStore: pip install 'psycopg[binary]'") def upsert(self, finding: Finding) -> None: - raise NotImplementedError("scaffold") + """Store/refresh the normalized finding; if it has been scored, append a + contextual result to its history.""" + self._conn.execute( + """ + INSERT INTO finding (finding_id, env_id, scan_id, category, fingerprint, data) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (finding_id) DO UPDATE SET + scan_id = EXCLUDED.scan_id, + category = EXCLUDED.category, + fingerprint = EXCLUDED.fingerprint, + data = EXCLUDED.data + """, + (finding.finding_id, self._env_id, self._scan_uuid(finding), + finding.category.type, finding.fingerprint, self._json(finding.to_dict())), + ) + if finding.contextual is not None: + self.record_score(finding) + self._conn.commit() + + def record_score(self, finding: Finding) -> None: + """Append one contextual result (a scoring run) to the score history.""" + c = finding.contextual + if c is None: + raise ValueError("finding has no contextual block to record") + self._conn.execute( + """ + INSERT INTO finding_score + (finding_id, env_id, policy_version, severity, delta, suppressed, contextual, scored_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, COALESCE(%s::timestamptz, now())) + """, + (finding.finding_id, self._env_id, c.policy_version or "0", + c.severity.name, c.delta, c.suppressed, + self._json(self._contextual_dict(c)), c.scored_at), + ) + + def by_scan(self, scan_id: str, *, include_suppressed: bool = True) -> list[Finding]: + rows = self._conn.execute( + "SELECT data FROM finding WHERE env_id = %s AND scan_id = %s", + (self._env_id, _uuid.UUID(scan_id) if _is_uuid(scan_id) else scan_id), + ).fetchall() + findings = [Finding.from_dict(r["data"]) for r in rows] + if not include_suppressed: + findings = [f for f in findings if not (f.contextual and f.contextual.suppressed)] + return findings + + def score_history(self, finding_id: str) -> list[dict]: + """Every scoring run for a finding, oldest first — proves a re-score under + a new policy is distinguishable in history (04 §Trust).""" + return self._conn.execute( + "SELECT policy_version, severity, delta, suppressed, scored_at " + "FROM finding_score WHERE finding_id = %s ORDER BY scored_at, id", + (finding_id,), + ).fetchall() + + # --- helpers ---------------------------------------------------------- + @staticmethod + def _json(value): + return Jsonb(value) if _HAS_PSYCOPG else value + + @staticmethod + def _scan_uuid(finding: Finding): + return _uuid.UUID(finding.scan_id) if _is_uuid(finding.scan_id) else finding.scan_id + + @staticmethod + def _contextual_dict(c) -> dict: + return { + "severity": c.severity.name, "delta": c.delta, "rationale": c.rationale, + "chains": c.chains, "tags": c.tags, "suppressed": c.suppressed, + "scored_at": c.scored_at, "policy_version": c.policy_version, + } + - def by_scan(self, scan_id: str) -> list[Finding]: - raise NotImplementedError("scaffold") +def _is_uuid(value: str) -> bool: + try: + _uuid.UUID(str(value)) + return True + except (ValueError, AttributeError, TypeError): + return False diff --git a/core/tests/conftest.py b/core/tests/conftest.py index 11c6813..ce0a595 100644 --- a/core/tests/conftest.py +++ b/core/tests/conftest.py @@ -4,9 +4,74 @@ a live scanner (03-adapter-interface.md). """ +from __future__ import annotations + +from pathlib import Path + import pytest +from reachability.graphstore import InMemoryGraphStore +from reachability.ingestion import questionnaire +from reachability.models.finding import ( + AssetRef, + BaseSeverity, + Category, + Exploitability, + Finding, + Severity, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +EXAMPLE_ANSWERS = REPO_ROOT / "context" / "questionnaire.answers.example.yaml" + @pytest.fixture def sample_policy_path() -> str: - return "config/policy.default.yaml" + return str(REPO_ROOT / "config" / "policy.default.yaml") + + +@pytest.fixture +def example_graph() -> InMemoryGraphStore: + """The shipped example questionnaire loaded into an in-memory graph.""" + graph = InMemoryGraphStore() + nodes, edges = questionnaire.load(str(EXAMPLE_ANSWERS)) + questionnaire.populate(graph, nodes, edges) + return graph + + +def make_finding( + node_id: str | None, + category: str = "xss", + level: Severity = Severity.high, + *, + kev: bool = False, + epss: float | None = None, + exploit_public: bool = False, + confidence: str | None = None, + pre: list[str] | None = None, + post: list[str] | None = None, + component: str | None = None, + host: str | None = None, + url: str | None = None, +) -> Finding: + """Minimal finding pointed at a graph node (or, for SCA/DAST shape, an + unresolved component/host/url locator with node_id=None), for reasoning tests.""" + locator = {} + if component: + locator["component"] = component + if host: + locator["host"] = host + if url: + locator["url"] = url + return Finding( + finding_id=f"finding:{node_id or component}:{category}", + scan_id="scan:test", + adapter={"name": "test", "kind": "dast", "version": "0"}, + category=Category(type=category), + base_severity=BaseSeverity(level=level), + asset_ref=AssetRef(node_id=node_id, locator=locator), + exploitability=Exploitability(epss_score=epss, kev=kev, exploit_public=exploit_public), + evidence={"confidence": confidence} if confidence else {}, + preconditions=pre or [], + postconditions=post or [], + ) diff --git a/core/tests/fixtures/cloudformation.json b/core/tests/fixtures/cloudformation.json new file mode 100644 index 0000000..5b073d6 --- /dev/null +++ b/core/tests/fixtures/cloudformation.json @@ -0,0 +1,52 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "PublicSubnet": { + "Type": "AWS::EC2::Subnet", + "Properties": {"MapPublicIpOnLaunch": true, "CidrBlock": "10.0.1.0/24"} + }, + "PrivateSubnet": { + "Type": "AWS::EC2::Subnet", + "Properties": {"MapPublicIpOnLaunch": false, "CidrBlock": "10.0.2.0/24"} + }, + "WebLB": { + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", + "Properties": {"Scheme": "internet-facing", "Subnets": [{"Ref": "PublicSubnet"}]} + }, + "AppInstance": { + "Type": "AWS::EC2::Instance", + "Properties": {"SubnetId": {"Ref": "PrivateSubnet"}} + }, + "EdgeWaf": { + "Type": "AWS::WAFv2::WebACL", + "Properties": {"Name": "edge-waf"} + }, + "WafAssoc": { + "Type": "AWS::WAFv2::WebACLAssociation", + "Properties": {"ResourceArn": {"Ref": "WebLB"}, "WebACLArn": {"Ref": "EdgeWaf"}} + }, + "AppTargetGroup": { + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", + "Properties": {"Targets": [{"Id": {"Ref": "AppInstance"}}]} + }, + "HttpsListener": { + "Type": "AWS::ElasticLoadBalancingV2::Listener", + "Properties": { + "LoadBalancerArn": {"Ref": "WebLB"}, + "DefaultActions": [{"TargetGroupArn": {"Ref": "AppTargetGroup"}}] + } + }, + "DbSubnets": { + "Type": "AWS::RDS::DBSubnetGroup", + "Properties": {"SubnetIds": [{"Ref": "PrivateSubnet"}]} + }, + "CustomerDb": { + "Type": "AWS::RDS::DBInstance", + "Properties": { + "DBInstanceIdentifier": "customer-db", "Engine": "postgres", + "StorageEncrypted": true, "DBSubnetGroupName": {"Ref": "DbSubnets"}, + "Tags": [{"Key": "sensitivity", "Value": "crown_jewel"}] + } + } + } +} diff --git a/core/tests/fixtures/cloudformation.yaml b/core/tests/fixtures/cloudformation.yaml new file mode 100644 index 0000000..39bab11 --- /dev/null +++ b/core/tests/fixtures/cloudformation.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion: "2010-09-09" +Resources: + PublicSubnet: + Type: AWS::EC2::Subnet + Properties: + MapPublicIpOnLaunch: true + CidrBlock: 10.0.1.0/24 + WebLB: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Scheme: internet-facing + Subnets: + - !Ref PublicSubnet + EdgeWaf: + Type: AWS::WAFv2::WebACL + Properties: + Name: edge-waf + WafAssoc: + Type: AWS::WAFv2::WebACLAssociation + Properties: + ResourceArn: !Ref WebLB + WebACLArn: !Ref EdgeWaf diff --git a/core/tests/fixtures/grype_sample.json b/core/tests/fixtures/grype_sample.json new file mode 100644 index 0000000..a0a746e --- /dev/null +++ b/core/tests/fixtures/grype_sample.json @@ -0,0 +1,30 @@ +{ + "matches": [ + { + "vulnerability": { + "id": "CVE-2021-23337", + "severity": "High", + "description": "lodash before 4.17.21: command injection via template." + }, + "artifact": { + "name": "lodash", + "version": "4.17.20", + "type": "npm", + "purl": "pkg:npm/lodash@4.17.20" + } + }, + { + "vulnerability": { + "id": "CVE-2020-8203", + "severity": "Medium", + "description": "Prototype pollution in lodash before 4.17.19." + }, + "artifact": { + "name": "lodash", + "version": "4.17.20", + "type": "npm", + "purl": "pkg:npm/lodash@4.17.20" + } + } + ] +} diff --git a/core/tests/fixtures/sbom_cyclonedx.json b/core/tests/fixtures/sbom_cyclonedx.json new file mode 100644 index 0000000..83c3964 --- /dev/null +++ b/core/tests/fixtures/sbom_cyclonedx.json @@ -0,0 +1,17 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": {"type": "application", "name": "web-frontend", "version": "1.0.0", "bom-ref": "app"} + }, + "components": [ + {"type": "library", "name": "lodash", "version": "4.17.20", + "purl": "pkg:npm/lodash@4.17.20", "bom-ref": "comp-lodash"}, + {"type": "library", "name": "express", "version": "4.18.2", + "purl": "pkg:npm/express@4.18.2", "bom-ref": "comp-express"} + ], + "dependencies": [ + {"ref": "app", "dependsOn": ["comp-lodash"]}, + {"ref": "comp-lodash", "dependsOn": []} + ] +} diff --git a/core/tests/fixtures/sbom_spdx.json b/core/tests/fixtures/sbom_spdx.json new file mode 100644 index 0000000..a16b5a6 --- /dev/null +++ b/core/tests/fixtures/sbom_spdx.json @@ -0,0 +1,18 @@ +{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "web-frontend-sbom", + "documentDescribes": ["SPDXRef-App"], + "packages": [ + {"SPDXID": "SPDXRef-App", "name": "web-frontend", "versionInfo": "1.0.0"}, + {"SPDXID": "SPDXRef-lodash", "name": "lodash", "versionInfo": "4.17.20", + "externalRefs": [ + {"referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", + "referenceLocator": "pkg:npm/lodash@4.17.20"} + ]} + ], + "relationships": [ + {"spdxElementId": "SPDXRef-App", "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-lodash"} + ] +} diff --git a/core/tests/fixtures/terraform_aws.tfstate.json b/core/tests/fixtures/terraform_aws.tfstate.json new file mode 100644 index 0000000..3e1ae76 --- /dev/null +++ b/core/tests/fixtures/terraform_aws.tfstate.json @@ -0,0 +1,40 @@ +{ + "format_version": "1.0", + "terraform_version": "1.7.0", + "values": { + "root_module": { + "resources": [ + {"type": "aws_subnet", "name": "public", + "values": {"id": "subnet-pub", "map_public_ip_on_launch": true, "cidr_block": "10.0.1.0/24", + "tags": {"Name": "public"}}}, + {"type": "aws_subnet", "name": "private", + "values": {"id": "subnet-priv", "map_public_ip_on_launch": false, "cidr_block": "10.0.2.0/24", + "tags": {"Name": "private"}}}, + {"type": "aws_lb", "name": "web", + "values": {"arn": "arn:aws:elasticloadbalancing:lb/web", "name": "web-lb", + "internal": false, "subnets": ["subnet-pub"]}}, + {"type": "aws_instance", "name": "app", + "values": {"id": "i-app", "subnet_id": "subnet-priv", "associate_public_ip_address": false, + "ami": "ami-123", "tags": {"Name": "app"}}}, + {"type": "aws_wafv2_web_acl", "name": "edge", + "values": {"arn": "arn:aws:wafv2:web-acl/edge", "name": "edge-waf"}}, + {"type": "aws_wafv2_web_acl_association", "name": "assoc", + "values": {"resource_arn": "arn:aws:elasticloadbalancing:lb/web", + "web_acl_arn": "arn:aws:wafv2:web-acl/edge"}}, + {"type": "aws_lb_target_group", "name": "tg", + "values": {"arn": "arn:aws:elasticloadbalancing:tg/app"}}, + {"type": "aws_lb_target_group_attachment", "name": "att", + "values": {"target_group_arn": "arn:aws:elasticloadbalancing:tg/app", "target_id": "i-app"}}, + {"type": "aws_lb_listener", "name": "https", + "values": {"load_balancer_arn": "arn:aws:elasticloadbalancing:lb/web", + "default_action": [{"target_group_arn": "arn:aws:elasticloadbalancing:tg/app"}]}}, + {"type": "aws_db_subnet_group", "name": "db_sng", + "values": {"name": "customer-db-subnets", "subnet_ids": ["subnet-priv"]}}, + {"type": "aws_db_instance", "name": "customer", + "values": {"id": "db-1", "identifier": "customer-db", "engine": "postgres", + "db_subnet_group_name": "customer-db-subnets", + "storage_encrypted": true, "tags": {"sensitivity": "crown_jewel"}}} + ] + } + } +} diff --git a/core/tests/fixtures/terraform_azure.tfstate.json b/core/tests/fixtures/terraform_azure.tfstate.json new file mode 100644 index 0000000..156d316 --- /dev/null +++ b/core/tests/fixtures/terraform_azure.tfstate.json @@ -0,0 +1,18 @@ +{ + "format_version": "1.0", + "terraform_version": "1.7.0", + "values": { + "root_module": { + "resources": [ + {"type": "azurerm_subnet", "name": "app", + "values": {"name": "app-subnet", "address_prefixes": ["10.1.0.0/24"]}}, + {"type": "azurerm_frontdoor", "name": "edge", + "values": {"name": "edge-fd"}}, + {"type": "azurerm_linux_virtual_machine", "name": "api", + "values": {"name": "api-vm"}}, + {"type": "azurerm_mssql_database", "name": "customer", + "values": {"name": "customer-db", "tags": {"sensitivity": "crown_jewel"}}} + ] + } + } +} diff --git a/core/tests/fixtures/terraform_azure_frontdoor.tfstate.json b/core/tests/fixtures/terraform_azure_frontdoor.tfstate.json new file mode 100644 index 0000000..405c7f5 --- /dev/null +++ b/core/tests/fixtures/terraform_azure_frontdoor.tfstate.json @@ -0,0 +1,23 @@ +{ + "format_version": "1.0", + "terraform_version": "1.7.0", + "values": { + "root_module": { + "resources": [ + {"type": "azurerm_linux_web_app", "name": "api", + "values": {"name": "api-app", "default_hostname": "api-app.azurewebsites.net"}}, + {"type": "azurerm_frontdoor", "name": "classicfd", + "values": {"name": "classic-fd", "host_name": "classic.example.com", + "backend_pool": [ + {"name": "pool", "backend": [{"address": "api-app.azurewebsites.net"}]} + ]}}, + {"type": "azurerm_cdn_frontdoor_profile", "name": "stdfd", + "values": {"id": "prof-1", "name": "std-fd"}}, + {"type": "azurerm_cdn_frontdoor_origin_group", "name": "og", + "values": {"id": "og-1", "cdn_frontdoor_profile_id": "prof-1"}}, + {"type": "azurerm_cdn_frontdoor_origin", "name": "origin", + "values": {"host_name": "api-app.azurewebsites.net", "cdn_frontdoor_origin_group_id": "og-1"}} + ] + } + } +} diff --git a/core/tests/fixtures/terraform_azure_rich.tfstate.json b/core/tests/fixtures/terraform_azure_rich.tfstate.json new file mode 100644 index 0000000..363c792 --- /dev/null +++ b/core/tests/fixtures/terraform_azure_rich.tfstate.json @@ -0,0 +1,61 @@ +{ + "format_version": "1.0", + "terraform_version": "1.7.0", + "values": { + "root_module": { + "resources": [ + {"type": "azurerm_subnet", "name": "web", + "values": {"id": "subnet-web", "name": "web", "address_prefixes": ["10.1.1.0/24"]}}, + {"type": "azurerm_subnet", "name": "app", + "values": {"id": "subnet-app", "name": "app", "address_prefixes": ["10.1.2.0/24"]}}, + {"type": "azurerm_network_security_group", "name": "web_nsg", + "values": {"id": "nsg-web", "name": "web-nsg", + "security_rule": [ + {"name": "https-in", "access": "Allow", "direction": "Inbound", + "source_address_prefix": "Internet", "destination_port_range": "443"} + ]}}, + {"type": "azurerm_network_security_group", "name": "app_nsg", + "values": {"id": "nsg-app", "name": "app-nsg", + "security_rule": [ + {"name": "from-web", "access": "Allow", "direction": "Inbound", + "source_address_prefix": "10.1.1.0/24", "destination_port_range": "8080"} + ]}}, + {"type": "azurerm_subnet_network_security_group_association", "name": "web_assoc", + "values": {"subnet_id": "subnet-web", "network_security_group_id": "nsg-web"}}, + {"type": "azurerm_subnet_network_security_group_association", "name": "app_assoc", + "values": {"subnet_id": "subnet-app", "network_security_group_id": "nsg-app"}}, + {"type": "azurerm_public_ip", "name": "agw_pip", + "values": {"id": "pip-agw", "name": "agw-pip", "fqdn": "app.example.com"}}, + {"type": "azurerm_web_application_firewall_policy", "name": "edge_waf", + "values": {"id": "wafpol-1", "name": "edge-waf"}}, + {"type": "azurerm_application_gateway", "name": "appgw", + "values": {"name": "app-gw", "firewall_policy_id": "wafpol-1", + "frontend_ip_configuration": [{"name": "fe", "public_ip_address_id": "pip-agw"}], + "backend_address_pool": [{"name": "be", "ip_addresses": ["10.1.2.4"]}]}}, + {"type": "azurerm_network_interface", "name": "worker_nic", + "values": {"id": "nic-worker", + "ip_configuration": [{"name": "ipcfg", "subnet_id": "subnet-app", + "private_ip_address": "10.1.2.4", + "public_ip_address_id": null}]}}, + {"type": "azurerm_linux_virtual_machine", "name": "worker", + "values": {"name": "worker-vm", "network_interface_ids": ["nic-worker"]}}, + {"type": "azurerm_lb", "name": "ilb", + "values": {"id": "lb-ilb", "name": "internal-lb"}}, + {"type": "azurerm_lb_backend_address_pool", "name": "ilb_pool", + "values": {"id": "pool-1", "loadbalancer_id": "lb-ilb"}}, + {"type": "azurerm_network_interface_backend_address_pool_association", "name": "ilb_assoc", + "values": {"network_interface_id": "nic-worker", "backend_address_pool_id": "pool-1"}}, + {"type": "azurerm_linux_web_app", "name": "api", + "values": {"name": "api-app"}}, + {"type": "azurerm_key_vault", "name": "secrets", + "values": {"id": "kv-1", "name": "app-secrets", "tags": {"sensitivity": "crown_jewel"}}}, + {"type": "azurerm_private_endpoint", "name": "kv_pe", + "values": {"subnet_id": "subnet-app", + "private_service_connection": [{"private_connection_resource_id": "kv-1"}]}}, + {"type": "azurerm_postgresql_flexible_server", "name": "db", + "values": {"name": "customer-db", "delegated_subnet_id": "subnet-app", + "tags": {"sensitivity": "crown_jewel"}}} + ] + } + } +} diff --git a/core/tests/fixtures/topology.mmd b/core/tests/fixtures/topology.mmd new file mode 100644 index 0000000..4d55990 --- /dev/null +++ b/core/tests/fixtures/topology.mmd @@ -0,0 +1,10 @@ +%% Example architecture diagram (no IaC available) +flowchart LR + net((Internet)) --> lb[Load Balancer] + subgraph dmz + lb --> web[Web Frontend] + end + subgraph internal + web --> db[(Customer DB)] + end + dmz --> internal diff --git a/core/tests/fixtures/zap_alerts.json b/core/tests/fixtures/zap_alerts.json new file mode 100644 index 0000000..ab6e0b9 --- /dev/null +++ b/core/tests/fixtures/zap_alerts.json @@ -0,0 +1,24 @@ +{ + "alerts": [ + { + "id": "0", "pluginId": "10038", "name": "Content Security Policy (CSP) Header Not Set", + "alert": "Content Security Policy (CSP) Header Not Set", + "risk": "Medium", "confidence": "High", "cweid": "693", "wascid": "15", + "method": "GET", "url": "http://target.local:8080/", + "param": "", "evidence": "", + "description": "Content Security Policy (CSP) is an added layer of security ...", + "solution": "Ensure that your web server sets the Content-Security-Policy header.", + "reference": "https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP" + }, + { + "id": "1", "pluginId": "40012", "name": "Cross Site Scripting (Reflected)", + "alert": "Cross Site Scripting (Reflected)", + "risk": "High", "confidence": "Medium", "cweid": "79", "wascid": "8", + "method": "GET", "url": "http://target.local:8080/search?q=%3Cscript%3E", + "param": "q", "evidence": "", + "description": "Cross-site Scripting (XSS) is an attack technique ...", + "solution": "Validate and encode all user input.", + "reference": "https://owasp.org/www-community/attacks/xss/" + } + ] +} diff --git a/core/tests/test_chaining.py b/core/tests/test_chaining.py index 39c97e4..3ed5294 100644 --- a/core/tests/test_chaining.py +++ b/core/tests/test_chaining.py @@ -2,23 +2,96 @@ Contract: 06-attack-path-chaining.md. """ -import pytest +from reachability.engine import chaining +from reachability.graphstore import InMemoryGraphStore +from reachability.models.finding import Severity +from reachability.models.graph import Edge, Node + +from .conftest import make_finding + + +def _node(node_id, type_, **attrs): + return Node(id=node_id, type=type_, label=node_id, attributes=attrs) + + +def _crown_jewel_graph() -> InMemoryGraphStore: + """Internet -> web (DMZ); DMZ connects to internal; crown-jewel db is internal.""" + g = InMemoryGraphStore() + for n in [ + _node("entrypoint:pub", "entrypoint", public=True), + _node("asset:web", "asset", internet_reachable=True), + _node("zone:dmz", "zone"), + _node("zone:internal", "zone", internet_facing=False), + _node("asset:db", "asset", data_sensitivity="crown_jewel"), + ]: + g.upsert_node(n) + for e in [ + Edge(type="routes_to", src="entrypoint:pub", dst="asset:web"), + Edge(type="resides_in", src="asset:web", dst="zone:dmz"), + Edge(type="connects_to", src="zone:dmz", dst="zone:internal"), + Edge(type="resides_in", src="asset:db", dst="zone:internal"), + ]: + g.upsert_edge(e) + return g + + +def _ssrf_then_authz(): + # Two individually-modest findings that combine into a breach. + ssrf = make_finding( + "asset:web", category="ssrf", level=Severity.medium, + pre=["network:internet_reachable"], post=["position:pivot_to:internal"], + ) + authz = make_finding( + "asset:db", category="missing_authentication", level=Severity.low, + pre=["network:reachable_from:internal"], post=["access:db_read"], + ) + return ssrf, authz -@pytest.mark.skip(reason="scaffold") def test_two_mediums_chain_to_crown_jewel(): - # The canonical case: an SSRF (pivot) + an internal-only unauthenticated admin - # endpoint chain into a crown-jewel datastore -> one reportable critical chain. - raise NotImplementedError + g = _crown_jewel_graph() + ssrf, authz = _ssrf_then_authz() + chains = chaining.find_chains([ssrf, authz], g) + assert len(chains) == 1 + chain = chains[0] + assert chain.steps == [ssrf.finding_id, authz.finding_id] + assert chain.reaches_crown_jewel is True + assert chain.target == "asset:db" + + +def test_no_chain_when_pivot_is_missing(): + # Without the SSRF's pivot, the internal authz finding is never applicable. + g = _crown_jewel_graph() + _, authz = _ssrf_then_authz() + assert chaining.find_chains([authz], g) == [] -@pytest.mark.skip(reason="scaffold") def test_termination_no_finding_reused_and_max_depth(): - # No finding appears twice in a path; search respects MAX_DEPTH=6. - raise NotImplementedError + # A long sequential pipeline where every hop lands on a crown-jewel asset: + # every prefix >= 2 is reportable, so the only thing bounding length is MAX_DEPTH. + g = InMemoryGraphStore() + g.upsert_node(_node("entrypoint:pub", "entrypoint", public=True)) + findings = [] + prev = "network:internet_reachable" + for i in range(8): + aid = f"asset:s{i}" + g.upsert_node(_node(aid, "asset", internet_reachable=True, data_sensitivity="crown_jewel")) + g.upsert_edge(Edge(type="routes_to", src="entrypoint:pub", dst=aid)) + findings.append(make_finding( + aid, category="info_disclosure", level=Severity.low, + pre=[prev], post=[f"access:step{i}"], + )) + prev = f"access:step{i}" + + chains = chaining.find_chains(findings, g) + assert chains + assert max(len(c.steps) for c in chains) == chaining.MAX_DEPTH # hard depth bound + assert all(len(set(c.steps)) == len(c.steps) for c in chains) # no finding reused -@pytest.mark.skip(reason="scaffold") def test_chaining_is_deterministic(): - # Same inputs twice -> identical chains and scores (non-negotiable, 06 §boundary). - raise NotImplementedError + g = _crown_jewel_graph() + ssrf, authz = _ssrf_then_authz() + a = chaining.find_chains([ssrf, authz], g) + b = chaining.find_chains([ssrf, authz], g) + assert [(c.id, c.score) for c in a] == [(c.id, c.score) for c in b] diff --git a/core/tests/test_cli.py b/core/tests/test_cli.py new file mode 100644 index 0000000..938fa2c --- /dev/null +++ b/core/tests/test_cli.py @@ -0,0 +1,101 @@ +"""End-to-end CLI flow: ingest -> reason -> report, on file artifacts (no DB).""" + +import json +from pathlib import Path + +from reachability import cli +from reachability.graphstore import InMemoryGraphStore +from reachability.models.graph import Edge, Node +from reachability.report.render import render_report + +from .conftest import make_finding + +REPO_ROOT = Path(__file__).resolve().parents[2] +EXAMPLE_ANSWERS = REPO_ROOT / "context" / "questionnaire.answers.example.yaml" + + +def _write_jsonl(path: Path, findings): + path.write_text("\n".join(json.dumps(f.to_dict()) for f in findings), encoding="utf-8") + + +def test_cli_ingest_writes_graph_artifact(tmp_path): + out = tmp_path / "graph.json" + summary = cli.cmd_ingest(str(EXAMPLE_ANSWERS), str(out)) + assert out.exists() + assert summary["nodes"] > 0 + reloaded = InMemoryGraphStore.from_dict(json.loads(out.read_text())) + assert reloaded.get_node("asset:web-frontend") is not None + + +def test_cli_end_to_end_downgrade(tmp_path): + # ingest the example, score an xss on the WAF-protected frontend, report it. + graph = tmp_path / "graph.json" + findings = tmp_path / "findings.jsonl" + scored = tmp_path / "scored.jsonl" + cli.cmd_ingest(str(EXAMPLE_ANSWERS), str(graph)) + _write_jsonl(findings, [make_finding("asset:web-frontend", category="xss")]) + + out_findings, chains = cli.cmd_reason(str(graph), str(findings), + str(REPO_ROOT / "config" / "policy.default.yaml"), + str(scored), str(tmp_path / "chains.json")) + assert out_findings[0].contextual.severity.name == "low" # high -> low behind WAF + + report = cli.cmd_report(str(scored), str(tmp_path / "chains.json"), "text") + assert "LOW <- high" in report + assert "xss" in report + assert "azure-front-door" in report + assert "1 downgraded" in report + + +def test_cli_report_renders_attack_chain(tmp_path): + # Build a crown-jewel topology directly, dump it, then score bare findings. + g = InMemoryGraphStore() + for n in [ + Node(id="entrypoint:pub", type="entrypoint", label="pub", attributes={"public": True}), + Node(id="asset:web", type="asset", label="web", attributes={"internet_reachable": True}), + Node(id="zone:dmz", type="zone", label="dmz"), + Node(id="zone:internal", type="zone", label="internal"), + Node(id="asset:db", type="asset", label="db", attributes={"data_sensitivity": "crown_jewel"}), + ]: + g.upsert_node(n) + for e in [ + Edge(type="routes_to", src="entrypoint:pub", dst="asset:web"), + Edge(type="resides_in", src="asset:web", dst="zone:dmz"), + Edge(type="connects_to", src="zone:dmz", dst="zone:internal"), + Edge(type="resides_in", src="asset:db", dst="zone:internal"), + ]: + g.upsert_edge(e) + + graph = tmp_path / "graph.json" + graph.write_text(json.dumps(g.to_dict()), encoding="utf-8") + findings = tmp_path / "findings.jsonl" + scored = tmp_path / "scored.jsonl" + chains = tmp_path / "chains.json" + _write_jsonl(findings, [ + make_finding("asset:web", category="ssrf"), + make_finding("asset:db", category="missing_authentication"), + ]) + + _, found_chains = cli.cmd_reason(str(graph), str(findings), + str(REPO_ROOT / "config" / "policy.default.yaml"), + str(scored), str(chains)) + assert len(found_chains) == 1 + + report = cli.cmd_report(str(scored), str(chains), "text") + assert "Attack chains (1)" in report + assert "crown-jewel asset:db" in report + assert "CRITICAL" in report + + +def test_cli_init_db_applies_migrations_via_store(): + class _FakeStore: + def init_schema(self): + return ["0001_init.sql", "0002_edge_unique.sql", "0003_findings.sql"] + assert cli.cmd_init_db(None, store=_FakeStore())[0] == "0001_init.sql" + + +def test_report_json_format_is_valid(): + findings = [make_finding("asset:x", category="xss")] + findings[0].contextual = None + parsed = json.loads(render_report(findings, [], fmt="json")) + assert "findings" in parsed and "chains" in parsed diff --git a/core/tests/test_cloudformation.py b/core/tests/test_cloudformation.py new file mode 100644 index 0000000..ae9ad63 --- /dev/null +++ b/core/tests/test_cloudformation.py @@ -0,0 +1,74 @@ +"""CloudFormation ingestion (JSON + YAML short-form). Contract: 05 §1.""" + +from pathlib import Path + +from reachability.engine import chaining, reachability +from reachability.graphstore import InMemoryGraphStore +from reachability.ingestion import iac, questionnaire +from reachability.models.finding import Severity +from reachability.models.graph import Confidence + +from .conftest import make_finding +from reachability.adapters import conditions_map + +FIX = Path(__file__).resolve().parents[0] / "fixtures" + + +def _parse_json(): + return iac.load(str(FIX / "cloudformation.json")) + + +def _by_id(nodes): + return {n.id: n for n in nodes} + + +def test_cfn_dialect_is_autodetected(): + # iac.load must pick CloudFormation over the Terraform providers. + nodes, _ = _parse_json() + n = _by_id(nodes) + assert n["zone:publicsubnet"].attributes["internet_facing"] is True + assert all(x.source == "cloudformation" for x in nodes) + assert all(x.confidence is Confidence.ground_truth for x in nodes) + + +def test_cfn_ref_resolves_lb_waf_and_routing(): + nodes, edges = _parse_json() + n = _by_id(nodes) + assert n["asset:weblb"].attributes["internet_reachable"] is True + assert n["control:edgewaf"].attributes["control_type"] == "waf" + # WebACLAssociation Ref -> protected_by; listener+target group Ref -> routes_to + assert any(e.type == "protected_by" and e.src == "asset:weblb" and e.dst == "control:edgewaf" + for e in edges) + assert any(e.type == "routes_to" and e.src == "entrypoint:weblb" and e.dst == "asset:weblb" + for e in edges) + assert any(e.type == "routes_to" and e.src == "asset:weblb" and e.dst == "asset:appinstance" + for e in edges) + + +def test_cfn_rds_placed_in_db_subnet_group_zone(): + _, edges = _parse_json() + assert any(e.type == "resides_in" and e.src == "datastore:customer-db" + and e.dst == "zone:privatesubnet" for e in edges) + + +def test_cfn_yaml_short_form_intrinsics(): + nodes, edges = iac.load(str(FIX / "cloudformation.yaml")) + # !Ref WebLB / !Ref EdgeWaf must resolve just like the JSON Ref form + assert any(e.type == "protected_by" and e.src == "asset:weblb" and e.dst == "control:edgewaf" + for e in edges) + + +def test_cfn_private_instance_reachable_through_lb_and_chain_to_crown_jewel(): + # Full parity with Terraform: a private instance is internet-reachable via the + # internet-facing LB, and a chain assembled from the CFN graph reaches the RDS. + g = InMemoryGraphStore() + questionnaire.populate(g, *_parse_json()) + assert reachability.compute_one( + make_finding("asset:appinstance", category="xss"), g).reachable_from_internet is True + + rce = make_finding("asset:appinstance", category="command_injection", level=Severity.high) + authz = make_finding("datastore:customer-db", category="missing_authentication", level=Severity.low) + conditions_map.enrich(rce, g) + conditions_map.enrich(authz, g) + chains = chaining.find_chains([rce, authz], g) + assert any(c.reaches_crown_jewel and c.target == "datastore:customer-db" for c in chains) diff --git a/core/tests/test_conditions.py b/core/tests/test_conditions.py new file mode 100644 index 0000000..5303363 --- /dev/null +++ b/core/tests/test_conditions.py @@ -0,0 +1,32 @@ +"""namespace:value vocabulary parsing. Contract: 01-finding-schema.md vocabulary.""" + +import pytest + +from reachability.models.conditions import Condition, Namespace, parse_condition + + +def test_simple_token(): + c = parse_condition("network:internet_reachable") + assert c == Condition(Namespace.network, "internet_reachable", None) + assert str(c) == "network:internet_reachable" + + +def test_parameterized_token_roundtrips(): + c = parse_condition("network:reachable_from:vlan-20") + assert c == Condition(Namespace.network, "reachable_from", "vlan-20") + assert str(c) == "network:reachable_from:vlan-20" + + +def test_position_foothold(): + c = parse_condition("position:foothold:vlan-20") + assert (c.namespace, c.value, c.param) == (Namespace.position, "foothold", "vlan-20") + + +def test_unknown_namespace_rejected(): + with pytest.raises(ValueError): + parse_condition("bogus:thing") + + +def test_not_a_token_rejected(): + with pytest.raises(ValueError): + parse_condition("internet_reachable") diff --git a/core/tests/test_conditions_map.py b/core/tests/test_conditions_map.py new file mode 100644 index 0000000..55f2531 --- /dev/null +++ b/core/tests/test_conditions_map.py @@ -0,0 +1,91 @@ +"""Category -> default pre/postconditions table + enrichment. +Contract: schema/categories.md. +""" + +from reachability.adapters import conditions_map +from reachability.graphstore import InMemoryGraphStore +from reachability.models.graph import Edge, Node + +from .conftest import make_finding + + +def test_defaults_transcribe_categories_md(): + ssrf = conditions_map.defaults_for("ssrf") + assert ssrf is not None + assert "exec:ssrf_outbound" in ssrf.postconditions + assert "access:cloud_creds" in ssrf.postconditions + assert "position:pivot_to:" in ssrf.postconditions + assert "segmentation" in ssrf.mitigated_by + assert ssrf.default_base_severity == "high" + + +def test_aliases_share_defaults(): + assert conditions_map.defaults_for("idor") is not None + assert conditions_map.defaults_for("idor").postconditions == \ + conditions_map.defaults_for("broken_access_control").postconditions + assert conditions_map.defaults_for("lfi") is conditions_map.defaults_for("lfi") + assert conditions_map.defaults_for("deserialization") is not None + + +def test_vulnerable_dependency_is_under_specified(): + # The categories.md invariant: no flat pre/post, severity stays scanner-driven. + d = conditions_map.defaults_for("vulnerable_dependency") + assert d.preconditions == () + assert d.postconditions == () + assert d.default_base_severity is None + + +def test_unknown_category_has_no_defaults(): + assert conditions_map.defaults_for("unknown") is None + assert conditions_map.defaults_for("something_exotic") is None + + +def test_enrich_fills_empty_conditions(): + f = make_finding("asset:x", category="xss") # no pre/post supplied + conditions_map.enrich(f) + assert "network:internet_reachable" in f.preconditions + assert "access:session_token" in f.postconditions + + +def test_enrich_does_not_overwrite_existing(): + f = make_finding("asset:x", category="xss", pre=["network:internal_only"]) + conditions_map.enrich(f) + assert f.preconditions == ["network:internal_only"] # adapter's value preserved + + +def test_enrich_is_noop_for_vulnerable_dependency(): + f = make_finding("asset:lib", category="vulnerable_dependency") + conditions_map.enrich(f) + assert f.preconditions == [] + assert f.postconditions == [] + + +def test_enrich_resolves_asset_zone_placeholder(): + g = InMemoryGraphStore() + g.upsert_node(Node(id="asset:worker", type="asset", label="worker")) + g.upsert_node(Node(id="zone:batch", type="zone", label="batch")) + g.upsert_edge(Edge(type="resides_in", src="asset:worker", dst="zone:batch")) + f = make_finding("asset:worker", category="command_injection") + conditions_map.enrich(f, g) + assert "position:foothold:batch" in f.postconditions # -> batch + assert "exec:code_on_host" in f.postconditions + + +def test_enrich_resolves_internal_zone_from_connectivity(): + g = InMemoryGraphStore() + g.upsert_node(Node(id="asset:web", type="asset", label="web")) + g.upsert_node(Node(id="zone:dmz", type="zone", label="dmz")) + g.upsert_node(Node(id="zone:internal", type="zone", label="internal")) + g.upsert_edge(Edge(type="resides_in", src="asset:web", dst="zone:dmz")) + g.upsert_edge(Edge(type="connects_to", src="zone:dmz", dst="zone:internal")) + f = make_finding("asset:web", category="ssrf") + conditions_map.enrich(f, g) + assert "position:pivot_to:internal" in f.postconditions + + +def test_enrich_drops_unresolvable_placeholder(): + # No graph -> the token can't resolve and is dropped, not emitted. + f = make_finding("asset:x", category="command_injection") + conditions_map.enrich(f) + assert "exec:code_on_host" in f.postconditions + assert not any("<" in t for t in f.postconditions) diff --git a/core/tests/test_diagrams.py b/core/tests/test_diagrams.py new file mode 100644 index 0000000..05087a5 --- /dev/null +++ b/core/tests/test_diagrams.py @@ -0,0 +1,68 @@ +"""Diagrams-as-code (Mermaid) ingestion. Contract: 05-context-ingestion.md §4.""" + +from pathlib import Path + +from reachability.engine import reachability +from reachability.graphstore import InMemoryGraphStore +from reachability.ingestion import diagrams, questionnaire +from reachability.models.graph import Confidence + +from .conftest import make_finding + +FIX = Path(__file__).resolve().parents[0] / "fixtures" + + +def _load(): + return diagrams.load(str(FIX / "topology.mmd")) + + +def _by_id(nodes): + return {n.id: n for n in nodes} + + +def test_node_shapes_map_to_types(): + n = _by_id(_load()[0]) + assert n["entrypoint:net"].attributes["public"] is True # ((Internet)) circle + assert n["entrypoint:net"].label == "Internet" + assert n["asset:web"].type == "asset" # [Web Frontend] + assert n["asset:web"].label == "Web Frontend" + assert n["datastore:db"].type == "datastore" # [(Customer DB)] cylinder + + +def test_everything_is_declared_confidence(): + nodes, edges = _load() + assert all(x.confidence is Confidence.declared for x in nodes) + assert all(x.confidence is Confidence.declared for x in edges) + assert all(x.source == "diagram" for x in nodes) + + +def test_directed_edges_become_routes_to(): + edges = _load()[1] + assert any(e.type == "routes_to" and e.src == "entrypoint:net" and e.dst == "asset:lb" + for e in edges) + assert any(e.type == "routes_to" and e.src == "asset:web" and e.dst == "datastore:db" + for e in edges) + + +def test_subgraphs_become_zones_with_membership(): + nodes, edges = _load() + n = _by_id(nodes) + assert n["zone:dmz"].type == "zone" + # lb first appears inside the dmz subgraph block -> resides_in dmz (Mermaid semantics) + assert any(e.type == "resides_in" and e.src == "asset:lb" and e.dst == "zone:dmz" for e in edges) + assert any(e.type == "resides_in" and e.src == "datastore:db" and e.dst == "zone:internal" + for e in edges) + + +def test_subgraph_to_subgraph_edge_is_connects_to(): + edges = _load()[1] + assert any(e.type == "connects_to" and e.src == "zone:dmz" and e.dst == "zone:internal" + for e in edges) + + +def test_diagram_topology_drives_reachability(): + # With no IaC, the diagram alone makes the web frontend internet-reachable. + g = InMemoryGraphStore() + questionnaire.populate(g, *_load()) + facts = reachability.compute_one(make_finding("asset:web", category="xss"), g) + assert facts.reachable_from_internet is True diff --git a/core/tests/test_enrichment.py b/core/tests/test_enrichment.py new file mode 100644 index 0000000..108d7a2 --- /dev/null +++ b/core/tests/test_enrichment.py @@ -0,0 +1,80 @@ +"""LLM pre/postcondition proposals (advisory, human-reviewed). Contract: 06 §LLM.""" + +import json + +from reachability.llm.enrichment import Proposal, apply_proposal, propose_conditions + +from .conftest import make_finding + + +class _FakeProvider: + def __init__(self, text, avail=True, raises=False): + self._text, self._avail, self._raises = text, avail, raises + + def available(self): + return self._avail + + def complete(self, prompt, *, system=None, max_tokens=512): + if self._raises: + raise RuntimeError("model error") + return self._text + + +def _finding(): + return make_finding("asset:web", category="ssrf") + + +def test_no_provider_means_no_proposal(): + # The deterministic defaults stand; the model is never required. + assert propose_conditions(_finding(), provider=None) is None + + +def test_unavailable_provider_means_no_proposal(): + assert propose_conditions(_finding(), provider=_FakeProvider("", avail=False)) is None + + +def test_proposal_parses_and_validates_tokens(): + payload = json.dumps({ + "preconditions": ["network:internet_reachable"], + "postconditions": ["access:cloud_creds", "position:pivot_to:metadata"], + "rationale": "SSRF to the metadata endpoint yields cloud creds.", + }) + p = propose_conditions(_finding(), provider=_FakeProvider(payload)) + assert isinstance(p, Proposal) + assert p.preconditions == ["network:internet_reachable"] + assert "access:cloud_creds" in p.postconditions + assert p.status == "proposed" + assert p.rationale + + +def test_invalid_tokens_are_dropped(): + payload = json.dumps({ + "preconditions": ["magic:foo", "network:internet_reachable", "not-a-token"], + "postconditions": ["access:secrets"], + }) + p = propose_conditions(_finding(), provider=_FakeProvider(payload)) + assert p.preconditions == ["network:internet_reachable"] # bogus namespaces dropped + assert p.postconditions == ["access:secrets"] + + +def test_prose_wrapped_json_is_tolerated(): + payload = 'Sure! Here you go:\n{"preconditions": [], "postconditions": ["exec:code_on_host"]}\nHope that helps.' + p = propose_conditions(_finding(), provider=_FakeProvider(payload)) + assert p.postconditions == ["exec:code_on_host"] + + +def test_malformed_output_yields_no_proposal(): + assert propose_conditions(_finding(), provider=_FakeProvider("not json at all")) is None + assert propose_conditions(_finding(), provider=_FakeProvider("{}", raises=True)) is None + + +def test_apply_proposal_is_the_only_path_into_the_finding(): + f = _finding() + f.preconditions = [] # bare finding + proposal = Proposal(finding_id=f.finding_id, + preconditions=["network:internet_reachable"], + postconditions=["access:cloud_creds"]) + apply_proposal(f, proposal) + assert f.preconditions == ["network:internet_reachable"] + assert f.postconditions == ["access:cloud_creds"] + assert proposal.status == "applied" diff --git a/core/tests/test_grype_live.py b/core/tests/test_grype_live.py new file mode 100644 index 0000000..894656b --- /dev/null +++ b/core/tests/test_grype_live.py @@ -0,0 +1,35 @@ +"""Live Grype scan, gated on grype being on PATH (runs in CI; skipped otherwise). + +Exercises the real declarative-adapter path end to end: invoke grype, normalize its +output, and assert genuine vulnerable_dependency findings — the complement to the +recorded-fixture tests in test_normalize/test_grype.py. +""" + +import shutil +from pathlib import Path + +import pytest +import yaml + +from reachability.adapters.protocol import ScanTarget +from reachability.adapters.yaml_adapter import YamlAdapter + +REPO_ROOT = Path(__file__).resolve().parents[2] +grype_live = pytest.mark.skipif(shutil.which("grype") is None, reason="grype not on PATH") + + +@grype_live +def test_grype_live_scans_sbom_and_normalizes(): + spec = yaml.safe_load((REPO_ROOT / "adapters" / "grype" / "adapter.yaml").read_text()) + adapter = YamlAdapter(spec) + assert adapter.healthcheck() is True + assert adapter.version != "unknown" # parsed from `grype version -o json` + + sbom = REPO_ROOT / "core" / "tests" / "fixtures" / "sbom_cyclonedx.json" + findings = list(adapter.run(ScanTarget(env_id="", targets=[f"sbom:{sbom}"]))) + + assert findings, "grype should report vulnerabilities for lodash@4.17.20" + assert all(f["category"]["type"] == "vulnerable_dependency" for f in findings) + assert any(f["cve"] for f in findings) + # the recorded fixture intentionally pins a known-vulnerable lodash + assert any("lodash" in (f["asset_ref"]["locator"].get("component") or "") for f in findings) diff --git a/core/tests/test_narrative.py b/core/tests/test_narrative.py new file mode 100644 index 0000000..4f86450 --- /dev/null +++ b/core/tests/test_narrative.py @@ -0,0 +1,75 @@ +"""Chain narratives (advisory, runs last). Contract: 06-attack-path-chaining.md §LLM.""" + +from reachability.engine.chaining import Chain +from reachability.llm.narrative import narrate_all, narrate_chain +from reachability.llm.provider import ClaudeProvider, NullProvider, default_provider + +from .conftest import make_finding + + +def _chain_and_findings(): + ssrf = make_finding("asset:web", category="ssrf", pre=["network:internet_reachable"], + post=["position:pivot_to:internal", "access:cloud_creds"]) + authz = make_finding("asset:db", category="missing_authentication", + post=["access:db_read"]) + chain = Chain(id="chain:1", steps=[ssrf.finding_id, authz.finding_id], + target="asset:db", reaches_crown_jewel=True, score=3.0) + return chain, [ssrf, authz] + + +class _FakeProvider: + def __init__(self, text="LLM PROSE", avail=True, raises=False): + self._text, self._avail, self._raises = text, avail, raises + + def available(self): + return self._avail + + def complete(self, prompt, *, system=None, max_tokens=512): + if self._raises: + raise RuntimeError("model error") + return self._text + + +def test_template_narrative_is_deterministic_and_concrete(): + chain, findings = _chain_and_findings() + by_id = {f.finding_id: f for f in findings} + a = narrate_chain(chain, by_id) # no provider -> template + b = narrate_chain(chain, by_id) + assert a == b # deterministic + assert "ssrf on asset:web" in a + assert "pivoting to internal" in a + assert "database read access" in a + assert "crown-jewel asset:db across 2 hops" in a + + +def test_llm_provider_output_is_used_when_available(): + chain, findings = _chain_and_findings() + by_id = {f.finding_id: f for f in findings} + assert narrate_chain(chain, by_id, provider=_FakeProvider("An attacker ...")) == "An attacker ..." + + +def test_falls_back_to_template_when_provider_errors(): + chain, findings = _chain_and_findings() + by_id = {f.finding_id: f for f in findings} + out = narrate_chain(chain, by_id, provider=_FakeProvider(raises=True)) + assert "ssrf on asset:web" in out # template, not an exception + + +def test_unavailable_provider_uses_template(): + chain, findings = _chain_and_findings() + by_id = {f.finding_id: f for f in findings} + out = narrate_chain(chain, by_id, provider=_FakeProvider(avail=False)) + assert "This chain reaches" in out + + +def test_narrate_all_keys_by_chain_id(): + chain, findings = _chain_and_findings() + result = narrate_all([chain], findings) + assert set(result) == {"chain:1"} + + +def test_provider_availability_without_key(monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + assert ClaudeProvider(api_key=None).available() is False + assert NullProvider().available() is False + assert isinstance(default_provider(), NullProvider) diff --git a/core/tests/test_normalize/test_grype.py b/core/tests/test_normalize/test_grype.py index 4c660eb..2e36c1b 100644 --- a/core/tests/test_normalize/test_grype.py +++ b/core/tests/test_normalize/test_grype.py @@ -1,15 +1,77 @@ -"""normalize() unit tests against recorded Grype output — no live scanner. +"""Grype normalize() against recorded output — no live scanner. Contract: 03-adapter-interface.md ("normalize() kept public + separable so it can -be unit-tested against captured scanner output"). +be unit-tested against captured scanner output"). Drives the YamlAdapter +interpreter against the *actual* shipped adapters/grype/adapter.yaml. """ -import pytest +import json +from pathlib import Path +import yaml -@pytest.mark.skip(reason="scaffold: implement YamlAdapter.normalize() then unskip") -def test_grype_maps_cve_and_component(): - # Given a recorded matches[*] item, normalize() should yield category.type= - # vulnerable_dependency, the CVE id, and asset_ref.locator.component name@version, - # preserving the native item under 'raw'. - raise NotImplementedError +from reachability.adapters.yaml_adapter import YamlAdapter +from reachability.models.finding import Finding, Severity + +REPO_ROOT = Path(__file__).resolve().parents[3] +GRYPE_YAML = REPO_ROOT / "adapters" / "grype" / "adapter.yaml" +FIXTURE = REPO_ROOT / "core" / "tests" / "fixtures" / "grype_sample.json" + + +def _adapter() -> YamlAdapter: + return YamlAdapter(yaml.safe_load(GRYPE_YAML.read_text(encoding="utf-8"))) + + +def _raw() -> dict: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def _first_match() -> dict: + return _raw()["matches"][0] + + +def test_iterator_extracts_all_matches(): + assert len(_adapter().iterate(_raw())) == 2 + + +def test_normalize_maps_grype_fields(): + f = _adapter().normalize(_first_match()) + assert f["category"]["type"] == "vulnerable_dependency" # from defaults + assert f["cve"] == ["CVE-2021-23337"] # scalar coerced to list + assert f["asset_ref"]["locator"]["component"] == "lodash@4.17.20" # concat expr + assert f["base_severity"]["level"] == "high" # "High" | lower + assert f["title"] == "CVE-2021-23337" + assert f["adapter"] == {"name": "grype", "kind": "sca", "version": "unknown"} + + +def test_normalize_preserves_raw_untouched(): + item = _first_match() + assert _adapter().normalize(item)["raw"] == item + + +def test_finding_id_is_stable_and_deterministic(): + ad = _adapter() + a = ad.normalize(_first_match()) + b = ad.normalize(_first_match()) + assert a["fingerprint"].startswith("sha256-") + assert a["finding_id"] == b["finding_id"] # reproducible + # distinct findings get distinct ids + assert a["finding_id"] != ad.normalize(_raw()["matches"][1])["finding_id"] + + +def test_version_extraction_handles_json_and_plain(): + from reachability.adapters.yaml_adapter import _extract_version + # grype version -o json prints a JSON object with a "version" field + assert _extract_version('{"application":"grype","version":"0.114.0"}') == "0.114.0" + # plain tools: first line + assert _extract_version("nmap 7.94\nPlatform: x86_64") == "nmap 7.94" + + +def test_from_dict_roundtrips_severity_and_category(): + f = _adapter().normalize(_first_match()) + finding = Finding.from_dict(f) + assert finding.base_severity.level is Severity.high + assert finding.category.type == "vulnerable_dependency" + assert finding.cve == ["CVE-2021-23337"] + # to_dict emits the level back as its ladder name + assert finding.to_dict()["base_severity"]["level"] == "high" diff --git a/core/tests/test_normalize/test_zap.py b/core/tests/test_normalize/test_zap.py new file mode 100644 index 0000000..719846f --- /dev/null +++ b/core/tests/test_normalize/test_zap.py @@ -0,0 +1,62 @@ +"""ZAP normalize() against recorded alerts — no live scanner. + +Contract: 03-adapter-interface.md (normalize() pure + testable on captured output). +Loads the actual shipped adapters/zap/adapter.py via the registry's code path. +""" + +import importlib.util +import json +from pathlib import Path + +from reachability.models.finding import Finding, Severity + +REPO_ROOT = Path(__file__).resolve().parents[3] +FIXTURE = REPO_ROOT / "core" / "tests" / "fixtures" / "zap_alerts.json" + + +def _load_zap_adapter(): + path = REPO_ROOT / "adapters" / "zap" / "adapter.py" + spec = importlib.util.spec_from_file_location("_zap_adapter", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.ADAPTER() + + +def _alerts(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["alerts"] + + +def test_csp_alert_maps_to_security_misconfiguration(): + f = _load_zap_adapter().normalize(_alerts()[0]) + assert f["category"]["type"] == "security_misconfiguration" + assert f["category"]["cwe"] == ["CWE-693"] + assert f["base_severity"]["level"] == "medium" + assert f["evidence"]["confidence"] == "confirmed" # ZAP High confidence + assert f["asset_ref"]["locator"]["host"] == "target.local" + assert f["asset_ref"]["locator"]["port"] == 8080 + assert f["adapter"]["kind"] == "dast" + + +def test_reflected_xss_maps_to_xss_with_evidence(): + f = _load_zap_adapter().normalize(_alerts()[1]) + assert f["category"]["type"] == "xss" + assert f["category"]["cwe"] == ["CWE-79"] + assert f["base_severity"]["level"] == "high" + assert f["evidence"]["confidence"] == "firm" # ZAP Medium confidence + assert "