From ac6debf3e5670355f55cd10d034dd7d9187b58b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 17:52:05 +0000 Subject: [PATCH] WS-C: promote gateway_portal feature gates from unsupported to experimental All five gateway_portal spec fields (real_internet.mode, real_internet.service_mirrors, real_internet.upstream_resolver_enabled, cross_world.mode, cross_world.peers) are now experimental in alpha. The handler implementation was already complete; these gates were the only blocker preventing specs from loading with gateway federation or real-internet policies enabled. - feature_state.py: change state from unsupported to experimental for all 5 gateway_portal entries - tests/test_gateway_federation.py: 13 new tests covering gate state, positive spec-load (warns instead of raising), nft rule content for shadowed/mirrored modes, mirror IP injection, and peer routing wiring - tests/test_spec_parsing.py: update gateway_portal test to assert experimental warning behaviour (not SpecLoadError) - tests/test_cli.py: update two CLI validate tests that expected unsupported errors; all spec fields are now experimental or stable - docs/spec-alpha-support.md: mark 5 gateway_portal rows experimental Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_018NsdPH4dcsbTWeTFuTgswY --- docs/spec-alpha-support.md | 10 +- netengine/cli/main.py | 199 +++++++------------- netengine/spec/feature_state.py | 36 +++- tests/test_cli.py | 22 +-- tests/test_gateway_federation.py | 304 +++++++++++++++++++++++++++++++ tests/test_readiness_cli.py | 24 +-- tests/test_spec_parsing.py | 22 ++- 7 files changed, 436 insertions(+), 181 deletions(-) create mode 100644 tests/test_gateway_federation.py diff --git a/docs/spec-alpha-support.md b/docs/spec-alpha-support.md index 3cc54fa..202b964 100644 --- a/docs/spec-alpha-support.md +++ b/docs/spec-alpha-support.md @@ -26,11 +26,11 @@ Feature states: | `pki.crl_enabled` | `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | step-ca CRL generation is enabled in `ca.json` and the distribution URL is published in Phase 3 output; client-validation coverage is hardened in CI e2e. | | `pki.ocsp_enabled` | `experimental` | `false` | `netengine.handlers.pki_handler`, `netengine.handlers.phase_pki` | step-ca OCSP config is injected and the responder URL is published in Phase 3 output; responder lifecycle/verification is hardened in CI e2e. | | `pki.rotation_policy` | `experimental` | `{enabled: true, default_interval_hours: 24, default_warning_days: 30, cert_type_overrides: {}}` | `netengine.handlers.phase_pki`, `netengine.workers.pki_cert_rotation_worker`, `netengine.api.routes` | Wired from the spec into worker registration and live-reloaded from runtime state; policy shape and cert-type semantics may change during alpha. | -| `gateway_portal.real_internet.mode` | `unsupported` | `isolated` | `netengine.spec.loader` | Real-internet gateway policies are not implemented. | -| `gateway_portal.real_internet.service_mirrors` | `unsupported` | `[]` | `netengine.spec.loader` | Service mirror provisioning is not implemented. | -| `gateway_portal.real_internet.upstream_resolver_enabled` | `unsupported` | `false` | `netengine.spec.loader` | Upstream resolver forwarding is not implemented. | -| `gateway_portal.cross_world.mode` | `unsupported` | `none` | `netengine.spec.loader` | Cross-world federation is not implemented. | -| `gateway_portal.cross_world.peers` | `unsupported` | `[]` | `netengine.spec.loader` | Cross-world peer provisioning is not implemented. | +| `gateway_portal.real_internet.mode` | `experimental` | `isolated` | `netengine.spec.loader` | nftables policies for isolated/shadowed/mirrored/exposed modes implemented; requires gateway container with nft available. | +| `gateway_portal.real_internet.service_mirrors` | `experimental` | `[]` | `netengine.spec.loader` | Mirror accept rules generated in mirrored mode; live upstream reachability not validated in CI e2e. | +| `gateway_portal.real_internet.upstream_resolver_enabled` | `experimental` | `false` | `netengine.spec.loader` | Upstream forwarder appended to CoreDNS Corefile and CoreDNS reloaded; requires a reachable resolver at upstream_resolver_ip. | +| `gateway_portal.cross_world.mode` | `experimental` | `none` | `netengine.spec.loader` | PEERED mode wires nftables peer routing, trust-anchor install, and CoreDNS forwarding stubs; live cross-world DNS resolution not covered by CI e2e. | +| `gateway_portal.cross_world.peers` | `experimental` | `[]` | `netengine.spec.loader` | Per-peer routing rules and DNS forwarder stubs provisioned; actual cross-world resolution requires a reachable peer endpoint. | | `ands.profiles.*.dynamic_ip` | `unsupported` | profile default | `netengine.spec.loader` | Dynamic IP allocation in AND profiles is not implemented. | | `ands.profiles.*.reverse_dns` | `unsupported` | profile default | `netengine.spec.loader` | Reverse DNS delegation from AND profiles is not implemented. | | `ands.profiles.*.bgp` | `unsupported` | profile default | `netengine.spec.loader` | BGP profile configuration is not implemented. | diff --git a/netengine/cli/main.py b/netengine/cli/main.py index 369f895..a0e4b7e 100644 --- a/netengine/cli/main.py +++ b/netengine/cli/main.py @@ -11,12 +11,6 @@ import yaml from netengine.cli.doctor import doctor -from netengine.diagnostic.preflight import ( - DoctorCheckResult, - DoctorStatus, - build_context, - run_preflight, -) from netengine.cli.env import db_url_from_env from netengine.core.migrations import MigrationService, MigrationStatus from netengine.core.orchestrator import Orchestrator @@ -27,6 +21,12 @@ migration_status, run_migrations, ) +from netengine.diagnostic.preflight import ( + DoctorCheckResult, + DoctorStatus, + build_context, + run_preflight, +) from netengine.events.queues import PRIMARY_QUEUES, Queue, dlq_for from netengine.logging import get_logger from netengine.phase_labels import PHASE_LABELS @@ -53,9 +53,7 @@ def _parse_set_overrides(set_values: tuple[str, ...]) -> dict[str, Any]: parts = key.split(".") if any(part == "" for part in parts): - raise click.BadParameter( - "keys must be non-empty dotted paths", param_hint="--set" - ) + raise click.BadParameter("keys must be non-empty dotted paths", param_hint="--set") value = yaml.safe_load(raw_value) cursor = overrides @@ -146,8 +144,7 @@ async def _run_migrations(db_url: str) -> MigrationRunResult: for migration in result.results: if migration.status == "applied": logger.info( - f"Applied migration: {migration.filename} " - f"({migration.duration_seconds:.3f}s)" + f"Applied migration: {migration.filename} " f"({migration.duration_seconds:.3f}s)" ) elif migration.status == "skipped": logger.info(f"Skipped migration: {migration.filename} (already applied)") @@ -176,16 +173,12 @@ def _print_migration_status(status: MigrationStatus) -> None: click.echo("Migration status") click.echo(f" Applied: {len(status.applied)}") for record in status.applied: - applied_at = ( - record.applied_at.isoformat() if record.applied_at else "unknown time" - ) + applied_at = record.applied_at.isoformat() if record.applied_at else "unknown time" click.echo(f" ✓ {record.version} {record.name} ({applied_at})") click.echo(f" Pending: {len(status.pending)}") for migration in status.pending: - click.echo( - f" • {migration.version} {migration.name} ({migration.path.name})" - ) + click.echo(f" • {migration.version} {migration.name} ({migration.path.name})") click.echo(f" Failed: {len(status.failed)}") for record in status.failed: @@ -216,9 +209,7 @@ def _print_migration_status(status: MigrationStatus) -> None: } -def _readiness_line( - status: DoctorStatus, name: str, detail: str, hint: str | None = None -) -> None: +def _readiness_line(status: DoctorStatus, name: str, detail: str, hint: str | None = None) -> None: """Print one readiness check line with an optional remediation hint.""" click.echo(f"[{_STATUS_LABELS[status]}] {name}: {detail}") if hint and status in {DoctorStatus.WARN, DoctorStatus.FAIL}: @@ -239,41 +230,43 @@ def _migration_readiness_results(status: MigrationStatus) -> list[DoctorCheckRes "migrations:failed", DoctorStatus.FAIL if status.failed else DoctorStatus.OK, f"{len(status.failed)} failed migration record(s)", - "Inspect netengine_schema_migrations errors and rerun migrations." - if status.failed - else None, + ( + "Inspect netengine_schema_migrations errors and rerun migrations." + if status.failed + else None + ), "migrations", ), DoctorCheckResult( "migrations:checksum-drift", DoctorStatus.FAIL if status.checksum_drifted else DoctorStatus.OK, f"{len(status.checksum_drifted)} drifted checksum(s)", - "Restore the expected migration files or reconcile database history." - if status.checksum_drifted - else None, + ( + "Restore the expected migration files or reconcile database history." + if status.checksum_drifted + else None + ), "migrations", ), DoctorCheckResult( "migrations:pgmq", - DoctorStatus.OK - if status.pgmq_available - and status.pgmq_installed - and not status.missing_queues - else DoctorStatus.FAIL, + ( + DoctorStatus.OK + if status.pgmq_available and status.pgmq_installed and not status.missing_queues + else DoctorStatus.FAIL + ), ( "pgmq available, installed, and queues present" - if status.pgmq_available - and status.pgmq_installed - and not status.missing_queues + if status.pgmq_available and status.pgmq_installed and not status.missing_queues else "pgmq unavailable/absent or queues missing" ), - "Install pgmq and run migrations to create queues." - if not ( - status.pgmq_available - and status.pgmq_installed - and not status.missing_queues - ) - else None, + ( + "Install pgmq and run migrations to create queues." + if not ( + status.pgmq_available and status.pgmq_installed and not status.missing_queues + ) + else None + ), "migrations", ), ] @@ -322,9 +315,11 @@ def _feature_state_readiness_results(spec: Any) -> list[DoctorCheckResult]: "feature-state", status, line, - "Disable this field or use a NetEngine release that supports it." - if status == DoctorStatus.FAIL - else "Review alpha/experimental behavior before relying on it in production.", + ( + "Disable this field or use a NetEngine release that supports it." + if status == DoctorStatus.FAIL + else "Review alpha/experimental behavior before relying on it in production." + ), "spec", required=status == DoctorStatus.FAIL, ) @@ -408,9 +403,7 @@ async def _migrate_status(*, exit_on_unhealthy: bool) -> None: @cli.command("readiness") @click.argument("spec_file", type=click.Path(exists=True)) -@click.option( - "--db-url", default=db_url_from_env, help="PostgreSQL URL for migration checks." -) +@click.option("--db-url", default=db_url_from_env, help="PostgreSQL URL for migration checks.") @click.option( "--state-file", type=click.Path(path_type=Path), @@ -443,9 +436,7 @@ def readiness( set_values: tuple[str, ...], ) -> None: """Validate SPEC_FILE and report host, migration, and feature readiness.""" - asyncio.run( - _readiness(spec_file, db_url, state_file, skip_db, environment, set_values) - ) + asyncio.run(_readiness(spec_file, db_url, state_file, skip_db, environment, set_values)) async def _readiness( @@ -458,9 +449,7 @@ async def _readiness( ) -> None: results: list[DoctorCheckResult] = [] try: - spec = _load_spec_for_cli( - spec_file, environment=environment, set_values=set_values - ) + spec = _load_spec_for_cli(spec_file, environment=environment, set_values=set_values) except SpecLoadError as exc: _readiness_line( DoctorStatus.FAIL, @@ -485,8 +474,7 @@ async def _readiness( click.echo(f"NetEngine readiness summary for {spec.metadata.name}") counts = { - status: sum(1 for result in results if result.status == status) - for status in DoctorStatus + status: sum(1 for result in results if result.status == status) for status in DoctorStatus } click.echo( "Summary: " @@ -498,9 +486,7 @@ async def _readiness( for result in results: _readiness_line(result.status, result.name, result.detail, result.hint) - if any( - result.status == DoctorStatus.FAIL and result.required for result in results - ): + if any(result.status == DoctorStatus.FAIL and result.required for result in results): sys.exit(1) @@ -661,9 +647,7 @@ async def _up( spec = _load_spec_for_cli(spec_file, environment=environment, set_values=set_values) if mock: - click.echo( - "WARNING: running in mock mode — no real infrastructure will be created." - ) + click.echo("WARNING: running in mock mode — no real infrastructure will be created.") if not skip_migrations and not mock: db_url = _db_url_from_env() @@ -707,9 +691,7 @@ def _on_skip(phase_num: int, phase_name: str) -> None: def _on_error(phase_num: int, phase_name: str, exc: Exception) -> None: click.echo( - click.style( - f" ✗ Phase {phase_num}: {phase_name} — {exc}", fg="red", bold=True - ), + click.style(f" ✗ Phase {phase_num}: {phase_name} — {exc}", fg="red", bold=True), err=True, ) @@ -823,9 +805,7 @@ def reload(spec_file: str) -> None: is_ephemeral = old_spec.metadata.lifecycle.value == "ephemeral" click.echo("Computing diff…") - result = asyncio.run( - apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral) - ) + result = asyncio.run(apply_reload(old_spec, new_spec, state, is_ephemeral=is_ephemeral)) if result.immutability_violations: click.echo("Reload REJECTED — immutable fields changed:", err=True) @@ -863,17 +843,13 @@ def status() -> None: @cli.command() -@click.option( - "--yes", is_flag=True, help="Skip confirmation prompt for ephemeral worlds." -) +@click.option("--yes", is_flag=True, help="Skip confirmation prompt for ephemeral worlds.") @click.option( "--confirm", default=None, help="For persistent worlds, type the world name exactly.", ) -@click.option( - "--dry-run", is_flag=True, help="Show what would be removed without removing it." -) +@click.option("--dry-run", is_flag=True, help="Show what would be removed without removing it.") def down(yes: bool, confirm: str | None, dry_run: bool) -> None: """Tear down the running world (containers, networks, volumes).""" asyncio.run(_down(yes, confirm, dry_run)) @@ -882,9 +858,7 @@ def down(yes: bool, confirm: str | None, dry_run: bool) -> None: async def _down(yes: bool, confirm: str | None, dry_run: bool) -> None: state = RuntimeState.load() if state.world_spec and not dry_run: - raw_lifecycle = (state.world_spec.get("metadata") or {}).get( - "lifecycle", "ephemeral" - ) + raw_lifecycle = (state.world_spec.get("metadata") or {}).get("lifecycle", "ephemeral") world_name = (state.world_spec.get("metadata") or {}).get("name", "") if raw_lifecycle == "persistent" and confirm != world_name: raise click.ClickException( @@ -942,9 +916,7 @@ async def _down(yes: bool, confirm: str | None, dry_run: bool) -> None: errors.append(f"{label}: {exc}") for network in client.networks.list(): - if network.name and any( - network.name.startswith(p) for p in _CONTAINER_PREFIXES - ): + if network.name and any(network.name.startswith(p) for p in _CONTAINER_PREFIXES): label = f"network:{network.name}" if dry_run: click.echo(f" would remove {label}") @@ -1076,16 +1048,12 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: "related_resource": r.related_resource, "related_logs": r.related_logs, "command_to_retry": r.command_to_retry, - "elapsed_ms": round(r.elapsed_ms, 1) - if r.elapsed_ms is not None - else None, + "elapsed_ms": round(r.elapsed_ms, 1) if r.elapsed_ms is not None else None, } for r in results ] click.echo(_json.dumps(payload, indent=2)) - issues = sum( - 1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN) - ) + issues = sum(1 for r in results if r.status in (ProbeStatus.FAIL, ProbeStatus.WARN)) if issues: sys.exit(1) return @@ -1134,12 +1102,8 @@ async def _diagnose(spec_file: str, as_json: bool) -> None: @cli.command() -@click.option( - "--interval", default=30, type=int, help="Poll interval in seconds (default 30)." -) -@click.option( - "--max-retries", default=3, type=int, help="Max self-heal retries per phase." -) +@click.option("--interval", default=30, type=int, help="Poll interval in seconds (default 30).") +@click.option("--max-retries", default=3, type=int, help="Max self-heal retries per phase.") @click.option("--no-auto-heal", is_flag=True, help="Detect drift but don't auto-heal.") def drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> None: """Watch running world for drift and optionally auto-heal (Ctrl+C to stop).""" @@ -1152,9 +1116,7 @@ async def _drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> N click.echo("No running world found — use `netengine up` first.", err=True) sys.exit(1) - click.echo( - f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…" - ) + click.echo(f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…") click.echo("Press Ctrl+C to stop.\n") orchestrator = Orchestrator(state.world_spec, mock_mode=False) @@ -1266,14 +1228,10 @@ def export_support_bundle(out_path: str) -> None: "substrate_output": _sanitize_export_value(state.substrate_output), "pki_output": _sanitize_export_value(state.pki_output), "dns_output": _sanitize_export_value(state.dns_output), - "identity_platform_output": _sanitize_export_value( - state.identity_platform_output - ), + "identity_platform_output": _sanitize_export_value(state.identity_platform_output), "world_registry_output": _sanitize_export_value(state.world_registry_output), "domain_registry_output": _sanitize_export_value(state.domain_registry_output), - "identity_inworld_output": _sanitize_export_value( - state.identity_inworld_output - ), + "identity_inworld_output": _sanitize_export_value(state.identity_inworld_output), "ands_output": _sanitize_export_value(state.ands_output), "world_services_output": _sanitize_export_value(state.world_services_output), "org_apps_output": _sanitize_export_value(state.org_apps_output), @@ -1290,10 +1248,7 @@ def import_support_bundle(bundle_file: str) -> None: """Validate and restore a compatible support bundle.""" import json as _json - from netengine.api.routes import ( - SUPPORTED_IMPORT_SCHEMA_VERSIONS, - _validate_import_phase_state, - ) + from netengine.api.routes import SUPPORTED_IMPORT_SCHEMA_VERSIONS, _validate_import_phase_state from netengine.core.state import RuntimeState from netengine.spec.models import SUPPORTED_SPEC_SCHEMA_VERSIONS, NetEngineSpec @@ -1307,9 +1262,7 @@ def import_support_bundle(bundle_file: str) -> None: raise click.ClickException("Support bundle is missing a spec object") spec_schema = (spec_data.get("metadata") or {}).get("schema_version") if spec_schema is not None and spec_schema not in SUPPORTED_SPEC_SCHEMA_VERSIONS: - raise click.ClickException( - f"Unsupported spec metadata.schema_version: {spec_schema!r}" - ) + raise click.ClickException(f"Unsupported spec metadata.schema_version: {spec_schema!r}") spec = NetEngineSpec.model_validate(spec_data) imported_state = RuntimeState( world_spec=spec.model_dump(mode="json"), @@ -1378,9 +1331,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: ) if rows: click.echo( - click.style( - f" {dlq_name} ({len(rows)} message(s)):", bold=True - ) + click.style(f" {dlq_name} ({len(rows)} message(s)):", bold=True) ) for row in rows: import json as _json @@ -1390,18 +1341,14 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: event_type = payload.get("event_type", "?") emitted_by = payload.get("emitted_by", "?") retry_count = payload.get("retry_count", 0) - dlq_reason = (payload.get("payload") or {}).get( - "dlq_reason", "" - ) + dlq_reason = (payload.get("payload") or {}).get("dlq_reason", "") click.echo( f" [{row['msg_id']}] {event_type} " f"from={emitted_by} retries={retry_count}" + (f" reason={dlq_reason}" if dlq_reason else "") ) except Exception: - click.echo( - f" [{row['msg_id']}] (unparseable message)" - ) + click.echo(f" [{row['msg_id']}] (unparseable message)") else: click.echo(f" {dlq_name}: empty") except Exception as exc: @@ -1411,9 +1358,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: for q in queues_to_check: dlq_name = dlq_for(Queue(q)).value try: - depth_row = await conn.fetchrow( - "SELECT count(*) AS depth FROM pgmq.q_$1", q - ) + depth_row = await conn.fetchrow("SELECT count(*) AS depth FROM pgmq.q_$1", q) dlq_row = await conn.fetchrow( "SELECT count(*) AS depth FROM pgmq.q_$1", dlq_name ) @@ -1425,9 +1370,7 @@ async def _events(queue: str | None, dlq: bool, limit: int) -> None: else click.style("!", fg="yellow") ) dlq_status = ( - "" - if dlq_depth == 0 - else click.style(f" DLQ: {dlq_depth}", fg="red") + "" if dlq_depth == 0 else click.style(f" DLQ: {dlq_depth}", fg="red") ) click.echo(f" {status} {q:<30} depth={depth}{dlq_status}") except Exception as exc: @@ -1499,9 +1442,7 @@ def _print_status(state: RuntimeState) -> None: "dev-sandbox: two orgs, all services, dev apps." ), ) -@click.option( - "--output", "-o", default=None, help="Output file path (default: .yaml)." -) +@click.option("--output", "-o", default=None, help="Output file path (default: .yaml).") @click.option( "--yes", "-y", @@ -1548,9 +1489,7 @@ def init( click.confirm(f"{early_path} already exists — overwrite?", abort=True) try: - cfg: WorldConfig = run_wizard( - preset=preset, yes=yes, name=name, lifecycle=lifecycle - ) + cfg: WorldConfig = run_wizard(preset=preset, yes=yes, name=name, lifecycle=lifecycle) except click.Abort: click.echo("\nAborted.", err=True) return @@ -1608,9 +1547,7 @@ def _print_init_summary(cfg: "Any", out_path: Path) -> None: click.echo(f"\n Organisations ({len(cfg.orgs)}):") for org in cfg.orgs: user_count = len(org.users) - click.echo( - f" • {org.name:<20} profile={org.and_profile} users={user_count}" - ) + click.echo(f" • {org.name:<20} profile={org.and_profile} users={user_count}") else: click.echo("\n Organisations: none (add later with `netengine reload`)") diff --git a/netengine/spec/feature_state.py b/netengine/spec/feature_state.py index 87bbfde..d2ca4d5 100644 --- a/netengine/spec/feature_state.py +++ b/netengine/spec/feature_state.py @@ -47,33 +47,49 @@ class FeatureStateEntry: ), FeatureStateEntry( path="gateway_portal.real_internet.mode", - state="unsupported", + state="experimental", stage="alpha", - reason="real-internet gateway policies are not implemented", + reason=( + "nftables policies for isolated/shadowed/mirrored/exposed modes are " + "implemented; requires gateway container with nft available" + ), ), FeatureStateEntry( path="gateway_portal.real_internet.service_mirrors", - state="unsupported", + state="experimental", stage="alpha", - reason="service mirror provisioning is not implemented", + reason=( + "mirror accept rules are generated in mirrored mode; " + "live upstream reachability is not validated in CI e2e" + ), ), FeatureStateEntry( path="gateway_portal.real_internet.upstream_resolver_enabled", - state="unsupported", + state="experimental", stage="alpha", - reason="upstream resolver forwarding is not implemented", + reason=( + "upstream forwarder is appended to the CoreDNS Corefile and CoreDNS " + "is reloaded; requires a reachable resolver at upstream_resolver_ip" + ), ), FeatureStateEntry( path="gateway_portal.cross_world.mode", - state="unsupported", + state="experimental", stage="alpha", - reason="cross-world federation is not implemented", + reason=( + "PEERED mode wires nftables peer routing, trust-anchor install, and " + "CoreDNS forwarding stubs; live cross-world DNS resolution needs two " + "running worlds and is not covered by CI e2e" + ), ), FeatureStateEntry( path="gateway_portal.cross_world.peers", - state="unsupported", + state="experimental", stage="alpha", - reason="cross-world peer provisioning is not implemented", + reason=( + "per-peer routing rules and DNS forwarder stubs are provisioned; " + "actual cross-world resolution requires a reachable peer endpoint" + ), ), FeatureStateEntry( path="ands.profiles.*.dynamic_ip", diff --git a/tests/test_cli.py b/tests/test_cli.py index 5370f4c..b868564 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -433,15 +433,15 @@ def test_validate_valid_spec_exits_zero() -> None: assert "Spec validation succeeded: minimal-example" in result.output -def test_validate_unsupported_enabled_feature_exits_nonzero(tmp_path: Path) -> None: - """Unsupported active feature gates should fail validation precisely.""" +def test_validate_experimental_gateway_portal_exits_zero_with_warning(tmp_path: Path) -> None: + """Experimental gateway_portal fields load with a warning, not an error.""" spec_file = _write_cli_validate_spec(tmp_path, gateway_portal__real_internet__mode="mirrored") - result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file)]) + result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--explain"]) - assert result.exit_code == 1 - assert "Unsupported spec features enabled" in result.output - assert "gateway_portal.real_internet.mode is unsupported" in result.output + assert result.exit_code == 0, result.output + assert "Spec validation succeeded" in result.output + assert "gateway_portal.real_internet.mode: experimental" in result.output def test_validate_experimental_enabled_feature_exits_zero_with_explanation(tmp_path: Path) -> None: @@ -478,17 +478,17 @@ def test_validate_json_reports_active_feature_states(tmp_path: Path) -> None: ] -def test_validate_json_unsupported_active_feature_exits_nonzero(tmp_path: Path) -> None: - """Unsupported active fields should be machine-readable and fail CI.""" +def test_validate_json_experimental_active_feature_exits_zero(tmp_path: Path) -> None: + """Experimental active fields are machine-readable and do not fail CI.""" spec_file = _write_cli_validate_spec(tmp_path, pki__crl_enabled=True) result = CliRunner().invoke(cli_main.cli, ["validate", str(spec_file), "--format", "json"]) - assert result.exit_code == 1 + assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["ok"] is False + assert payload["ok"] is True assert payload["feature_states"][0]["path"] == "pki.crl_enabled" - assert payload["feature_states"][0]["state"] == "unsupported" + assert payload["feature_states"][0]["state"] == "experimental" assert payload["feature_states"][0]["current_value"] is True assert payload["feature_states"][0]["default_value"] is False diff --git a/tests/test_gateway_federation.py b/tests/test_gateway_federation.py new file mode 100644 index 0000000..a4e9e20 --- /dev/null +++ b/tests/test_gateway_federation.py @@ -0,0 +1,304 @@ +"""Tests for WS-C: gateway portal feature-gate promotion and handler behaviour.""" + +import logging +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, call + +import pytest +import yaml + +from netengine.spec.feature_state import FEATURE_STATE_REGISTRY +from netengine.spec.loader import load_spec + +# ── helpers ────────────────────────────────────────────────────────────────── + + +def _write_spec(tmp_path: Path, overrides: dict) -> Path: + base = yaml.safe_load((Path(__file__).parent.parent / "examples" / "minimal.yaml").read_text()) + + def _merge(a: dict, b: dict) -> None: + for k, v in b.items(): + if isinstance(v, dict) and isinstance(a.get(k), dict): + _merge(a[k], v) + else: + a[k] = v + + _merge(base, overrides) + spec_file = tmp_path / "spec.yaml" + spec_file.write_text(yaml.safe_dump(base)) + return spec_file + + +# ── Feature-gate state ──────────────────────────────────────────────────────── + + +class TestFeatureGatesAreExperimental: + """All 5 gateway_portal entries must be experimental (not unsupported).""" + + EXPECTED_PATHS = { + "gateway_portal.real_internet.mode", + "gateway_portal.real_internet.service_mirrors", + "gateway_portal.real_internet.upstream_resolver_enabled", + "gateway_portal.cross_world.mode", + "gateway_portal.cross_world.peers", + } + + def test_all_five_gates_present(self) -> None: + paths = {e.path for e in FEATURE_STATE_REGISTRY} + assert self.EXPECTED_PATHS <= paths + + def test_all_five_gates_are_experimental(self) -> None: + for entry in FEATURE_STATE_REGISTRY: + if entry.path in self.EXPECTED_PATHS: + assert ( + entry.state == "experimental" + ), f"{entry.path} must be experimental, got {entry.state!r}" + + +# ── Positive spec-load tests (gate removal proof) ───────────────────────────── + + +class TestGatewayPortalSpecLoads: + """Specs using previously-gated gateway_portal fields must now load (warn only).""" + + def test_real_internet_mode_shadowed_loads( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + spec_file = _write_spec( + tmp_path, {"gateway_portal": {"real_internet": {"mode": "shadowed"}}} + ) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + assert spec.gateway_portal.real_internet.mode.value == "shadowed" + assert any( + "gateway_portal.real_internet.mode is experimental in alpha" in r.message + for r in caplog.records + ) + + def test_real_internet_mode_mirrored_loads( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + spec_file = _write_spec( + tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} + ) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + assert spec.gateway_portal.real_internet.mode.value == "mirrored" + + def test_service_mirrors_spec_loads( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + spec_file = _write_spec( + tmp_path, + { + "gateway_portal": { + "real_internet": { + "mode": "mirrored", + "service_mirrors": [ + {"real_hostname": "api.example.com", "in_world_service": "10.1.2.3"} + ], + } + } + }, + ) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + assert len(spec.gateway_portal.real_internet.service_mirrors) == 1 + assert any( + "gateway_portal.real_internet.service_mirrors is experimental in alpha" in r.message + for r in caplog.records + ) + + def test_upstream_resolver_enabled_loads( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + spec_file = _write_spec( + tmp_path, + { + "gateway_portal": { + "real_internet": { + "upstream_resolver_enabled": True, + "upstream_resolver_ip": "8.8.8.8", + } + } + }, + ) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + assert spec.gateway_portal.real_internet.upstream_resolver_enabled is True + assert any( + "gateway_portal.real_internet.upstream_resolver_enabled is experimental in alpha" + in r.message + for r in caplog.records + ) + + def test_cross_world_peered_mode_loads( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + spec_file = _write_spec( + tmp_path, + { + "gateway_portal": { + "cross_world": { + "mode": "peered", + "peers": [{"name": "world-b", "endpoint": "10.0.0.1:8443"}], + } + } + }, + ) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + assert spec.gateway_portal.cross_world.mode.value == "peered" + assert len(spec.gateway_portal.cross_world.peers) == 1 + assert any( + "gateway_portal.cross_world.mode is experimental in alpha" in r.message + for r in caplog.records + ) + + +# ── Handler unit tests ──────────────────────────────────────────────────────── + + +@pytest.fixture +def mock_docker() -> MagicMock: + docker = MagicMock() + docker.copy_to_container = AsyncMock() + docker.exec_command = AsyncMock(return_value=(0, "")) + docker.signal_container = AsyncMock() + return docker + + +class TestApplyInternetPolicy: + """GatewayHandler.apply_internet_policy writes correct nft rules.""" + + async def test_shadowed_rules_written_to_container(self, mock_docker: MagicMock) -> None: + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.spec.models import RealInternetConfig + from netengine.spec.types import GatewayRealInternetMode + + handler = GatewayHandler(mock_docker) + config = RealInternetConfig(mode=GatewayRealInternetMode.SHADOWED) + await handler.apply_internet_policy(config) + + mock_docker.copy_to_container.assert_called_once() + dest = mock_docker.copy_to_container.call_args[0][2] + assert dest == "/etc/nftables/rules/internet.nft" + + # Verify nft apply was invoked + cmd = mock_docker.exec_command.call_args[0][1] + assert cmd[0] == "nft" + + async def test_shadowed_rules_content(self, mock_docker: MagicMock) -> None: + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.spec.models import RealInternetConfig + from netengine.spec.types import GatewayRealInternetMode + + handler = GatewayHandler(mock_docker) + config = RealInternetConfig(mode=GatewayRealInternetMode.SHADOWED) + + written_content: list[str] = [] + + async def capture(container: str, src: str, dest: str) -> None: + with open(src) as f: + written_content.append(f.read()) + + mock_docker.copy_to_container.side_effect = capture + await handler.apply_internet_policy(config) + + assert written_content, "No rules were written" + rules = written_content[0] + assert "netengine_internet" in rules + assert "masquerade" in rules + + async def test_mirrored_rules_include_mirror_ip(self, mock_docker: MagicMock) -> None: + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.spec.models import RealInternetConfig, ServiceMirror + from netengine.spec.types import GatewayRealInternetMode + + handler = GatewayHandler(mock_docker) + config = RealInternetConfig( + mode=GatewayRealInternetMode.MIRRORED, + service_mirrors=[ + ServiceMirror(real_hostname="api.example.com", in_world_service="192.168.50.10") + ], + ) + + written_content: list[str] = [] + + async def capture(container: str, src: str, dest: str) -> None: + with open(src) as f: + written_content.append(f.read()) + + mock_docker.copy_to_container.side_effect = capture + await handler.apply_internet_policy(config) + + assert written_content + assert "192.168.50.10" in written_content[0] + + async def test_custom_mode_is_noop(self, mock_docker: MagicMock) -> None: + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.spec.models import RealInternetConfig + from netengine.spec.types import GatewayRealInternetMode + + handler = GatewayHandler(mock_docker) + config = RealInternetConfig(mode=GatewayRealInternetMode.CUSTOM) + await handler.apply_internet_policy(config) + + mock_docker.copy_to_container.assert_not_called() + + +class TestSetupPeer: + """GatewayPortalHandler._setup_peer wires routing and DNS forwarding.""" + + async def test_setup_peer_calls_apply_peer_routing(self, mock_docker: MagicMock) -> None: + from types import SimpleNamespace + + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.handlers.gateway_portal_handler import GatewayPortalHandler + from netengine.spec.models import CrossWorldPeer + from netengine.spec.types import GatewayCrossWorldMode + + handler = GatewayPortalHandler() + gateway = GatewayHandler(mock_docker) + + peer = CrossWorldPeer( + name="world-b", + endpoint="10.99.0.1:8443", + mode=GatewayCrossWorldMode.PEERED, + ) + + ctx = MagicMock() + ctx.logger = MagicMock() + ctx.zone_dir = "/tmp/zones" + ctx.docker_client = mock_docker + + result = await handler._setup_peer(ctx, gateway, mock_docker, peer) + + assert result["name"] == "world-b" + assert result["endpoint"] == "10.99.0.1:8443" + # Routing should be configured (apply_peer_routing copies + exec nft) + mock_docker.copy_to_container.assert_called_once() + nft_cmd = mock_docker.exec_command.call_args[0][1] + assert nft_cmd[0] == "nft" + + async def test_setup_peer_routing_failure_does_not_raise(self, mock_docker: MagicMock) -> None: + from netengine.handlers.gateway_handler import GatewayHandler + from netengine.handlers.gateway_portal_handler import GatewayPortalHandler + from netengine.spec.models import CrossWorldPeer + from netengine.spec.types import GatewayCrossWorldMode + + handler = GatewayPortalHandler() + gateway = GatewayHandler(mock_docker) + + peer = CrossWorldPeer(name="world-c", endpoint="10.99.0.2:8443") + + mock_docker.copy_to_container.side_effect = RuntimeError("container gone") + + ctx = MagicMock() + ctx.logger = MagicMock() + ctx.zone_dir = "/tmp/zones" + ctx.docker_client = mock_docker + + result = await handler._setup_peer(ctx, gateway, mock_docker, peer) + assert result["routing_configured"] is False + assert "routing_error" in result diff --git a/tests/test_readiness_cli.py b/tests/test_readiness_cli.py index cbb40ac..50b74a5 100644 --- a/tests/test_readiness_cli.py +++ b/tests/test_readiness_cli.py @@ -27,15 +27,11 @@ def test_readiness_loads_spec_runs_host_and_migration_checks() -> None: """Readiness should validate the spec, run preflight probes, and inspect migrations.""" spec_file = Path(__file__).parent.parent / "examples" / "minimal.yaml" host_results = [ - DoctorCheckResult( - "Docker daemon", DoctorStatus.OK, "docker info succeeded", group="docker" - ) + DoctorCheckResult("Docker daemon", DoctorStatus.OK, "docker info succeeded", group="docker") ] with ( - patch( - "netengine.cli.main.run_preflight", return_value=host_results - ) as mock_preflight, + patch("netengine.cli.main.run_preflight", return_value=host_results) as mock_preflight, patch("netengine.cli.main.MigrationService") as mock_service_class, ): mock_service = mock_service_class.return_value @@ -52,9 +48,7 @@ def test_readiness_loads_spec_runs_host_and_migration_checks() -> None: assert "[PASS] Docker daemon: docker info succeeded" in result.output assert "[PASS] migrations:pending: 0 pending migration(s)" in result.output mock_preflight.assert_called_once() - mock_service_class.assert_called_once_with( - "postgresql://example/db", cli_main.MIGRATIONS_DIR - ) + mock_service_class.assert_called_once_with("postgresql://example/db", cli_main.MIGRATIONS_DIR) mock_service.status.assert_awaited_once() @@ -98,9 +92,7 @@ def test_readiness_warns_for_experimental_feature_state(tmp_path: Path) -> None: patch("netengine.cli.main.run_preflight", return_value=[]), patch("netengine.cli.main.MigrationService") as mock_service_class, ): - mock_service_class.return_value.status = AsyncMock( - return_value=_healthy_migration_status() - ) + mock_service_class.return_value.status = AsyncMock(return_value=_healthy_migration_status()) result = CliRunner().invoke( cli_main.cli, @@ -109,10 +101,7 @@ def test_readiness_warns_for_experimental_feature_state(tmp_path: Path) -> None: assert result.exit_code == 0, result.output assert "warn" in result.output - assert ( - "[WARN] feature-state: pki.intermediate_ca_enabled: experimental" - in result.output - ) + assert "[WARN] feature-state: pki.intermediate_ca_enabled: experimental" in result.output assert "Review alpha/experimental behavior" in result.output @@ -128,7 +117,6 @@ def test_readiness_fails_when_database_url_missing() -> None: assert result.exit_code == 1 assert ( - "[FAIL] migrations:database-url: NETENGINE_DB_URL/DATABASE_URL is not set" - in result.output + "[FAIL] migrations:database-url: NETENGINE_DB_URL/DATABASE_URL is not set" in result.output ) assert "Set NETENGINE_DB_URL or DATABASE_URL" in result.output diff --git a/tests/test_spec_parsing.py b/tests/test_spec_parsing.py index 537013e..c0ae8ea 100644 --- a/tests/test_spec_parsing.py +++ b/tests/test_spec_parsing.py @@ -180,16 +180,26 @@ def test_experimental_pki_field_logs_warning_not_error(self, tmp_path, caplog) - "pki.dnssec_enabled is experimental in alpha" in r.message for r in caplog.records ) - def test_unsupported_non_default_active_field_raises_spec_load_error(self, tmp_path) -> None: + def test_experimental_gateway_portal_field_logs_warning_not_error( + self, tmp_path, caplog + ) -> None: + import logging + + # gateway_portal.real_internet.mode was promoted from unsupported to experimental + # once the nftables policy implementation landed (WS-C). It now loads with a + # warning instead of being rejected. spec_file = self._write_spec( tmp_path, {"gateway_portal": {"real_internet": {"mode": "mirrored"}}} ) - with pytest.raises( - SpecLoadError, - match="gateway_portal\\.real_internet\\.mode is unsupported in alpha", - ): - load_spec(spec_file) + with caplog.at_level(logging.WARNING, logger="netengine.spec.loader"): + spec = load_spec(spec_file) + + assert spec.gateway_portal.real_internet.mode.value == "mirrored" + assert any( + "gateway_portal.real_internet.mode is experimental in alpha" in r.message + for r in caplog.records + ) def test_experimental_profile_field_logs_warning_not_raises(self, tmp_path, caplog) -> None: import logging