Skip to content

Commit b14d4fd

Browse files
committed
Add state controller with continuous drift detection and self-healing
Implements a Kubernetes-style reconciliation controller that: - Continuously polls Docker + DNS + Keycloak state via handler healthchecks - Detects drift when actual state diverges from desired spec - Automatically re-applies diverged components (self-healing) - Emits events for monitoring/alerting integration - Persists drift history to RuntimeState for auditability Key changes: 1. netengine/core/drift_controller.py: Main DriftDetectionController that polls healthchecks and triggers self-healing on drift detection 2. netengine/core/self_healing.py: SelfHealingStrategy for phase re-execution 3. netengine/core/state.py: Added drift_history, last_drift_check_at, current_drift_phases fields to RuntimeState 4. netengine/core/orchestrator.py: Added start_drift_detection() method 5. netengine/cli/main.py: Added drift-watch and drift-status CLI commands 6. tests/test_drift_controller.py: Unit tests for drift detection logic 7. tests/integration/test_drift_detection.py: Integration tests The controller runs as a background consumer via ConsumerSupervisor with: - Configurable poll interval (default 30s) - Optional auto-healing (enabled by default, --no-auto-heal to disable) - Dependency-aware re-execution when upstream phases heal - Graceful restart on crashes with exponential backoff Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DyX7e2PSJ4zELLGHkQv6F1
1 parent aa49f40 commit b14d4fd

7 files changed

Lines changed: 1108 additions & 1 deletion

File tree

netengine/cli/main.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,91 @@ async def _diagnose(spec_file: str, as_json: bool) -> None:
410410
sys.exit(1)
411411

412412

413+
@cli.command()
414+
@click.option("--interval", default=30, type=int, help="Poll interval in seconds (default 30).")
415+
@click.option("--max-retries", default=3, type=int, help="Max self-heal retries per phase.")
416+
@click.option("--no-auto-heal", is_flag=True, help="Detect drift but don't auto-heal.")
417+
def drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> None:
418+
"""Watch running world for drift and optionally auto-heal (Ctrl+C to stop)."""
419+
asyncio.run(_drift_watch(interval, max_retries, no_auto_heal))
420+
421+
422+
async def _drift_watch(interval: int, max_retries: int, no_auto_heal: bool) -> None:
423+
state = RuntimeState.load()
424+
if not state.world_spec:
425+
click.echo("No running world found — use `netengine up` first.", err=True)
426+
sys.exit(1)
427+
428+
click.echo(f"Starting drift detection (interval={interval}s, auto-heal={not no_auto_heal})…")
429+
click.echo("Press Ctrl+C to stop.\n")
430+
431+
orchestrator = Orchestrator(state.world_spec, mock_mode=False)
432+
433+
orchestrator.start_drift_detection(
434+
poll_interval_seconds=interval,
435+
max_drift_retries=max_retries,
436+
auto_heal=not no_auto_heal,
437+
)
438+
439+
try:
440+
await orchestrator.start_consumers()
441+
try:
442+
await asyncio.sleep(float("inf"))
443+
except (KeyboardInterrupt, asyncio.CancelledError):
444+
pass
445+
finally:
446+
await orchestrator.consumer_supervisor.stop_all()
447+
click.echo("\nDrift detection stopped.")
448+
except Exception as exc:
449+
click.echo(f"Drift detection error: {exc}", err=True)
450+
sys.exit(1)
451+
452+
453+
@cli.command()
454+
def drift_status() -> None:
455+
"""Show current drift status and history."""
456+
state = RuntimeState.load()
457+
458+
if not state.world_spec:
459+
click.echo("No running world found — use `netengine up` first.", err=True)
460+
sys.exit(1)
461+
462+
click.echo("\nDrift Status\n")
463+
464+
if state.last_drift_check_at:
465+
check_time = state.last_drift_check_at.isoformat() if hasattr(
466+
state.last_drift_check_at, 'isoformat'
467+
) else str(state.last_drift_check_at)
468+
click.echo(f"Last check: {check_time}")
469+
else:
470+
click.echo("Last check: (no checks yet)")
471+
472+
if state.current_drift_phases:
473+
click.echo(f"\nCurrently drifted phases: {', '.join(str(p) for p in state.current_drift_phases)}")
474+
else:
475+
click.echo("\nCurrently drifted phases: none")
476+
477+
if state.drift_history:
478+
click.echo("\nRecent drift history (last 10 events):")
479+
for event in state.drift_history[-10:]:
480+
phase = event.get("phase_num", "?")
481+
detected = event.get("detected_at", "?")
482+
healed = event.get("healed_at")
483+
failed = event.get("healing_failed", False)
484+
485+
if healed:
486+
status = f"✓ healed at {healed}"
487+
elif failed:
488+
error = event.get("error", "unknown error")
489+
status = f"✗ healing failed: {error}"
490+
else:
491+
status = "⧗ healing in progress"
492+
493+
click.echo(f" Phase {phase}: detected at {detected}, {status}")
494+
else:
495+
click.echo("\nDrift history: (no events)")
496+
497+
413498
def _print_status(state: RuntimeState) -> None:
414499
world_name = None
415500
if state.world_spec and isinstance(state.world_spec, dict):

0 commit comments

Comments
 (0)