diff --git a/.env.example b/.env.example index 153d3d3e..8898b64e 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.0.2 +EDGEOPS_CONSOLE_VERSION=v1.0.3 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index 71381fa0..7c831f55 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.0.2"; fi + if [ -z "$VERSION" ]; then VERSION="1.0.3"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index c7de671f..3bb86a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - Agent status: removed **`processedMessages`**, **`messageSpeed`**; added **`availableRuntimes`**, optional **`runtimeAgentPhase`**, **`controlPlaneQuiesced`**. - Default container registry: **`docker.io`** (was `registry.hub.docker.com`). - Reserved ports: **54321**, **54322**, **53**. -- New field-agent endpoint: **`POST /api/v3/agent/controller/register`** (system fogs only; Edgelet rc.1+). +- New field-agent endpoint: **`POST /api/v3/agent/controller/register`** (system fogs only; Edgelet rc.1+). Register body accepts full container workload fields (`cmd`, `isPrivileged`, `healthCheck`, `capAdd`/`capDrop`, `extraHosts`, resources, etc.) with the same semantics as user microservice deploy; excludes `serviceAccount`, `natsConfig`, and ownership fields. #### API — architectures and applications @@ -42,7 +42,7 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * #### WebSocket exec — multi-session - **Microservice exec REST removed** — `POST/DELETE /api/v3/microservices/:uuid/exec` and `…/system/:uuid/exec` no longer exist. Open exec with **direct WebSocket** only: `WS /api/v3/microservices/exec/:uuid` (or `…/system/exec/:uuid`). -- **3 concurrent exec sessions** per microservice (was 1 user exec WS per MS in Plan 16). +- **3 concurrent exec sessions** per microservice (was 1 user exec WS per MS). - **Per-session lifecycle** — closing one exec session deletes only that session row only (no microservice-level exec flag). - **`execEnabled` removed** — dropped `microservices.exec_enabled` column and agent MS list field; exec attach is poll-driven only (`GET /agent/exec/sessions`). - **Agent exec discovery** — new `GET /api/v3/agent/exec/sessions` when change tracking reports `execSessions: true`. @@ -108,11 +108,15 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - **K8s control plane:** hub **`iofog-router`** ConfigMap patches serialized via DB lock; K8s Service create/update/delete with LoadBalancer watch timeout. - **`service-bridge-config.js`** — full recompute of service-derived TCP bridge config per fog on reconcile (preserves router base config). - **SQLite single-node production hardening** — WAL + `busy_timeout` pragmas, reconcile task claim retry on `SQLITE_BUSY`, staggered startup for reconcile-heavy background jobs (`settings.jobStartupDelaySeconds`). -- **WebSocket exec & log session hardening** — quotas (**3 exec** / 3 log WS per resource), per-session exec lifecycle (Plan 17), 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). -- **Multi exec sessions (Plan 17)** — `GET /api/v3/agent/exec/sessions`; agent exec WS `…/agent/exec/microservice/:uuid/:sessionId`; user ACTIVATION with `sessionId`; `MicroserviceExecSessions` table; `execMaxConcurrentPerResource` config (default **3**). +- **WebSocket exec & log session hardening** — quotas (**3 exec** / 3 log WS per resource), per-session exec lifecycle, 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). +- **Multi exec sessions** — `GET /api/v3/agent/exec/sessions`; agent exec WS `…/agent/exec/microservice/:uuid/:sessionId`; user ACTIVATION with `sessionId`; `MicroserviceExecSessions` table; `execMaxConcurrentPerResource` config (default **3**). +- **WebSocket relay production** — unified **`WsRelayTransport`** abstraction; cross-replica exec/log relay backend selected at startup by **`nats.enabled`** (`NATS_ENABLED`): **AMQP** router pool (8 connections per replica, overflow recovery, sendable gating) when `false`, **NATS Core** pub/sub on platform hub (`controller-relay` account) when `true`. Fail-fast activation on both transports; log backpressure drops `LOG_LINE` under pressure. Config: `server.webSocket.relay.amqp.*`, `server.webSocket.relay.nats.*`. No new relay env var; HA swagger/docs updated per R112. ### Fixed +- NATS relay and AMQP router connection resolvers — **Remote CP** uses Edgelet bridge DNS then DB host only (no `*.svc.cluster.local`); **Kubernetes CP** uses `nats-server.{namespace}.svc.cluster.local` / `router.{namespace}.svc.cluster.local` with DB host fallback; connect failures log and throw aggregate errors for all attempts; relay log messages are transport-aware. +- NATS relay **`controller-relay` creds loading** — read Opaque secret values as plain UTF-8 `.creds` text (matches `nats-service.js` and DB storage); fixes **`unable to parse credentials`** on hub connect when `NATS_ENABLED=true`. +- NATS platform relay identity renamed to account/user **`controller`** with rules **`controller-account`** / **`controller-user`**; **`GET /nats/accounts/controller/users/controller/creds`** supported for operator cred export. - Exec AMQP relay re-attaches queue receivers whenever user or agent WebSocket connects (fixes user-first sessions where ACTIVATION and STDIN never reached Edgelet); ACTIVATION is resent on agent WS reconnect. - Controller register accepts optional **`schedule: 0`**; server always enforces schedule **0** on create, re-register, and **`PATCH /api/v3/microservices/system/:uuid`** for controller workloads. - Agent version command (**`GET /api/v3/agent/version`**) refreshes the provision key on each pull instead of returning a stale or deleted key. diff --git a/Dockerfile b/Dockerfile index e6c9e7d7..3cfe9399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # ioFog overrides: EDGEOPS_CONSOLE_REPO=https://github.com/eclipse-iofog/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=iofog # node:24-bookworm — pin manifest list digest for reproducible multi-arch builds -FROM node:24-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS console-builder +FROM node:24-bookworm@sha256:fdddfb3e688158251943d52eba361de991548f6814007acba4917ae6b512d6be AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.3 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -28,7 +28,7 @@ RUN test -f build/index.html \ && cp -a build /tmp/console/build -FROM node:24-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS builder +FROM node:24-bookworm@sha256:fdddfb3e688158251943d52eba361de991548f6814007acba4917ae6b512d6be AS builder ARG PKG_VERSION @@ -48,9 +48,9 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e76548c58c4a29907cef1e01cb6a4cab426eb071ffe28ac1f25dd58d14a89569 +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:cc7648f8e1c7d628e4334328a712f30ea0820787bb92836cc93e349674c689bf -ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.3 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 0757d507..49685283 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.2 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.3 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.0.2 +EDGEOPS_CONSOLE_VERSION ?= v1.0.3 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/docs/architecture.md b/docs/architecture.md index 786e92f8..e19adfd3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -206,7 +206,37 @@ Full spec: [`.cursor/controllerv3.8/docs/15-fog-platform-reconcile.md`](../.curs ## WebSocket exec & log sessions -Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens log sessions and shared WS infra (HA, drain, OTEL). **Plan 17** redesigns **microservice exec** to log-style multi-session flow (3 concurrent per MS, agent poll + session-scoped WS) — **Edgelet agent wire change required** for exec (see [edgelet-invariants.md §10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md)). +Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens log sessions and shared WS infra (HA, drain, OTEL). **Plan 17** redesigns **microservice exec** to log-style multi-session flow (3 concurrent per MS, agent poll + session-scoped WS). **Plan 18** production-hardens cross-replica relay via **`WsRelayTransport`** — AMQP pool + recovery when `nats.enabled=false`, NATS Core when `nats.enabled=true` (R102–R113). **Edgelet agent wire change required** for exec only (see [edgelet-invariants.md §10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md)). + +```mermaid +flowchart TB + subgraph ws [WebSocketServer] + U[User WS] + A[Agent WS] + end + + subgraph factory [WsRelayTransportFactory] + SEL{nats.enabled?} + end + + subgraph amqp [AmqpRelayTransport] + POOL[RouterConnectionManager pool x8] + Q[agent/user queues per sessionId] + end + + subgraph nats [NatsRelayTransport] + NC[NatsRelayConnectionManager] + SUB[Core pub/sub per sessionId] + end + + U --> ws + A --> ws + ws --> factory + SEL -->|false| amqp + SEL -->|true| nats + POOL --> Q + NC --> SUB +``` ```mermaid sequenceDiagram @@ -214,7 +244,7 @@ sequenceDiagram participant C as Controller participant DB as MicroserviceExecSessions participant CT as Change tracking - participant Q as AMQP + participant R as WsRelayTransport participant A as Edgelet agent Note over U,A: MS exec (R92–R98) — log-style @@ -228,11 +258,11 @@ sequenceDiagram A->>C: GET /agent/exec/sessions A->>C: WS /agent/exec/microservice/:uuid/:sessionId C->>DB: ACTIVE agentConnected - C->>Q: agent-{sessionId} user-{sessionId} + C->>R: enable bridge if cross-replica U->>C: STDIN - C->>Q->>A: relay + C->>R->>A: relay A->>C: STDOUT - C->>Q->>U: relay + C->>R->>U: relay U-->>C: close C->>DB: DELETE row C->>CT: execSessions if needed @@ -242,12 +272,12 @@ sequenceDiagram C->>C: provision debug system MS U->>C: WS /microservices/system/exec/:debugMsUuid - Note over U,A: Logs (R82–R83, R84) — unchanged Plan 16 + Note over U,A: Logs (R82–R83, R84/R112) U->>C: WS logs + tail params C->>DB: PENDING sessionId A->>C: WS agent/logs/:sessionId A->>C: LOG_LINE - C->>Q->>U: logs-user-{sessionId} + C->>R->>U: relay ``` | Topic | Normative value | @@ -263,7 +293,7 @@ sequenceDiagram | Log concurrency | **3** user log WS per microservice (or per fog for node logs) | | Log limits | Tail max **5,000** lines; **120s** pending; **2h** idle | | Log content | Live relay only — no log line persistence; audit connect/disconnect | -| HA relay | Cross-replica sessions **require** AMQP (`WebSocketQueueService`); same-replica may use direct WS; **fail fast** when router down | +| HA relay | Cross-replica sessions **require** a **relay backend** (R112): **AMQP** router queues when `nats.enabled=false`; **NATS Core** subjects on hub when `nats.enabled=true`. Same-replica may use direct WS; **fail fast** close **1013** when active backend unavailable | | Graceful drain | **30s** on SIGTERM / k8s `preStop` — CLOSE frames, queue cleanup, session row delete | | Security | Agent handlers validate fog token **before** message processing; **50** upgrades/min/IP; **100** active WS/IP; JWT in `?token=` (ingress log redaction required) | | Scale SLO | **500** concurrent WS per replica; **p99 pairing < 5s** | @@ -271,13 +301,15 @@ sequenceDiagram **OTEL metric names (R87):** `ws_exec_sessions_active`, `ws_log_sessions_active`, `ws_pending_pairings`, `ws_pairing_duration_ms` (histogram), `ws_amqp_publish_errors`, `ws_router_connected` (gauge). Emitted when `ENABLE_TELEMETRY=true`; see `src/websocket/ws-metrics.js`. -**HA config (`server.webSocket.ha`):** `crossReplicaRequiresAmqp` (default `true`), `failFastOnRouterUnavailable` (default `true`). Env: `WS_HA_CROSS_REPLICA_REQUIRES_AMQP`, `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE`. Graceful drain timeout: `server.webSocket.session.drainTimeoutMs` (default **30s**, env `WS_DRAIN_TIMEOUT_MS`). +**Relay transport (Plan 18, R102):** Selected once at startup from existing `nats.enabled` / `NATS_ENABLED` — **no new relay env var**. `false` → AMQP pool (8 connections, sticky by `sessionId`); `true` → NATS Core on hub with **`controller`** NATS account. Relay connect is **lazy** — does not block Controller startup. + +**HA config (`server.webSocket.ha`):** `crossReplicaRequiresAmqp` (default `true`; semantics: cross-replica requires **active relay backend** per R112), `failFastOnRouterUnavailable` (default `true`; applies to selected backend). Env: `WS_HA_CROSS_REPLICA_REQUIRES_AMQP`, `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE`. Graceful drain timeout: `server.webSocket.session.drainTimeoutMs` (default **30s**, env `WS_DRAIN_TIMEOUT_MS`). -**Core modules:** `src/websocket/server.js`, `exec-session-manager.js` (Plan 17), `log-session-manager.js`, `src/services/websocket-queue-service.js`, `src/services/router-connection-service.js`. +**Core modules:** `src/websocket/server.js`, `exec-session-manager.js` (Plan 17), `log-session-manager.js`, `ws-relay-transport-factory.js` / `amqp-relay-transport.js` / `nats-relay-transport.js` (Plan 18), `src/services/websocket-queue-service.js`, `src/services/router-connection-manager.js` (Plan 18), `src/services/nats-relay-connection-manager.js` (Plan 18). -**Operator guide:** [operations/ws-sessions.md](operations/ws-sessions.md) — ingress `?token=` log redaction, HTTPS/WSS, multi-replica AMQP requirement, k8s preStop drain, load SLO probe. +**Operator guide:** [operations/ws-sessions.md](operations/ws-sessions.md) — ingress `?token=` log redaction, HTTPS/WSS, multi-replica relay backend (`nats.enabled`), k8s preStop drain, load SLO probe. -Full spec: Plan 16 [logs + shared infra](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · Plan 17 [MS exec](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md) · RFC R80–R91, R92–R101 · Edgelet contract: [edgelet-invariants.md §10–§10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md). +Full spec: Plan 16 [logs + shared infra](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · Plan 17 [MS exec](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md) · Plan 18 [WS relay production](../.cursor/controllerv3.8/docs/18-ws-relay-production.md) · RFC R80–R91, R92–R101, R102–R113 · Edgelet contract: [edgelet-invariants.md §10–§10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md). --- diff --git a/docs/operations/ws-sessions.md b/docs/operations/ws-sessions.md index 29c95802..34e268a1 100644 --- a/docs/operations/ws-sessions.md +++ b/docs/operations/ws-sessions.md @@ -6,7 +6,7 @@ ## Overview -Controller exposes **interactive exec** and **log streaming** over WebSocket on the API port (default **51121**). Sessions pair an operator browser/CLI client (Bearer JWT) with an Edgelet agent (fog token). In multi-replica deployments, cross-replica relay requires the **Skupper-style AMQP router** microservice. +Controller exposes **interactive exec** and **log streaming** over WebSocket on the API port (default **51121**). Sessions pair an operator browser/CLI client (Bearer JWT) with an Edgelet agent (fog token). In multi-replica deployments, cross-replica relay uses a **relay backend** selected at startup by **`nats.enabled`** (Plan 18, R102): **AMQP** router queues when `false`, **NATS Core** on the platform hub when `true`. --- @@ -38,18 +38,41 @@ Without redaction, long-lived bearer tokens may appear in load balancer logs. ## Multi-replica HA +Relay transport is selected **once at startup** from existing platform config — **no separate relay env var** (R102): + +| `nats.enabled` | Cross-replica relay backend | +|----------------|----------------------------| +| `false` (default) | **AMQP** — Skupper-style router queues via `WebSocketQueueService` | +| `true` | **NATS Core** — hub pub/sub subjects `controller.relay.v1.*` via `NatsRelayTransport` | + +Set `NATS_ENABLED=true` only when the platform NATS hub is deployed and all Controller replicas share the same value. + | Setting | Default | Env | |---------|---------|-----| -| Cross-replica requires AMQP | `true` | `WS_HA_CROSS_REPLICA_REQUIRES_AMQP` | -| Fail fast when router down | `true` | `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE` | +| Cross-replica requires relay backend | `true` | `WS_HA_CROSS_REPLICA_REQUIRES_AMQP` | +| Fail fast when relay backend down | `true` | `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE | -**Requirements:** +> Env names retain `AMQP`/`ROUTER` for backward compatibility; semantics apply to the **active relay backend** (AMQP or NATS) per R112. -1. Deploy the **router** system microservice and ensure Controller can reach AMQP (`RouterConnectionService`). +### AMQP relay (`nats.enabled=false`) + +1. Deploy the **router** system microservice and ensure Controller can reach AMQP (`RouterConnectionManager` pool). 2. Run **2+ Controller replicas** behind a load balancer with **sticky sessions optional** — cross-replica exec/log uses AMQP queues (`agent-{sessionId}`, `user-{sessionId}`, `logs-user-{sessionId}`). -3. When the router is unavailable, new cross-replica sessions close with WebSocket code **1013** (`Router unavailable for cross-replica session`). +3. When the router/AMQP backend is unavailable, new cross-replica sessions close with WebSocket code **1013** (`Router unavailable for cross-replica session`). + +Plan 18 adds an **8-connection AMQP pool** per replica with overflow recovery — intense log streams must not poison other sessions (no router restart required). **Remote CP** resolves **`router.default.svc.bridge.local`** then default router `host`; **Kubernetes CP** resolves **`router.{namespace}.svc.cluster.local`** then default router `host`. Port from `Routers.messagingPort` (default **5671**). -Same-replica sessions may relay directly without AMQP when both user and agent land on the same pod. +### NATS relay (`nats.enabled=true`) + +1. Platform NATS hub must be running with `NatsInstances.isHub=true`. +2. Controller provisions dedicated **`controller`** NATS account/user (not SYS / `admin-hub`) via NATS auth reconcile. +3. Cross-replica exec uses subjects `controller.relay.v1.exec.{sessionId}.agent` / `.user`; logs use `controller.relay.v1.log.{sessionId}.user`. Plain TCP to hub — port from `NatsInstances.serverPort` (default **4222**) for every host in the resolver list. +4. **Remote CP:** Controller resolves **`nats.default.svc.bridge.local`** (Edgelet internal DNS) then hub `host`; both use hub `serverPort`. +5. **Kubernetes CP:** Controller resolves **`nats-server.{namespace}.svc.cluster.local`** then hub `host`. +6. Remote ControlPlane replicas connect to the **hub** NATS only — not local fog NATS leaf. +7. When NATS relay is unavailable, fail-fast semantics match AMQP (close **1013** when configured). + +Same-replica sessions may relay directly without AMQP or NATS when both user and agent land on the same pod. --- @@ -59,7 +82,7 @@ On shutdown, Controller drains WebSocket sessions for up to **`WS_DRAIN_TIMEOUT_ 1. Reject new upgrades (`verifyClient` → draining). 2. Close pending users with code **1001** (`Server draining`). -3. Send CLOSE frames, clean exec/log session DB rows, tear down AMQP bridges. +3. Send CLOSE frames, clean exec/log session DB rows, tear down relay bridges (AMQP or NATS). ### Kubernetes manifest example @@ -113,6 +136,15 @@ node test/load/ws-pairing-load.js --multi-ms 100 The `--multi-ms` mode creates **3 exec sessions per microservice** (100 MS × 3 = 300 pairs) to validate multi-session pairing latency under the same p99 SLO. +**AMQP profile** (`nats.enabled=false`): run the probe above on a dev machine — it exercises in-process `ExecSessionManager` pairing only (no router required). Record p99 from stdout; target **< 5000 ms**. + +**NATS profile** (`nats.enabled=true`): the same probe validates session-manager pairing latency (transport-agnostic SLO). For end-to-end NATS relay validation in staging: + +1. Deploy Controller with **`NATS_ENABLED=true`** on **2+ replicas** and a platform NATS hub (`NatsInstances.isHub=true`). +2. Confirm **`controller`** NATS account reconcile succeeded (NATS auth logs). +3. Run cross-replica exec/log sessions (user on replica A, agent on replica B) while recording OTEL **`ws_pairing_duration_ms`** p99. +4. Optionally repeat `node test/load/ws-pairing-load.js --pairs 500` against staging API with agent simulators — same **p99 < 5s** SLO applies. + For production validation, repeat against a staging cluster with real agent simulators and record p99 from Controller OTEL histogram `ws_pairing_duration_ms`. --- @@ -128,7 +160,9 @@ Enable `ENABLE_TELEMETRY=true`. Key metrics (`src/websocket/ws-metrics.js`): | `ws_pending_pairings` | gauge | | `ws_pairing_duration_ms` | histogram | | `ws_amqp_publish_errors` | counter | -| `ws_router_connected` | gauge | +| `ws_amqp_session_saturated` | counter (Plan 18 overflow/backpressure) | +| `ws_router_pool_connections` | gauge (Plan 18 AMQP pool health) | +| `ws_router_pool_unsettled` | gauge (Plan 18 AMQP unsettled deliveries) | --- diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c0ff8311..abd65f4d 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2774,8 +2774,7 @@ paths: `{ sessionId, microserviceUuid }` in the `data` field and top-level `sessionId`, followed by STDERR "waiting for agent…". Relay frames use `execId` equal to `sessionId`. - **HA:** Multi-replica deployments require AMQP router (`WebSocketQueueService`). - Cross-replica sessions fail fast with close code **1013** when router is unavailable. + **HA (R112):** Multi-replica deployments require a **relay backend** selected at startup by **`nats.enabled`**: AMQP router queues when `false` (default), NATS Core pub/sub on the platform hub when `true`. Cross-replica sessions fail fast with close code **1013** when the active relay backend is unavailable. See `docs/operations/ws-sessions.md`. See `#/components/schemas/WsExecMessageTypes` and `#/components/schemas/WsCloseCodes`. operationId: userMicroserviceExecWebSocket @@ -6593,11 +6592,16 @@ paths: tags: - NATS summary: Gets NATS creds for specific account user + description: | + Returns base64-encoded `.creds` file content for the user. + + Path `{appName}` is an **application name** for app-linked accounts, or a **NATS account name** for platform/system accounts (`SYS`, leaf accounts, **`controller`**). operationId: getNatsUserCreds parameters: - in: path name: appName required: true + description: Application name or NATS account name (e.g. `SYS`, `controller`) schema: type: string - in: path @@ -7033,7 +7037,7 @@ tags: - name: WebSocketSessions description: | Interactive exec and log streaming WebSocket endpoints (MessagePack binary). - Multi-replica HA requires AMQP router — see operations/ws-sessions.md. + Multi-replica HA requires a relay backend per **`nats.enabled`** (AMQP when `false`, NATS Core when `true`) — see operations/ws-sessions.md. - name: User description: Manage your users - name: Secrets @@ -8276,7 +8280,11 @@ components: description: True when microservice is the ControlPlane controller workload (DB column) ControllerRegisterRequest: type: object - description: Slim register body for Edgelet ControlPlane controller workload + description: >- + Edgelet ControlPlane controller workload register body. Accepts the same + container/workload fields as user microservice deploy (excluding application, + serviceAccount, and natsConfig). Ownership fields (iofogUuid, application) + are derived from the authenticated system fog. required: - uuid - images @@ -8298,6 +8306,31 @@ components: $ref: "#/components/schemas/MicroserviceContainerImage" registryId: type: integer + config: + type: string + annotations: + type: string + hostNetworkMode: + type: boolean + isPrivileged: + type: boolean + logSize: + type: integer + minimum: 0 + runtime: + type: string + pidMode: + type: string + ipcMode: + type: string + runAsUser: + type: string + platform: + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer ports: type: array items: @@ -8306,16 +8339,57 @@ components: type: array items: $ref: "#/components/schemas/VolumeMappingRequest" + extraHosts: + type: array + items: + type: object + required: + - name + - address + properties: + name: + type: string + address: + type: string env: type: array items: $ref: "#/components/schemas/AgentEnvRequest" - config: - type: string - hostNetworkMode: - type: boolean - runtime: - type: string + cmd: + type: array + items: + type: string + cdiDevices: + type: array + items: + type: string + capAdd: + type: array + items: + type: string + capDrop: + type: array + items: + type: string + healthCheck: + type: object + required: + - test + properties: + test: + type: array + items: + type: string + interval: + type: integer + timeout: + type: integer + startPeriod: + type: integer + startInterval: + type: integer + retries: + type: integer schedule: type: integer enum: diff --git a/package-lock.json b/package-lock.json index a8d91af5..1a20b718 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@msgpack/msgpack": "^3.1.2", "@nats-io/jwt": "^0.0.10-5", "@nats-io/nkeys": "^2.0.3", + "@nats-io/transport-node": "^3.4.0", "@node-rs/argon2": "^2.0.2", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-trace-otlp-http": "^0.219.0", @@ -1662,6 +1663,16 @@ "node": ">=16.0.0" } }, + "node_modules/@nats-io/nats-core": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@nats-io/nats-core/-/nats-core-3.4.0.tgz", + "integrity": "sha512-QMDM86EUNm+wudlKC67HLar/KHHQUvJGLfb4jbahje3pUk3K9afeck9fwsxBZF0eUFox6T/TA6m/t+lQqf+QsQ==", + "license": "Apache-2.0", + "dependencies": { + "@nats-io/nkeys": "2.0.3", + "@nats-io/nuid": "3.0.0" + } + }, "node_modules/@nats-io/nkeys": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@nats-io/nkeys/-/nkeys-2.0.3.tgz", @@ -1674,6 +1685,29 @@ "node": ">=18.0.0" } }, + "node_modules/@nats-io/nuid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nats-io/nuid/-/nuid-3.0.0.tgz", + "integrity": "sha512-QbXZDrxmYlrn5AD06gYcUTmEHwxn96HBQMIk7XTjDlEcx6FPzVBBPjp4AMRh1lEv6B4iJ6Xb/Uz61JugPXFRQg==", + "license": "Apache-2.0", + "engines": { + "node": ">= 22.x" + } + }, + "node_modules/@nats-io/transport-node": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@nats-io/transport-node/-/transport-node-3.4.0.tgz", + "integrity": "sha512-hH7u7ejIBTFEJIZ8rIcMrHJI6wl+HhpO5sVFs1+ppmXa8RuB2+Lh1+UwTzZ5xTNNm1TKcRkYy+2qCV56qp8RxA==", + "license": "Apache-2.0", + "dependencies": { + "@nats-io/nats-core": "3.4.0", + "@nats-io/nkeys": "2.0.3", + "@nats-io/nuid": "3.0.0" + }, + "engines": { + "node": ">= 18.0.0" + } + }, "node_modules/@noble/hashes": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", @@ -3106,6 +3140,7 @@ "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "dev": true, "license": "MIT", "bin": { "acorn": "bin/acorn" diff --git a/package.json b/package.json index fa4a6f86..048c2f59 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "moment": "2.30.1", "multer": "1.4.5-lts.1", "mysql2": "3.10.1", + "@nats-io/transport-node": "^3.4.0", "nconf": "0.12.1", "node-fetch-npm": "^2.0.4", "node-forge": "^1.4.0", diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 46ab99f2..118b572f 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.2' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.3' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { diff --git a/src/config/config.yaml b/src/config/config.yaml index 0e39643f..3ee59f6b 100644 --- a/src/config/config.yaml +++ b/src/config/config.yaml @@ -32,6 +32,15 @@ server: logTailMaxLines: 5000 # Log: tail query param max (R82) replicaMaxConcurrentWs: 500 # Scale SLO per replica (R88) drainTimeoutMs: 30000 # Graceful drain on SIGTERM/preStop (R85) + relay: + amqp: + poolSize: 8 + sendTimeoutMs: 5000 + unsettledWarnThreshold: 1800 + nats: + maxPendingBytes: 33554432 + maxPendingMessages: 8192 + publishTimeoutMs: 5000 ha: crossReplicaRequiresAmqp: true # Cross-replica exec/log relay requires AMQP router (R84) failFastOnRouterUnavailable: true diff --git a/src/config/nats-system-rules.js b/src/config/nats-system-rules.js index f7ef9403..67e06ab3 100644 --- a/src/config/nats-system-rules.js +++ b/src/config/nats-system-rules.js @@ -5,6 +5,9 @@ const APPLICATION_ACCOUNT_RULE_NAME = 'default-account' const MICROSERVICE_USER_RULE_NAME = 'default-user' const MQTT_BEARER_USER_RULE_NAME = 'default-mqtt-user' const DEFAULT_LEAF_USER_RULE_NAME = 'default-leaf-user' +const CONTROLLER_ACCOUNT_RULE_NAME = 'controller-account' +const CONTROLLER_USER_RULE_NAME = 'controller-user' +const CONTROLLER_NATS_RELAY_SUBJECT_ALLOW = 'controller.relay.v1.>' const SYS_ACCOUNT_EXPORTS_INLINE = Object.freeze([ Object.freeze({ @@ -58,6 +61,25 @@ const ACCOUNT_RULES = Object.freeze([ maxMsgPayload: -1, maxSubscriptions: -1, exportsAllowWildcards: true + }), + Object.freeze({ + name: CONTROLLER_ACCOUNT_RULE_NAME, + description: 'Controller WebSocket relay account with standard app limits, no exports', + memStorage: -1, + diskStorage: -1, + streams: -1, + maxAckPending: -1, + memMaxStreamBytes: -1, + diskMaxStreamBytes: -1, + consumer: -1, + maxLeafNodeConnections: -1, + maxImports: -1, + maxExports: -1, + maxConnections: -1, + maxData: -1, + maxMsgPayload: -1, + maxSubscriptions: -1, + exportsAllowWildcards: true }) ]) @@ -88,6 +110,17 @@ const USER_RULES = Object.freeze([ maxSubscriptions: -1, maxPayload: -1, allowedConnectionTypes: ['LEAFNODE', 'WEBSOCKET'] + }), + Object.freeze({ + name: CONTROLLER_USER_RULE_NAME, + description: 'Controller WebSocket relay user with standard app limits, scoped to controller.relay.v1.>', + bearerToken: false, + allowedConnectionTypes: ['STANDARD'], + maxData: -1, + maxSubscriptions: -1, + maxPayload: -1, + pubAllow: [CONTROLLER_NATS_RELAY_SUBJECT_ALLOW], + subAllow: [CONTROLLER_NATS_RELAY_SUBJECT_ALLOW] }) ]) @@ -125,6 +158,9 @@ module.exports = { MICROSERVICE_USER_RULE_NAME, MQTT_BEARER_USER_RULE_NAME, DEFAULT_LEAF_USER_RULE_NAME, + CONTROLLER_ACCOUNT_RULE_NAME, + CONTROLLER_USER_RULE_NAME, + CONTROLLER_NATS_RELAY_SUBJECT_ALLOW, isReservedRuleName, getSystemAccountRuleDefinitions, getSystemUserRuleDefinitions, diff --git a/src/config/telemetry.js b/src/config/telemetry.js index 8c2c51f8..646e8ea7 100644 --- a/src/config/telemetry.js +++ b/src/config/telemetry.js @@ -44,9 +44,9 @@ async function startTelemetry () { try { await sdk.start() - const RouterConnectionService = require('../services/router-connection-service') + const RouterConnectionManager = require('../services/router-connection-manager') const { initWsMetrics } = require('../websocket/ws-metrics') - initWsMetrics(RouterConnectionService) + initWsMetrics(RouterConnectionManager) logger.info('OpenTelemetry initialized successfully') } catch (error) { logger.error('Error initializing OpenTelemetry:', error) diff --git a/src/helpers/connect-endpoint-utils.js b/src/helpers/connect-endpoint-utils.js new file mode 100644 index 00000000..f3d93602 --- /dev/null +++ b/src/helpers/connect-endpoint-utils.js @@ -0,0 +1,32 @@ +function createDedupeHostList () { + const hosts = [] + const seen = new Set() + const addHost = (candidate) => { + if (!candidate) return + const trimmed = String(candidate).trim() + if (trimmed.length === 0 || seen.has(trimmed)) return + seen.add(trimmed) + hosts.push(trimmed) + } + return { hosts, addHost } +} + +function formatConnectAttempts (attempts, port) { + if (!attempts || attempts.length === 0) { + return 'no hosts configured' + } + return attempts.map((entry) => `${entry.host}:${port} (${entry.error})`).join('; ') +} + +function aggregateConnectError (message, attempts, port) { + const detail = formatConnectAttempts(attempts, port) + const error = new Error(`${message}: ${detail}`) + error.connectAttempts = attempts + return error +} + +module.exports = { + createDedupeHostList, + formatConnectAttempts, + aggregateConnectError +} diff --git a/src/helpers/constants.js b/src/helpers/constants.js index 7abfe6b7..fb4f06b1 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -67,6 +67,7 @@ module.exports = { ROUTER_BRIDGE_DNS_SAN: 'router.default.svc.bridge.local', NATS_BRIDGE_DNS_SAN: 'nats.default.svc.bridge.local', DEFAULT_ROUTER_K8S_SERVICE: 'router', + DEFAULT_NATS_K8S_SERVICE: 'nats-server', RESERVED_PORTS: [54321, 54322, 53], diff --git a/src/schemas/controller-register.js b/src/schemas/controller-register.js index f5f5ca5b..bd1b9bd1 100644 --- a/src/schemas/controller-register.js +++ b/src/schemas/controller-register.js @@ -1,3 +1,5 @@ +const { microserviceContainerSpecProperties } = require('./microservice-container-spec') + const controllerRegister = { id: '/controllerRegister', type: 'object', @@ -11,25 +13,11 @@ const controllerRegister = { items: { $ref: '/image' } }, registryId: { type: 'integer' }, - ports: { - type: 'array', - items: { $ref: '/ports' } - }, - volumeMappings: { - type: 'array', - items: { $ref: '/volumeMappings' } - }, - env: { - type: 'array', - items: { $ref: '/env' } - }, - config: { type: 'string' }, - hostNetworkMode: { type: 'boolean' }, - runtime: { type: 'string' }, schedule: { type: 'integer', enum: [0] - } + }, + ...microserviceContainerSpecProperties }, required: ['uuid', 'images', 'registryId'], additionalProperties: false diff --git a/src/schemas/microservice-container-spec.js b/src/schemas/microservice-container-spec.js new file mode 100644 index 00000000..6cf86b63 --- /dev/null +++ b/src/schemas/microservice-container-spec.js @@ -0,0 +1,55 @@ +const microserviceContainerSpecProperties = { + config: { type: 'string' }, + annotations: { type: 'string' }, + hostNetworkMode: { type: 'boolean' }, + isPrivileged: { type: 'boolean' }, + logSize: { type: 'integer', minimum: 0 }, + runtime: { type: 'string' }, + pidMode: { type: 'string' }, + ipcMode: { type: 'string' }, + runAsUser: { type: 'string' }, + platform: { type: 'string' }, + cpuSetCpus: { type: 'string' }, + memoryLimit: { type: 'integer' }, + ports: { + type: 'array', + items: { $ref: '/ports' } + }, + volumeMappings: { + type: 'array', + items: { $ref: '/volumeMappings' } + }, + extraHosts: { + type: 'array', + items: { $ref: '/extraHosts' } + }, + env: { + type: 'array', + items: { $ref: '/env' } + }, + cmd: { + type: 'array', + items: { type: 'string' } + }, + cdiDevices: { + type: 'array', + items: { type: 'string' } + }, + capAdd: { + type: 'array', + items: { type: 'string' } + }, + capDrop: { + type: 'array', + items: { type: 'string' } + }, + healthCheck: { + type: 'object', + properties: { $ref: '/microserviceHealthCheck' } + } +} + +module.exports = { + mainSchemas: [], + microserviceContainerSpecProperties +} diff --git a/src/services/amqp-relay-transport.js b/src/services/amqp-relay-transport.js new file mode 100644 index 00000000..0bfb95a3 --- /dev/null +++ b/src/services/amqp-relay-transport.js @@ -0,0 +1,77 @@ +const WsRelayTransport = require('./ws-relay-transport') +const WebSocketQueueService = require('./websocket-queue-service') +const RouterConnectionManager = require('./router-connection-manager') + +class AmqpRelayTransport extends WsRelayTransport { + constructor (queueService = WebSocketQueueService) { + super() + this._queueService = queueService + this._recoveryCallbacks = [] + + this._queueService.onRecovery((sessionId, meta) => { + for (const cb of this._recoveryCallbacks) { + try { + cb(sessionId, meta) + } catch (error) { + // Queue service already logs; swallow to protect other callbacks. + } + } + }) + } + + getTransport () { + return 'amqp' + } + + async isAvailable () { + return RouterConnectionManager.isRouterAvailable() + } + + async enableForSession (session, cleanupCb) { + return this._queueService.enableForSession(session, cleanupCb) + } + + async enableForLogSession (session, cleanupCb) { + return this._queueService.enableForLogSession(session, cleanupCb) + } + + async publishToAgent (execId, buffer, opts) { + return this._queueService.publishToAgent(execId, buffer, opts) + } + + async publishToUser (execId, buffer, opts) { + return this._queueService.publishToUser(execId, buffer, opts) + } + + async publishLogToUser (sessionId, buffer) { + return this._queueService.publishLogToUser(sessionId, buffer) + } + + shouldUseRelay (execId) { + return this._queueService.shouldUseQueue(execId) + } + + shouldUseRelayForLogs (sessionId) { + return this._queueService.shouldUseQueueForLogs(sessionId) + } + + async cleanup (execId) { + return this._queueService.cleanup(execId) + } + + async cleanupLogSession (sessionId) { + return this._queueService.cleanupLogSession(sessionId) + } + + async shutdown () { + return this._queueService.shutdown() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this._recoveryCallbacks.push(cb) + } + } +} + +module.exports = AmqpRelayTransport diff --git a/src/services/controller-ms-service.js b/src/services/controller-ms-service.js index 1e02017f..9d2e92d5 100644 --- a/src/services/controller-ms-service.js +++ b/src/services/controller-ms-service.js @@ -8,7 +8,6 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const CatalogItemImageManager = require('../data/managers/catalog-item-image-manager') -const MicroserviceEnvManager = require('../data/managers/microservice-env-manager') const RegistryManager = require('../data/managers/registry-manager') const VolumeMappingManager = require('../data/managers/volume-mapping-manager') const ConfigMapManager = require('../data/managers/config-map-manager') @@ -16,7 +15,7 @@ const SecretManager = require('../data/managers/secret-manager') const MicroservicesService = require('./microservices-service') const MicroservicePortService = require('./microservice-ports/microservice-port') const VolumeMountService = require('./volume-mount-service') -const constants = require('../helpers/constants') +const WorkloadSpec = require('./microservice-workload-spec') const isEqual = require('lodash/isEqual') const Op = require('sequelize').Op const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') @@ -58,13 +57,6 @@ function _validateImageArch (name, fog, images) { validateImageMatchesFogArch(name, fog, images) } -function _validateMicroserviceConfig (config) { - if (config) { - return config.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape - } - return '{}' -} - function _rejectServiceAccountVolumeMappings (volumeMappings) { if (!volumeMappings) { return @@ -230,17 +222,6 @@ async function _updateVolumeMappings (microserviceUuid, volumeMappings, fogUuid, await _createVolumeMappings(microserviceUuid, volumeMappings, transaction) } -async function _updateEnv (env, microserviceUuid, transaction) { - await MicroserviceEnvManager.delete({ microserviceUuid }, transaction) - for (const envData of env) { - await MicroserviceEnvManager.create({ - microserviceUuid, - key: envData.key, - value: envData.value - }, transaction) - } -} - async function _updateImages (images, microserviceUuid, transaction) { await CatalogItemImageManager.delete({ microserviceUuid }, transaction) await _createMicroserviceImages(microserviceUuid, images, transaction) @@ -253,21 +234,18 @@ async function _updatePorts (ports, microservice, transaction) { } } -async function _createControllerMicroservice (registerData, fog, application, transaction) { +async function _createControllerMicroservice (registerData, fog, application, validatedExtraHosts, transaction) { await _checkForDuplicateName(CONTROLLER_MS_NAME, registerData.uuid, application.id, transaction) const microserviceData = { uuid: registerData.uuid, name: CONTROLLER_MS_NAME, - config: _validateMicroserviceConfig(registerData.config), iofogUuid: fog.uuid, - hostNetworkMode: registerData.hostNetworkMode, - runtime: registerData.runtime, registryId: registerData.registryId, schedule: 0, - logSize: constants.MICROSERVICE_DEFAULT_LOG_SIZE * 1, applicationId: application.id, - isController: true + isController: true, + ...WorkloadSpec.buildCreateScalarColumns(registerData) } const microservice = await MicroserviceManager.create( @@ -284,18 +262,13 @@ async function _createControllerMicroservice (registerData, fog, application, tr } } - if (registerData.env) { - for (const env of registerData.env) { - await MicroserviceEnvManager.create({ - microserviceUuid: microservice.uuid, - key: env.key, - value: env.value - }, transaction) - } - } - await _createVolumeMappings(microservice.uuid, registerData.volumeMappings, transaction) + await WorkloadSpec.createWorkloadRelations(microservice, registerData, { + validatedExtraHosts, + transaction + }) + await MicroserviceStatusManager.create({ microserviceUuid: microservice.uuid }, transaction) await MicroserviceExecStatusManager.create({ microserviceUuid: microservice.uuid }, transaction) @@ -304,38 +277,41 @@ async function _createControllerMicroservice (registerData, fog, application, tr return microservice } -async function _updateControllerMicroservice (existing, registerData, fog, transaction) { +async function _updateControllerMicroservice (existing, registerData, fog, validatedExtraHosts, transaction) { const existingImages = await CatalogItemImageManager.findAll({ microserviceUuid: existing.uuid }, transaction) const config = registerData.config !== undefined - ? _validateMicroserviceConfig(registerData.config) + ? WorkloadSpec.validateMicroserviceConfig(registerData.config) : undefined + const annotations = registerData.annotations !== undefined + ? WorkloadSpec.validateMicroserviceAnnotations(registerData.annotations) + : undefined + + const imagesChanged = registerData.images && + registerData.images.length > 0 && + _imagesChanged(registerData.images, existingImages) const microserviceUpdate = AppHelper.deleteUndefinedFields({ isController: true, schedule: 0, registryId: registerData.registryId, - hostNetworkMode: registerData.hostNetworkMode, - runtime: registerData.runtime, config, - rebuild: false + annotations, + rebuild: false, + ...WorkloadSpec.buildScalarColumns(registerData) }) - if (registerData.images && registerData.images.length > 0 && _imagesChanged(registerData.images, existingImages)) { + if (imagesChanged) { await _updateImages(registerData.images, existing.uuid, transaction) microserviceUpdate.rebuild = true } - microserviceUpdate.rebuild = microserviceUpdate.rebuild || !!( - existing.schedule !== 0 || - (registerData.hostNetworkMode !== undefined && existing.hostNetworkMode !== registerData.hostNetworkMode) || - (registerData.runtime !== undefined && existing.runtime !== registerData.runtime) || - (config !== undefined && existing.config !== config) || - registerData.env || - registerData.volumeMappings || - registerData.ports + microserviceUpdate.rebuild = microserviceUpdate.rebuild || WorkloadSpec.shouldRebuildForWorkloadChange( + existing, + registerData, + { config, annotations, imagesChanged } ) const updatedMicroservice = await MicroserviceManager.updateAndFind( @@ -353,9 +329,10 @@ async function _updateControllerMicroservice (existing, registerData, fog, trans await _updateVolumeMappings(existing.uuid, registerData.volumeMappings, fog.uuid, transaction) } - if (registerData.env) { - await _updateEnv(registerData.env, existing.uuid, transaction) - } + await WorkloadSpec.updateWorkloadRelations(existing.uuid, registerData, { + validatedExtraHosts: registerData.extraHosts ? validatedExtraHosts : undefined, + transaction + }) await MicroservicesService.updateChangeTracking(true, fog.uuid, transaction) @@ -385,6 +362,10 @@ async function registerControllerMicroservice (registerData, fog, transaction) { await MicroservicePortService.validatePortMappings({ ports: registerData.ports, iofogUuid: fog.uuid }, transaction) } + const validatedExtraHosts = registerData.extraHosts + ? await WorkloadSpec.validateExtraHosts(registerData, fog.uuid, transaction) + : undefined + const existing = await MicroserviceManager.findOne({ uuid: registerData.uuid }, transaction) if (existing && existing.iofogUuid !== fog.uuid) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, registerData.uuid)) @@ -393,9 +374,9 @@ async function registerControllerMicroservice (registerData, fog, transaction) { const application = await ensureSystemApplication(fog, transaction) if (existing) { - await _updateControllerMicroservice(existing, registerData, fog, transaction) + await _updateControllerMicroservice(existing, registerData, fog, validatedExtraHosts, transaction) } else { - await _createControllerMicroservice(registerData, fog, application, transaction) + await _createControllerMicroservice(registerData, fog, application, validatedExtraHosts, transaction) } return { uuid: registerData.uuid } diff --git a/src/services/microservice-workload-spec.js b/src/services/microservice-workload-spec.js new file mode 100644 index 00000000..599ab119 --- /dev/null +++ b/src/services/microservice-workload-spec.js @@ -0,0 +1,420 @@ +const AppHelper = require('../helpers/app-helper') +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const constants = require('../helpers/constants') +const FogManager = require('../data/managers/iofog-manager') +const ApplicationManager = require('../data/managers/application-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') +const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') +const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') +const MicroserviceEnvManager = require('../data/managers/microservice-env-manager') +const MicroserviceExtraHostManager = require('../data/managers/microservice-extra-host-manager') +const MicroserviceHealthCheckManager = require('../data/managers/microservice-healthcheck-manager') +const ConfigMapManager = require('../data/managers/config-map-manager') +const SecretManager = require('../data/managers/secret-manager') + +function validateMicroserviceConfig (config) { + if (config) { + return config.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape + } + return '{}' +} + +function validateMicroserviceAnnotations (annotations) { + if (annotations) { + return annotations.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape + } + return undefined +} + +function _validateMicroserviceHealthCheck (healthCheck) { + if (healthCheck) { + return JSON.stringify(healthCheck) + } + return undefined +} + +function processHealthCheckForDB (healthCheckData) { + if (!healthCheckData) { + return null + } + + return { + test: _validateMicroserviceHealthCheck(healthCheckData.test), + interval: healthCheckData.interval, + timeout: healthCheckData.timeout, + startPeriod: healthCheckData.startPeriod, + startInterval: healthCheckData.startInterval, + retries: healthCheckData.retries + } +} + +function buildScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEFAULT_LOG_SIZE } = {}) { + return AppHelper.deleteUndefinedFields({ + config: spec.config !== undefined ? validateMicroserviceConfig(spec.config) : undefined, + annotations: spec.annotations !== undefined ? validateMicroserviceAnnotations(spec.annotations) : undefined, + hostNetworkMode: spec.hostNetworkMode, + isPrivileged: spec.isPrivileged, + logSize: spec.logSize != null ? spec.logSize * 1 : undefined, + runtime: spec.runtime, + pidMode: spec.pidMode, + ipcMode: spec.ipcMode, + runAsUser: spec.runAsUser, + platform: spec.platform, + cpuSetCpus: spec.cpuSetCpus, + memoryLimit: spec.memoryLimit + }) +} + +function buildCreateScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEFAULT_LOG_SIZE } = {}) { + const columns = buildScalarColumns(spec, { defaultLogSize }) + if (columns.logSize == null) { + columns.logSize = defaultLogSize * 1 + } + if (columns.config == null) { + columns.config = validateMicroserviceConfig(spec.config) + } + return columns +} + +async function _validateLocalAppHostTemplate (extraHost, templateArgs, msvc, fogUuid, transaction) { + if (templateArgs.length !== 4) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + const fog = await FogManager.findOne({ uuid: msvc.iofogUuid }, transaction) + if (!fog) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[2])) + } + if (fogUuid !== fog.uuid) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_APPS_TEMPLATE, msvc.name)) + } + + extraHost.targetFogUuid = fog.uuid + extraHost.value = `iofog_${msvc.uuid}` + + return extraHost +} + +async function _validateAppHostTemplate (extraHost, templateArgs, fogUuid, transaction) { + if (templateArgs.length < 4) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + const application = await ApplicationManager.findOne({ name: templateArgs[1] }, transaction) + if (!application) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[1])) + } + const msvc = await MicroserviceManager.findOne({ applicationId: application.id, name: templateArgs[2] }, transaction) + if (!msvc) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[2])) + } + extraHost.templateType = 'Apps' + extraHost.targetMicroserviceUuid = msvc.uuid + if (templateArgs[3] === 'local') { + return _validateLocalAppHostTemplate(extraHost, templateArgs, msvc, fogUuid, transaction) + } + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) +} + +async function _validateAgentHostTemplate (extraHost, templateArgs, transaction) { + if (templateArgs.length !== 2) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + + extraHost.templateType = 'Agents' + const fog = await FogManager.findOne({ name: templateArgs[1] }, transaction) + if (!fog) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[1])) + } + extraHost.targetFogUuid = fog.uuid + extraHost.value = fog.host + + return extraHost +} + +async function _validateExtraHost (extraHostData, fogUuid, transaction) { + if (extraHostData.name === 'service.local') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, 'Extra Host name cannot be service.local')) + } + const extraHost = { + templateType: 'Litteral', + name: extraHostData.name, + template: extraHostData.address, + value: extraHostData.address + } + if (!(extraHost.template.startsWith('${') && extraHost.template.endsWith('}'))) { + return extraHost + } + const template = extraHost.value.slice(2, extraHost.value.length - 1) + const templateArgs = template.split('.') + extraHost.templateType = templateArgs[0] + if (templateArgs[0] === 'Apps') { + return _validateAppHostTemplate(extraHost, templateArgs, fogUuid, transaction) + } else if (templateArgs[0] === 'Agents') { + return _validateAgentHostTemplate(extraHost, templateArgs, transaction) + } + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, template)) +} + +async function validateExtraHosts (spec, fogUuid, transaction) { + if (!spec.extraHosts || spec.extraHosts.length === 0) { + return [] + } + const extraHosts = [] + for (const extraHost of spec.extraHosts) { + extraHosts.push(await _validateExtraHost(extraHost, fogUuid, transaction)) + } + return extraHosts +} + +async function _buildEnvRecord (microserviceUuid, envData, transaction) { + const envObj = { + microserviceUuid, + key: envData.key, + value: envData.value + } + + if (envData.valueFromSecret) { + const [secretName, dataKey] = envData.valueFromSecret.split('/') + if (!secretName || !dataKey) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_SECRET_REFERENCE, envData.valueFromSecret)) + } + const secret = await SecretManager.getSecret(secretName, transaction) + if (!secret) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.SECRET_NOT_FOUND, secretName)) + } + if (!secret.data[dataKey]) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SECRET_KEY_NOT_FOUND, dataKey, secretName)) + } + if (secret.type === 'tls') { + try { + envObj.value = Buffer.from(secret.data[dataKey], 'base64').toString('utf-8') + } catch (error) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_BASE64_VALUE, dataKey, secretName)) + } + } else { + envObj.value = secret.data[dataKey] + } + envObj.valueFromSecret = envData.valueFromSecret + } + + if (envData.valueFromConfigMap) { + const [configMapName, dataKey] = envData.valueFromConfigMap.split('/') + if (!configMapName || !dataKey) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_CONFIGMAP_REFERENCE, envData.valueFromConfigMap)) + } + const configMap = await ConfigMapManager.getConfigMap(configMapName, transaction) + if (!configMap) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.CONFIGMAP_NOT_FOUND, configMapName)) + } + if (!configMap.data[dataKey]) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.CONFIGMAP_KEY_NOT_FOUND, dataKey, configMapName)) + } + envObj.value = configMap.data[dataKey] + envObj.valueFromConfigMap = envData.valueFromConfigMap + } + + return envObj +} + +async function _createEnv (microserviceUuid, envData, transaction) { + const envObj = await _buildEnvRecord(microserviceUuid, envData, transaction) + await MicroserviceEnvManager.create(envObj, transaction) +} + +async function _updateEnv (env, microserviceUuid, transaction) { + await MicroserviceEnvManager.delete({ microserviceUuid }, transaction) + for (const envData of env) { + await _createEnv(microserviceUuid, envData, transaction) + } +} + +async function _createExtraHost (microserviceUuid, extraHostData, transaction) { + await MicroserviceExtraHostManager.create({ + ...extraHostData, + microserviceUuid + }, transaction) +} + +async function _updateExtraHosts (extraHosts, microserviceUuid, transaction) { + await MicroserviceExtraHostManager.delete({ microserviceUuid }, transaction) + for (const extraHost of extraHosts) { + await _createExtraHost(microserviceUuid, extraHost, transaction) + } +} + +async function _createArg (microserviceUuid, arg, transaction) { + await MicroserviceArgManager.create({ cmd: arg, microserviceUuid }, transaction) +} + +async function _updateArg (arg, microserviceUuid, transaction) { + await MicroserviceArgManager.delete({ microserviceUuid }, transaction) + for (const argData of arg) { + await _createArg(microserviceUuid, argData, transaction) + } +} + +async function _createCdiDevice (microserviceUuid, cdiDevice, transaction) { + await MicroserviceCdiDevManager.create({ cdiDevices: cdiDevice, microserviceUuid }, transaction) +} + +async function _updateCdiDevices (cdiDevices, microserviceUuid, transaction) { + await MicroserviceCdiDevManager.delete({ microserviceUuid }, transaction) + for (const cdiDevice of cdiDevices) { + await _createCdiDevice(microserviceUuid, cdiDevice, transaction) + } +} + +async function _createCapAdd (microserviceUuid, capAdd, transaction) { + await MicroserviceCapAddManager.create({ capAdd, microserviceUuid }, transaction) +} + +async function _updateCapAdd (capAdd, microserviceUuid, transaction) { + await MicroserviceCapAddManager.delete({ microserviceUuid }, transaction) + for (const capAddData of capAdd) { + await _createCapAdd(microserviceUuid, capAddData, transaction) + } +} + +async function _createCapDrop (microserviceUuid, capDrop, transaction) { + await MicroserviceCapDropManager.create({ capDrop, microserviceUuid }, transaction) +} + +async function _updateCapDrop (capDrop, microserviceUuid, transaction) { + await MicroserviceCapDropManager.delete({ microserviceUuid }, transaction) + for (const capDropData of capDrop) { + await _createCapDrop(microserviceUuid, capDropData, transaction) + } +} + +async function _createHealthCheck (microserviceUuid, healthCheck, transaction) { + const healthCheckData = { + microserviceUuid, + ...processHealthCheckForDB(healthCheck) + } + if (healthCheckData.test && healthCheckData.test.length > 0) { + await MicroserviceHealthCheckManager.create(healthCheckData, transaction) + } +} + +async function _updateHealthCheck (microserviceUuid, healthCheck, transaction) { + await MicroserviceHealthCheckManager.delete({ microserviceUuid }, transaction) + if (healthCheck) { + await _createHealthCheck(microserviceUuid, healthCheck, transaction) + } +} + +async function createWorkloadRelations (microservice, spec, { validatedExtraHosts, transaction }) { + const microserviceUuid = microservice.uuid + + if (validatedExtraHosts && validatedExtraHosts.length) { + for (const extraHost of validatedExtraHosts) { + await _createExtraHost(microserviceUuid, extraHost, transaction) + } + } + + if (spec.env) { + for (const env of spec.env) { + await _createEnv(microserviceUuid, env, transaction) + } + } + + if (spec.cmd) { + for (const arg of spec.cmd) { + await _createArg(microserviceUuid, arg, transaction) + } + } + + if (spec.cdiDevices) { + for (const cdiDevice of spec.cdiDevices) { + await _createCdiDevice(microserviceUuid, cdiDevice, transaction) + } + } + + if (spec.capAdd) { + for (const capAdd of spec.capAdd) { + await _createCapAdd(microserviceUuid, capAdd, transaction) + } + } + + if (spec.capDrop) { + for (const capDrop of spec.capDrop) { + await _createCapDrop(microserviceUuid, capDrop, transaction) + } + } + + if (spec.healthCheck) { + await _createHealthCheck(microserviceUuid, spec.healthCheck, transaction) + } +} + +async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtraHosts, transaction }) { + if (validatedExtraHosts) { + await _updateExtraHosts(validatedExtraHosts, microserviceUuid, transaction) + } + + if (spec.env) { + await _updateEnv(spec.env, microserviceUuid, transaction) + } + + if (spec.cmd) { + await _updateArg(spec.cmd, microserviceUuid, transaction) + } + + if (spec.cdiDevices) { + await _updateCdiDevices(spec.cdiDevices, microserviceUuid, transaction) + } + + if (spec.capAdd) { + await _updateCapAdd(spec.capAdd, microserviceUuid, transaction) + } + + if (spec.capDrop) { + await _updateCapDrop(spec.capDrop, microserviceUuid, transaction) + } + + if (spec.healthCheck) { + await _updateHealthCheck(microserviceUuid, spec.healthCheck, transaction) + } +} + +function shouldRebuildForWorkloadChange (existing, spec, { config, annotations, imagesChanged }) { + return !!( + imagesChanged || + existing.schedule !== 0 || + (spec.hostNetworkMode !== undefined && existing.hostNetworkMode !== spec.hostNetworkMode) || + (spec.isPrivileged !== undefined && existing.isPrivileged !== spec.isPrivileged) || + (spec.logSize !== undefined && existing.logSize !== spec.logSize * 1) || + (spec.runtime !== undefined && existing.runtime !== spec.runtime) || + (spec.pidMode !== undefined && existing.pidMode !== spec.pidMode) || + (spec.ipcMode !== undefined && existing.ipcMode !== spec.ipcMode) || + (spec.runAsUser !== undefined && existing.runAsUser !== spec.runAsUser) || + (spec.platform !== undefined && existing.platform !== spec.platform) || + (spec.cpuSetCpus !== undefined && existing.cpuSetCpus !== spec.cpuSetCpus) || + (spec.memoryLimit !== undefined && existing.memoryLimit !== spec.memoryLimit) || + (config !== undefined && existing.config !== config) || + (annotations !== undefined && existing.annotations !== annotations) || + spec.env || + spec.volumeMappings || + spec.ports || + spec.extraHosts || + spec.cmd || + spec.cdiDevices || + spec.capAdd || + spec.capDrop || + spec.healthCheck + ) +} + +module.exports = { + validateMicroserviceConfig, + validateMicroserviceAnnotations, + processHealthCheckForDB, + buildScalarColumns, + buildCreateScalarColumns, + validateExtraHosts, + createWorkloadRelations, + updateWorkloadRelations, + shouldRebuildForWorkloadChange +} diff --git a/src/services/nats-api-service.js b/src/services/nats-api-service.js index 4d9dbfbf..8d07cb98 100644 --- a/src/services/nats-api-service.js +++ b/src/services/nats-api-service.js @@ -341,24 +341,31 @@ async function createUser (appName, payload, transaction) { } } -async function getUserCreds (appName, userName, transaction) { - const application = await ApplicationManager.findOne({ name: appName }, transaction) +async function _resolveAccountIdForCredsApi (appName, transaction) { const sysAccount = await NatsAccountManager.findOne({ name: appName }, transaction) - if (!application && (!sysAccount || (!sysAccount.isSystem && !sysAccount.isLeafSystem))) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_NAME, appName)) + if (NatsAuthService.isPlatformControllerNatsAccount(sysAccount)) { + return sysAccount.id } - let accountId = null + + const application = await ApplicationManager.findOne({ name: appName }, transaction) if (application) { const account = await NatsAccountManager.findOne({ applicationId: application.id }, transaction) if (!account) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_ID, application.id)) } - accountId = account.id + return account.id } + if (sysAccount && (sysAccount.isSystem || sysAccount.isLeafSystem)) { - accountId = sysAccount.id + return sysAccount.id } + + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_NAME, appName)) +} + +async function getUserCreds (appName, userName, transaction) { + const accountId = await _resolveAccountIdForCredsApi(appName, transaction) const user = await NatsUserManager.findOne({ accountId, name: userName }, transaction) if (!user) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_NAME, userName)) diff --git a/src/services/nats-auth-service.js b/src/services/nats-auth-service.js index 623bd1eb..30892991 100644 --- a/src/services/nats-auth-service.js +++ b/src/services/nats-auth-service.js @@ -28,6 +28,17 @@ const leafSystemAccountUserName = (fog) => `admin-leaf-${slugifyName(fog.name)}` const accountSeedSecretName = (applicationName) => `nats-account-seed-${slugifyName(applicationName)}` const mqttBearerSecretName = (applicationName, userName) => `nats-mqtt-creds-${slugifyName(applicationName)}-${slugifyName(userName)}` const microserviceCredsSecretName = (applicationName, microserviceName) => `nats-creds-${slugifyName(applicationName)}-${slugifyName(microserviceName)}` +const CONTROLLER_NATS_ACCOUNT_NAME = 'controller' +const CONTROLLER_NATS_USER_NAME = 'controller' +const controllerNatsCredsSecretName = () => `nats-creds-${slugifyName(CONTROLLER_NATS_ACCOUNT_NAME)}-${slugifyName(CONTROLLER_NATS_USER_NAME)}` + +function isPlatformControllerNatsAccount (account) { + return account && + account.name === CONTROLLER_NATS_ACCOUNT_NAME && + account.applicationId == null && + !account.isSystem && + !account.isLeafSystem +} function _parseJsonText (value, fallback) { if (!value) { @@ -174,6 +185,19 @@ async function _resolveNatsUserRule (ruleName, defaultRuleName, transaction) { return NatsUserRuleManager.findOne({ name: defaultRuleName }, transaction) } +async function _resolveAccountRuleForAccount (account, app, transaction) { + if (account.isSystem) { + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) + } + if (account.name === CONTROLLER_NATS_ACCOUNT_NAME && account.applicationId == null) { + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME }, transaction) + } + if (app && app.natsRuleId) { + return NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) + } + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction) +} + async function _encodeAccountJwtWithRuleAndRevocations (accountName, accountKp, operatorKp, rule, existingAccountJwt) { const existingRevocations = _extractAccountRevocations(existingAccountJwt) return encodeAccount( @@ -188,10 +212,16 @@ async function _encodeAccountJwtWithRuleAndRevocations (accountName, accountKp, } function _normalizeSystemUserRuleForPersistence (rule) { - return { - ...rule, - allowedConnectionTypes: rule.allowedConnectionTypes ? JSON.stringify(rule.allowedConnectionTypes) : undefined + const out = { ...rule } + if (out.allowedConnectionTypes) { + out.allowedConnectionTypes = JSON.stringify(out.allowedConnectionTypes) } + for (const field of ['pubAllow', 'pubDeny', 'subAllow', 'subDeny', 'src', 'tags']) { + if (Array.isArray(out[field])) { + out[field] = JSON.stringify(out[field]) + } + } + return out } /** @@ -316,11 +346,7 @@ async function rotateOperator (transaction) { const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const newAccountJwt = await _encodeAccountJwtWithRuleAndRevocations( account.name, accountKp, @@ -507,6 +533,78 @@ async function deleteServerSysUserForFog (fog, isHub, transaction) { _triggerResolverArtifactsReconcile({ fogUuids: [fog.uuid] }) } +async function ensureControllerNatsAccount (transaction, ...rest) { + const options = _triggerOptionsFromArgs(rest) + await ensureDefaultRules(transaction) + + const existingAccount = await NatsAccountManager.findOne({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (existingAccount) { + const existingUser = await NatsUserManager.findOne({ + accountId: existingAccount.id, + name: CONTROLLER_NATS_USER_NAME + }, transaction) + if (existingUser) { + return { account: existingAccount, user: existingUser } + } + const result = await createUserForAccount( + existingAccount.id, + CONTROLLER_NATS_USER_NAME, + null, + NatsSystemRules.CONTROLLER_USER_RULE_NAME, + null, + transaction + ) + _triggerResolverArtifactsReconcile(options) + return result + } + + const operator = await ensureOperator(transaction, options) + const operatorSeed = await _loadSeedFromSecret(operator.seedSecretName, transaction) + const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) + + const accountKp = createAccount() + const accountRule = await NatsAccountRuleManager.findOne({ + name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME + }, transaction) + const accountJwt = await encodeAccount( + CONTROLLER_NATS_ACCOUNT_NAME, + accountKp, + _buildAccountRuleClaims(accountRule), + { signer: operatorKp } + ) + const accountSeed = new TextDecoder().decode(accountKp.getSeed()) + + const seedSecretName = accountSeedSecretName(CONTROLLER_NATS_ACCOUNT_NAME) + await _upsertOpaqueSecret(seedSecretName, { seed: accountSeed }, transaction) + + const account = await NatsAccountManager.create({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + publicKey: accountKp.getPublicKey(), + jwt: accountJwt, + seedSecretName, + operatorId: operator.id, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + + const result = await createUserForAccount( + account.id, + CONTROLLER_NATS_USER_NAME, + null, + NatsSystemRules.CONTROLLER_USER_RULE_NAME, + null, + transaction + ) + _triggerResolverArtifactsReconcile(options) + return result +} + async function ensureAccountForApplication (applicationId, transaction) { await ensureDefaultRules(transaction) const existing = await NatsAccountManager.findOne({ applicationId }, transaction) @@ -891,6 +989,7 @@ async function ensureLeafUserForAccount (accountId, fogName, transaction, natsIn } async function reissueForAccountRule (accountRuleId, transaction) { + const rule = await NatsAccountRuleManager.findOne({ id: accountRuleId }, transaction) const applications = await ApplicationManager.findAll({ natsRuleId: accountRuleId }, transaction) logger.info(`Reissuing account JWTs for rule ${accountRuleId}`) for (const app of applications) { @@ -903,7 +1002,6 @@ async function reissueForAccountRule (accountRuleId, transaction) { const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) - const rule = await NatsAccountRuleManager.findOne({ id: accountRuleId }, transaction) const accountJwt = await _encodeAccountJwtWithRuleAndRevocations( app.name, accountKp, @@ -913,6 +1011,29 @@ async function reissueForAccountRule (accountRuleId, transaction) { ) await NatsAccountManager.update({ id: account.id }, { jwt: accountJwt }, transaction) } + if (rule && rule.name === NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME) { + const relayAccount = await NatsAccountManager.findOne({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (relayAccount) { + const operator = await ensureOperator(transaction) + const operatorSeed = await _loadSeedFromSecret(operator.seedSecretName, transaction) + const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) + const accountSeed = await _loadSeedFromSecret(relayAccount.seedSecretName, transaction) + const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) + const accountJwt = await _encodeAccountJwtWithRuleAndRevocations( + CONTROLLER_NATS_ACCOUNT_NAME, + accountKp, + operatorKp, + rule, + relayAccount.jwt + ) + await NatsAccountManager.update({ id: relayAccount.id }, { jwt: accountJwt }, transaction) + } + } _triggerResolverArtifactsReconcile({ reason: 'account-rule-updated', accountRuleId }) } @@ -926,11 +1047,7 @@ async function _addRevocationToAccount (account, publicKey, transaction) { const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -949,11 +1066,7 @@ async function _reissueOneUserForRule (user, userRuleId, operatorKp, transaction const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1018,11 +1131,7 @@ async function revokeMicroserviceUser (microserviceUuid, transaction) { const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1104,11 +1213,7 @@ async function revokeUserByAccountAndName (accountId, userName, transaction) { const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1177,12 +1282,17 @@ function scheduleReissueUsersForMicroservices (microserviceUuids = []) { module.exports = { SYSTEM_ACCOUNT_NAME, + CONTROLLER_NATS_ACCOUNT_NAME, + CONTROLLER_NATS_USER_NAME, + controllerNatsCredsSecretName, + isPlatformControllerNatsAccount, sysUserNameForServer, leafSystemAccountName, leafSystemAccountUserName, ensureOperator: TransactionDecorator.generateTransaction(ensureOperator), rotateOperator: TransactionDecorator.generateTransaction(rotateOperator), ensureSystemAccount: TransactionDecorator.generateTransaction(ensureSystemAccount), + ensureControllerNatsAccount: TransactionDecorator.generateTransaction(ensureControllerNatsAccount), ensureSysUserForServer: TransactionDecorator.generateTransaction(ensureSysUserForServer), ensureLeafSystemAccount: TransactionDecorator.generateTransaction(ensureLeafSystemAccount), ensureLeafSystemAccountUser: TransactionDecorator.generateTransaction(ensureLeafSystemAccountUser), diff --git a/src/services/nats-relay-connection-manager.js b/src/services/nats-relay-connection-manager.js new file mode 100644 index 00000000..a85a0d25 --- /dev/null +++ b/src/services/nats-relay-connection-manager.js @@ -0,0 +1,336 @@ +const { connect, credsAuthenticator } = require('@nats-io/transport-node') +const config = require('../config') +const logger = require('../logger') +const Constants = require('../helpers/constants') +const { createDedupeHostList, aggregateConnectError } = require('../helpers/connect-endpoint-utils') +const NatsInstanceManager = require('../data/managers/nats-instance-manager') +const NatsAccountManager = require('../data/managers/nats-account-manager') +const NatsUserManager = require('../data/managers/nats-user-manager') +const NatsAuthService = require('./nats-auth-service') +const SecretService = require('./secret-service') + +const NATS_DEFAULT_PORT = 4222 + +class NatsRelayConnectionManager { + constructor (deps = {}) { + this._connectFn = deps.connectFn || connect + this._config = deps.config || config + this.maxReconnectAttempts = deps.maxReconnectAttempts ?? -1 + this.fakeTransaction = { fakeTransaction: true } + this.connection = null + this.connectionPromise = null + this.cachedHubRecord = null + this.cachedCreds = null + this.credsPromise = null + this.recoveryListeners = [] + this.statusTask = null + this.shuttingDown = false + } + + onReconnect (cb) { + if (typeof cb === 'function') { + this.recoveryListeners.push(cb) + } + } + + _notifyReconnect () { + for (const listener of this.recoveryListeners) { + try { + listener() + } catch (error) { + logger.error({ error: error.message }, '[NATS][RELAY] Reconnect listener failed') + } + } + } + + isConnected () { + return !!(this.connection && !this.connection.isClosed()) + } + + async isAvailable () { + if (this.isConnected()) { + return true + } + try { + await this.getConnection() + return this.isConnected() + } catch (error) { + logger.debug({ error: error.message }, '[NATS][RELAY] Hub unavailable for relay') + return false + } + } + + async getConnection () { + if (this.shuttingDown) { + throw new Error('NATS relay connection manager is shutting down') + } + if (this.connection && !this.connection.isClosed()) { + return this.connection + } + if (this.connectionPromise) { + return this.connectionPromise + } + + this.connectionPromise = this._createConnection() + return this.connectionPromise + } + + async _createConnection () { + try { + const { hosts, port } = await this._resolveHubEndpoint() + const creds = await this._loadControllerRelayCreds() + const connectAttempts = [] + + for (let attempt = 0; attempt < hosts.length; attempt++) { + const host = hosts[attempt] + const servers = `nats://${host}:${port}` + try { + const nc = await this._connectFn({ + servers, + authenticator: credsAuthenticator(creds), + maxReconnectAttempts: this.maxReconnectAttempts, + reconnect: true, + reconnectTimeWait: 1000, + name: 'controller-ws-relay' + }) + + this.connection = nc + this.connectionPromise = null + this._watchConnectionStatus(nc) + + logger.info({ + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' + }, '[NATS][RELAY] Hub connection established') + return nc + } catch (error) { + const errorMessage = error.message || String(error) + connectAttempts.push({ host, error: errorMessage }) + logger.warn({ + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + error: errorMessage + }, '[NATS][RELAY] Hub connect attempt failed') + } + } + + const aggregateError = aggregateConnectError( + 'Unable to connect NATS hub after all fallback hosts', + connectAttempts, + port + ) + logger.error({ + hosts, + port, + connectAttempts, + error: aggregateError.message + }, '[NATS][RELAY] Unable to connect hub after all fallback hosts') + throw aggregateError + } catch (error) { + this.connectionPromise = null + throw error + } + } + + _watchConnectionStatus (nc) { + if (this.statusTask) { + return + } + + this.statusTask = (async () => { + try { + for await (const status of nc.status()) { + if (status.type === 'disconnect') { + logger.warn({ + error: status.data ? status.data.message : undefined + }, '[NATS][RELAY] Hub connection disconnected') + } else if (status.type === 'reconnect') { + logger.info('[NATS][RELAY] Hub connection reconnected') + this._notifyReconnect() + } + } + } catch (error) { + if (!this.shuttingDown) { + logger.debug({ error: error.message }, '[NATS][RELAY] Connection status watcher ended') + } + } finally { + this.statusTask = null + } + })() + } + + async _resolveHubEndpoint () { + const hub = await this._getHubRecord() + const port = hub.serverPort || NATS_DEFAULT_PORT + const hosts = this._buildHubHostList(hub) + return { hosts, port, hubUuid: hub.iofogUuid } + } + + _buildHubHostList (hub) { + if (this._isKubernetes()) { + return this._kubernetesHubHosts(hub) + } + return this._remoteHubHosts(hub) + } + + _kubernetesHubHosts (hub) { + const { hosts, addHost } = createDedupeHostList() + + const namespace = process.env.CONTROLLER_NAMESPACE || this._config.get('app.namespace') + if (namespace && namespace.trim().length > 0) { + addHost(`${Constants.DEFAULT_NATS_K8S_SERVICE}.${namespace.trim()}.svc.cluster.local`) + } + if (hub.host) { + addHost(hub.host) + } + if (hosts.length === 0) { + addHost(Constants.DEFAULT_NATS_K8S_SERVICE) + } + return hosts + } + + _remoteHubHosts (hub) { + const { hosts, addHost } = createDedupeHostList() + + addHost(Constants.NATS_BRIDGE_DNS_SAN) + + if (hub.host) { + addHost(hub.host) + } + + return hosts + } + + async _getHubRecord () { + if (this.cachedHubRecord) { + return this.cachedHubRecord + } + const hub = await NatsInstanceManager.findOne({ isHub: true }, this.fakeTransaction) + if (!hub) { + throw new Error('NATS hub not found. Ensure a hub NatsInstances row with isHub=true exists.') + } + this.cachedHubRecord = hub + return hub + } + + async _loadControllerRelayCreds () { + if (this.cachedCreds) { + return this.cachedCreds + } + if (this.credsPromise) { + return this.credsPromise + } + + this.credsPromise = (async () => { + try { + await this._ensureControllerNatsAccount() + const creds = await this._fetchControllerRelayCreds() + this.cachedCreds = creds + return creds + } finally { + this.credsPromise = null + } + })() + return this.credsPromise + } + + async _ensureControllerNatsAccount () { + const hub = await NatsInstanceManager.findOne({ isHub: true }, this.fakeTransaction) + if (!hub) { + return + } + await NatsAuthService.ensureControllerNatsAccount() + } + + async _fetchControllerRelayCreds () { + const account = await NatsAccountManager.findOne({ + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, this.fakeTransaction) + + let credsSecretName = NatsAuthService.controllerNatsCredsSecretName() + if (account) { + const user = await NatsUserManager.findOne({ + accountId: account.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME + }, this.fakeTransaction) + if (user && user.credsSecretName) { + credsSecretName = user.credsSecretName + } + } + + const secret = await this._safeGetSecret(credsSecretName) + if (!secret || !secret.data) { + throw new Error(`Controller relay NATS creds secret not found: ${credsSecretName}`) + } + + const credsKey = Object.keys(secret.data).find((key) => key.endsWith('.creds')) || 'creds' + const raw = secret.data[credsKey] + if (!raw) { + throw new Error(`Missing creds payload in secret ${credsSecretName}`) + } + + const credsText = typeof raw === 'string' + ? raw + : Buffer.from(raw, 'base64').toString('utf8') + + return new TextEncoder().encode(credsText) + } + + async _safeGetSecret (name) { + try { + return await SecretService.getSecretEndpoint(name) + } catch (error) { + if (error.name === 'NotFoundError') { + logger.debug({ secret: name }, '[NATS][RELAY] Secret not found') + return null + } + throw error + } + } + + _isKubernetes () { + const controlPlane = process.env.CONTROL_PLANE || this._config.get('app.ControlPlane') + return controlPlane && controlPlane.toLowerCase() === 'kubernetes' + } + + async shutdown () { + this.shuttingDown = true + if (this.connection && !this.connection.isClosed()) { + try { + await this.connection.drain() + } catch (error) { + logger.debug({ error: error.message }, '[NATS][RELAY] Drain failed during shutdown') + try { + await this.connection.close() + } catch (closeError) { + logger.debug({ error: closeError.message }, '[NATS][RELAY] Close failed during shutdown') + } + } + } + this.connection = null + this.connectionPromise = null + logger.info('[NATS][RELAY] Connection manager shut down') + } + + resetForTests () { + this.connection = null + this.connectionPromise = null + this.cachedHubRecord = null + this.cachedCreds = null + this.credsPromise = null + this.statusTask = null + this.shuttingDown = false + this.recoveryListeners = [] + } +} + +module.exports = new NatsRelayConnectionManager() +module.exports.NatsRelayConnectionManager = NatsRelayConnectionManager diff --git a/src/services/nats-relay-transport-impl.js b/src/services/nats-relay-transport-impl.js new file mode 100644 index 00000000..3929b488 --- /dev/null +++ b/src/services/nats-relay-transport-impl.js @@ -0,0 +1,627 @@ +const WebSocket = require('ws') +const { headers: natsHeaders } = require('@nats-io/transport-node') +const msgpack = require('@msgpack/msgpack') +const config = require('../config') +const logger = require('../logger') +const NatsRelayConnectionManager = require('./nats-relay-connection-manager') + +const SUBJECT_PREFIX = 'controller.relay.v1' +const LOG_BACKPRESSURE_BUFFER_BYTES = 256 * 1024 +const LOG_MESSAGE_TYPES = { LOG_ERROR: 9, LOG_LINE: 6 } + +const MESSAGE_TYPES = { + CLOSE: 4 +} + +const HEADER_MESSAGE_TYPE = 'messageType' +const HEADER_CLOSE_ACK = 'closeAck' + +function execAgentSubject (sessionId) { + return `${SUBJECT_PREFIX}.exec.${sessionId}.agent` +} + +function execUserSubject (sessionId) { + return `${SUBJECT_PREFIX}.exec.${sessionId}.user` +} + +function logUserSubject (sessionId) { + return `${SUBJECT_PREFIX}.log.${sessionId}.user` +} + +function decodeLogMessageType (buffer) { + try { + const msg = msgpack.decode(buffer) + return typeof msg.type === 'number' ? msg.type : null + } catch (error) { + return null + } +} + +function getBufferFromData (data) { + if (!data) return Buffer.alloc(0) + if (Buffer.isBuffer(data)) return data + if (data instanceof Uint8Array) return Buffer.from(data) + return Buffer.from(data) +} + +class NatsRelayTransportImpl { + constructor (connectionManager = NatsRelayConnectionManager, configOverride = null) { + this._connectionManager = connectionManager + this._config = configOverride || config + this.maxPendingBytes = this._config.get('server.webSocket.relay.nats.maxPendingBytes', 33554432) + this.maxPendingMessages = this._config.get('server.webSocket.relay.nats.maxPendingMessages', 8192) + this.publishTimeoutMs = this._config.get('server.webSocket.relay.nats.publishTimeoutMs', 5000) + this.execBridges = new Map() + this.logBridges = new Map() + this.recoveryCallbacks = [] + this.rebinding = new Set() + + this._connectionManager.onReconnect(() => { + return this._handleReconnect().catch((error) => { + logger.error('[NATS][RELAY] Reconnect handling failed', { error: error.message }) + }) + }) + } + + async isAvailable () { + return this._connectionManager.isAvailable() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryCallbacks.push(cb) + } + } + + async enableForSession (session, cleanupCallback) { + const execId = session.execId + if (!execId) { + logger.warn('[NATS][RELAY] Missing execId for session, skipping bridge enablement') + return false + } + + const bridge = this.execBridges.get(execId) || { + execId, + session: null, + subscriptions: {}, + sockets: {}, + cleanupCallback: null + } + + bridge.session = session + if (cleanupCallback) { + bridge.cleanupCallback = cleanupCallback + } + + const nc = await this._connectionManager.getConnection() + + if (session.user) { + await this._ensureExecSubscription(bridge, 'user', session.user, nc) + } + if (session.agent) { + await this._ensureExecSubscription(bridge, 'agent', session.agent, nc) + } + + this.execBridges.set(execId, bridge) + return true + } + + shouldUseRelay (execId) { + return this.execBridges.has(execId) + } + + async publishToAgent (execId, buffer, options = {}) { + await this._publishExec(execId, execAgentSubject(execId), buffer, options) + } + + async publishToUser (execId, buffer, options = {}) { + await this._publishExec(execId, execUserSubject(execId), buffer, options) + } + + async cleanup (execId) { + const bridge = this.execBridges.get(execId) + if (!bridge) return + this._closeExecBridge(bridge) + this.execBridges.delete(execId) + } + + async enableForLogSession (session, cleanupCallback) { + const sessionId = session.sessionId + if (!sessionId) { + logger.warn('[NATS][RELAY] Missing sessionId for log session, skipping bridge enablement') + return false + } + + const bridge = this.logBridges.get(sessionId) || { + sessionId, + session: null, + subscription: null, + cleanupCallback: null, + backpressureNotified: false, + pendingBytes: 0, + pendingMessages: 0 + } + + bridge.session = session + if (cleanupCallback) { + bridge.cleanupCallback = cleanupCallback + } + + if (session.user) { + const nc = await this._connectionManager.getConnection() + await this._ensureLogUserSubscription(bridge, session.user, nc) + } + + this.logBridges.set(sessionId, bridge) + return true + } + + shouldUseRelayForLogs (sessionId) { + return this.logBridges.has(sessionId) + } + + async publishLogToUser (sessionId, buffer) { + const bridge = this.logBridges.get(sessionId) + if (!bridge) { + throw new Error(`Log bridge missing for sessionId=${sessionId}`) + } + + const messageType = decodeLogMessageType(buffer) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + + try { + await this._publish(logUserSubject(sessionId), buffer, {}, sessionId) + } catch (error) { + if (isLogLine) { + this._dropLogLineForPublishBackpressure(bridge, sessionId) + return + } + logger.error('[NATS][RELAY] Failed to publish log message', { + sessionId, + error: error.message + }) + throw error + } + } + + async cleanupLogSession (sessionId) { + const bridge = this.logBridges.get(sessionId) + if (!bridge) return + + if (bridge.subscription) { + try { + await bridge.subscription.drain() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to drain log subscription during cleanup', { + sessionId, + error: error.message + }) + } + bridge.subscription = null + } + + if (bridge.cleanupCallback) { + bridge.cleanupCallback = null + } + + this.logBridges.delete(sessionId) + } + + async shutdown () { + for (const execId of this.execBridges.keys()) { + await this.cleanup(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.cleanupLogSession(sessionId) + } + await this._connectionManager.shutdown() + } + + async _handleReconnect () { + for (const execId of this.execBridges.keys()) { + await this.rebindSessionBridges(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.rebindLogSessionBridges(sessionId) + } + } + + async rebindSessionBridges (execId) { + if (this.rebinding.has(execId)) return + this.rebinding.add(execId) + + try { + const bridge = this.execBridges.get(execId) + if (!bridge || !bridge.session) return + + logger.info('[NATS][RELAY] Rebinding exec session subscriptions after reconnect', { execId }) + + this._closeExecBridge(bridge) + bridge.subscriptions = {} + bridge.sockets = {} + + const nc = await this._connectionManager.getConnection() + const session = bridge.session + if (session.user) { + await this._ensureExecSubscription(bridge, 'user', session.user, nc) + } + if (session.agent) { + await this._ensureExecSubscription(bridge, 'agent', session.agent, nc) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(execId, { kind: 'exec' }) + } catch (error) { + logger.error('[NATS][RELAY] Exec recovery callback failed', { + execId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(execId) + } + } + + async rebindLogSessionBridges (sessionId) { + const key = `log:${sessionId}` + if (this.rebinding.has(key)) return + this.rebinding.add(key) + + try { + const bridge = this.logBridges.get(sessionId) + if (!bridge || !bridge.session) return + + logger.info('[NATS][RELAY] Rebinding log session subscription after reconnect', { sessionId }) + + if (bridge.subscription) { + try { + await bridge.subscription.drain() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to drain log subscription during rebind', { + sessionId, + error: error.message + }) + } + bridge.subscription = null + } + + const session = bridge.session + if (session.user) { + const nc = await this._connectionManager.getConnection() + await this._ensureLogUserSubscription(bridge, session.user, nc) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(sessionId, { kind: 'log' }) + } catch (error) { + logger.error('[NATS][RELAY] Log recovery callback failed', { + sessionId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(key) + } + } + + _closeExecBridge (bridge) { + for (const side of Object.keys(bridge.subscriptions || {})) { + const sub = bridge.subscriptions[side] + if (!sub) continue + try { + sub.unsubscribe() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to unsubscribe exec side during cleanup', { + execId: bridge.execId, + side, + error: error.message + }) + } + } + bridge.subscriptions = {} + } + + async _ensureExecSubscription (bridge, side, socket, nc) { + if (!socket) return + + bridge.sockets[side] = socket + if (bridge.subscriptions[side]) { + return + } + + const subject = side === 'agent' ? execAgentSubject(bridge.execId) : execUserSubject(bridge.execId) + logger.info('[NATS][RELAY] Subscribing exec relay subject', { + execId: bridge.execId, + side, + subject + }) + + const sub = nc.subscribe(subject, { + callback: (err, msg) => { + if (err) { + logger.error('[NATS][RELAY] Exec subscription error', { + execId: bridge.execId, + side, + error: err.message + }) + return + } + this._handleExecMessage(bridge, side, msg).catch((error) => { + logger.error('[NATS][RELAY] Failed to handle exec relay message', { + execId: bridge.execId, + side, + error: error.message + }) + }) + } + }) + + bridge.subscriptions[side] = sub + } + + async _handleExecMessage (bridge, side, msg) { + const currentBridge = this.execBridges.get(bridge.execId) + if (!currentBridge) return + + const ws = currentBridge.sockets[side] + const body = getBufferFromData(msg.data) + const msgTypeHeader = msg.headers ? msg.headers.get(HEADER_MESSAGE_TYPE) : null + const msgType = msgTypeHeader != null && msgTypeHeader !== '' ? Number(msgTypeHeader) : null + + if (msgType === MESSAGE_TYPES.CLOSE) { + await this._handleCloseMessage({ + bridge: currentBridge, + session: currentBridge.session, + side, + ws, + msg, + body + }) + return + } + + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(body, { binary: true, compress: false, mask: false, fin: true }) + logger.debug('[NATS][RELAY] Delivered exec message to socket', { + execId: bridge.execId, + side, + messageSize: body.length + }) + } else { + logger.debug('[NATS][RELAY] No socket available for exec delivery', { + execId: bridge.execId, + side, + hasSocket: !!ws, + socketState: ws ? ws.readyState : 'N/A' + }) + } + } + + async _handleCloseMessage ({ bridge, session, side, ws, msg, body }) { + const execId = session.execId + const closeInitiator = side === 'user' ? 'agent' : 'user' + const closeAck = Boolean(msg.headers && msg.headers.get(HEADER_CLOSE_ACK) === 'true') + + logger.info('[NATS][RELAY] Received CLOSE message via relay', { + execId, + side, + closeInitiator, + closeAck + }) + + if (ws && ws.readyState === WebSocket.OPEN) { + try { + const reason = closeInitiator === 'agent' ? 'Agent closed connection' : 'User closed connection' + ws.close(1000, reason) + } catch (error) { + logger.warn('[NATS][RELAY] Failed to close WebSocket after CLOSE message', { + execId, + side, + error: error.message + }) + } + } + + if (!closeAck && this.execBridges.has(execId)) { + const ackSide = side === 'user' ? 'agent' : 'user' + try { + const hdrs = natsHeaders() + hdrs.set(HEADER_MESSAGE_TYPE, String(MESSAGE_TYPES.CLOSE)) + hdrs.set(HEADER_CLOSE_ACK, 'true') + const subject = ackSide === 'agent' ? execAgentSubject(execId) : execUserSubject(execId) + await this._publish(subject, body, { headers: hdrs }, execId) + } catch (error) { + logger.warn('[NATS][RELAY] Failed to send CLOSE acknowledgement', { + execId, + ackSide, + error: error.message + }) + } + } + + if (bridge && bridge.cleanupCallback) { + try { + await bridge.cleanupCallback(execId) + } catch (error) { + logger.error('[NATS][RELAY] Error in cleanup callback during CLOSE handling', { + execId, + error: error.message + }) + } + } + } + + async _ensureLogUserSubscription (bridge, userWs, nc) { + bridge.session.user = userWs + if (bridge.subscription) { + return + } + + const subject = logUserSubject(bridge.sessionId) + logger.info('[NATS][RELAY] Subscribing log relay subject', { + sessionId: bridge.sessionId, + subject + }) + + bridge.pendingBytes = 0 + bridge.pendingMessages = 0 + + const sub = nc.subscribe(subject, { + callback: (err, msg) => { + if (err) { + logger.error('[NATS][RELAY] Log subscription error', { + sessionId: bridge.sessionId, + error: err.message + }) + return + } + this._handleLogMessage(bridge, msg).catch((error) => { + logger.error('[NATS][RELAY] Failed to handle log relay message', { + sessionId: bridge.sessionId, + error: error.message + }) + }) + } + }) + + bridge.subscription = sub + } + + async _handleLogMessage (bridge, msg) { + const currentBridge = this.logBridges.get(bridge.sessionId) + if (!currentBridge) return + + const ws = currentBridge.session && currentBridge.session.user + const body = getBufferFromData(msg.data) + const messageType = decodeLogMessageType(body) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + + currentBridge.pendingBytes += body.length + currentBridge.pendingMessages += 1 + + const overPendingLimits = + currentBridge.pendingBytes > this.maxPendingBytes || + currentBridge.pendingMessages > this.maxPendingMessages + + if (isLogLine && overPendingLimits) { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + this._dropLogLineForReceiveBackpressure(currentBridge, bridge.sessionId) + return + } + + if (ws && ws.readyState === WebSocket.OPEN) { + if (isLogLine && ws.bufferedAmount > LOG_BACKPRESSURE_BUFFER_BYTES) { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + this._dropLogLineForReceiveBackpressure(currentBridge, bridge.sessionId) + return + } + + ws.send(body, { binary: true }) + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + } else { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + } + } + + _dropLogLineForReceiveBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + logger.warn('[NATS][RELAY] Dropping log line due to subscription backpressure', { sessionId }) + this._notifyLogBackpressure(bridge, sessionId) + } + } + + _dropLogLineForPublishBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + logger.warn('[NATS][RELAY] Dropping log line due to publish-side backpressure', { sessionId }) + this._notifyLogBackpressure(bridge, sessionId) + } + } + + _notifyLogBackpressure (bridge, sessionId) { + const user = bridge.session && bridge.session.user + if (user && user.readyState === WebSocket.OPEN) { + try { + const errorBody = msgpack.encode({ + type: LOG_MESSAGE_TYPES.LOG_ERROR, + data: Buffer.from('Log stream backpressure: dropping lines until client catches up\n'), + sessionId, + timestamp: Date.now() + }) + user.send(errorBody, { binary: true }) + } catch (error) { + logger.debug('[NATS][RELAY] Failed to notify user of log backpressure', { + sessionId, + error: error.message + }) + } + } + } + + async _publishExec (execId, subject, buffer, options = {}) { + const hdrs = natsHeaders() + const hasMessageType = Object.prototype.hasOwnProperty.call(options, 'messageType') + const messageType = hasMessageType ? options.messageType : null + if (messageType !== null && messageType !== undefined) { + hdrs.set(HEADER_MESSAGE_TYPE, String(messageType)) + } + + const applicationProperties = (options && options.applicationProperties) || {} + if (applicationProperties.closeAck) { + hdrs.set(HEADER_CLOSE_ACK, 'true') + } + + await this._publish(subject, buffer, { headers: hdrs }, execId) + } + + async _publish (subject, buffer, opts = {}, sessionKey = null) { + const nc = await this._connectionManager.getConnection() + nc.publish(subject, buffer, opts) + + const flushPromise = nc.flush() + const timeoutMs = this.publishTimeoutMs + let timer = null + try { + await Promise.race([ + flushPromise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new Error(`NATS publish not flushed within ${timeoutMs}ms`)) + }, timeoutMs) + }) + ]) + } catch (error) { + logger.error('[NATS][RELAY] Failed to publish message', { + subject, + sessionKey, + error: error.message + }) + throw error + } finally { + if (timer) clearTimeout(timer) + } + + logger.debug('[NATS][RELAY] Published relay message', { + subject, + sessionKey, + messageSize: buffer.length + }) + } +} + +const singleton = new NatsRelayTransportImpl() + +module.exports = singleton +module.exports.NatsRelayTransportImpl = NatsRelayTransportImpl +module.exports.execAgentSubject = execAgentSubject +module.exports.execUserSubject = execUserSubject +module.exports.logUserSubject = logUserSubject diff --git a/src/services/nats-relay-transport.js b/src/services/nats-relay-transport.js new file mode 100644 index 00000000..e7d47e51 --- /dev/null +++ b/src/services/nats-relay-transport.js @@ -0,0 +1,76 @@ +const WsRelayTransport = require('./ws-relay-transport') +const NatsRelayTransportImpl = require('./nats-relay-transport-impl') + +class NatsRelayTransport extends WsRelayTransport { + constructor (transportImpl = NatsRelayTransportImpl) { + super() + this._impl = transportImpl + this._recoveryCallbacks = [] + + this._impl.onRecovery((sessionId, meta) => { + for (const cb of this._recoveryCallbacks) { + try { + cb(sessionId, meta) + } catch (error) { + // Impl already logs; protect other callbacks. + } + } + }) + } + + getTransport () { + return 'nats' + } + + async isAvailable () { + return this._impl.isAvailable() + } + + async enableForSession (session, cleanupCb) { + return this._impl.enableForSession(session, cleanupCb) + } + + async enableForLogSession (session, cleanupCb) { + return this._impl.enableForLogSession(session, cleanupCb) + } + + async publishToAgent (execId, buffer, opts) { + return this._impl.publishToAgent(execId, buffer, opts) + } + + async publishToUser (execId, buffer, opts) { + return this._impl.publishToUser(execId, buffer, opts) + } + + async publishLogToUser (sessionId, buffer) { + return this._impl.publishLogToUser(sessionId, buffer) + } + + shouldUseRelay (execId) { + return this._impl.shouldUseRelay(execId) + } + + shouldUseRelayForLogs (sessionId) { + return this._impl.shouldUseRelayForLogs(sessionId) + } + + async cleanup (execId) { + return this._impl.cleanup(execId) + } + + async cleanupLogSession (sessionId) { + return this._impl.cleanupLogSession(sessionId) + } + + async shutdown () { + return this._impl.shutdown() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this._recoveryCallbacks.push(cb) + } + } +} + +module.exports = NatsRelayTransport diff --git a/src/services/nats-service.js b/src/services/nats-service.js index c372c5a5..c33d88dd 100644 --- a/src/services/nats-service.js +++ b/src/services/nats-service.js @@ -336,6 +336,17 @@ async function _buildHubJwtBundle (transaction) { jwtBundle[`${account.publicKey}.jwt`] = account.jwt } } + if (config.getBoolean('nats.enabled', false)) { + const relayAccount = await NatsAccountManager.findOne({ + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (relayAccount) { + jwtBundle[`${relayAccount.publicKey}.jwt`] = relayAccount.jwt + } + } return jwtBundle } @@ -1367,6 +1378,12 @@ async function _reconcileResolverArtifactsOnce (options = {}, transaction) { const reconcileTriggerOptions = { triggerReconcile: false } await NatsAuthServiceRuntime.ensureSystemAccount(transaction, reconcileTriggerOptions) + if (config.getBoolean('nats.enabled', false)) { + const hubInstance = (natsInstances || []).find((ni) => ni.isHub) + if (hubInstance) { + await NatsAuthServiceRuntime.ensureControllerNatsAccount(transaction, reconcileTriggerOptions) + } + } const systemAccount = await NatsAccountManager.findOne({ isSystem: true }, transaction) const systemNatsMicroservices = candidateFogUuids.length > 0 diff --git a/src/services/router-connection-manager.js b/src/services/router-connection-manager.js new file mode 100644 index 00000000..22d7aaf6 --- /dev/null +++ b/src/services/router-connection-manager.js @@ -0,0 +1,625 @@ +const rhea = require('rhea') +const config = require('../config') +const logger = require('../logger') +const Constants = require('../helpers/constants') +const { createDedupeHostList, aggregateConnectError } = require('../helpers/connect-endpoint-utils') +const RouterManager = require('../data/managers/router-manager') +const CertificateService = require('./certificate-service') +const SecretService = require('./secret-service') +const os = require('os') + +const CONTROLLER_CERT_PREFIX = 'controller-exec-session-client' +const hostname = process.env.HOSTNAME || os.hostname() +const CONTROLLER_CERT_NAME = hostname ? `${CONTROLLER_CERT_PREFIX}-${hostname}` : CONTROLLER_CERT_PREFIX + +const AMQP_DEFAULT_PORT = 5671 +const RELAY_CONTAINER_PREFIX = 'controller-relay' + +function hashSessionId (sessionId) { + let hash = 5381 + const value = String(sessionId) + for (let i = 0; i < value.length; i++) { + hash = ((hash << 5) + hash) ^ value.charCodeAt(i) + } + return Math.abs(hash >>> 0) +} + +function isCircularBufferOverflow (error) { + const msg = error && error.message ? error.message : String(error) + return /circular buffer overflow/i.test(msg) +} + +function isTlsError (error) { + if (!error) return false + const msg = error.message ? error.message : String(error) + return /tls|certificate|cert|ssl|handshake/i.test(msg) +} + +class PoolSlot { + constructor (manager, slotId) { + this.manager = manager + this.slotId = slotId + this.containerId = `${RELAY_CONTAINER_PREFIX}-${hostname || 'local'}-${slotId}` + this.container = rhea.create_container({ + id: this.containerId, + enable_sasl_external: true + }) + this.connection = null + this.connectionPromise = null + this.healthy = true + this.unsettledCount = 0 + } +} + +class RouterConnectionManager { + constructor () { + this.poolSize = config.get('server.webSocket.relay.amqp.poolSize', 8) + this.sendTimeoutMs = config.get('server.webSocket.relay.amqp.sendTimeoutMs', 5000) + this.unsettledWarnThreshold = config.get('server.webSocket.relay.amqp.unsettledWarnThreshold', 1800) + this.certificatePromise = null + this.cachedCertificate = null + this.cachedRouterRecord = null + this.fakeTransaction = { fakeTransaction: true } + this.slots = Array.from({ length: this.poolSize }, (_, slotId) => new PoolSlot(this, slotId)) + this.recoveryListeners = [] + this.saturationCount = 0 + this.shuttingDown = false + } + + slotIdForSession (sessionId) { + return hashSessionId(sessionId) % this.poolSize + } + + async acquire (sessionId) { + if (this.shuttingDown) { + throw new Error('Router connection pool is shutting down') + } + const slotId = this.slotIdForSession(sessionId) + return this._getSlotConnection(slotId) + } + + async getConnection () { + return this.acquire('__legacy__') + } + + getSlot (slotId) { + return this.slots[slotId] + } + + isConnected () { + return this.slots.some((slot) => slot.connection && slot.connection.is_open && slot.connection.is_open()) + } + + getHealthyPoolCount () { + return this.slots.filter((slot) => + slot.healthy && + slot.connection && + slot.connection.is_open && + slot.connection.is_open() + ).length + } + + getTotalUnsettled () { + return this.slots.reduce((sum, slot) => sum + (slot.unsettledCount || 0), 0) + } + + recordSessionSaturation () { + this.saturationCount += 1 + } + + getSaturationCount () { + return this.saturationCount + } + + onSlotRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryListeners.push(cb) + } + } + + _notifySlotRecovery (slotId) { + for (const listener of this.recoveryListeners) { + try { + listener(slotId) + } catch (error) { + logger.error('[AMQP] Slot recovery listener failed', { + slotId, + error: error.message + }) + } + } + } + + async isRouterAvailable () { + if (this.isConnected()) { + return true + } + try { + await this.acquire('__healthcheck__') + return this.isConnected() + } catch (error) { + return false + } + } + + async waitForSendable (sender, timeoutMs = this.sendTimeoutMs) { + if (!sender) { + throw new Error('AMQP sender is missing') + } + if (typeof sender.sendable === 'function' && sender.sendable()) { + return + } + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup() + reject(new Error(`AMQP sender not sendable within ${timeoutMs}ms`)) + }, timeoutMs) + + const onSendable = () => { + cleanup() + resolve() + } + + const onError = (context) => { + cleanup() + reject(context.error || new Error('AMQP sender error while waiting for sendable')) + } + + const cleanup = () => { + clearTimeout(timer) + sender.removeListener('sendable', onSendable) + sender.removeListener('error', onError) + } + + sender.once('sendable', onSendable) + sender.once('error', onError) + }) + } + + async markSlotUnhealthy (slotId, reason) { + const slot = this.slots[slotId] + if (!slot) return + + slot.healthy = false + this.recordSessionSaturation() + logger.warn('[AMQP] Marking router pool slot unhealthy', { + slotId, + containerId: slot.containerId, + reason: reason || 'unknown' + }) + + await this.reconnectSlot(slotId) + } + + async reconnectSlot (slotId) { + const slot = this.slots[slotId] + if (!slot || this.shuttingDown) return + + this._closeSlotConnection(slot) + + try { + await this._createSlotConnection(slot) + slot.healthy = true + logger.info('[AMQP] Router pool slot reconnected', { + slotId, + containerId: slot.containerId + }) + this._notifySlotRecovery(slotId) + } catch (error) { + slot.healthy = false + logger.error('[AMQP] Router pool slot reconnect failed', { + slotId, + containerId: slot.containerId, + error: error.message + }) + throw error + } + } + + handleSendError (sessionId, error) { + if (isCircularBufferOverflow(error)) { + const slotId = this.slotIdForSession(sessionId) + this.markSlotUnhealthy(slotId, error.message).catch((reconnectError) => { + logger.error('[AMQP] Failed to recover slot after overflow', { + slotId, + error: reconnectError.message + }) + }) + return true + } + return false + } + + updateUnsettledCount (slotId, count) { + const slot = this.slots[slotId] + if (!slot) return + slot.unsettledCount = count + if (count >= this.unsettledWarnThreshold) { + logger.warn('[AMQP] Router pool slot unsettled deliveries high', { + slotId, + unsettled: count, + threshold: this.unsettledWarnThreshold + }) + } + } + + async _getSlotConnection (slotId) { + const slot = this.slots[slotId] + if (!slot) { + throw new Error(`Invalid router pool slot: ${slotId}`) + } + + if (slot.connection && slot.connection.is_open && slot.connection.is_open() && slot.healthy) { + return slot.connection + } + + if (slot.connectionPromise) { + return slot.connectionPromise + } + + slot.connectionPromise = this._createSlotConnection(slot) + return slot.connectionPromise + } + + async _createSlotConnection (slot) { + try { + const { hosts, port } = await this._resolveRouterEndpoint() + const certBundle = await this._ensureControllerCertificate() + + const connectAttempts = [] + for (let attempt = 0; attempt < hosts.length; attempt++) { + const host = hosts[attempt] + const options = this._buildConnectOptions(host, port, certBundle, slot.containerId) + try { + const connection = await this._connectToHost(slot, host, port, options) + this.cachedCertificate = certBundle + logger.info({ + slotId: slot.slotId, + containerId: slot.containerId, + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length + }, '[AMQP] Router pool slot connection established') + return connection + } catch (error) { + const errorMessage = error.message || String(error) + connectAttempts.push({ host, error: errorMessage }) + logger.warn({ + slotId: slot.slotId, + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + error: errorMessage + }, '[AMQP] Router pool slot connect attempt failed') + } + } + + const aggregateError = aggregateConnectError( + 'Unable to connect router pool slot after all fallback hosts', + connectAttempts, + port + ) + logger.error({ + transport: 'amqp', + slotId: slot.slotId, + hosts, + port, + connectAttempts, + error: aggregateError.message + }, '[AMQP] Unable to connect router pool slot after all fallback hosts') + throw aggregateError + } catch (error) { + slot.connectionPromise = null + throw error + } + } + + _buildConnectOptions (host, port, certBundle, containerId) { + return { + transport: 'tls', + host, + hostname: host, + port, + rejectUnauthorized: true, + idle_time_out: 300000, + reconnect: false, + username: '', + password: '', + container_id: containerId, + cert: certBundle.cert, + key: certBundle.key, + ca: [certBundle.ca] + } + } + + _connectToHost (slot, host, port, options) { + return new Promise((resolve, reject) => { + const connection = slot.container.connect(options) + let settled = false + + const settle = (handler) => (context) => { + if (settled) return + settled = true + handler(context) + } + + const cleanupPromise = () => { + slot.connection = null + slot.connectionPromise = null + slot.healthy = false + } + + connection.once('connection_open', settle(() => { + slot.connection = connection + slot.connectionPromise = null + slot.healthy = true + + connection.on('connection_error', (context) => { + logger.error({ + err: context.error, + transport: 'amqp', + msg: '[AMQP] Pool slot connection error event', + slotId: slot.slotId, + host, + port + }) + if (isTlsError(context.error)) { + this.cachedCertificate = null + this.reconnectSlot(slot.slotId).catch((error) => { + logger.error('[AMQP] TLS reconnect failed for pool slot', { + slotId: slot.slotId, + error: error.message + }) + }) + } + }) + + connection.on('connection_close', () => { + logger.warn('[AMQP] Router pool slot connection closed', { + slotId: slot.slotId, + host, + port + }) + cleanupPromise() + }) + + connection.on('disconnected', (context) => { + logger.warn('[AMQP] Router pool slot disconnected', { + slotId: slot.slotId, + host, + port, + error: context.error ? context.error.message : 'unknown' + }) + cleanupPromise() + if (context.error && isTlsError(context.error)) { + this.cachedCertificate = null + } + }) + + resolve(connection) + })) + + connection.once('connection_close', settle((context) => { + reject(context.error || new Error('Router connection closed before opening')) + })) + + connection.once('disconnected', settle((context) => { + reject(context.error || new Error('Router disconnected during connect')) + })) + }) + } + + _closeSlotConnection (slot) { + if (!slot.connection) return + try { + slot.connection.removeAllListeners() + slot.connection.close() + } catch (error) { + logger.debug('[AMQP] Failed to close pool slot connection during reconnect', { + slotId: slot.slotId, + error: error.message + }) + } + slot.connection = null + slot.connectionPromise = null + } + + async shutdown () { + this.shuttingDown = true + for (const slot of this.slots) { + this._closeSlotConnection(slot) + } + logger.info('[AMQP] Router connection pool shut down', { poolSize: this.poolSize }) + } + + async _resolveRouterEndpoint () { + logger.debug({ msg: '[AMQP] Resolving default router endpoint' }) + try { + const router = await this._getDefaultRouterRecord() + const port = router.messagingPort || AMQP_DEFAULT_PORT + const hosts = this._buildRouterHostList(router) + const host = hosts[0] + logger.debug({ + msg: '[AMQP] Default router resolved', + routerHost: router.host, + hosts, + host, + port, + routerUuid: router.iofogUuid, + controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' + }) + return { + host, + hosts, + port, + routerUuid: router.iofogUuid + } + } catch (error) { + logger.error({ err: error, msg: '[AMQP] Failed while resolving router endpoint' }) + throw error + } + } + + _buildRouterHostList (router) { + if (this._isKubernetes()) { + return this._kubernetesRouterHosts(router) + } + return this._remoteRouterHosts(router) + } + + _kubernetesRouterHosts (router) { + const { hosts, addHost } = createDedupeHostList() + + const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') + if (namespace && namespace.trim().length > 0) { + addHost(`${Constants.DEFAULT_ROUTER_K8S_SERVICE}.${namespace.trim()}.svc.cluster.local`) + } + if (router.host) { + addHost(router.host) + } + if (hosts.length === 0) { + addHost(Constants.DEFAULT_ROUTER_K8S_SERVICE) + } + return hosts + } + + _remoteRouterHosts (router) { + const { hosts, addHost } = createDedupeHostList() + + addHost(Constants.ROUTER_BRIDGE_DNS_SAN) + + if (router.host) { + addHost(router.host) + } + + return hosts + } + + async _getDefaultRouterRecord () { + if (this.cachedRouterRecord) { + return this.cachedRouterRecord + } + const router = await RouterManager.findOne({ isDefault: true }, this.fakeTransaction) + if (!router) { + throw new Error('Default router not found. Please ensure default router is provisioned.') + } + this.cachedRouterRecord = router + return router + } + + _isKubernetes () { + const controlPlane = process.env.CONTROL_PLANE || config.get('app.ControlPlane') + return controlPlane && controlPlane.toLowerCase() === 'kubernetes' + } + + async _ensureControllerCertificate () { + if (this.cachedCertificate) { + return this.cachedCertificate + } + if (this.certificatePromise) { + return this.certificatePromise + } + this.certificatePromise = (async () => { + try { + const bundle = await this._createControllerCertificate() + this.cachedCertificate = bundle + return bundle + } finally { + this.certificatePromise = null + } + })() + return this.certificatePromise + } + + async _createControllerCertificate () { + logger.debug('[AMQP] Ensuring controller certificate secret exists', { name: CONTROLLER_CERT_NAME }) + await CertificateService.ensureRouterLocalCA(this.fakeTransaction) + const existingSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) + const caName = Constants.DEFAULT_ROUTER_LOCAL_CA + if (existingSecret) { + const caSecret = await this._safeGetSecret(caName) + const bundle = this._decodeCertificate(existingSecret, caSecret) + logger.debug({ msg: '[AMQP] Using existing controller-exec-session-client certificate', ca: caName }) + return bundle + } + + const hosts = this._buildControllerHosts() + logger.debug({ msg: '[AMQP] Generating controller-exec-session-client certificate', hosts, ca: caName }) + + try { + await CertificateService.createCertificateEndpoint({ + name: CONTROLLER_CERT_NAME, + subject: CONTROLLER_CERT_NAME, + hosts: hosts.join(','), + ca: { + type: 'direct', + secretName: caName + }, + expiration: 36 + }) + } catch (error) { + logger.error({ err: error, ca: caName, msg: '[AMQP] Failed to create controller certificate' }) + throw error + } + + const certSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) + const caSecret = await this._safeGetSecret(caName) + if (!certSecret || !caSecret) { + throw new Error('Controller certificate creation succeeded but secret not found') + } + logger.debug({ msg: '[AMQP] controller-exec-session-client certificate generated successfully', ca: caName }) + return this._decodeCertificate(certSecret, caSecret) + } + + _buildControllerHosts () { + const hosts = new Set(['localhost', '127.0.0.1']) + if (hostname) hosts.add(hostname) + if (this._isKubernetes() && (process.env.CONTROLLER_NAMESPACE != null && process.env.CONTROLLER_NAMESPACE !== '')) { + hosts.add(`controller.${process.env.CONTROLLER_NAMESPACE}.svc.cluster.local`) + } + if (process.env.CONTROLLER_HOST) hosts.add(process.env.CONTROLLER_HOST) + return Array.from(hosts) + } + + _decodeCertificate (certSecret, caSecret) { + if (!certSecret || !certSecret.data) { + throw new Error(`Secret ${CONTROLLER_CERT_NAME} is empty or missing.`) + } + if (!caSecret || !caSecret.data) { + throw new Error('CA secret not found for router connection.') + } + const decode = (value, label) => { + if (!value) { + throw new Error(`Missing ${label} in certificate secret`) + } + return Buffer.from(value, 'base64') + } + return { + cert: decode(certSecret.data['tls.crt'], 'tls.crt'), + key: decode(certSecret.data['tls.key'], 'tls.key'), + ca: decode(caSecret.data['tls.crt'], 'ca.crt') + } + } + + async _safeGetSecret (name) { + try { + return await SecretService.getSecretEndpoint(name) + } catch (error) { + if (error.name === 'NotFoundError') { + logger.debug('[AMQP] Secret not found', { secret: name }) + return null + } + logger.error('[AMQP] Unexpected error while fetching secret', { + secret: name, + error: error.message, + stack: error.stack + }) + throw error + } + } +} + +module.exports = new RouterConnectionManager() diff --git a/src/services/router-connection-service.js b/src/services/router-connection-service.js index 77f70412..d9b7376d 100644 --- a/src/services/router-connection-service.js +++ b/src/services/router-connection-service.js @@ -1,392 +1,4 @@ -const rhea = require('rhea') -const config = require('../config') -const logger = require('../logger') -const Constants = require('../helpers/constants') -const RouterManager = require('../data/managers/router-manager') -const CertificateService = require('./certificate-service') -const SecretService = require('./secret-service') -const os = require('os') - -const CONTROLLER_CERT_PREFIX = 'controller-exec-session-client' -const hostname = process.env.HOSTNAME || os.hostname() -const CONTROLLER_CERT_NAME = hostname ? `${CONTROLLER_CERT_PREFIX}-${hostname}` : CONTROLLER_CERT_PREFIX - -const DEFAULT_ROUTER_SERVICE = 'router' -const AMQP_DEFAULT_PORT = 5671 - -class RouterConnectionService { - constructor () { - this.connection = null - this.connectionPromise = null - this.certificatePromise = null - this.cachedCertificate = null - this.connectionOptions = null - this.cachedRouterRecord = null - this.fakeTransaction = { fakeTransaction: true } - this.container = rhea.create_container({ - id: 'controller-exec-session-client', - enable_sasl_external: true - }) - } - - async getConnection () { - if (this.connection && this.connection.is_open && this.connection.is_open()) { - return this.connection - } - if (this.connectionPromise) { - return this.connectionPromise - } - this.connectionPromise = this._createConnection() - return this.connectionPromise - } - - isConnected () { - return !!(this.connection && this.connection.is_open && this.connection.is_open()) - } - - async isRouterAvailable () { - if (this.isConnected()) { - return true - } - try { - await this.getConnection() - return this.isConnected() - } catch (error) { - return false - } - } - - async _createConnection () { - try { - logger.debug({ msg: '[AMQP] Preparing router connection options' }) - - const { hosts, port } = await this._resolveRouterEndpoint() - logger.debug({ msg: '[AMQP] Router endpoint resolved', hosts, port }) - const certBundle = await this._ensureControllerCertificate() - - let lastError = null - for (let attempt = 0; attempt < hosts.length; attempt++) { - const host = hosts[attempt] - const options = this._buildConnectOptions(host, port, certBundle) - try { - const connection = await this._connectToHost(host, port, options) - this.connectionOptions = { - transport: 'tls', - host, - hostname: host, - port, - rejectUnauthorized: true, - idle_time_out: 300000, - reconnect: true, - reconnect_limit: 100, - username: '', - password: '', - container_id: 'controller-exec-session-client' - } - this.cachedCertificate = certBundle - logger.info({ - msg: '[AMQP] Router connection established', - host, - port, - attempt: attempt + 1, - totalAttempts: hosts.length - }) - return connection - } catch (error) { - lastError = error - logger.warn({ - msg: '[AMQP] Router connect attempt failed', - host, - port, - attempt: attempt + 1, - totalAttempts: hosts.length, - err: error.message || String(error) - }) - } - } - - const aggregateError = lastError || new Error('No router hosts available for connection') - logger.error({ - err: aggregateError, - transport: 'amqp', - msg: '[AMQP] Unable to connect to router after all fallback hosts', - hosts, - port - }) - throw aggregateError - } catch (error) { - this.connectionPromise = null - logger.error('[AMQP] Failed to create router connection', { - error: error.message, - stack: error.stack - }) - throw error - } - } - - _buildConnectOptions (host, port, certBundle) { - logger.debug({ msg: '[AMQP] Router connection options built', host, port }) - return { - transport: 'tls', - host, - hostname: host, - port, - rejectUnauthorized: true, - idle_time_out: 300000, - reconnect: false, - username: '', - password: '', - container_id: 'controller-exec-session-client', - cert: certBundle.cert, - key: certBundle.key, - ca: [certBundle.ca] - } - } - - _connectToHost (host, port, options) { - return new Promise((resolve, reject) => { - const connection = this.container.connect(options) - let settled = false - - const settle = (handler) => (context) => { - if (settled) return - settled = true - handler(context) - } - - const cleanupPromise = () => { - this.connection = null - this.connectionPromise = null - } - - connection.once('connection_open', settle(() => { - this.connection = connection - this.connectionPromise = null - connection.on('connection_error', (context) => { - logger.error({ - err: context.error, - transport: 'amqp', - msg: '[AMQP] Connection error event', - host, - port - }) - }) - connection.on('connection_close', () => { - logger.warn('[AMQP] Router connection closed', { host, port }) - cleanupPromise() - }) - connection.on('disconnected', (context) => { - logger.warn('[AMQP] Router connection disconnected', { - host, - port, - error: context.error ? context.error.message : 'unknown' - }) - cleanupPromise() - }) - resolve(connection) - })) - - connection.once('connection_close', settle((context) => { - reject(context.error || new Error('Router connection closed before opening')) - })) - - connection.once('disconnected', settle((context) => { - reject(context.error || new Error('Router disconnected during connect')) - })) - }) - } - - async _resolveRouterEndpoint () { - logger.debug({ msg: '[AMQP] Resolving default router endpoint' }) - try { - const router = await this._getDefaultRouterRecord() - const port = router.messagingPort || AMQP_DEFAULT_PORT - const hosts = this._buildRouterHostList(router) - const host = hosts[0] - logger.debug({ - msg: '[AMQP] Default router resolved', - routerHost: router.host, - hosts, - host, - port, - routerUuid: router.iofogUuid, - controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' - }) - return { - host, - hosts, - port, - routerUuid: router.iofogUuid - } - } catch (error) { - logger.error({ err: error, msg: '[AMQP] Failed while resolving router endpoint' }) - throw error - } - } - - _buildRouterHostList (router) { - if (this._isKubernetes()) { - return [this._kubernetesRouterHost(router)] - } - return this._remoteRouterHosts(router) - } - - _kubernetesRouterHost (router) { - const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') - if (namespace && namespace.trim().length > 0) { - return `${DEFAULT_ROUTER_SERVICE}.${namespace.trim()}.svc.cluster.local` - } - const dbHost = router.host && router.host.trim().length > 0 ? router.host.trim() : '' - return dbHost || DEFAULT_ROUTER_SERVICE - } - - _remoteRouterHosts (router) { - const hosts = [] - const seen = new Set() - const addHost = (candidate) => { - if (!candidate) return - const trimmed = String(candidate).trim() - if (trimmed.length === 0 || seen.has(trimmed)) return - seen.add(trimmed) - hosts.push(trimmed) - } - - addHost(Constants.ROUTER_BRIDGE_DNS_SAN) - - if (router.host) { - addHost(router.host) - } - - const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') - if (namespace && namespace.trim().length > 0) { - addHost(`${DEFAULT_ROUTER_SERVICE}.${namespace.trim()}.svc.cluster.local`) - } - - return hosts - } - - async _getDefaultRouterRecord () { - if (this.cachedRouterRecord) { - return this.cachedRouterRecord - } - const router = await RouterManager.findOne({ isDefault: true }, this.fakeTransaction) - if (!router) { - throw new Error('Default router not found. Please ensure default router is provisioned.') - } - this.cachedRouterRecord = router - return router - } - - _isKubernetes () { - const controlPlane = process.env.CONTROL_PLANE || config.get('app.ControlPlane') - return controlPlane && controlPlane.toLowerCase() === 'kubernetes' - } - - async _ensureControllerCertificate () { - if (this.cachedCertificate) { - return this.cachedCertificate - } - if (this.certificatePromise) { - return this.certificatePromise - } - this.certificatePromise = (async () => { - try { - const bundle = await this._createControllerCertificate() - this.cachedCertificate = bundle - return bundle - } finally { - this.certificatePromise = null - } - })() - return this.certificatePromise - } - - async _createControllerCertificate () { - logger.debug('[AMQP] Ensuring controller certificate secret exists', { name: CONTROLLER_CERT_NAME }) - await CertificateService.ensureRouterLocalCA(this.fakeTransaction) - const existingSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) - const caName = Constants.DEFAULT_ROUTER_LOCAL_CA - if (existingSecret) { - const caSecret = await this._safeGetSecret(caName) - const bundle = this._decodeCertificate(existingSecret, caSecret) - logger.debug({ msg: '[AMQP] Using existing controller-exec-session-client certificate', ca: caName }) - return bundle - } - - const hosts = this._buildControllerHosts() - logger.debug({ msg: '[AMQP] Generating controller-exec-session-client certificate', hosts, ca: caName }) - - try { - await CertificateService.createCertificateEndpoint({ - name: CONTROLLER_CERT_NAME, - subject: CONTROLLER_CERT_NAME, - hosts: hosts.join(','), - ca: { - type: 'direct', - secretName: caName - }, - expiration: 36 // months - }) - } catch (error) { - logger.error({ err: error, ca: caName, msg: '[AMQP] Failed to create controller certificate' }) - throw error - } - - const certSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) - const caSecret = await this._safeGetSecret(caName) - if (!certSecret || !caSecret) { - throw new Error('Controller certificate creation succeeded but secret not found') - } - logger.debug({ msg: '[AMQP] controller-exec-session-client certificate generated successfully', ca: caName }) - return this._decodeCertificate(certSecret, caSecret) - } - - _buildControllerHosts () { - const hosts = new Set(['localhost', '127.0.0.1']) - if (hostname) hosts.add(hostname) - if (this._isKubernetes() && (process.env.CONTROLLER_NAMESPACE != null && process.env.CONTROLLER_NAMESPACE !== '')) { - hosts.add(`controller.${process.env.CONTROLLER_NAMESPACE}.svc.cluster.local`) - } - if (process.env.CONTROLLER_HOST) hosts.add(process.env.CONTROLLER_HOST) - return Array.from(hosts) - } - - _decodeCertificate (certSecret, caSecret) { - if (!certSecret || !certSecret.data) { - throw new Error(`Secret ${CONTROLLER_CERT_NAME} is empty or missing.`) - } - if (!caSecret || !caSecret.data) { - throw new Error('CA secret not found for router connection.') - } - const decode = (value, label) => { - if (!value) { - throw new Error(`Missing ${label} in certificate secret`) - } - return Buffer.from(value, 'base64') - } - return { - cert: decode(certSecret.data['tls.crt'], 'tls.crt'), - key: decode(certSecret.data['tls.key'], 'tls.key'), - ca: decode(caSecret.data['tls.crt'], 'ca.crt') - } - } - - async _safeGetSecret (name) { - try { - return await SecretService.getSecretEndpoint(name) - } catch (error) { - if (error.name === 'NotFoundError') { - logger.debug('[AMQP] Secret not found', { secret: name }) - return null - } - logger.error('[AMQP] Unexpected error while fetching secret', { - secret: name, - error: error.message, - stack: error.stack - }) - throw error - } - } -} - -module.exports = new RouterConnectionService() +/** + * Backward-compatible export — implementation is RouterConnectionManager (Plan 18-B pool). + */ +module.exports = require('./router-connection-manager') diff --git a/src/services/websocket-queue-service.js b/src/services/websocket-queue-service.js index 1052e31c..595c9124 100644 --- a/src/services/websocket-queue-service.js +++ b/src/services/websocket-queue-service.js @@ -1,12 +1,14 @@ const WebSocket = require('ws') const logger = require('../logger') -const RouterConnectionService = require('./router-connection-service') -const { recordAmqpPublishError } = require('../websocket/ws-metrics') +const RouterConnectionManager = require('./router-connection-manager') +const { + recordAmqpPublishError, + recordAmqpSessionSaturation +} = require('../websocket/ws-metrics') const msgpack = require('@msgpack/msgpack') -// Plan 16-C: drop LOG_LINE when user WS buffer exceeds threshold (see forwardLogToUser). const LOG_BACKPRESSURE_BUFFER_BYTES = 256 * 1024 -const LOG_MESSAGE_TYPES = { LOG_ERROR: 9 } +const LOG_MESSAGE_TYPES = { LOG_ERROR: 9, LOG_LINE: 6 } const MESSAGE_TYPES = { STDIN: 0, @@ -38,10 +40,49 @@ function getBufferFromBody (body) { return Buffer.from(body) } +function decodeLogMessageType (buffer) { + try { + const msg = msgpack.decode(buffer) + return typeof msg.type === 'number' ? msg.type : null + } catch (error) { + return null + } +} + class WebSocketQueueService { constructor () { this.execBridges = new Map() - this.logBridges = new Map() // New: for log sessions + this.logBridges = new Map() + this.recoveryCallbacks = [] + this.rebinding = new Set() + + RouterConnectionManager.onSlotRecovery((slotId) => { + this._handleSlotRecovery(slotId).catch((error) => { + logger.error('[AMQP][QUEUE] Slot recovery handling failed', { + slotId, + error: error.message + }) + }) + }) + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryCallbacks.push(cb) + } + } + + async _handleSlotRecovery (slotId) { + for (const [execId, bridge] of this.execBridges.entries()) { + if (bridge.slotId === slotId) { + await this.rebindSessionBridges(execId) + } + } + for (const [sessionId, bridge] of this.logBridges.entries()) { + if (bridge.slotId === slotId) { + await this.rebindLogSessionBridges(sessionId) + } + } } async enableForSession (session, cleanupCallback) { @@ -51,14 +92,19 @@ class WebSocketQueueService { return false } + const slotId = RouterConnectionManager.slotIdForSession(execId) const bridge = this.execBridges.get(execId) || { execId, + slotId, + session: null, senders: {}, receivers: {}, + linkRefs: {}, cleanupCallback: null } - // Store cleanup callback for CLOSE message handling + bridge.slotId = slotId + bridge.session = session if (cleanupCallback) { bridge.cleanupCallback = cleanupCallback } @@ -93,6 +139,47 @@ class WebSocketQueueService { this.execBridges.delete(execId) } + async rebindSessionBridges (execId) { + if (this.rebinding.has(execId)) return + this.rebinding.add(execId) + + try { + const bridge = this.execBridges.get(execId) + if (!bridge || !bridge.session) return + + logger.info('[AMQP][QUEUE] Rebinding exec session bridges after slot recovery', { + execId, + slotId: bridge.slotId + }) + + this._closeBridgeLinks(bridge, execId) + bridge.senders = {} + bridge.receivers = {} + bridge.linkRefs = {} + + const session = bridge.session + if (session.user) { + await this._ensureReceiver(bridge, 'user', session.user, session) + } + if (session.agent) { + await this._ensureReceiver(bridge, 'agent', session.agent, session) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(execId, { kind: 'exec', slotId: bridge.slotId }) + } catch (error) { + logger.error('[AMQP][QUEUE] Exec recovery callback failed', { + execId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(execId) + } + } + _closeBridgeLinks (bridge, sessionKey) { const closeLink = (linkWrapper) => { if (!linkWrapper) return @@ -116,18 +203,25 @@ class WebSocketQueueService { closeLink(bridge.receivers?.user) closeLink(bridge.senders?.agent) closeLink(bridge.senders?.user) + bridge.linkRefs = {} } _invalidateExecSender (bridge, side) { if (bridge.senders[side]) { bridge.senders[side] = null } + if (bridge.linkRefs) { + bridge.linkRefs[`sender:${side}`] = null + } } _invalidateExecReceiver (bridge, side) { if (bridge.receivers[side]) { bridge.receivers[side] = null } + if (bridge.linkRefs) { + bridge.linkRefs[`receiver:${side}`] = null + } } _attachSenderLifecycle (bridge, side, sender, execId) { @@ -169,11 +263,15 @@ class WebSocketQueueService { } async _send (execId, side, buffer, options = {}) { - const bridge = await this._ensureSender(execId, side) - if (!bridge) { - throw new Error('Queue bridge missing for execId=' + execId) - } + let bridge try { + bridge = await this._ensureSender(execId, side) + if (!bridge) { + throw new Error('Queue bridge missing for execId=' + execId) + } + + await RouterConnectionManager.waitForSendable(bridge.sender) + const message = { body: buffer, content_type: 'application/octet-stream' @@ -199,6 +297,7 @@ class WebSocketQueueService { } catch (error) { recordAmqpPublishError({ sessionType: 'exec', side }) logger.error('[AMQP][QUEUE] Failed to publish message', { execId, side, error: error.message }) + RouterConnectionManager.handleSendError(execId, error) const execBridge = this.execBridges.get(execId) if (execBridge) { this._invalidateExecSender(execBridge, side) @@ -219,7 +318,7 @@ class WebSocketQueueService { side === 'agent' ? MESSAGE_QUEUE_PREFIX.agent : MESSAGE_QUEUE_PREFIX.user, execId ) - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(execId) const sender = await new Promise((resolve, reject) => { const link = connection.open_sender({ target: { @@ -238,13 +337,13 @@ class WebSocketQueueService { this._attachSenderLifecycle(bridge, side, sender, execId) bridge.senders[side] = { sender } + bridge.linkRefs[`sender:${side}`] = sender return bridge.senders[side] } async _ensureReceiver (bridge, side, socket, session) { if (!socket) return if (bridge.receivers[side]) { - // Update socket reference if receiver already exists bridge.receivers[side].socket = socket logger.debug('[AMQP][QUEUE] Updated socket reference for existing receiver', { execId: session.execId, @@ -263,7 +362,7 @@ class WebSocketQueueService { side, queueName }) - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.execId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -290,7 +389,6 @@ class WebSocketQueueService { receiver.on('message', async (context) => { try { - // Always get the latest socket reference from the bridge const currentBridge = this.execBridges.get(session.execId) const ws = currentBridge && currentBridge.receivers[side] ? currentBridge.receivers[side].socket : null const body = getBufferFromBody(context.message.body) @@ -298,7 +396,6 @@ class WebSocketQueueService { ? context.message.application_properties.messageType : null - // Handle CLOSE messages (works for both user and agent sides) if (msgType === MESSAGE_TYPES.CLOSE) { await this._handleCloseMessage({ bridge: currentBridge, @@ -311,7 +408,6 @@ class WebSocketQueueService { return } - // Forward message to socket (normal message or non-CLOSE message) if (ws && ws.readyState === WebSocket.OPEN) { try { ws.send(body, { @@ -363,6 +459,7 @@ class WebSocketQueueService { }) bridge.receivers[side] = { receiver, socket } + bridge.linkRefs[`receiver:${side}`] = receiver logger.info('[AMQP][QUEUE] Receiver setup complete', { execId: session.execId, side, @@ -385,7 +482,6 @@ class WebSocketQueueService { closeAck }) - // Attempt to close the socket gracefully (if present) if (ws && ws.readyState === WebSocket.OPEN) { try { const reason = closeInitiator === 'agent' ? 'Agent closed connection' : 'User closed connection' @@ -432,7 +528,6 @@ class WebSocketQueueService { } } - // Invoke cleanup callback to remove session/queue resources if (bridge && bridge.cleanupCallback) { try { await bridge.cleanupCallback(execId) @@ -445,11 +540,6 @@ class WebSocketQueueService { } } - // ========== Log Session Queue Methods ========== - - // Enable queue bridge for log session (unidirectional: agent → user, one-to-one) - // Following exec session pattern: Use queues for ALL scenarios - // Each sessionId has its own queues (one-to-one, like exec sessions) async enableForLogSession (session, cleanupCallback) { const sessionId = session.sessionId if (!sessionId) { @@ -457,25 +547,30 @@ class WebSocketQueueService { return false } + const slotId = RouterConnectionManager.slotIdForSession(sessionId) const bridge = this.logBridges.get(sessionId) || { sessionId, + slotId, + session: null, agentSender: null, agentReceiver: null, - userReceiver: null, // Single user receiver (one-to-one) - userSender: null, // User queue sender - cleanupCallback: null + userReceiver: null, + userSender: null, + linkRefs: {}, + cleanupCallback: null, + backpressureNotified: false } + bridge.slotId = slotId + bridge.session = session if (cleanupCallback) { bridge.cleanupCallback = cleanupCallback } - // Agent side: single receiver (agent receives from queue) if (session.agent) { await this._ensureLogAgentReceiver(bridge, session.agent, session) } - // User side: single receiver (one-to-one, like exec sessions) if (session.user) { await this._ensureLogUserReceiver(bridge, session.user, session) } @@ -488,51 +583,46 @@ class WebSocketQueueService { return this.logBridges.has(sessionId) } - // Forward to user via queue (one-to-one, like exec sessions) - // Note: Exec sessions use queues for BOTH single and multi-replica - // We follow the same pattern for consistency - // Buffer is already MessagePack encoded from agent async publishLogToUser (sessionId, buffer, options = {}) { const bridge = this.logBridges.get(sessionId) if (!bridge) { throw new Error(`Log bridge missing for sessionId=${sessionId}`) } - // Ensure user queue sender exists + const messageType = decodeLogMessageType(buffer) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + if (!bridge.userSender) { - const userQueueName = `logs-user-${sessionId}` - const connection = await RouterConnectionService.getConnection() - const sender = await new Promise((resolve, reject) => { - const link = connection.open_sender({ - target: { - address: userQueueName, - durable: 0, - expiry_policy: 'link-detach' - }, - autosettle: true - }) + await this._ensureLogUserSender(sessionId) + } - link.once('sender_open', () => resolve(link)) - link.once('sender_close', reject) - link.once('error', reject) - }) - bridge.userSender = { sender } + if (isLogLine && bridge.userSender && bridge.userSender.sender) { + const sender = bridge.userSender.sender + if (typeof sender.sendable === 'function' && !sender.sendable()) { + return this._dropLogLineForPublishBackpressure(bridge, sessionId) + } } - // Buffer is already MessagePack encoded, send as binary const message = { - body: buffer, // MessagePack encoded buffer + body: buffer, content_type: 'application/octet-stream', application_properties: options.applicationProperties || {} } try { + if (bridge.userSender && bridge.userSender.sender) { + await RouterConnectionManager.waitForSendable(bridge.userSender.sender) + } bridge.userSender.sender.send(message) logger.debug('[AMQP][QUEUE] Published log message to user queue', { sessionId, messageSize: buffer.length }) } catch (error) { + if (isLogLine) { + RouterConnectionManager.handleSendError(sessionId, error) + return this._dropLogLineForPublishBackpressure(bridge, sessionId) + } recordAmqpPublishError({ sessionType: 'log', side: 'user' }) logger.error('[AMQP][QUEUE] Failed to publish log message to user queue', { sessionId, @@ -543,16 +633,64 @@ class WebSocketQueueService { } } - // Setup receiver for agent queue (one-to-one per sessionId) + _dropLogLineForPublishBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + recordAmqpSessionSaturation() + logger.warn('[AMQP][QUEUE] Dropping log line due to publish-side backpressure', { sessionId }) + const session = bridge.session + const user = session && session.user + if (user && user.readyState === WebSocket.OPEN) { + try { + const errorBody = msgpack.encode({ + type: LOG_MESSAGE_TYPES.LOG_ERROR, + data: Buffer.from('Log stream backpressure: dropping lines until client catches up\n'), + sessionId, + timestamp: Date.now() + }) + user.send(errorBody, { binary: true }) + } catch (sendError) { + logger.debug('[AMQP][QUEUE] Failed to notify user of log publish backpressure', { + sessionId, + error: sendError.message + }) + } + } + } + } + + async _ensureLogUserSender (sessionId) { + const bridge = this.logBridges.get(sessionId) + if (!bridge || bridge.userSender) return + + const userQueueName = `logs-user-${sessionId}` + const connection = await RouterConnectionManager.acquire(sessionId) + const sender = await new Promise((resolve, reject) => { + const link = connection.open_sender({ + target: { + address: userQueueName, + durable: 0, + expiry_policy: 'link-detach' + }, + autosettle: true + }) + + link.once('sender_open', () => resolve(link)) + link.once('sender_close', reject) + link.once('error', reject) + }) + bridge.userSender = { sender } + bridge.linkRefs.userSender = sender + } + async _ensureLogAgentReceiver (bridge, agentWs, session) { if (bridge.agentReceiver) { bridge.agentReceiver.socket = agentWs return } - // Queue name per sessionId (one-to-one) const queueName = `logs-agent-${session.sessionId}` - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.sessionId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -574,8 +712,6 @@ class WebSocketQueueService { const ws = currentBridge && currentBridge.agentReceiver ? currentBridge.agentReceiver.socket : null const body = getBufferFromBody(context.message.body) - // Body is already MessagePack encoded from agent - // Forward directly to agent WebSocket (binary) if (ws && ws.readyState === WebSocket.OPEN) { ws.send(body, { binary: true }) context.delivery.accept() @@ -585,47 +721,17 @@ class WebSocketQueueService { }) bridge.agentReceiver = { receiver, socket: agentWs } + bridge.linkRefs.agentReceiver = receiver } - // Setup sender for agent queue (one-to-one per sessionId) - async _ensureLogAgentSender (sessionId) { - const bridge = this.logBridges.get(sessionId) - if (!bridge) return null - if (bridge.agentSender) return bridge.agentSender - - // Queue name per sessionId (one-to-one) - const queueName = `logs-agent-${sessionId}` - const connection = await RouterConnectionService.getConnection() - - const sender = await new Promise((resolve, reject) => { - const link = connection.open_sender({ - target: { - address: queueName, - durable: 0, - expiry_policy: 'link-detach' - }, - autosettle: true - }) - - link.once('sender_open', () => resolve(link)) - link.once('sender_close', reject) - link.once('error', reject) - }) - - bridge.agentSender = { sender } - return bridge.agentSender - } - - // Setup receiver for user queue (one-to-one, like exec session pattern) async _ensureLogUserReceiver (bridge, userWs, session) { if (bridge.userReceiver) { bridge.userReceiver.socket = userWs return } - // Queue name per sessionId (one-to-one) const queueName = `logs-user-${session.sessionId}` - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.sessionId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -677,9 +783,71 @@ class WebSocketQueueService { }) bridge.userReceiver = { receiver, socket: userWs } + bridge.linkRefs.userReceiver = receiver + } + + async rebindLogSessionBridges (sessionId) { + if (this.rebinding.has(`log:${sessionId}`)) return + this.rebinding.add(`log:${sessionId}`) + + try { + const bridge = this.logBridges.get(sessionId) + if (!bridge || !bridge.session) return + + logger.info('[AMQP][QUEUE] Rebinding log session bridges after slot recovery', { + sessionId, + slotId: bridge.slotId + }) + + const closeLink = (linkWrapper) => { + if (!linkWrapper) return + try { + if (linkWrapper.receiver) { + linkWrapper.receiver.removeAllListeners() + linkWrapper.receiver.close() + } else if (linkWrapper.sender) { + linkWrapper.sender.removeAllListeners() + linkWrapper.sender.close() + } + } catch (error) { + logger.debug('[AMQP][QUEUE] Failed to close log link during rebind', { + sessionId, + error: error.message + }) + } + } + + closeLink(bridge.agentReceiver) + closeLink(bridge.userReceiver) + closeLink(bridge.userSender) + bridge.agentReceiver = null + bridge.userReceiver = null + bridge.userSender = null + bridge.linkRefs = {} + + const session = bridge.session + if (session.agent) { + await this._ensureLogAgentReceiver(bridge, session.agent, session) + } + if (session.user) { + await this._ensureLogUserReceiver(bridge, session.user, session) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(sessionId, { kind: 'log', slotId: bridge.slotId }) + } catch (error) { + logger.error('[AMQP][QUEUE] Log recovery callback failed', { + sessionId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(`log:${sessionId}`) + } } - // Cleanup log session (one-to-one) async cleanupLogSession (sessionId) { const bridge = this.logBridges.get(sessionId) if (!bridge) return @@ -710,6 +878,16 @@ class WebSocketQueueService { this.logBridges.delete(sessionId) } + + async shutdown () { + for (const execId of this.execBridges.keys()) { + await this.cleanup(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.cleanupLogSession(sessionId) + } + await RouterConnectionManager.shutdown() + } } module.exports = new WebSocketQueueService() diff --git a/src/services/ws-relay-transport-factory.js b/src/services/ws-relay-transport-factory.js new file mode 100644 index 00000000..578889af --- /dev/null +++ b/src/services/ws-relay-transport-factory.js @@ -0,0 +1,38 @@ +const config = require('../config') +const logger = require('../logger') +const AmqpRelayTransport = require('./amqp-relay-transport') +const NatsRelayTransport = require('./nats-relay-transport') + +let transportInstance = null +let resolvedTransportName = null + +/** + * Resolve singleton WsRelayTransport from nats.enabled (fixed at first call). + * @param {Object} [configOverride] - Optional config for tests. + * @returns {import('./ws-relay-transport')} + */ +function resolveTransport (configOverride) { + const cfg = configOverride || config + const useNats = cfg.getBoolean('nats.enabled', false) + const transportName = useNats ? 'nats' : 'amqp' + + if (!transportInstance || resolvedTransportName !== transportName) { + logger.info(`[RELAY] transport=${transportName}`) + transportInstance = useNats + ? new NatsRelayTransport() + : new AmqpRelayTransport() + resolvedTransportName = transportName + } + + return transportInstance +} + +function resetTransportForTests () { + transportInstance = null + resolvedTransportName = null +} + +module.exports = { + resolveTransport, + resetTransportForTests +} diff --git a/src/services/ws-relay-transport.js b/src/services/ws-relay-transport.js new file mode 100644 index 00000000..0c84aa15 --- /dev/null +++ b/src/services/ws-relay-transport.js @@ -0,0 +1,111 @@ +/** + * Cross-replica exec/log relay transport interface (Plan 18). + * Implementations: AmqpRelayTransport, NatsRelayTransport (18-D). + */ +class WsRelayTransport { + /** + * @returns {'amqp' | 'nats'} + */ + getTransport () { + throw new Error('WsRelayTransport.getTransport() not implemented') + } + + /** + * Lazy health check for session fail-fast. + * @returns {Promise} + */ + async isAvailable () { + throw new Error('WsRelayTransport.isAvailable() not implemented') + } + + /** + * @param {Object} session + * @param {Function} [cleanupCb] + * @returns {Promise} + */ + async enableForSession (session, cleanupCb) { + throw new Error('WsRelayTransport.enableForSession() not implemented') + } + + /** + * @param {Object} session + * @param {Function} [cleanupCb] + * @returns {Promise} + */ + async enableForLogSession (session, cleanupCb) { + throw new Error('WsRelayTransport.enableForLogSession() not implemented') + } + + /** + * @param {string} execId + * @param {Buffer} buffer + * @param {Object} [opts] + */ + async publishToAgent (execId, buffer, opts) { + throw new Error('WsRelayTransport.publishToAgent() not implemented') + } + + /** + * @param {string} execId + * @param {Buffer} buffer + * @param {Object} [opts] + */ + async publishToUser (execId, buffer, opts) { + throw new Error('WsRelayTransport.publishToUser() not implemented') + } + + /** + * @param {string} sessionId + * @param {Buffer} buffer + */ + async publishLogToUser (sessionId, buffer) { + throw new Error('WsRelayTransport.publishLogToUser() not implemented') + } + + /** + * @param {string} execId + * @returns {boolean} + */ + shouldUseRelay (execId) { + throw new Error('WsRelayTransport.shouldUseRelay() not implemented') + } + + /** + * @param {string} sessionId + * @returns {boolean} + */ + shouldUseRelayForLogs (sessionId) { + throw new Error('WsRelayTransport.shouldUseRelayForLogs() not implemented') + } + + /** + * @param {string} execId + */ + async cleanup (execId) { + throw new Error('WsRelayTransport.cleanup() not implemented') + } + + /** + * @param {string} sessionId + */ + async cleanupLogSession (sessionId) { + throw new Error('WsRelayTransport.cleanupLogSession() not implemented') + } + + /** + * Drain on process shutdown. + */ + async shutdown () { + // Default no-op; implementations may override. + } + + /** + * Optional callback registration for bridge rebind after reconnect (18-B / 18-D). + * @param {Function} _cb + */ + onRecovery (_cb) { + // Default no-op. + } +} + +module.exports = WsRelayTransport diff --git a/src/websocket/server.js b/src/websocket/server.js index 8dcd0477..88754bf2 100644 --- a/src/websocket/server.js +++ b/src/websocket/server.js @@ -12,8 +12,7 @@ const { microserviceState } = require('../enums/microservice-state') const AuthDecorator = require('../decorators/authorization-decorator') const TransactionDecorator = require('../decorators/transaction-decorator') const msgpack = require('@msgpack/msgpack') -const WebSocketQueueService = require('../services/websocket-queue-service') -const RouterConnectionService = require('../services/router-connection-service') +const { resolveTransport } = require('../services/ws-relay-transport-factory') const { recordExecSessionActive, recordLogSessionActive @@ -39,8 +38,8 @@ const MESSAGE_TYPES = { LOG_ERROR: 9 // Log streaming error } -const ROUTER_UNAVAILABLE_CLOSE_CODE = 1013 -const ROUTER_UNAVAILABLE_CLOSE_REASON = 'Router unavailable for cross-replica session' +const RELAY_UNAVAILABLE_CLOSE_CODE = 1013 +const RELAY_UNAVAILABLE_CLOSE_REASON = 'Relay unavailable for cross-replica session' const DRAIN_CLOSE_CODE = 1001 const DRAIN_CLOSE_REASON = 'Server draining' // when user WS bufferedAmount exceeds this, drop LOG_LINE silently and emit LOG_ERROR once. @@ -168,7 +167,23 @@ class WebSocketServer { this.logSessionManager = new LogSessionManager(config.get('server.webSocket')) this.execSessionManager = new ExecSessionManager(config.get('server.webSocket')) this.sessionConfig = config.get('server.webSocket.session') - this.queueService = WebSocketQueueService + this.relayTransport = resolveTransport() + this.relayTransport.onRecovery(async (sessionId, meta) => { + if (!meta || meta.kind !== 'exec') return + const session = this.execSessionManager.getExecSession(sessionId) + if (!session || !session.user || !session.agent) return + session.activationSent = false + try { + await TransactionDecorator.generateTransaction(async (tx) => { + await this.sendExecActivationToExecSession(session, sessionId, tx) + })() + } catch (error) { + logger.error('[RELAY] Failed to resend exec activation after relay recovery', { + sessionId, + error: error.message + }) + } + }) this.pendingCloseTimeouts = new Map() // Track pending CLOSE messages in cross-replica scenarios this.haConfig = config.get('server.webSocket.ha') || {} this.isDraining = false @@ -376,15 +391,17 @@ class WebSocketServer { return !!(session && (!session.agent || !session.user)) } - async requireRouterForCrossReplica (ws) { + async requireRelayForCrossReplica (ws) { if (this.haConfig.failFastOnRouterUnavailable === false) { return true } - const available = await RouterConnectionService.isRouterAvailable() + const available = await this.relayTransport.isAvailable() if (!available) { - logger.warn('[WS-HA] Router unavailable for cross-replica session') + logger.warn('[RELAY] Relay backend unavailable for cross-replica session', { + transport: this.relayTransport.getTransport() + }) if (ws && ws.readyState === WebSocket.OPEN) { - ws.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + ws.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } return false } @@ -1020,7 +1037,7 @@ class WebSocketServer { let session = this.execSessionManager.getExecSession(sessionId) if (!session) { - if (!(await this.requireRouterForCrossReplica(ws))) { + if (!(await this.requireRelayForCrossReplica(ws))) { return } session = this.execSessionManager.createExecSession( @@ -1106,12 +1123,12 @@ class WebSocketServer { closeTransaction ) - const queueEnabled = this.queueService.shouldUseQueue(sessionId) + const relayEnabled = this.relayTransport.shouldUseRelay(sessionId) if (!currentSession.user) { await this.cleanupExecSession(sessionId, closeTransaction) } else { - if (queueEnabled) { + if (relayEnabled) { try { const closeMsg = { type: MESSAGE_TYPES.CLOSE, @@ -1122,7 +1139,7 @@ class WebSocketServer { data: Buffer.from('Agent closed connection') } const encoded = this.encodeMessage(closeMsg) - await this.queueService.publishToUser(sessionId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) + await this.relayTransport.publishToUser(sessionId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) } catch (error) { logger.error('[WS-CLOSE] Failed to send CLOSE to user via queue after agent exec disconnect', { sessionId, @@ -1236,7 +1253,7 @@ class WebSocketServer { logger.info('[RELAY] Exec session activation sent to agent:' + JSON.stringify({ sessionId, microserviceUuid: session.microserviceUuid, - queueEnabled: this.queueService.shouldUseQueue(sessionId) + relayEnabled: this.relayTransport.shouldUseRelay(sessionId) })) } else { logger.error('[RELAY] Exec session activation to agent failed:' + JSON.stringify({ @@ -1465,6 +1482,10 @@ class WebSocketServer { execSessions: execSessionIds.length, logSessions: logSessionIds.length }) + + await this.relayTransport.shutdown().catch((error) => { + logger.warn('[WS-DRAIN] Relay transport shutdown failed', { error: error.message }) + }) })() return this.drainPromise @@ -1582,11 +1603,11 @@ class WebSocketServer { async sendMessageToAgent (agent, message, execId, microserviceUuid) { try { const encoded = this.encodeMessage(message) - const isQueueEnabled = this.queueService.shouldUseQueue(execId) + const relayEnabled = this.relayTransport.shouldUseRelay(execId) const messageType = typeof message.type === 'number' ? message.type : null - if (isQueueEnabled) { - await this.queueService.publishToAgent(execId, encoded, { messageType }) + if (relayEnabled) { + await this.relayTransport.publishToAgent(execId, encoded, { messageType }) logger.debug('[RELAY] Queued message for agent via AMQP:' + JSON.stringify({ execId, microserviceUuid, @@ -2103,7 +2124,7 @@ class WebSocketServer { let session = this.logSessionManager.getLogSession(sessionId) if (!session) { // Session might be on different replica, create it - if (!(await this.requireRouterForCrossReplica(ws))) { + if (!(await this.requireRelayForCrossReplica(ws))) { return } session = this.logSessionManager.createLogSession( @@ -2339,7 +2360,7 @@ class WebSocketServer { } // Enable queue bridge for cross-replica support (one-to-one, like exec sessions) - await this.queueService.enableForLogSession(session, (sessionId) => { + await this.relayTransport.enableForLogSession(session, (sessionId) => { this.cleanupLogSession(sessionId, transaction) }) @@ -2445,18 +2466,18 @@ class WebSocketServer { // Buffer is already MessagePack encoded from agent // Following exec session pattern: Use queue for ALL scenarios (single and multi-replica) // One-to-one forwarding (agent → user) via queue - const useQueue = this.queueService.shouldUseQueueForLogs(sessionId) + const useRelay = this.relayTransport.shouldUseRelayForLogs(sessionId) logger.debug('forwardLogToUser:' + JSON.stringify({ sessionId, - useQueue, + useRelay, hasUser: !!session.user, userState: session.user ? session.user.readyState : 'N/A', bufferLength: buffer.length })) - if (useQueue) { - // Publish MessagePack encoded buffer to user queue - await this.queueService.publishLogToUser(sessionId, buffer) + if (useRelay) { + // Publish MessagePack encoded buffer to user relay + await this.relayTransport.publishLogToUser(sessionId, buffer) } else { // Fallback: Direct WebSocket (only if queue not enabled) if (this._shouldDropLogLineForBackpressure(session, sessionId)) { @@ -2499,7 +2520,7 @@ class WebSocketServer { } this.logBackpressureNotified.delete(sessionId) await this.logSessionManager.removeLogSession(sessionId, transaction) - await this.queueService.cleanupLogSession(sessionId) + await this.relayTransport.cleanupLogSession(sessionId) } async setupExecMessageForwarding (sessionId, transaction) { @@ -2514,7 +2535,7 @@ class WebSocketServer { const wasQueueBridgeEnabled = session.queueBridgeEnabled try { - await this.queueService.enableForSession(session, async (closeExecId) => { + await this.relayTransport.enableForSession(session, async (closeExecId) => { const timeout = this.pendingCloseTimeouts.get(closeExecId) if (timeout) { clearTimeout(timeout) @@ -2524,9 +2545,10 @@ class WebSocketServer { }) session.queueBridgeEnabled = true if (!wasQueueBridgeEnabled) { - logger.info('[RELAY] AMQP queue bridge enabled for exec session', { + logger.info('[RELAY] Relay bridge enabled for exec session', { sessionId, - microserviceUuid: session.microserviceUuid + microserviceUuid: session.microserviceUuid, + transport: this.relayTransport.getTransport() }) } } catch (error) { @@ -2534,27 +2556,36 @@ class WebSocketServer { const requireQueue = this.isCrossReplicaSession(session) && this.haConfig.failFastOnRouterUnavailable !== false if (requireQueue && !wasQueueBridgeEnabled) { - logger.error('[RELAY] AMQP required for cross-replica exec session but router bridge failed', { + logger.error('[RELAY] Relay required for cross-replica exec session but bridge failed', { sessionId, + transport: this.relayTransport.getTransport(), error: error.message }) if (session.user && session.user.readyState === WebSocket.OPEN) { - session.user.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + session.user.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } if (session.agent && session.agent.readyState === WebSocket.OPEN) { - session.agent.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + session.agent.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } await this.cleanupExecSession(sessionId, transaction) return } - logger.warn('[RELAY] Failed to enable AMQP queue bridge for exec session', { + logger.warn('[RELAY] Failed to enable relay bridge for exec session', { sessionId, + transport: this.relayTransport.getTransport(), error: error.message }) } if (user && agent) { - await this.sendExecActivationToExecSession(session, sessionId, transaction) + const activated = await this.sendExecActivationToExecSession(session, sessionId, transaction) + if (!activated) { + logger.error('[RELAY] Exec session activation failed; aborting message forwarding setup', { + sessionId, + microserviceUuid: session.microserviceUuid + }) + return + } } if (user) { @@ -2576,7 +2607,11 @@ class WebSocketServer { sessionId, timestamp: Date.now() } - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + const sent = await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + if (!sent && this.relayTransport.shouldUseRelay(execId)) { + logger.error('[RELAY] Exec relay publish failed; closing session', { sessionId: execId }) + await this.cleanupExecSession(execId, transaction) + } return } @@ -2591,8 +2626,8 @@ class WebSocketServer { if (msg.type === MESSAGE_TYPES.CLOSE) { await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { const timeout = setTimeout(async () => { const currentSession = this.execSessionManager.getExecSession(execId) if (currentSession && currentSession.user && currentSession.user.readyState === WebSocket.OPEN) { @@ -2635,7 +2670,11 @@ class WebSocketServer { } } - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + const sent = await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + if (!sent && this.relayTransport.shouldUseRelay(execId)) { + logger.error('[RELAY] Exec relay publish failed; closing session', { sessionId: execId }) + await this.cleanupExecSession(execId, transaction) + } } catch (error) { logger.error('[RELAY] Failed to process exec user message:' + JSON.stringify({ sessionId: execId, @@ -2657,10 +2696,10 @@ class WebSocketServer { const msg = this.decodeMessage(buffer) if (msg.type === MESSAGE_TYPES.CLOSE) { - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { try { - await this.queueService.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) + await this.relayTransport.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) } catch (error) { logger.error('[RELAY] Failed to enqueue exec CLOSE for user', { sessionId: execId, @@ -2674,9 +2713,17 @@ class WebSocketServer { return } - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { - await this.queueService.publishToUser(execId, buffer) + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { + try { + await this.relayTransport.publishToUser(execId, buffer) + } catch (error) { + logger.error('[RELAY] Exec relay publish to user failed; closing session', { + sessionId: execId, + error: error.message + }) + await this.cleanupExecSession(execId, transaction) + } } else if (session.user && session.user.readyState === WebSocket.OPEN) { if (msg.type === MESSAGE_TYPES.STDOUT || msg.type === MESSAGE_TYPES.STDERR) { if (msg.data && msg.data.length > 0) { @@ -2744,7 +2791,7 @@ class WebSocketServer { } await this.execSessionManager.removeExecSession(sessionId, transaction) - await this.queueService.cleanup(sessionId) + await this.relayTransport.cleanup(sessionId) .catch(error => { logger.warn('[RELAY] Failed to cleanup exec queue bridge during session cleanup', { sessionId, diff --git a/src/websocket/ws-metrics.js b/src/websocket/ws-metrics.js index 0aa9f5f6..d1aa5822 100644 --- a/src/websocket/ws-metrics.js +++ b/src/websocket/ws-metrics.js @@ -9,7 +9,9 @@ let logSessionsActive = null let pendingPairings = null let pairingDurationMs = null let amqpPublishErrors = null -let routerConnected = null +let routerPoolConnections = null +let routerPoolUnsettled = null +let amqpSessionSaturated = null function getMeter () { if (!meter) { @@ -18,7 +20,7 @@ function getMeter () { return meter } -function initWsMetrics (routerConnectionService) { +function initWsMetrics (routerConnectionManager) { const m = getMeter() execSessionsActive = m.createUpDownCounter('ws_exec_sessions_active', { @@ -37,14 +39,29 @@ function initWsMetrics (routerConnectionService) { amqpPublishErrors = m.createCounter('ws_amqp_publish_errors', { description: 'AMQP publish failures for exec/log relay' }) + amqpSessionSaturated = m.createCounter('ws_amqp_session_saturated', { + description: 'AMQP relay saturation events (overflow / publish backpressure)' + }) + + if (routerConnectionManager) { + routerPoolConnections = m.createObservableGauge('ws_router_pool_connections', { + description: 'Healthy router AMQP pool connections on this replica' + }) + routerPoolConnections.addCallback((result) => { + const healthy = typeof routerConnectionManager.getHealthyPoolCount === 'function' + ? routerConnectionManager.getHealthyPoolCount() + : (routerConnectionManager.isConnected() ? 1 : 0) + result.observe(healthy) + }) - if (routerConnectionService) { - routerConnected = m.createObservableGauge('ws_router_connected', { - description: 'Router AMQP connection availability (1=connected, 0=disconnected)' + routerPoolUnsettled = m.createObservableGauge('ws_router_pool_unsettled', { + description: 'Total unsettled AMQP deliveries across router pool slots' }) - routerConnected.addCallback((result) => { - const connected = routerConnectionService.isConnected() ? 1 : 0 - result.observe(connected) + routerPoolUnsettled.addCallback((result) => { + const unsettled = typeof routerConnectionManager.getTotalUnsettled === 'function' + ? routerConnectionManager.getTotalUnsettled() + : 0 + result.observe(unsettled) }) } } @@ -71,11 +88,16 @@ function recordAmqpPublishError (attributes = {}) { amqpPublishErrors?.add(1, attributes) } +function recordAmqpSessionSaturation () { + amqpSessionSaturated?.add(1) +} + module.exports = { initWsMetrics, recordExecSessionActive, recordLogSessionActive, recordPendingPairing, recordPairingDurationMs, - recordAmqpPublishError + recordAmqpPublishError, + recordAmqpSessionSaturation } diff --git a/test/src/services/controller-ms-service.test.js b/test/src/services/controller-ms-service.test.js index 2dfc4cdf..dfa2f2f9 100644 --- a/test/src/services/controller-ms-service.test.js +++ b/test/src/services/controller-ms-service.test.js @@ -13,6 +13,7 @@ const VolumeMappingManager = require('../../../src/data/managers/volume-mapping- const MicroservicesService = require('../../../src/services/microservices-service') const MicroservicePortService = require('../../../src/services/microservice-ports/microservice-port') const ApplicationManager = require('../../../src/data/managers/application-manager') +const WorkloadSpec = require('../../../src/services/microservice-workload-spec') const Errors = require('../../../src/helpers/errors') describe('Controller MS Service', () => { @@ -97,6 +98,9 @@ describe('Controller MS Service', () => { $sandbox.stub(MicroservicesService, 'updateChangeTracking').resolves() $sandbox.stub(MicroservicesService, 'injectServiceAccountVolume').resolves() $sandbox.stub(MicroservicesService, 'createOrUpdateServiceAccountForMicroservice').resolves() + $sandbox.stub(WorkloadSpec, 'createWorkloadRelations').resolves() + $sandbox.stub(WorkloadSpec, 'updateWorkloadRelations').resolves() + $sandbox.stub(WorkloadSpec, 'validateExtraHosts').resolves([]) }) context('on non-system fog', () => { @@ -162,6 +166,49 @@ describe('Controller MS Service', () => { expect(MicroservicesService.createOrUpdateServiceAccountForMicroservice).to.not.have.been.called }) + context('when container workload fields are sent on create', () => { + def('body', () => ({ + ...registerData, + capAdd: ['NET_ADMIN'], + isPrivileged: true, + cmd: ['/bin/controller'] + })) + + it('persists container workload relations on create', async () => { + await $subject + expect(WorkloadSpec.createWorkloadRelations).to.have.been.calledWith( + sinon.match({ uuid: msUuid }), + sinon.match({ + capAdd: ['NET_ADMIN'], + isPrivileged: true, + cmd: ['/bin/controller'] + }), + sinon.match({ transaction }) + ) + }) + }) + + context('when container scalar fields are sent on create', () => { + def('body', () => ({ + ...registerData, + isPrivileged: true, + pidMode: 'host', + memoryLimit: 1073741824 + })) + + it('creates microservice with container scalar fields', async () => { + await $subject + expect(MicroserviceManager.create).to.have.been.calledWith( + sinon.match({ + isPrivileged: true, + pidMode: 'host', + memoryLimit: 1073741824 + }), + transaction + ) + }) + }) + context('when microservice already exists', () => { const existing = { uuid: msUuid, @@ -234,6 +281,26 @@ describe('Controller MS Service', () => { expect(MicroservicesService.updateChangeTracking).to.have.been.calledWith(true, fogUuid, transaction) }) + context('when container workload fields are sent on update', () => { + def('body', () => ({ + ...registerData, + capAdd: ['NET_BIND_SERVICE'], + healthCheck: { test: ['CMD', 'true'] } + })) + + it('updates container workload relations on upsert', async () => { + await $subject + expect(WorkloadSpec.updateWorkloadRelations).to.have.been.calledWith( + msUuid, + sinon.match({ + capAdd: ['NET_BIND_SERVICE'], + healthCheck: { test: ['CMD', 'true'] } + }), + sinon.match({ transaction }) + ) + }) + }) + it('rejects uuid registered on a different fog', async () => { MicroserviceManager.findOne.callsFake((where) => { if (where.uuid === msUuid) { diff --git a/test/src/services/microservice-workload-spec.test.js b/test/src/services/microservice-workload-spec.test.js new file mode 100644 index 00000000..1df3a713 --- /dev/null +++ b/test/src/services/microservice-workload-spec.test.js @@ -0,0 +1,95 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const WorkloadSpec = require('../../../src/services/microservice-workload-spec') +const MicroserviceCapAddManager = require('../../../src/data/managers/microservice-cap-add-manager') +const MicroserviceCapDropManager = require('../../../src/data/managers/microservice-cap-drop-manager') +const MicroserviceArgManager = require('../../../src/data/managers/microservice-arg-manager') +const MicroserviceHealthCheckManager = require('../../../src/data/managers/microservice-healthcheck-manager') +const MicroserviceEnvManager = require('../../../src/data/managers/microservice-env-manager') + +describe('Microservice Workload Spec', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('.shouldRebuildForWorkloadChange()', () => { + const existing = { + schedule: 0, + hostNetworkMode: false, + isPrivileged: false, + logSize: 2048, + runtime: 'io.containerd.runc.v2', + config: '{}', + annotations: '{}' + } + + it('returns true when capAdd is present in update', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, { capAdd: ['NET_ADMIN'] }, {})).to.equal(true) + }) + + it('returns true when isPrivileged changes', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, { isPrivileged: true }, {})).to.equal(true) + }) + + it('returns false when no workload fields are present', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, {}, {})).to.equal(false) + }) + }) + + describe('.createWorkloadRelations()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceCapAddManager, 'create').resolves() + $sandbox.stub(MicroserviceCapDropManager, 'create').resolves() + $sandbox.stub(MicroserviceArgManager, 'create').resolves() + $sandbox.stub(MicroserviceHealthCheckManager, 'create').resolves() + $sandbox.stub(MicroserviceEnvManager, 'create').resolves() + }) + + it('persists capAdd, capDrop, cmd, and healthCheck on create', async () => { + await WorkloadSpec.createWorkloadRelations( + { uuid: 'ms-uuid' }, + { + capAdd: ['NET_ADMIN'], + capDrop: ['MKNOD'], + cmd: ['/bin/controller'], + healthCheck: { test: ['CMD', 'true'], interval: 30 } + }, + { transaction } + ) + + expect(MicroserviceCapAddManager.create).to.have.been.calledWith( + { capAdd: 'NET_ADMIN', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceCapDropManager.create).to.have.been.calledWith( + { capDrop: 'MKNOD', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + { cmd: '/bin/controller', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceHealthCheckManager.create).to.have.been.called + }) + }) + + describe('.updateWorkloadRelations()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceCapAddManager, 'delete').resolves() + $sandbox.stub(MicroserviceCapAddManager, 'create').resolves() + }) + + it('clears capAdd when an empty array is sent', async () => { + await WorkloadSpec.updateWorkloadRelations( + 'ms-uuid', + { capAdd: [] }, + { transaction } + ) + + expect(MicroserviceCapAddManager.delete).to.have.been.calledWith({ microserviceUuid: 'ms-uuid' }, transaction) + expect(MicroserviceCapAddManager.create).to.not.have.been.called + }) + }) +}) diff --git a/test/src/services/nats-api-service.test.js b/test/src/services/nats-api-service.test.js new file mode 100644 index 00000000..d56da614 --- /dev/null +++ b/test/src/services/nats-api-service.test.js @@ -0,0 +1,101 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const NatsApiService = require('../../../src/services/nats-api-service') +const NatsAuthService = require('../../../src/services/nats-auth-service') +const ApplicationManager = require('../../../src/data/managers/application-manager') +const NatsAccountManager = require('../../../src/data/managers/nats-account-manager') +const NatsUserManager = require('../../../src/data/managers/nats-user-manager') +const SecretService = require('../../../src/services/secret-service') + +describe('NatsApiService', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + const sampleCredsText = [ + '-----BEGIN NATS USER JWT-----', + 'test-jwt', + '------END NATS USER JWT------', + '' + ].join('\n') + + afterEach(() => $sandbox.restore()) + + describe('getUserCreds()', () => { + it('returns creds for platform controller NATS account', async () => { + const platformAccount = { + id: 42, + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + } + const platformUser = { + id: 7, + accountId: platformAccount.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME, + credsSecretName: NatsAuthService.controllerNatsCredsSecretName() + } + + $sandbox.stub(NatsAccountManager, 'findOne').resolves(platformAccount) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + $sandbox.stub(NatsUserManager, 'findOne').resolves(platformUser) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: { + 'controller/controller.creds': sampleCredsText + } + }) + + const result = await NatsApiService.getUserCreds( + NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + NatsAuthService.CONTROLLER_NATS_USER_NAME, + transaction + ) + + expect(result.credsBase64).to.equal(Buffer.from(sampleCredsText, 'utf8').toString('base64')) + expect(ApplicationManager.findOne).to.not.have.been.called + }) + + it('returns creds for SYS account by account name', async () => { + const sysAccount = { + id: 1, + name: 'SYS', + applicationId: null, + isSystem: true, + isLeafSystem: false + } + const sysUser = { + id: 2, + accountId: sysAccount.id, + name: 'admin-hub', + credsSecretName: 'nats-creds-sys-admin-hub' + } + + $sandbox.stub(NatsAccountManager, 'findOne').resolves(sysAccount) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + $sandbox.stub(NatsUserManager, 'findOne').resolves(sysUser) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: { + 'sys/admin-hub.creds': sampleCredsText + } + }) + + const result = await NatsApiService.getUserCreds('SYS', 'admin-hub', transaction) + + expect(result.credsBase64).to.equal(Buffer.from(sampleCredsText, 'utf8').toString('base64')) + }) + + it('rejects unknown account names that are not applications or system accounts', async () => { + $sandbox.stub(NatsAccountManager, 'findOne').resolves(null) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + + try { + await NatsApiService.getUserCreds('missing-account', 'some-user', transaction) + expect.fail('expected getUserCreds to fail') + } catch (error) { + expect(error.name).to.equal('NotFoundError') + expect(error.message).to.include('missing-account') + } + }) + }) +}) diff --git a/test/src/services/nats-auth-service.test.js b/test/src/services/nats-auth-service.test.js index 2642dee0..a665254f 100644 --- a/test/src/services/nats-auth-service.test.js +++ b/test/src/services/nats-auth-service.test.js @@ -7,11 +7,18 @@ const NatsUserManager = require('../../../src/data/managers/nats-user-manager') const NatsUserRuleManager = require('../../../src/data/managers/nats-user-rule-manager') const NatsAccountRuleManager = require('../../../src/data/managers/nats-account-rule-manager') const ApplicationManager = require('../../../src/data/managers/application-manager') +const NatsOperatorManager = require('../../../src/data/managers/nats-operator-manager') const SecretService = require('../../../src/services/secret-service') const NatsService = require('../../../src/services/nats-service') const NatsAuthService = require('../../../src/services/nats-auth-service') +const NatsSystemRules = require('../../../src/config/nats-system-rules') const { createOperator, createAccount } = require('@nats-io/nkeys') +function decodeJwtPayload (jwt) { + const parts = jwt.split('.') + return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) +} + describe('NATS Auth Service', () => { def('sandbox', () => sinon.createSandbox()) const transaction = {} @@ -135,4 +142,144 @@ describe('NATS Auth Service', () => { }) }) }) + + describe('ensureControllerNatsAccount', () => { + const operatorKp = createOperator() + const operatorSeed = new TextDecoder().decode(operatorKp.getSeed()) + const operator = { + id: 1, + name: 'test-operator', + seedSecretName: 'nats-operator-seed', + publicKey: operatorKp.getPublicKey() + } + const relayAccountRule = { + id: 50, + name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME, + maxConnections: -1, + maxLeafNodeConnections: -1, + maxData: -1, + maxExports: -1, + maxImports: -1, + maxMsgPayload: -1, + maxSubscriptions: -1, + exportsAllowWildcards: true + } + const relayUserRule = { + id: 51, + name: NatsSystemRules.CONTROLLER_USER_RULE_NAME, + bearerToken: false, + allowedConnectionTypes: JSON.stringify(['STANDARD']), + maxData: -1, + maxSubscriptions: -1, + maxPayload: -1, + pubAllow: JSON.stringify([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]), + subAllow: JSON.stringify([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + } + let createdAccount + let createdUser + + beforeEach(() => { + createdAccount = null + createdUser = null + $sandbox.stub(NatsAccountRuleManager, 'updateOrCreate').resolves() + $sandbox.stub(NatsUserRuleManager, 'updateOrCreate').resolves() + $sandbox.stub(NatsService, 'enqueueReconcileTask').resolves() + $sandbox.stub(NatsOperatorManager, 'findOne').resolves(operator) + $sandbox.stub(SecretService, 'getSecretEndpoint').callsFake((secretName) => { + if (secretName === operator.seedSecretName) { + return Promise.resolve({ data: { seed: operatorSeed } }) + } + if (secretName && secretName.startsWith('nats-account-seed-')) { + const accountKp = createAccount() + return Promise.resolve({ data: { seed: new TextDecoder().decode(accountKp.getSeed()) } }) + } + return Promise.resolve(null) + }) + $sandbox.stub(SecretService, 'createSecretEndpoint').resolves() + $sandbox.stub(SecretService, 'updateSecretEndpointIfChanged').resolves() + $sandbox.stub(NatsAccountRuleManager, 'findOne').callsFake(({ name }) => { + if (name === NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME) { + return Promise.resolve(relayAccountRule) + } + if (name === NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }) + } + if (name === NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsUserRuleManager, 'findOne').callsFake(({ name }) => { + if (name === NatsSystemRules.CONTROLLER_USER_RULE_NAME) { + return Promise.resolve(relayUserRule) + } + if (name === NatsSystemRules.MICROSERVICE_USER_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.MICROSERVICE_USER_RULE_NAME }) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsAccountManager, 'findOne').callsFake((query) => { + if (createdAccount && query.name === NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME) { + return Promise.resolve(createdAccount) + } + if (createdAccount && query.id === createdAccount.id) { + return Promise.resolve(createdAccount) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsAccountManager, 'create').callsFake((data) => { + createdAccount = { id: 100, ...data } + return Promise.resolve(createdAccount) + }) + $sandbox.stub(NatsUserManager, 'findOne').callsFake(({ accountId, name }) => { + if (createdUser && accountId === createdAccount.id && name === NatsAuthService.CONTROLLER_NATS_USER_NAME) { + return Promise.resolve(createdUser) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsUserManager, 'create').callsFake((data) => { + createdUser = { id: 200, ...data } + return Promise.resolve(createdUser) + }) + }) + + it('creates controller NATS account and user with scoped pub/sub JWT claims', async () => { + const result = await NatsAuthService.ensureControllerNatsAccount(transaction, { triggerReconcile: false }) + + expect(result.account.name).to.equal(NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME) + expect(result.user.name).to.equal(NatsAuthService.CONTROLLER_NATS_USER_NAME) + expect(result.user.credsSecretName).to.equal(NatsAuthService.controllerNatsCredsSecretName()) + + const payload = decodeJwtPayload(result.user.jwt) + expect(payload.nats.pub.allow).to.deep.equal([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + expect(payload.nats.sub.allow).to.deep.equal([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + expect(NatsAccountManager.create).to.have.been.calledOnce + expect(NatsUserManager.create).to.have.been.calledOnce + }) + + it('is idempotent when account and user already exist', async () => { + createdAccount = { + id: 100, + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false, + seedSecretName: 'nats-account-seed-controller' + } + createdUser = { + id: 200, + accountId: createdAccount.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME, + credsSecretName: NatsAuthService.controllerNatsCredsSecretName(), + jwt: 'existing.jwt.token' + } + + const result = await NatsAuthService.ensureControllerNatsAccount(transaction, { triggerReconcile: false }) + + expect(result.account).to.equal(createdAccount) + expect(result.user).to.equal(createdUser) + expect(NatsAccountManager.create).to.not.have.been.called + expect(NatsUserManager.create).to.not.have.been.called + }) + }) }) diff --git a/test/src/services/nats-relay-connection-manager.test.js b/test/src/services/nats-relay-connection-manager.test.js new file mode 100644 index 00000000..49913ba8 --- /dev/null +++ b/test/src/services/nats-relay-connection-manager.test.js @@ -0,0 +1,198 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const { + NatsRelayConnectionManager +} = require('../../../src/services/nats-relay-connection-manager') +const NatsInstanceManager = require('../../../src/data/managers/nats-instance-manager') +const NatsAccountManager = require('../../../src/data/managers/nats-account-manager') +const NatsUserManager = require('../../../src/data/managers/nats-user-manager') +const SecretService = require('../../../src/services/secret-service') +const NatsAuthService = require('../../../src/services/nats-auth-service') +const Constants = require('../../../src/helpers/constants') + +describe('NatsRelayConnectionManager', () => { + def('sandbox', () => sinon.createSandbox()) + + const hubRecord = { + host: 'hub.example.com', + serverPort: 4222 + } + + const sampleCredsText = [ + '-----BEGIN NATS USER JWT-----', + 'test-jwt', + '------END NATS USER JWT------', + '', + '-----BEGIN USER NKEY SEED-----', + 'SUATESTSEED', + '------END USER NKEY SEED------', + '' + ].join('\n') + + function relayCredsSecretData (credsText = sampleCredsText) { + return { + 'controller/controller.creds': credsText + } + } + + function createManager (overrides = {}) { + const cfg = { + get: sinon.stub().callsFake((key, defaultValue) => { + if (key === 'app.namespace') return 'pot-ns' + if (key === 'app.ControlPlane') return 'Remote' + return defaultValue + }), + getBoolean: sinon.stub().returns(false) + } + + return new NatsRelayConnectionManager({ + connectFn: overrides.connectFn || sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }), + config: cfg, + maxReconnectAttempts: 0 + }) + } + + afterEach(() => { + $sandbox.restore() + }) + + it('resolves remote hub hosts in fallback order', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + manager._connectFn = connectFn + + await manager.getConnection() + + expect(connectFn).to.have.been.calledOnce + const options = connectFn.firstCall.args[0] + expect(options.servers).to.equal(`nats://${Constants.NATS_BRIDGE_DNS_SAN}:4222`) + expect(NatsAuthService.ensureControllerNatsAccount).to.have.been.calledOnce + }) + + it('ensures controller NATS account before loading creds when hub exists', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + const ensureStub = $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + manager._connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + await manager.getConnection() + + expect(ensureStub).to.have.been.calledBefore(SecretService.getSecretEndpoint) + }) + + it('skips ensure when no hub record exists', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(null) + const ensureStub = $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + + await manager._ensureControllerNatsAccount() + + expect(ensureStub).to.not.have.been.called + }) + + it('resolves kubernetes hub service host first', async () => { + const manager = createManager() + manager._config.get = sinon.stub().callsFake((key, defaultValue) => { + if (key === 'app.namespace') return 'prod' + if (key === 'app.ControlPlane') return 'kubernetes' + return defaultValue + }) + + const hosts = manager._buildHubHostList(hubRecord) + expect(hosts[0]).to.equal('nats-server.prod.svc.cluster.local') + expect(hosts).to.deep.equal([ + 'nats-server.prod.svc.cluster.local', + 'hub.example.com' + ]) + }) + + it('resolves remote hub hosts without cluster.local fallback', () => { + const manager = createManager() + const hosts = manager._remoteHubHosts(hubRecord) + + expect(hosts).to.deep.equal([ + Constants.NATS_BRIDGE_DNS_SAN, + hubRecord.host + ]) + }) + + it('uses hub serverPort for all remote connect attempts', async () => { + const manager = createManager() + const customPort = 14222 + $sandbox.stub(NatsInstanceManager, 'findOne').resolves({ + ...hubRecord, + serverPort: customPort + }) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + manager._connectFn = connectFn + + await manager.getConnection() + + expect(connectFn.firstCall.args[0].servers).to.equal(`nats://${Constants.NATS_BRIDGE_DNS_SAN}:${customPort}`) + }) + + it('throws aggregate error listing all connect attempts', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + manager._connectFn = sinon.stub().rejects(new Error('ECONNREFUSED')) + + try { + await manager.getConnection() + expect.fail('expected connect to fail') + } catch (error) { + expect(error.message).to.include(Constants.NATS_BRIDGE_DNS_SAN) + expect(error.message).to.include(hubRecord.host) + expect(error.message).to.include('ECONNREFUSED') + } + }) + + it('loads opaque creds secret as plain UTF-8 text', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const creds = await manager._loadControllerRelayCreds() + expect(Buffer.from(creds).toString('utf8')).to.equal(sampleCredsText) + }) + + it('reports unavailable when hub record is missing', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(null) + + const available = await manager.isAvailable() + expect(available).to.equal(false) + }) +}) diff --git a/test/src/services/nats-relay-transport.test.js b/test/src/services/nats-relay-transport.test.js new file mode 100644 index 00000000..2c872ad1 --- /dev/null +++ b/test/src/services/nats-relay-transport.test.js @@ -0,0 +1,217 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const msgpack = require('@msgpack/msgpack') +const WebSocket = require('ws') + +const { + NatsRelayTransportImpl, + execAgentSubject, + execUserSubject, + logUserSubject +} = require('../../../src/services/nats-relay-transport-impl') + +function createMockNatsConnection () { + const subscriptions = new Map() + + return { + subscriptions, + publish (subject, data, opts = {}) { + const handlers = subscriptions.get(subject) || [] + for (const handler of handlers) { + handler(null, { + subject, + data, + headers: opts.headers + }) + } + }, + subscribe (subject, opts = {}) { + if (!subscriptions.has(subject)) { + subscriptions.set(subject, []) + } + subscriptions.get(subject).push(opts.callback) + return { + unsubscribe () { + const list = subscriptions.get(subject) || [] + subscriptions.set(subject, list.filter((fn) => fn !== opts.callback)) + }, + drain: async () => { + const list = subscriptions.get(subject) || [] + subscriptions.set(subject, list.filter((fn) => fn !== opts.callback)) + } + } + }, + flush: async () => {}, + drain: async () => {}, + close: async () => {}, + isClosed: () => false + } +} + +function createMockConnectionManager (nc) { + const listeners = [] + return { + onReconnect (cb) { + listeners.push(cb) + }, + async getConnection () { + return nc + }, + async isAvailable () { + return true + }, + async shutdown () {}, + triggerReconnect () { + return Promise.all(listeners.map((cb) => Promise.resolve(cb()))) + } + } +} + +function createMockWebSocket () { + const ws = { + readyState: WebSocket.OPEN, + bufferedAmount: 0, + _sent: [], + send (data) { + ws._sent.push(Buffer.from(data)) + }, + close: sinon.stub() + } + return ws +} + +describe('NatsRelayTransportImpl', () => { + def('sandbox', () => sinon.createSandbox()) + + let nc + let connectionManager + let transport + const execId = 'exec-session-1' + const logSessionId = 'log-session-1' + + beforeEach(() => { + nc = createMockNatsConnection() + connectionManager = createMockConnectionManager(nc) + transport = new NatsRelayTransportImpl(connectionManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.maxPendingBytes') return 33554432 + if (key === 'server.webSocket.relay.nats.maxPendingMessages') return 8192 + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 50 + return defaultValue + } + }) + }) + + afterEach(() => { + $sandbox.restore() + }) + + it('relays exec stdin from user publish to agent subscription', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + + await transport.enableForSession({ execId, user: userWs }) + await transport.enableForSession({ execId, agent: agentWs }) + + const frame = Buffer.from('hello-from-user') + await transport.publishToAgent(execId, frame) + + expect(agentWs._sent).to.have.length(1) + expect(agentWs._sent[0].toString()).to.equal('hello-from-user') + }) + + it('relays exec stdout from agent publish to user subscription', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + + await transport.enableForSession({ execId, user: userWs }) + await transport.enableForSession({ execId, agent: agentWs }) + + const frame = Buffer.from('hello-from-agent') + await transport.publishToUser(execId, frame) + + expect(userWs._sent).to.have.length(1) + expect(userWs._sent[0].toString()).to.equal('hello-from-agent') + }) + + it('forwards log lines to user websocket by sessionId', async () => { + const userWs = createMockWebSocket() + + await transport.enableForLogSession({ sessionId: logSessionId, user: userWs }) + + const line = msgpack.encode({ + type: 6, + data: Buffer.from('line-1\n'), + sessionId: logSessionId + }) + + nc.publish(logUserSubject(logSessionId), line) + + expect(userWs._sent).to.have.length(1) + expect(msgpack.decode(userWs._sent[0]).data.toString()).to.equal('line-1\n') + }) + + it('drops log lines and emits LOG_ERROR when pending limits exceeded', async () => { + const userWs = createMockWebSocket() + const limitedTransport = new NatsRelayTransportImpl(connectionManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.maxPendingBytes') return 32 + if (key === 'server.webSocket.relay.nats.maxPendingMessages') return 1 + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 50 + return defaultValue + } + }) + + await limitedTransport.enableForLogSession({ sessionId: logSessionId, user: userWs }) + const bridge = limitedTransport.logBridges.get(logSessionId) + bridge.pendingMessages = 1 + bridge.pendingBytes = 32 + + const line = msgpack.encode({ + type: 6, + data: Buffer.from('overflow-line\n'), + sessionId: logSessionId + }) + + nc.publish(logUserSubject(logSessionId), line) + + expect(userWs._sent).to.have.length(1) + expect(msgpack.decode(userWs._sent[0]).type).to.equal(9) + }) + + it('re-subscribes active exec sessions after reconnect', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const recovery = sinon.stub() + + transport.onRecovery(recovery) + await transport.enableForSession({ execId, user: userWs, agent: agentWs }) + + expect(nc.subscriptions.get(execUserSubject(execId))).to.have.length(1) + expect(nc.subscriptions.get(execAgentSubject(execId))).to.have.length(1) + + nc.subscriptions.clear() + await connectionManager.triggerReconnect() + + expect(nc.subscriptions.get(execUserSubject(execId))).to.have.length(1) + expect(nc.subscriptions.get(execAgentSubject(execId))).to.have.length(1) + expect(recovery).to.have.been.calledOnceWith(execId, { kind: 'exec' }) + }) + + it('fails exec publish when flush exceeds timeout', async () => { + const slowNc = createMockNatsConnection() + slowNc.flush = () => new Promise(() => {}) + const slowManager = createMockConnectionManager(slowNc) + const slowTransport = new NatsRelayTransportImpl(slowManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 20 + return defaultValue + } + }) + + await slowTransport.enableForSession({ execId, user: createMockWebSocket() }) + + await expect(slowTransport.publishToAgent(execId, Buffer.from('x'))) + .to.be.rejectedWith(/not flushed within 20ms/) + }) +}) diff --git a/test/src/services/router-connection-service.test.js b/test/src/services/router-connection-service.test.js index 24b64fc3..bdaf9b66 100644 --- a/test/src/services/router-connection-service.test.js +++ b/test/src/services/router-connection-service.test.js @@ -6,9 +6,9 @@ const config = require('../../../src/config') const RouterManager = require('../../../src/data/managers/router-manager') const CertificateService = require('../../../src/services/certificate-service') const SecretService = require('../../../src/services/secret-service') -const RouterConnectionService = require('../../../src/services/router-connection-service') +const RouterConnectionManager = require('../../../src/services/router-connection-manager') -describe('Router Connection Service', () => { +describe('Router Connection Manager', () => { def('sandbox', () => sinon.createSandbox()) const defaultRouter = { @@ -21,12 +21,17 @@ describe('Router Connection Service', () => { const originalNamespace = process.env.CONTROLLER_NAMESPACE beforeEach(() => { - RouterConnectionService.connection = null - RouterConnectionService.connectionPromise = null - RouterConnectionService.cachedRouterRecord = null - RouterConnectionService.cachedCertificate = null - RouterConnectionService.connectionOptions = null - RouterConnectionService.certificatePromise = null + for (const slot of RouterConnectionManager.slots) { + slot.connection = null + slot.connectionPromise = null + slot.healthy = true + slot.unsettledCount = 0 + } + RouterConnectionManager.cachedRouterRecord = null + RouterConnectionManager.cachedCertificate = null + RouterConnectionManager.certificatePromise = null + RouterConnectionManager.shuttingDown = false + RouterConnectionManager.saturationCount = 0 $sandbox.stub(RouterManager, 'findOne').resolves(defaultRouter) }) @@ -47,33 +52,53 @@ describe('Router Connection Service', () => { } }) + describe('pool acquire — sticky assignment', () => { + it('maps the same sessionId to the same slot', () => { + const sessionId = 'exec-session-sticky-abc' + const first = RouterConnectionManager.slotIdForSession(sessionId) + const second = RouterConnectionManager.slotIdForSession(sessionId) + expect(first).to.equal(second) + expect(first).to.be.at.least(0).and.below(RouterConnectionManager.poolSize) + }) + + it('distributes different sessionIds across the pool range', () => { + const slots = new Set() + for (let i = 0; i < 64; i++) { + slots.add(RouterConnectionManager.slotIdForSession(`session-${i}`)) + } + expect(slots.size).to.be.at.least(2) + }) + }) + describe('_resolveRouterEndpoint() — Kubernetes control plane', () => { beforeEach(() => { process.env.CONTROL_PLANE = 'kubernetes' process.env.CONTROLLER_NAMESPACE = 'iofog' }) - it('returns a single cluster.local service host', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('returns cluster service host then DB host fallback', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() - expect(result.hosts).to.deep.equal(['router.iofog.svc.cluster.local']) + expect(result.hosts).to.deep.equal([ + 'router.iofog.svc.cluster.local', + defaultRouter.host + ]) expect(result.host).to.equal('router.iofog.svc.cluster.local') expect(result.port).to.equal(5671) expect(result.routerUuid).to.equal(defaultRouter.iofogUuid) }) - it('does not include bridge DNS or DB host in the fallback list', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('does not include bridge DNS in the fallback list', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.not.include(Constants.ROUTER_BRIDGE_DNS_SAN) - expect(result.hosts).to.not.include(defaultRouter.host) }) it('falls back to DB host when namespace is unset', async () => { delete process.env.CONTROLLER_NAMESPACE $sandbox.stub(config, 'get').withArgs('app.namespace').returns('') - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([defaultRouter.host]) }) @@ -86,7 +111,7 @@ describe('Router Connection Service', () => { host: '' }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal(['router']) }) @@ -98,13 +123,12 @@ describe('Router Connection Service', () => { process.env.CONTROLLER_NAMESPACE = 'edge-ns' }) - it('returns ordered fallback hosts: bridge DNS, DB host, cluster.local', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('returns ordered fallback hosts: bridge DNS then DB host', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ Constants.ROUTER_BRIDGE_DNS_SAN, - defaultRouter.host, - 'router.edge-ns.svc.cluster.local' + defaultRouter.host ]) expect(result.host).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) expect(result.port).to.equal(5671) @@ -116,11 +140,10 @@ describe('Router Connection Service', () => { host: Constants.ROUTER_BRIDGE_DNS_SAN }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ - Constants.ROUTER_BRIDGE_DNS_SAN, - 'router.edge-ns.svc.cluster.local' + Constants.ROUTER_BRIDGE_DNS_SAN ]) }) @@ -128,7 +151,7 @@ describe('Router Connection Service', () => { delete process.env.CONTROLLER_NAMESPACE $sandbox.stub(config, 'get').withArgs('app.namespace').returns('') - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ Constants.ROUTER_BRIDGE_DNS_SAN, @@ -142,7 +165,7 @@ describe('Router Connection Service', () => { messagingPort: null }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.port).to.equal(5671) }) @@ -172,7 +195,7 @@ describe('Router Connection Service', () => { }) it('ensures default-router-local-ca before creating the exec client certificate', async () => { - await RouterConnectionService._createControllerCertificate() + await RouterConnectionManager._createControllerCertificate() expect(CertificateService.ensureRouterLocalCA).to.have.been.calledOnce expect(CertificateService.createCertificateEndpoint).to.have.been.calledOnce @@ -183,7 +206,7 @@ describe('Router Connection Service', () => { }) }) - describe('_createConnection() — Remote connect fallback', () => { + describe('_createSlotConnection() — Remote connect fallback', () => { const certBundle = { cert: Buffer.from('cert'), key: Buffer.from('key'), @@ -193,44 +216,105 @@ describe('Router Connection Service', () => { def('mockConnection', () => ({ is_open: () => true })) beforeEach(() => { - $sandbox.stub(RouterConnectionService, '_ensureControllerCertificate').resolves(certBundle) + $sandbox.stub(RouterConnectionManager, '_ensureControllerCertificate').resolves(certBundle) }) it('tries hosts in order until one connects', async () => { - const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host, 'router.edge-ns.svc.cluster.local'] - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - const connectStub = $sandbox.stub(RouterConnectionService, '_connectToHost') + const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host] + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + const connectStub = $sandbox.stub(RouterConnectionManager, '_connectToHost') .onCall(0).rejects(new Error('ECONNREFUSED')) .onCall(1).resolves($mockConnection) - const connection = await RouterConnectionService._createConnection() + const slot = RouterConnectionManager.slots[0] + const connection = await RouterConnectionManager._createSlotConnection(slot) expect(connection).to.equal($mockConnection) expect(connectStub).to.have.been.calledTwice - expect(connectStub.firstCall.args[0]).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) - expect(connectStub.secondCall.args[0]).to.equal(defaultRouter.host) + expect(connectStub.firstCall.args[1]).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) + expect(connectStub.secondCall.args[1]).to.equal(defaultRouter.host) }) - it('throws after all hosts fail', async () => { + it('throws aggregate error after all hosts fail', async () => { const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host] - const lastError = new Error('all hosts down') - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - $sandbox.stub(RouterConnectionService, '_connectToHost').rejects(lastError) - - await expect(RouterConnectionService._createConnection()).to.be.rejectedWith('all hosts down') - expect(RouterConnectionService._connectToHost).to.have.been.calledTwice + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + $sandbox.stub(RouterConnectionManager, '_connectToHost') + .onCall(0).rejects(new Error('bridge down')) + .onCall(1).rejects(new Error('db host down')) + + const slot = RouterConnectionManager.slots[0] + try { + await RouterConnectionManager._createSlotConnection(slot) + expect.fail('expected connect to fail') + } catch (error) { + expect(error.message).to.include(Constants.ROUTER_BRIDGE_DNS_SAN) + expect(error.message).to.include(defaultRouter.host) + expect(error.message).to.include('bridge down') + expect(error.message).to.include('db host down') + } + expect(RouterConnectionManager._connectToHost).to.have.been.calledTwice }) - it('connects on first host for Kubernetes single-host list', async () => { - const hosts = ['router.iofog.svc.cluster.local'] - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - const connectStub = $sandbox.stub(RouterConnectionService, '_connectToHost').resolves($mockConnection) + it('connects on second host when first fails for Kubernetes host list', async () => { + const hosts = ['router.iofog.svc.cluster.local', defaultRouter.host] + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + const connectStub = $sandbox.stub(RouterConnectionManager, '_connectToHost') + .onCall(0).rejects(new Error('ECONNREFUSED')) + .onCall(1).resolves($mockConnection) - const connection = await RouterConnectionService._createConnection() + const slot = RouterConnectionManager.slots[0] + const connection = await RouterConnectionManager._createSlotConnection(slot) expect(connection).to.equal($mockConnection) - expect(connectStub).to.have.been.calledOnce - expect(connectStub.firstCall.args[0]).to.equal('router.iofog.svc.cluster.local') + expect(connectStub).to.have.been.calledTwice + expect(connectStub.firstCall.args[1]).to.equal('router.iofog.svc.cluster.local') + expect(connectStub.secondCall.args[1]).to.equal(defaultRouter.host) + }) + }) + + describe('waitForSendable()', () => { + it('resolves immediately when sender is already sendable', async () => { + const sender = { + sendable: () => true, + once: sinon.stub(), + removeListener: sinon.stub() + } + + await RouterConnectionManager.waitForSendable(sender, 100) + expect(sender.once).to.not.have.been.called + }) + + it('waits for sendable event when sender is not ready', async () => { + const sender = new (require('events').EventEmitter)() + sender.sendable = () => false + + const pending = RouterConnectionManager.waitForSendable(sender, 500) + setImmediate(() => sender.emit('sendable')) + await pending + }) + + it('rejects on timeout when sender never becomes sendable', async () => { + const sender = new (require('events').EventEmitter)() + sender.sendable = () => false + + await expect(RouterConnectionManager.waitForSendable(sender, 20)) + .to.be.rejectedWith(/not sendable/) + }) + }) + + describe('handleSendError()', () => { + it('marks slot unhealthy on circular buffer overflow', async () => { + const sessionId = 'overflow-session' + const slotId = RouterConnectionManager.slotIdForSession(sessionId) + const reconnectStub = $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + + const handled = RouterConnectionManager.handleSendError( + sessionId, + new Error('circular buffer overflow') + ) + + expect(handled).to.equal(true) + expect(reconnectStub).to.have.been.calledOnceWith(slotId, 'circular buffer overflow') }) }) }) diff --git a/test/src/services/websocket-queue-service.test.js b/test/src/services/websocket-queue-service.test.js new file mode 100644 index 00000000..cd80007f --- /dev/null +++ b/test/src/services/websocket-queue-service.test.js @@ -0,0 +1,140 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const msgpack = require('@msgpack/msgpack') + +const WebSocketQueueService = require('../../../src/services/websocket-queue-service') +const RouterConnectionManager = require('../../../src/services/router-connection-manager') + +describe('WebSocketQueueService — AMQP hardening', () => { + def('sandbox', () => sinon.createSandbox()) + + const sessionId = 'log-session-backpressure' + const execId = 'exec-session-overflow' + + afterEach(() => { + $sandbox.restore() + WebSocketQueueService.execBridges.clear() + WebSocketQueueService.logBridges.clear() + }) + + describe('publishLogToUser() publish-side backpressure', () => { + it('drops LOG_LINE when sender is not sendable', async () => { + const userWs = { + readyState: 1, + send: sinon.stub() + } + const sender = { + sendable: () => false, + send: sinon.stub() + } + + WebSocketQueueService.logBridges.set(sessionId, { + sessionId, + slotId: 0, + session: { sessionId, user: userWs }, + userSender: { sender }, + backpressureNotified: false + }) + + const buffer = msgpack.encode({ + type: 6, + data: Buffer.from('line\n'), + sessionId + }) + + await WebSocketQueueService.publishLogToUser(sessionId, buffer) + + expect(sender.send).to.not.have.been.called + expect(userWs.send).to.have.been.calledOnce + }) + }) + + describe('overflow recovery', () => { + it('recovers publish after slot reconnect without poisoning other sessions', async () => { + const bridge = { + execId, + slotId: RouterConnectionManager.slotIdForSession(execId), + session: { execId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(execId, bridge) + + const sender = { + sendable: () => true, + send: sinon.stub() + } + let ensureCount = 0 + $sandbox.stub(WebSocketQueueService, '_ensureSender').callsFake(async () => { + ensureCount += 1 + if (ensureCount === 1) { + throw new Error('circular buffer overflow') + } + return { sender } + }) + + const markStub = $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + $sandbox.stub(RouterConnectionManager, 'waitForSendable').resolves() + + await expect(WebSocketQueueService.publishToAgent(execId, Buffer.from('fail'))) + .to.be.rejectedWith(/circular buffer overflow/) + + expect(markStub).to.have.been.calledOnce + + await WebSocketQueueService.publishToAgent(execId, Buffer.from('ok')) + + expect(sender.send).to.have.been.calledOnceWith(sinon.match({ body: Buffer.from('ok') })) + }) + + it('does not poison unrelated exec sessions when one slot overflows', async () => { + const otherExecId = 'exec-session-healthy' + const healthyBridge = { + execId: otherExecId, + slotId: RouterConnectionManager.slotIdForSession(otherExecId), + session: { execId: otherExecId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(otherExecId, healthyBridge) + + const overflowBridge = { + execId, + slotId: RouterConnectionManager.slotIdForSession(execId), + session: { execId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(execId, overflowBridge) + + const healthySender = { + sendable: () => true, + send: sinon.stub() + } + const overflowSender = { + sendable: () => true, + send: sinon.stub().throws(new Error('circular buffer overflow')) + } + + $sandbox.stub(WebSocketQueueService, '_ensureSender').callsFake(async (id) => { + if (id === execId) { + return { sender: overflowSender } + } + return { sender: healthySender } + }) + + $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + $sandbox.stub(RouterConnectionManager, 'waitForSendable').resolves() + + await expect(WebSocketQueueService.publishToAgent(execId, Buffer.from('fail'))) + .to.be.rejectedWith(/circular buffer overflow/) + + await WebSocketQueueService.publishToAgent(otherExecId, Buffer.from('ok')) + + expect(healthySender.send).to.have.been.calledOnceWith(sinon.match({ body: Buffer.from('ok') })) + expect(overflowSender.send).to.have.been.calledOnce + }) + }) +}) diff --git a/test/src/services/ws-relay-transport-factory.test.js b/test/src/services/ws-relay-transport-factory.test.js new file mode 100644 index 00000000..8fd94961 --- /dev/null +++ b/test/src/services/ws-relay-transport-factory.test.js @@ -0,0 +1,52 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const config = require('../../../src/config') +const AmqpRelayTransport = require('../../../src/services/amqp-relay-transport') +const NatsRelayTransport = require('../../../src/services/nats-relay-transport') +const { + resolveTransport, + resetTransportForTests +} = require('../../../src/services/ws-relay-transport-factory') + +describe('ws-relay-transport-factory', () => { + def('sandbox', () => sinon.createSandbox()) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + }) + + it('selects amqp when nats.enabled is false', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return false + return defaultValue + }) + + const transport = resolveTransport() + expect(transport).to.be.instanceOf(AmqpRelayTransport) + expect(transport.getTransport()).to.equal('amqp') + }) + + it('selects nats transport when nats.enabled is true', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return true + return defaultValue + }) + + const transport = resolveTransport() + expect(transport).to.be.instanceOf(NatsRelayTransport) + expect(transport.getTransport()).to.equal('nats') + }) + + it('returns the same singleton instance on repeated calls', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return false + return defaultValue + }) + + const first = resolveTransport() + const second = resolveTransport() + expect(first).to.equal(second) + }) +}) diff --git a/test/src/websocket/ws-cross-replica-nats.test.js b/test/src/websocket/ws-cross-replica-nats.test.js new file mode 100644 index 00000000..277177ac --- /dev/null +++ b/test/src/websocket/ws-cross-replica-nats.test.js @@ -0,0 +1,211 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const WebSocketServerClass = require('../../../src/websocket/server') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const AppHelper = require('../../../src/helpers/app-helper') +const EventService = require('../../../src/services/event-service') +const { + MESSAGE_TYPES, + createMockWebSocket, + createMockRequest, + buildExecFrame, + decodeExecMessage, + createMockNatsRelayTransport, + resetWebSocketServerSingleton, + newTestIds, + waitForSent, + delay +} = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') + +describe('WebSocket exec/log — cross-replica mock NATS', () => { + def('sandbox', () => sinon.createSandbox()) + def('ids', () => newTestIds()) + + let wsServer + let mockRelay + let transaction + + beforeEach(() => { + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + wsServer = new WebSocketServerClass() + mockRelay = createMockNatsRelayTransport() + wsServer.relayTransport = mockRelay + transaction = { fakeTransaction: true } + + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(EventService, 'createWsConnectEvent').resolves() + $sandbox.stub(EventService, 'createWsDisconnectEvent').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'create').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').callsFake(async () => ({ + sessionId: $ids.sessionId, + microserviceUuid: $ids.microserviceUuid, + status: 'PENDING', + userConnected: true, + agentConnected: false + })) + $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) + $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(AppHelper, 'generateUUID').returns($ids.sessionId) + $sandbox.stub(wsServer, 'countExecSessionsInDb').resolves(0) + }) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + }) + + it('uses nats transport for cross-replica exec relay', async () => { + expect(mockRelay.getTransport()).to.equal('nats') + + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + null, + userWs, + transaction + ) + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(50) + + const session = wsServer.execSessionManager.getExecSession(sessionId) + session.agent = agentWs + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(50) + + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) + + const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, sessionId, $ids.microserviceUuid, 'echo hi\n') + userWs.emit('message', stdinFrame, true) + await waitForSent(agentWs, 1) + + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) + }) + + it('delivers agent STDOUT to user through mock NATS publishToUser', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + transaction + ) + await mockRelay.enableForSession( + wsServer.execSessionManager.getExecSession(sessionId), + () => {} + ) + + const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, sessionId, $ids.microserviceUuid, 'line\n') + await mockRelay.publishToUser(sessionId, stdoutFrame) + await waitForSent(userWs, 1) + + const userReceived = decodeExecMessage(lastSent(userWs)) + expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) + }) + + it('routes log lines through mock NATS bridge', async () => { + const userWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.logSessionManager.createLogSession( + sessionId, + $ids.microserviceUuid, + null, + userWs, + { lines: 100, follow: true, since: null, until: null }, + transaction + ) + + await mockRelay.enableForLogSession( + { sessionId, microserviceUuid: $ids.microserviceUuid, user: userWs, agent: null }, + () => {} + ) + + const logLine = buildExecFrame(MESSAGE_TYPES.LOG_LINE, sessionId, $ids.microserviceUuid, 'log entry\n') + await mockRelay.publishLogToUser(sessionId, logLine) + await delay(20) + + expect(mockRelay.logBridges.has(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelayForLogs(sessionId)).to.equal(true) + }) + + it('user-first then agent-second delivers ACTIVATION and STDIN via NATS bridge', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const userReq = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + userReq.headers.authorization = 'Bearer user-jwt' + + await wsServer.handleUserExecConnection( + userWs, + userReq, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + + expect(mockRelay.shouldUseRelay($ids.sessionId)).to.equal(true) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) + await delay(50) + + const activationFrames = agentWs._sentMessages.filter((entry) => { + try { + return decodeExecMessage(entry.data).type === MESSAGE_TYPES.ACTIVATION + } catch (e) { + return false + } + }) + expect(activationFrames.length).to.be.at.least(1) + + const stdinFrame = buildExecFrame( + MESSAGE_TYPES.STDIN, + $ids.sessionId, + $ids.microserviceUuid, + 'echo hi\n' + ) + userWs.emit('message', stdinFrame, true) + await waitForSent(agentWs, activationFrames.length + 1) + + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) + }) +}) + +function lastSent (ws) { + const entry = ws._sentMessages[ws._sentMessages.length - 1] + return entry.data +} diff --git a/test/src/websocket/ws-cross-replica.test.js b/test/src/websocket/ws-cross-replica.test.js index 689414ae..0712ffac 100644 --- a/test/src/websocket/ws-cross-replica.test.js +++ b/test/src/websocket/ws-cross-replica.test.js @@ -7,7 +7,6 @@ const MicroserviceManager = require('../../../src/data/managers/microservice-man const FogManager = require('../../../src/data/managers/iofog-manager') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const AppHelper = require('../../../src/helpers/app-helper') -const RouterConnectionService = require('../../../src/services/router-connection-service') const EventService = require('../../../src/services/event-service') const { MESSAGE_TYPES, @@ -15,29 +14,30 @@ const { createMockRequest, buildExecFrame, decodeExecMessage, - createMockQueueService, + createMockRelayTransport, resetWebSocketServerSingleton, newTestIds, waitForSent, delay } = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') describe('WebSocket exec/log — cross-replica mock AMQP', () => { def('sandbox', () => sinon.createSandbox()) def('ids', () => newTestIds()) let wsServer - let mockQueue + let mockRelay let transaction beforeEach(() => { + resetTransportForTests() resetWebSocketServerSingleton(WebSocketServerClass) wsServer = new WebSocketServerClass() - mockQueue = createMockQueueService() - wsServer.queueService = mockQueue + mockRelay = createMockRelayTransport() + wsServer.relayTransport = mockRelay transaction = { fakeTransaction: true } - $sandbox.stub(RouterConnectionService, 'isRouterAvailable').resolves(true) $sandbox.stub(MicroserviceManager, 'update').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) @@ -63,6 +63,7 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { afterEach(() => { $sandbox.restore() + resetTransportForTests() resetWebSocketServerSingleton(WebSocketServerClass) }) @@ -86,13 +87,13 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { await wsServer.setupExecMessageForwarding(sessionId, transaction) await delay(50) - expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, sessionId, $ids.microserviceUuid, 'echo hi\n') userWs.emit('message', stdinFrame, true) await waitForSent(agentWs, 1) - expect(mockQueue.execBridges.has(sessionId)).to.equal(true) + expect(mockRelay.execBridges.has(sessionId)).to.equal(true) const agentReceived = decodeExecMessage(lastSent(agentWs)) expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) }) @@ -109,16 +110,16 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { userWs, transaction ) - await mockQueue.enableForSession( + await mockRelay.enableForSession( wsServer.execSessionManager.getExecSession(sessionId), () => {} ) const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, sessionId, $ids.microserviceUuid, 'line\n') - await mockQueue.publishToUser(sessionId, stdoutFrame) + await mockRelay.publishToUser(sessionId, stdoutFrame) await waitForSent(userWs, 1) - expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) const userReceived = decodeExecMessage(lastSent(userWs)) expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) }) @@ -136,17 +137,17 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { transaction ) - await mockQueue.enableForLogSession( + await mockRelay.enableForLogSession( { sessionId, microserviceUuid: $ids.microserviceUuid, user: userWs, agent: null }, () => {} ) const logLine = buildExecFrame(MESSAGE_TYPES.LOG_LINE, sessionId, $ids.microserviceUuid, 'log entry\n') - await mockQueue.publishLogToUser(sessionId, logLine) + await mockRelay.publishLogToUser(sessionId, logLine) await delay(20) - expect(mockQueue.logBridges.has(sessionId)).to.equal(true) - expect(mockQueue.shouldUseLogQueue(sessionId)).to.equal(true) + expect(mockRelay.logBridges.has(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelayForLogs(sessionId)).to.equal(true) }) it('user-first then agent-second delivers ACTIVATION and STDIN via AMQP bridge', async () => { @@ -164,7 +165,7 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { transaction ) - expect(mockQueue.shouldUseQueue($ids.sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay($ids.sessionId)).to.equal(true) expect(wsServer.execSessionManager.getExecSession($ids.sessionId).agent).to.equal(null) const agentReq = createMockRequest( diff --git a/test/src/websocket/ws-drain.test.js b/test/src/websocket/ws-drain.test.js index ed94815c..85ca9992 100644 --- a/test/src/websocket/ws-drain.test.js +++ b/test/src/websocket/ws-drain.test.js @@ -28,8 +28,8 @@ describe('WebSocket graceful drain', () => { drainTimeoutMs: 500 } - $sandbox.stub(wsServer.queueService, 'cleanup').resolves() - $sandbox.stub(wsServer.queueService, 'cleanupLogSession').resolves() + $sandbox.stub(wsServer.relayTransport, 'cleanup').resolves() + $sandbox.stub(wsServer.relayTransport, 'cleanupLogSession').resolves() $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) diff --git a/test/src/websocket/ws-exec-activation-failfast.test.js b/test/src/websocket/ws-exec-activation-failfast.test.js new file mode 100644 index 00000000..4052a708 --- /dev/null +++ b/test/src/websocket/ws-exec-activation-failfast.test.js @@ -0,0 +1,75 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const logger = require('../../../src/logger') +const WebSocketServerClass = require('../../../src/websocket/server') +const { + createMockWebSocket, + resetWebSocketServerSingleton, + newTestIds, + delay +} = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') + +describe('WebSocket exec activation fail-fast', () => { + def('sandbox', () => sinon.createSandbox()) + def('ids', () => newTestIds()) + + let wsServer + let transaction + + beforeEach(() => { + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + wsServer = new WebSocketServerClass() + wsServer.relayTransport = { + getTransport: () => 'amqp', + isAvailable: async () => true, + enableForSession: async () => true, + shouldUseRelay: () => true, + publishToAgent: async () => {}, + publishToUser: async () => {}, + cleanup: async () => {}, + onRecovery: () => {}, + shutdown: async () => {} + } + transaction = { fakeTransaction: true } + + $sandbox.stub(logger, 'info') + $sandbox.stub(logger, 'error') + $sandbox.stub(logger, 'warn') + $sandbox.stub(logger, 'debug') + $sandbox.stub(wsServer, 'sendMessageToAgent').resolves(false) + $sandbox.stub(wsServer, 'cleanupExecSession').resolves() + }) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + }) + + it('does not log setup complete when activation fails', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + transaction + ) + + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(20) + + const setupCompleteCalls = logger.info.getCalls().filter((call) => { + const first = call.args[0] + return typeof first === 'string' && first.includes('Exec message forwarding setup complete') + }) + + expect(setupCompleteCalls).to.have.length(0) + }) +}) diff --git a/test/src/websocket/ws-exec-same-replica.test.js b/test/src/websocket/ws-exec-same-replica.test.js index 6f19b788..aea148b5 100644 --- a/test/src/websocket/ws-exec-same-replica.test.js +++ b/test/src/websocket/ws-exec-same-replica.test.js @@ -37,9 +37,9 @@ describe('WebSocket exec — same-replica integration (Plan 17)', () => { agentWs = createMockWebSocket() transaction = { fakeTransaction: true } - $sandbox.stub(wsServer.queueService, 'enableForSession').resolves(true) - $sandbox.stub(wsServer.queueService, 'shouldUseQueue').returns(false) - $sandbox.stub(wsServer.queueService, 'cleanup').resolves() + $sandbox.stub(wsServer.relayTransport, 'enableForSession').resolves(true) + $sandbox.stub(wsServer.relayTransport, 'shouldUseRelay').returns(false) + $sandbox.stub(wsServer.relayTransport, 'cleanup').resolves() $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) diff --git a/test/support/ws-session-harness.js b/test/support/ws-session-harness.js index 5eaec0a7..b2e0e26e 100644 --- a/test/support/ws-session-harness.js +++ b/test/support/ws-session-harness.js @@ -86,9 +86,10 @@ function buildExecFrame (type, execId, microserviceUuid, data) { } /** - * In-memory AMQP stub for cross-replica exec/log relay tests. + * In-memory relay stub for cross-replica exec/log tests. + * @param {'amqp'|'nats'} [transport='amqp'] */ -function createMockQueueService () { +function createMockRelayTransport (transport = 'amqp') { const execBridges = new Map() const logBridges = new Map() @@ -96,6 +97,14 @@ function createMockQueueService () { execBridges, logBridges, + getTransport () { + return transport + }, + + async isAvailable () { + return true + }, + async enableForSession (session, cleanupCallback) { const execId = session.execId if (!execId) return false @@ -111,7 +120,7 @@ function createMockQueueService () { return true }, - shouldUseQueue (execId) { + shouldUseRelay (execId) { return execBridges.has(execId) }, @@ -139,7 +148,7 @@ function createMockQueueService () { return true }, - shouldUseLogQueue (sessionId) { + shouldUseRelayForLogs (sessionId) { return logBridges.has(sessionId) }, @@ -152,10 +161,23 @@ function createMockQueueService () { async cleanupLogSession (sessionId) { logBridges.delete(sessionId) - } + }, + + async shutdown () {}, + + onRecovery () {} } } +function createMockNatsRelayTransport () { + return createMockRelayTransport('nats') +} + +/** @deprecated use createMockRelayTransport */ +function createMockQueueService () { + return createMockRelayTransport() +} + function resetWebSocketServerSingleton (WebSocketServerClass) { if (WebSocketServerClass.instance) { const instance = WebSocketServerClass.instance @@ -220,6 +242,8 @@ module.exports = { decodeExecMessage, buildAgentInitialMessage, buildExecFrame, + createMockRelayTransport, + createMockNatsRelayTransport, createMockQueueService, resetWebSocketServerSingleton, buildFakeJwt,