From 76461e4d25e99565b20f2dc994c6975b66bc01c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Mon, 29 Jun 2026 00:50:34 +0300 Subject: [PATCH 1/4] Extend controller register to persist full container workload fields via shared spec helper. Extract microservice workload persistence so agent controller register reuses the same scalar, env, port, and mapping semantics as user microservices. --- CHANGELOG.md | 12 +- src/schemas/controller-register.js | 20 +- src/schemas/microservice-container-spec.js | 55 +++ src/services/controller-ms-service.js | 91 ++-- src/services/microservice-workload-spec.js | 420 ++++++++++++++++++ .../services/controller-ms-service.test.js | 67 +++ .../microservice-workload-spec.test.js | 95 ++++ 7 files changed, 685 insertions(+), 75 deletions(-) create mode 100644 src/schemas/microservice-container-spec.js create mode 100644 src/services/microservice-workload-spec.js create mode 100644 test/src/services/microservice-workload-spec.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index c7de671f..3bb86a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - Agent status: removed **`processedMessages`**, **`messageSpeed`**; added **`availableRuntimes`**, optional **`runtimeAgentPhase`**, **`controlPlaneQuiesced`**. - Default container registry: **`docker.io`** (was `registry.hub.docker.com`). - Reserved ports: **54321**, **54322**, **53**. -- New field-agent endpoint: **`POST /api/v3/agent/controller/register`** (system fogs only; Edgelet rc.1+). +- New field-agent endpoint: **`POST /api/v3/agent/controller/register`** (system fogs only; Edgelet rc.1+). Register body accepts full container workload fields (`cmd`, `isPrivileged`, `healthCheck`, `capAdd`/`capDrop`, `extraHosts`, resources, etc.) with the same semantics as user microservice deploy; excludes `serviceAccount`, `natsConfig`, and ownership fields. #### API — architectures and applications @@ -42,7 +42,7 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * #### WebSocket exec — multi-session - **Microservice exec REST removed** — `POST/DELETE /api/v3/microservices/:uuid/exec` and `…/system/:uuid/exec` no longer exist. Open exec with **direct WebSocket** only: `WS /api/v3/microservices/exec/:uuid` (or `…/system/exec/:uuid`). -- **3 concurrent exec sessions** per microservice (was 1 user exec WS per MS in Plan 16). +- **3 concurrent exec sessions** per microservice (was 1 user exec WS per MS). - **Per-session lifecycle** — closing one exec session deletes only that session row only (no microservice-level exec flag). - **`execEnabled` removed** — dropped `microservices.exec_enabled` column and agent MS list field; exec attach is poll-driven only (`GET /agent/exec/sessions`). - **Agent exec discovery** — new `GET /api/v3/agent/exec/sessions` when change tracking reports `execSessions: true`. @@ -108,11 +108,15 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - **K8s control plane:** hub **`iofog-router`** ConfigMap patches serialized via DB lock; K8s Service create/update/delete with LoadBalancer watch timeout. - **`service-bridge-config.js`** — full recompute of service-derived TCP bridge config per fog on reconcile (preserves router base config). - **SQLite single-node production hardening** — WAL + `busy_timeout` pragmas, reconcile task claim retry on `SQLITE_BUSY`, staggered startup for reconcile-heavy background jobs (`settings.jobStartupDelaySeconds`). -- **WebSocket exec & log session hardening** — quotas (**3 exec** / 3 log WS per resource), per-session exec lifecycle (Plan 17), 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). -- **Multi exec sessions (Plan 17)** — `GET /api/v3/agent/exec/sessions`; agent exec WS `…/agent/exec/microservice/:uuid/:sessionId`; user ACTIVATION with `sessionId`; `MicroserviceExecSessions` table; `execMaxConcurrentPerResource` config (default **3**). +- **WebSocket exec & log session hardening** — quotas (**3 exec** / 3 log WS per resource), per-session exec lifecycle, 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). +- **Multi exec sessions** — `GET /api/v3/agent/exec/sessions`; agent exec WS `…/agent/exec/microservice/:uuid/:sessionId`; user ACTIVATION with `sessionId`; `MicroserviceExecSessions` table; `execMaxConcurrentPerResource` config (default **3**). +- **WebSocket relay production** — unified **`WsRelayTransport`** abstraction; cross-replica exec/log relay backend selected at startup by **`nats.enabled`** (`NATS_ENABLED`): **AMQP** router pool (8 connections per replica, overflow recovery, sendable gating) when `false`, **NATS Core** pub/sub on platform hub (`controller-relay` account) when `true`. Fail-fast activation on both transports; log backpressure drops `LOG_LINE` under pressure. Config: `server.webSocket.relay.amqp.*`, `server.webSocket.relay.nats.*`. No new relay env var; HA swagger/docs updated per R112. ### Fixed +- NATS relay and AMQP router connection resolvers — **Remote CP** uses Edgelet bridge DNS then DB host only (no `*.svc.cluster.local`); **Kubernetes CP** uses `nats-server.{namespace}.svc.cluster.local` / `router.{namespace}.svc.cluster.local` with DB host fallback; connect failures log and throw aggregate errors for all attempts; relay log messages are transport-aware. +- NATS relay **`controller-relay` creds loading** — read Opaque secret values as plain UTF-8 `.creds` text (matches `nats-service.js` and DB storage); fixes **`unable to parse credentials`** on hub connect when `NATS_ENABLED=true`. +- NATS platform relay identity renamed to account/user **`controller`** with rules **`controller-account`** / **`controller-user`**; **`GET /nats/accounts/controller/users/controller/creds`** supported for operator cred export. - Exec AMQP relay re-attaches queue receivers whenever user or agent WebSocket connects (fixes user-first sessions where ACTIVATION and STDIN never reached Edgelet); ACTIVATION is resent on agent WS reconnect. - Controller register accepts optional **`schedule: 0`**; server always enforces schedule **0** on create, re-register, and **`PATCH /api/v3/microservices/system/:uuid`** for controller workloads. - Agent version command (**`GET /api/v3/agent/version`**) refreshes the provision key on each pull instead of returning a stale or deleted key. diff --git a/src/schemas/controller-register.js b/src/schemas/controller-register.js index f5f5ca5b..bd1b9bd1 100644 --- a/src/schemas/controller-register.js +++ b/src/schemas/controller-register.js @@ -1,3 +1,5 @@ +const { microserviceContainerSpecProperties } = require('./microservice-container-spec') + const controllerRegister = { id: '/controllerRegister', type: 'object', @@ -11,25 +13,11 @@ const controllerRegister = { items: { $ref: '/image' } }, registryId: { type: 'integer' }, - ports: { - type: 'array', - items: { $ref: '/ports' } - }, - volumeMappings: { - type: 'array', - items: { $ref: '/volumeMappings' } - }, - env: { - type: 'array', - items: { $ref: '/env' } - }, - config: { type: 'string' }, - hostNetworkMode: { type: 'boolean' }, - runtime: { type: 'string' }, schedule: { type: 'integer', enum: [0] - } + }, + ...microserviceContainerSpecProperties }, required: ['uuid', 'images', 'registryId'], additionalProperties: false diff --git a/src/schemas/microservice-container-spec.js b/src/schemas/microservice-container-spec.js new file mode 100644 index 00000000..6cf86b63 --- /dev/null +++ b/src/schemas/microservice-container-spec.js @@ -0,0 +1,55 @@ +const microserviceContainerSpecProperties = { + config: { type: 'string' }, + annotations: { type: 'string' }, + hostNetworkMode: { type: 'boolean' }, + isPrivileged: { type: 'boolean' }, + logSize: { type: 'integer', minimum: 0 }, + runtime: { type: 'string' }, + pidMode: { type: 'string' }, + ipcMode: { type: 'string' }, + runAsUser: { type: 'string' }, + platform: { type: 'string' }, + cpuSetCpus: { type: 'string' }, + memoryLimit: { type: 'integer' }, + ports: { + type: 'array', + items: { $ref: '/ports' } + }, + volumeMappings: { + type: 'array', + items: { $ref: '/volumeMappings' } + }, + extraHosts: { + type: 'array', + items: { $ref: '/extraHosts' } + }, + env: { + type: 'array', + items: { $ref: '/env' } + }, + cmd: { + type: 'array', + items: { type: 'string' } + }, + cdiDevices: { + type: 'array', + items: { type: 'string' } + }, + capAdd: { + type: 'array', + items: { type: 'string' } + }, + capDrop: { + type: 'array', + items: { type: 'string' } + }, + healthCheck: { + type: 'object', + properties: { $ref: '/microserviceHealthCheck' } + } +} + +module.exports = { + mainSchemas: [], + microserviceContainerSpecProperties +} diff --git a/src/services/controller-ms-service.js b/src/services/controller-ms-service.js index 1e02017f..9d2e92d5 100644 --- a/src/services/controller-ms-service.js +++ b/src/services/controller-ms-service.js @@ -8,7 +8,6 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const CatalogItemImageManager = require('../data/managers/catalog-item-image-manager') -const MicroserviceEnvManager = require('../data/managers/microservice-env-manager') const RegistryManager = require('../data/managers/registry-manager') const VolumeMappingManager = require('../data/managers/volume-mapping-manager') const ConfigMapManager = require('../data/managers/config-map-manager') @@ -16,7 +15,7 @@ const SecretManager = require('../data/managers/secret-manager') const MicroservicesService = require('./microservices-service') const MicroservicePortService = require('./microservice-ports/microservice-port') const VolumeMountService = require('./volume-mount-service') -const constants = require('../helpers/constants') +const WorkloadSpec = require('./microservice-workload-spec') const isEqual = require('lodash/isEqual') const Op = require('sequelize').Op const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') @@ -58,13 +57,6 @@ function _validateImageArch (name, fog, images) { validateImageMatchesFogArch(name, fog, images) } -function _validateMicroserviceConfig (config) { - if (config) { - return config.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape - } - return '{}' -} - function _rejectServiceAccountVolumeMappings (volumeMappings) { if (!volumeMappings) { return @@ -230,17 +222,6 @@ async function _updateVolumeMappings (microserviceUuid, volumeMappings, fogUuid, await _createVolumeMappings(microserviceUuid, volumeMappings, transaction) } -async function _updateEnv (env, microserviceUuid, transaction) { - await MicroserviceEnvManager.delete({ microserviceUuid }, transaction) - for (const envData of env) { - await MicroserviceEnvManager.create({ - microserviceUuid, - key: envData.key, - value: envData.value - }, transaction) - } -} - async function _updateImages (images, microserviceUuid, transaction) { await CatalogItemImageManager.delete({ microserviceUuid }, transaction) await _createMicroserviceImages(microserviceUuid, images, transaction) @@ -253,21 +234,18 @@ async function _updatePorts (ports, microservice, transaction) { } } -async function _createControllerMicroservice (registerData, fog, application, transaction) { +async function _createControllerMicroservice (registerData, fog, application, validatedExtraHosts, transaction) { await _checkForDuplicateName(CONTROLLER_MS_NAME, registerData.uuid, application.id, transaction) const microserviceData = { uuid: registerData.uuid, name: CONTROLLER_MS_NAME, - config: _validateMicroserviceConfig(registerData.config), iofogUuid: fog.uuid, - hostNetworkMode: registerData.hostNetworkMode, - runtime: registerData.runtime, registryId: registerData.registryId, schedule: 0, - logSize: constants.MICROSERVICE_DEFAULT_LOG_SIZE * 1, applicationId: application.id, - isController: true + isController: true, + ...WorkloadSpec.buildCreateScalarColumns(registerData) } const microservice = await MicroserviceManager.create( @@ -284,18 +262,13 @@ async function _createControllerMicroservice (registerData, fog, application, tr } } - if (registerData.env) { - for (const env of registerData.env) { - await MicroserviceEnvManager.create({ - microserviceUuid: microservice.uuid, - key: env.key, - value: env.value - }, transaction) - } - } - await _createVolumeMappings(microservice.uuid, registerData.volumeMappings, transaction) + await WorkloadSpec.createWorkloadRelations(microservice, registerData, { + validatedExtraHosts, + transaction + }) + await MicroserviceStatusManager.create({ microserviceUuid: microservice.uuid }, transaction) await MicroserviceExecStatusManager.create({ microserviceUuid: microservice.uuid }, transaction) @@ -304,38 +277,41 @@ async function _createControllerMicroservice (registerData, fog, application, tr return microservice } -async function _updateControllerMicroservice (existing, registerData, fog, transaction) { +async function _updateControllerMicroservice (existing, registerData, fog, validatedExtraHosts, transaction) { const existingImages = await CatalogItemImageManager.findAll({ microserviceUuid: existing.uuid }, transaction) const config = registerData.config !== undefined - ? _validateMicroserviceConfig(registerData.config) + ? WorkloadSpec.validateMicroserviceConfig(registerData.config) : undefined + const annotations = registerData.annotations !== undefined + ? WorkloadSpec.validateMicroserviceAnnotations(registerData.annotations) + : undefined + + const imagesChanged = registerData.images && + registerData.images.length > 0 && + _imagesChanged(registerData.images, existingImages) const microserviceUpdate = AppHelper.deleteUndefinedFields({ isController: true, schedule: 0, registryId: registerData.registryId, - hostNetworkMode: registerData.hostNetworkMode, - runtime: registerData.runtime, config, - rebuild: false + annotations, + rebuild: false, + ...WorkloadSpec.buildScalarColumns(registerData) }) - if (registerData.images && registerData.images.length > 0 && _imagesChanged(registerData.images, existingImages)) { + if (imagesChanged) { await _updateImages(registerData.images, existing.uuid, transaction) microserviceUpdate.rebuild = true } - microserviceUpdate.rebuild = microserviceUpdate.rebuild || !!( - existing.schedule !== 0 || - (registerData.hostNetworkMode !== undefined && existing.hostNetworkMode !== registerData.hostNetworkMode) || - (registerData.runtime !== undefined && existing.runtime !== registerData.runtime) || - (config !== undefined && existing.config !== config) || - registerData.env || - registerData.volumeMappings || - registerData.ports + microserviceUpdate.rebuild = microserviceUpdate.rebuild || WorkloadSpec.shouldRebuildForWorkloadChange( + existing, + registerData, + { config, annotations, imagesChanged } ) const updatedMicroservice = await MicroserviceManager.updateAndFind( @@ -353,9 +329,10 @@ async function _updateControllerMicroservice (existing, registerData, fog, trans await _updateVolumeMappings(existing.uuid, registerData.volumeMappings, fog.uuid, transaction) } - if (registerData.env) { - await _updateEnv(registerData.env, existing.uuid, transaction) - } + await WorkloadSpec.updateWorkloadRelations(existing.uuid, registerData, { + validatedExtraHosts: registerData.extraHosts ? validatedExtraHosts : undefined, + transaction + }) await MicroservicesService.updateChangeTracking(true, fog.uuid, transaction) @@ -385,6 +362,10 @@ async function registerControllerMicroservice (registerData, fog, transaction) { await MicroservicePortService.validatePortMappings({ ports: registerData.ports, iofogUuid: fog.uuid }, transaction) } + const validatedExtraHosts = registerData.extraHosts + ? await WorkloadSpec.validateExtraHosts(registerData, fog.uuid, transaction) + : undefined + const existing = await MicroserviceManager.findOne({ uuid: registerData.uuid }, transaction) if (existing && existing.iofogUuid !== fog.uuid) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, registerData.uuid)) @@ -393,9 +374,9 @@ async function registerControllerMicroservice (registerData, fog, transaction) { const application = await ensureSystemApplication(fog, transaction) if (existing) { - await _updateControllerMicroservice(existing, registerData, fog, transaction) + await _updateControllerMicroservice(existing, registerData, fog, validatedExtraHosts, transaction) } else { - await _createControllerMicroservice(registerData, fog, application, transaction) + await _createControllerMicroservice(registerData, fog, application, validatedExtraHosts, transaction) } return { uuid: registerData.uuid } diff --git a/src/services/microservice-workload-spec.js b/src/services/microservice-workload-spec.js new file mode 100644 index 00000000..599ab119 --- /dev/null +++ b/src/services/microservice-workload-spec.js @@ -0,0 +1,420 @@ +const AppHelper = require('../helpers/app-helper') +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const constants = require('../helpers/constants') +const FogManager = require('../data/managers/iofog-manager') +const ApplicationManager = require('../data/managers/application-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') +const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') +const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') +const MicroserviceEnvManager = require('../data/managers/microservice-env-manager') +const MicroserviceExtraHostManager = require('../data/managers/microservice-extra-host-manager') +const MicroserviceHealthCheckManager = require('../data/managers/microservice-healthcheck-manager') +const ConfigMapManager = require('../data/managers/config-map-manager') +const SecretManager = require('../data/managers/secret-manager') + +function validateMicroserviceConfig (config) { + if (config) { + return config.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape + } + return '{}' +} + +function validateMicroserviceAnnotations (annotations) { + if (annotations) { + return annotations.split('\\"').join('"').split('"').join('\"') // eslint-disable-line no-useless-escape + } + return undefined +} + +function _validateMicroserviceHealthCheck (healthCheck) { + if (healthCheck) { + return JSON.stringify(healthCheck) + } + return undefined +} + +function processHealthCheckForDB (healthCheckData) { + if (!healthCheckData) { + return null + } + + return { + test: _validateMicroserviceHealthCheck(healthCheckData.test), + interval: healthCheckData.interval, + timeout: healthCheckData.timeout, + startPeriod: healthCheckData.startPeriod, + startInterval: healthCheckData.startInterval, + retries: healthCheckData.retries + } +} + +function buildScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEFAULT_LOG_SIZE } = {}) { + return AppHelper.deleteUndefinedFields({ + config: spec.config !== undefined ? validateMicroserviceConfig(spec.config) : undefined, + annotations: spec.annotations !== undefined ? validateMicroserviceAnnotations(spec.annotations) : undefined, + hostNetworkMode: spec.hostNetworkMode, + isPrivileged: spec.isPrivileged, + logSize: spec.logSize != null ? spec.logSize * 1 : undefined, + runtime: spec.runtime, + pidMode: spec.pidMode, + ipcMode: spec.ipcMode, + runAsUser: spec.runAsUser, + platform: spec.platform, + cpuSetCpus: spec.cpuSetCpus, + memoryLimit: spec.memoryLimit + }) +} + +function buildCreateScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEFAULT_LOG_SIZE } = {}) { + const columns = buildScalarColumns(spec, { defaultLogSize }) + if (columns.logSize == null) { + columns.logSize = defaultLogSize * 1 + } + if (columns.config == null) { + columns.config = validateMicroserviceConfig(spec.config) + } + return columns +} + +async function _validateLocalAppHostTemplate (extraHost, templateArgs, msvc, fogUuid, transaction) { + if (templateArgs.length !== 4) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + const fog = await FogManager.findOne({ uuid: msvc.iofogUuid }, transaction) + if (!fog) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[2])) + } + if (fogUuid !== fog.uuid) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_APPS_TEMPLATE, msvc.name)) + } + + extraHost.targetFogUuid = fog.uuid + extraHost.value = `iofog_${msvc.uuid}` + + return extraHost +} + +async function _validateAppHostTemplate (extraHost, templateArgs, fogUuid, transaction) { + if (templateArgs.length < 4) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + const application = await ApplicationManager.findOne({ name: templateArgs[1] }, transaction) + if (!application) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[1])) + } + const msvc = await MicroserviceManager.findOne({ applicationId: application.id, name: templateArgs[2] }, transaction) + if (!msvc) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[2])) + } + extraHost.templateType = 'Apps' + extraHost.targetMicroserviceUuid = msvc.uuid + if (templateArgs[3] === 'local') { + return _validateLocalAppHostTemplate(extraHost, templateArgs, msvc, fogUuid, transaction) + } + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) +} + +async function _validateAgentHostTemplate (extraHost, templateArgs, transaction) { + if (templateArgs.length !== 2) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, templateArgs.join('.'))) + } + + extraHost.templateType = 'Agents' + const fog = await FogManager.findOne({ name: templateArgs[1] }, transaction) + if (!fog) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_HOST_TEMPLATE, templateArgs[1])) + } + extraHost.targetFogUuid = fog.uuid + extraHost.value = fog.host + + return extraHost +} + +async function _validateExtraHost (extraHostData, fogUuid, transaction) { + if (extraHostData.name === 'service.local') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, 'Extra Host name cannot be service.local')) + } + const extraHost = { + templateType: 'Litteral', + name: extraHostData.name, + template: extraHostData.address, + value: extraHostData.address + } + if (!(extraHost.template.startsWith('${') && extraHost.template.endsWith('}'))) { + return extraHost + } + const template = extraHost.value.slice(2, extraHost.value.length - 1) + const templateArgs = template.split('.') + extraHost.templateType = templateArgs[0] + if (templateArgs[0] === 'Apps') { + return _validateAppHostTemplate(extraHost, templateArgs, fogUuid, transaction) + } else if (templateArgs[0] === 'Agents') { + return _validateAgentHostTemplate(extraHost, templateArgs, transaction) + } + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_HOST_TEMPLATE, template)) +} + +async function validateExtraHosts (spec, fogUuid, transaction) { + if (!spec.extraHosts || spec.extraHosts.length === 0) { + return [] + } + const extraHosts = [] + for (const extraHost of spec.extraHosts) { + extraHosts.push(await _validateExtraHost(extraHost, fogUuid, transaction)) + } + return extraHosts +} + +async function _buildEnvRecord (microserviceUuid, envData, transaction) { + const envObj = { + microserviceUuid, + key: envData.key, + value: envData.value + } + + if (envData.valueFromSecret) { + const [secretName, dataKey] = envData.valueFromSecret.split('/') + if (!secretName || !dataKey) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_SECRET_REFERENCE, envData.valueFromSecret)) + } + const secret = await SecretManager.getSecret(secretName, transaction) + if (!secret) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.SECRET_NOT_FOUND, secretName)) + } + if (!secret.data[dataKey]) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SECRET_KEY_NOT_FOUND, dataKey, secretName)) + } + if (secret.type === 'tls') { + try { + envObj.value = Buffer.from(secret.data[dataKey], 'base64').toString('utf-8') + } catch (error) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_BASE64_VALUE, dataKey, secretName)) + } + } else { + envObj.value = secret.data[dataKey] + } + envObj.valueFromSecret = envData.valueFromSecret + } + + if (envData.valueFromConfigMap) { + const [configMapName, dataKey] = envData.valueFromConfigMap.split('/') + if (!configMapName || !dataKey) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_CONFIGMAP_REFERENCE, envData.valueFromConfigMap)) + } + const configMap = await ConfigMapManager.getConfigMap(configMapName, transaction) + if (!configMap) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.CONFIGMAP_NOT_FOUND, configMapName)) + } + if (!configMap.data[dataKey]) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.CONFIGMAP_KEY_NOT_FOUND, dataKey, configMapName)) + } + envObj.value = configMap.data[dataKey] + envObj.valueFromConfigMap = envData.valueFromConfigMap + } + + return envObj +} + +async function _createEnv (microserviceUuid, envData, transaction) { + const envObj = await _buildEnvRecord(microserviceUuid, envData, transaction) + await MicroserviceEnvManager.create(envObj, transaction) +} + +async function _updateEnv (env, microserviceUuid, transaction) { + await MicroserviceEnvManager.delete({ microserviceUuid }, transaction) + for (const envData of env) { + await _createEnv(microserviceUuid, envData, transaction) + } +} + +async function _createExtraHost (microserviceUuid, extraHostData, transaction) { + await MicroserviceExtraHostManager.create({ + ...extraHostData, + microserviceUuid + }, transaction) +} + +async function _updateExtraHosts (extraHosts, microserviceUuid, transaction) { + await MicroserviceExtraHostManager.delete({ microserviceUuid }, transaction) + for (const extraHost of extraHosts) { + await _createExtraHost(microserviceUuid, extraHost, transaction) + } +} + +async function _createArg (microserviceUuid, arg, transaction) { + await MicroserviceArgManager.create({ cmd: arg, microserviceUuid }, transaction) +} + +async function _updateArg (arg, microserviceUuid, transaction) { + await MicroserviceArgManager.delete({ microserviceUuid }, transaction) + for (const argData of arg) { + await _createArg(microserviceUuid, argData, transaction) + } +} + +async function _createCdiDevice (microserviceUuid, cdiDevice, transaction) { + await MicroserviceCdiDevManager.create({ cdiDevices: cdiDevice, microserviceUuid }, transaction) +} + +async function _updateCdiDevices (cdiDevices, microserviceUuid, transaction) { + await MicroserviceCdiDevManager.delete({ microserviceUuid }, transaction) + for (const cdiDevice of cdiDevices) { + await _createCdiDevice(microserviceUuid, cdiDevice, transaction) + } +} + +async function _createCapAdd (microserviceUuid, capAdd, transaction) { + await MicroserviceCapAddManager.create({ capAdd, microserviceUuid }, transaction) +} + +async function _updateCapAdd (capAdd, microserviceUuid, transaction) { + await MicroserviceCapAddManager.delete({ microserviceUuid }, transaction) + for (const capAddData of capAdd) { + await _createCapAdd(microserviceUuid, capAddData, transaction) + } +} + +async function _createCapDrop (microserviceUuid, capDrop, transaction) { + await MicroserviceCapDropManager.create({ capDrop, microserviceUuid }, transaction) +} + +async function _updateCapDrop (capDrop, microserviceUuid, transaction) { + await MicroserviceCapDropManager.delete({ microserviceUuid }, transaction) + for (const capDropData of capDrop) { + await _createCapDrop(microserviceUuid, capDropData, transaction) + } +} + +async function _createHealthCheck (microserviceUuid, healthCheck, transaction) { + const healthCheckData = { + microserviceUuid, + ...processHealthCheckForDB(healthCheck) + } + if (healthCheckData.test && healthCheckData.test.length > 0) { + await MicroserviceHealthCheckManager.create(healthCheckData, transaction) + } +} + +async function _updateHealthCheck (microserviceUuid, healthCheck, transaction) { + await MicroserviceHealthCheckManager.delete({ microserviceUuid }, transaction) + if (healthCheck) { + await _createHealthCheck(microserviceUuid, healthCheck, transaction) + } +} + +async function createWorkloadRelations (microservice, spec, { validatedExtraHosts, transaction }) { + const microserviceUuid = microservice.uuid + + if (validatedExtraHosts && validatedExtraHosts.length) { + for (const extraHost of validatedExtraHosts) { + await _createExtraHost(microserviceUuid, extraHost, transaction) + } + } + + if (spec.env) { + for (const env of spec.env) { + await _createEnv(microserviceUuid, env, transaction) + } + } + + if (spec.cmd) { + for (const arg of spec.cmd) { + await _createArg(microserviceUuid, arg, transaction) + } + } + + if (spec.cdiDevices) { + for (const cdiDevice of spec.cdiDevices) { + await _createCdiDevice(microserviceUuid, cdiDevice, transaction) + } + } + + if (spec.capAdd) { + for (const capAdd of spec.capAdd) { + await _createCapAdd(microserviceUuid, capAdd, transaction) + } + } + + if (spec.capDrop) { + for (const capDrop of spec.capDrop) { + await _createCapDrop(microserviceUuid, capDrop, transaction) + } + } + + if (spec.healthCheck) { + await _createHealthCheck(microserviceUuid, spec.healthCheck, transaction) + } +} + +async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtraHosts, transaction }) { + if (validatedExtraHosts) { + await _updateExtraHosts(validatedExtraHosts, microserviceUuid, transaction) + } + + if (spec.env) { + await _updateEnv(spec.env, microserviceUuid, transaction) + } + + if (spec.cmd) { + await _updateArg(spec.cmd, microserviceUuid, transaction) + } + + if (spec.cdiDevices) { + await _updateCdiDevices(spec.cdiDevices, microserviceUuid, transaction) + } + + if (spec.capAdd) { + await _updateCapAdd(spec.capAdd, microserviceUuid, transaction) + } + + if (spec.capDrop) { + await _updateCapDrop(spec.capDrop, microserviceUuid, transaction) + } + + if (spec.healthCheck) { + await _updateHealthCheck(microserviceUuid, spec.healthCheck, transaction) + } +} + +function shouldRebuildForWorkloadChange (existing, spec, { config, annotations, imagesChanged }) { + return !!( + imagesChanged || + existing.schedule !== 0 || + (spec.hostNetworkMode !== undefined && existing.hostNetworkMode !== spec.hostNetworkMode) || + (spec.isPrivileged !== undefined && existing.isPrivileged !== spec.isPrivileged) || + (spec.logSize !== undefined && existing.logSize !== spec.logSize * 1) || + (spec.runtime !== undefined && existing.runtime !== spec.runtime) || + (spec.pidMode !== undefined && existing.pidMode !== spec.pidMode) || + (spec.ipcMode !== undefined && existing.ipcMode !== spec.ipcMode) || + (spec.runAsUser !== undefined && existing.runAsUser !== spec.runAsUser) || + (spec.platform !== undefined && existing.platform !== spec.platform) || + (spec.cpuSetCpus !== undefined && existing.cpuSetCpus !== spec.cpuSetCpus) || + (spec.memoryLimit !== undefined && existing.memoryLimit !== spec.memoryLimit) || + (config !== undefined && existing.config !== config) || + (annotations !== undefined && existing.annotations !== annotations) || + spec.env || + spec.volumeMappings || + spec.ports || + spec.extraHosts || + spec.cmd || + spec.cdiDevices || + spec.capAdd || + spec.capDrop || + spec.healthCheck + ) +} + +module.exports = { + validateMicroserviceConfig, + validateMicroserviceAnnotations, + processHealthCheckForDB, + buildScalarColumns, + buildCreateScalarColumns, + validateExtraHosts, + createWorkloadRelations, + updateWorkloadRelations, + shouldRebuildForWorkloadChange +} diff --git a/test/src/services/controller-ms-service.test.js b/test/src/services/controller-ms-service.test.js index 2dfc4cdf..dfa2f2f9 100644 --- a/test/src/services/controller-ms-service.test.js +++ b/test/src/services/controller-ms-service.test.js @@ -13,6 +13,7 @@ const VolumeMappingManager = require('../../../src/data/managers/volume-mapping- const MicroservicesService = require('../../../src/services/microservices-service') const MicroservicePortService = require('../../../src/services/microservice-ports/microservice-port') const ApplicationManager = require('../../../src/data/managers/application-manager') +const WorkloadSpec = require('../../../src/services/microservice-workload-spec') const Errors = require('../../../src/helpers/errors') describe('Controller MS Service', () => { @@ -97,6 +98,9 @@ describe('Controller MS Service', () => { $sandbox.stub(MicroservicesService, 'updateChangeTracking').resolves() $sandbox.stub(MicroservicesService, 'injectServiceAccountVolume').resolves() $sandbox.stub(MicroservicesService, 'createOrUpdateServiceAccountForMicroservice').resolves() + $sandbox.stub(WorkloadSpec, 'createWorkloadRelations').resolves() + $sandbox.stub(WorkloadSpec, 'updateWorkloadRelations').resolves() + $sandbox.stub(WorkloadSpec, 'validateExtraHosts').resolves([]) }) context('on non-system fog', () => { @@ -162,6 +166,49 @@ describe('Controller MS Service', () => { expect(MicroservicesService.createOrUpdateServiceAccountForMicroservice).to.not.have.been.called }) + context('when container workload fields are sent on create', () => { + def('body', () => ({ + ...registerData, + capAdd: ['NET_ADMIN'], + isPrivileged: true, + cmd: ['/bin/controller'] + })) + + it('persists container workload relations on create', async () => { + await $subject + expect(WorkloadSpec.createWorkloadRelations).to.have.been.calledWith( + sinon.match({ uuid: msUuid }), + sinon.match({ + capAdd: ['NET_ADMIN'], + isPrivileged: true, + cmd: ['/bin/controller'] + }), + sinon.match({ transaction }) + ) + }) + }) + + context('when container scalar fields are sent on create', () => { + def('body', () => ({ + ...registerData, + isPrivileged: true, + pidMode: 'host', + memoryLimit: 1073741824 + })) + + it('creates microservice with container scalar fields', async () => { + await $subject + expect(MicroserviceManager.create).to.have.been.calledWith( + sinon.match({ + isPrivileged: true, + pidMode: 'host', + memoryLimit: 1073741824 + }), + transaction + ) + }) + }) + context('when microservice already exists', () => { const existing = { uuid: msUuid, @@ -234,6 +281,26 @@ describe('Controller MS Service', () => { expect(MicroservicesService.updateChangeTracking).to.have.been.calledWith(true, fogUuid, transaction) }) + context('when container workload fields are sent on update', () => { + def('body', () => ({ + ...registerData, + capAdd: ['NET_BIND_SERVICE'], + healthCheck: { test: ['CMD', 'true'] } + })) + + it('updates container workload relations on upsert', async () => { + await $subject + expect(WorkloadSpec.updateWorkloadRelations).to.have.been.calledWith( + msUuid, + sinon.match({ + capAdd: ['NET_BIND_SERVICE'], + healthCheck: { test: ['CMD', 'true'] } + }), + sinon.match({ transaction }) + ) + }) + }) + it('rejects uuid registered on a different fog', async () => { MicroserviceManager.findOne.callsFake((where) => { if (where.uuid === msUuid) { diff --git a/test/src/services/microservice-workload-spec.test.js b/test/src/services/microservice-workload-spec.test.js new file mode 100644 index 00000000..1df3a713 --- /dev/null +++ b/test/src/services/microservice-workload-spec.test.js @@ -0,0 +1,95 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const WorkloadSpec = require('../../../src/services/microservice-workload-spec') +const MicroserviceCapAddManager = require('../../../src/data/managers/microservice-cap-add-manager') +const MicroserviceCapDropManager = require('../../../src/data/managers/microservice-cap-drop-manager') +const MicroserviceArgManager = require('../../../src/data/managers/microservice-arg-manager') +const MicroserviceHealthCheckManager = require('../../../src/data/managers/microservice-healthcheck-manager') +const MicroserviceEnvManager = require('../../../src/data/managers/microservice-env-manager') + +describe('Microservice Workload Spec', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('.shouldRebuildForWorkloadChange()', () => { + const existing = { + schedule: 0, + hostNetworkMode: false, + isPrivileged: false, + logSize: 2048, + runtime: 'io.containerd.runc.v2', + config: '{}', + annotations: '{}' + } + + it('returns true when capAdd is present in update', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, { capAdd: ['NET_ADMIN'] }, {})).to.equal(true) + }) + + it('returns true when isPrivileged changes', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, { isPrivileged: true }, {})).to.equal(true) + }) + + it('returns false when no workload fields are present', () => { + expect(WorkloadSpec.shouldRebuildForWorkloadChange(existing, {}, {})).to.equal(false) + }) + }) + + describe('.createWorkloadRelations()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceCapAddManager, 'create').resolves() + $sandbox.stub(MicroserviceCapDropManager, 'create').resolves() + $sandbox.stub(MicroserviceArgManager, 'create').resolves() + $sandbox.stub(MicroserviceHealthCheckManager, 'create').resolves() + $sandbox.stub(MicroserviceEnvManager, 'create').resolves() + }) + + it('persists capAdd, capDrop, cmd, and healthCheck on create', async () => { + await WorkloadSpec.createWorkloadRelations( + { uuid: 'ms-uuid' }, + { + capAdd: ['NET_ADMIN'], + capDrop: ['MKNOD'], + cmd: ['/bin/controller'], + healthCheck: { test: ['CMD', 'true'], interval: 30 } + }, + { transaction } + ) + + expect(MicroserviceCapAddManager.create).to.have.been.calledWith( + { capAdd: 'NET_ADMIN', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceCapDropManager.create).to.have.been.calledWith( + { capDrop: 'MKNOD', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + { cmd: '/bin/controller', microserviceUuid: 'ms-uuid' }, + transaction + ) + expect(MicroserviceHealthCheckManager.create).to.have.been.called + }) + }) + + describe('.updateWorkloadRelations()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceCapAddManager, 'delete').resolves() + $sandbox.stub(MicroserviceCapAddManager, 'create').resolves() + }) + + it('clears capAdd when an empty array is sent', async () => { + await WorkloadSpec.updateWorkloadRelations( + 'ms-uuid', + { capAdd: [] }, + { transaction } + ) + + expect(MicroserviceCapAddManager.delete).to.have.been.calledWith({ microserviceUuid: 'ms-uuid' }, transaction) + expect(MicroserviceCapAddManager.create).to.not.have.been.called + }) + }) +}) From d6d3dabca36e012615900cc6a82e48d4738183de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Mon, 29 Jun 2026 00:50:49 +0300 Subject: [PATCH 2/4] Add production cross-replica WebSocket relay with AMQP pool and optional NATS hub transport. Introduce WsRelayTransport, eight-connection AMQP router pool with recovery, NATS Core relay on hub, K8s/remote connect resolvers, platform controller NATS account with creds API support, and plain-text Opaque creds loading for hub connect. --- package-lock.json | 35 + package.json | 1 + src/config/config.yaml | 9 + src/config/nats-system-rules.js | 36 + src/config/telemetry.js | 4 +- src/helpers/connect-endpoint-utils.js | 32 + src/helpers/constants.js | 1 + src/services/amqp-relay-transport.js | 77 +++ src/services/nats-api-service.js | 21 +- src/services/nats-auth-service.js | 168 ++++- src/services/nats-relay-connection-manager.js | 336 ++++++++++ src/services/nats-relay-transport-impl.js | 627 ++++++++++++++++++ src/services/nats-relay-transport.js | 76 +++ src/services/nats-service.js | 17 + src/services/router-connection-manager.js | 625 +++++++++++++++++ src/services/router-connection-service.js | 396 +---------- src/services/websocket-queue-service.js | 356 +++++++--- src/services/ws-relay-transport-factory.js | 38 ++ src/services/ws-relay-transport.js | 111 ++++ src/websocket/server.js | 135 ++-- src/websocket/ws-metrics.js | 40 +- test/src/services/nats-api-service.test.js | 101 +++ test/src/services/nats-auth-service.test.js | 147 ++++ .../nats-relay-connection-manager.test.js | 198 ++++++ .../src/services/nats-relay-transport.test.js | 217 ++++++ .../router-connection-service.test.js | 180 +++-- .../services/websocket-queue-service.test.js | 140 ++++ .../ws-relay-transport-factory.test.js | 52 ++ .../websocket/ws-cross-replica-nats.test.js | 211 ++++++ test/src/websocket/ws-cross-replica.test.js | 33 +- test/src/websocket/ws-drain.test.js | 4 +- .../ws-exec-activation-failfast.test.js | 75 +++ .../websocket/ws-exec-same-replica.test.js | 6 +- test/support/ws-session-harness.js | 34 +- 34 files changed, 3893 insertions(+), 646 deletions(-) create mode 100644 src/helpers/connect-endpoint-utils.js create mode 100644 src/services/amqp-relay-transport.js create mode 100644 src/services/nats-relay-connection-manager.js create mode 100644 src/services/nats-relay-transport-impl.js create mode 100644 src/services/nats-relay-transport.js create mode 100644 src/services/router-connection-manager.js create mode 100644 src/services/ws-relay-transport-factory.js create mode 100644 src/services/ws-relay-transport.js create mode 100644 test/src/services/nats-api-service.test.js create mode 100644 test/src/services/nats-relay-connection-manager.test.js create mode 100644 test/src/services/nats-relay-transport.test.js create mode 100644 test/src/services/websocket-queue-service.test.js create mode 100644 test/src/services/ws-relay-transport-factory.test.js create mode 100644 test/src/websocket/ws-cross-replica-nats.test.js create mode 100644 test/src/websocket/ws-exec-activation-failfast.test.js diff --git a/package-lock.json b/package-lock.json index a8d91af5..1a20b718 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@msgpack/msgpack": "^3.1.2", "@nats-io/jwt": "^0.0.10-5", "@nats-io/nkeys": "^2.0.3", + "@nats-io/transport-node": "^3.4.0", "@node-rs/argon2": "^2.0.2", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-trace-otlp-http": "^0.219.0", @@ -1662,6 +1663,16 @@ "node": ">=16.0.0" } }, + "node_modules/@nats-io/nats-core": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@nats-io/nats-core/-/nats-core-3.4.0.tgz", + "integrity": "sha512-QMDM86EUNm+wudlKC67HLar/KHHQUvJGLfb4jbahje3pUk3K9afeck9fwsxBZF0eUFox6T/TA6m/t+lQqf+QsQ==", + "license": "Apache-2.0", + "dependencies": { + "@nats-io/nkeys": "2.0.3", + "@nats-io/nuid": "3.0.0" + } + }, "node_modules/@nats-io/nkeys": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@nats-io/nkeys/-/nkeys-2.0.3.tgz", @@ -1674,6 +1685,29 @@ "node": ">=18.0.0" } }, + "node_modules/@nats-io/nuid": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nats-io/nuid/-/nuid-3.0.0.tgz", + "integrity": "sha512-QbXZDrxmYlrn5AD06gYcUTmEHwxn96HBQMIk7XTjDlEcx6FPzVBBPjp4AMRh1lEv6B4iJ6Xb/Uz61JugPXFRQg==", + "license": "Apache-2.0", + "engines": { + "node": ">= 22.x" + } + }, + "node_modules/@nats-io/transport-node": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@nats-io/transport-node/-/transport-node-3.4.0.tgz", + "integrity": "sha512-hH7u7ejIBTFEJIZ8rIcMrHJI6wl+HhpO5sVFs1+ppmXa8RuB2+Lh1+UwTzZ5xTNNm1TKcRkYy+2qCV56qp8RxA==", + "license": "Apache-2.0", + "dependencies": { + "@nats-io/nats-core": "3.4.0", + "@nats-io/nkeys": "2.0.3", + "@nats-io/nuid": "3.0.0" + }, + "engines": { + "node": ">= 18.0.0" + } + }, "node_modules/@noble/hashes": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", @@ -3106,6 +3140,7 @@ "version": "8.11.3", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "dev": true, "license": "MIT", "bin": { "acorn": "bin/acorn" diff --git a/package.json b/package.json index fa4a6f86..048c2f59 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "moment": "2.30.1", "multer": "1.4.5-lts.1", "mysql2": "3.10.1", + "@nats-io/transport-node": "^3.4.0", "nconf": "0.12.1", "node-fetch-npm": "^2.0.4", "node-forge": "^1.4.0", diff --git a/src/config/config.yaml b/src/config/config.yaml index 0e39643f..3ee59f6b 100644 --- a/src/config/config.yaml +++ b/src/config/config.yaml @@ -32,6 +32,15 @@ server: logTailMaxLines: 5000 # Log: tail query param max (R82) replicaMaxConcurrentWs: 500 # Scale SLO per replica (R88) drainTimeoutMs: 30000 # Graceful drain on SIGTERM/preStop (R85) + relay: + amqp: + poolSize: 8 + sendTimeoutMs: 5000 + unsettledWarnThreshold: 1800 + nats: + maxPendingBytes: 33554432 + maxPendingMessages: 8192 + publishTimeoutMs: 5000 ha: crossReplicaRequiresAmqp: true # Cross-replica exec/log relay requires AMQP router (R84) failFastOnRouterUnavailable: true diff --git a/src/config/nats-system-rules.js b/src/config/nats-system-rules.js index f7ef9403..67e06ab3 100644 --- a/src/config/nats-system-rules.js +++ b/src/config/nats-system-rules.js @@ -5,6 +5,9 @@ const APPLICATION_ACCOUNT_RULE_NAME = 'default-account' const MICROSERVICE_USER_RULE_NAME = 'default-user' const MQTT_BEARER_USER_RULE_NAME = 'default-mqtt-user' const DEFAULT_LEAF_USER_RULE_NAME = 'default-leaf-user' +const CONTROLLER_ACCOUNT_RULE_NAME = 'controller-account' +const CONTROLLER_USER_RULE_NAME = 'controller-user' +const CONTROLLER_NATS_RELAY_SUBJECT_ALLOW = 'controller.relay.v1.>' const SYS_ACCOUNT_EXPORTS_INLINE = Object.freeze([ Object.freeze({ @@ -58,6 +61,25 @@ const ACCOUNT_RULES = Object.freeze([ maxMsgPayload: -1, maxSubscriptions: -1, exportsAllowWildcards: true + }), + Object.freeze({ + name: CONTROLLER_ACCOUNT_RULE_NAME, + description: 'Controller WebSocket relay account with standard app limits, no exports', + memStorage: -1, + diskStorage: -1, + streams: -1, + maxAckPending: -1, + memMaxStreamBytes: -1, + diskMaxStreamBytes: -1, + consumer: -1, + maxLeafNodeConnections: -1, + maxImports: -1, + maxExports: -1, + maxConnections: -1, + maxData: -1, + maxMsgPayload: -1, + maxSubscriptions: -1, + exportsAllowWildcards: true }) ]) @@ -88,6 +110,17 @@ const USER_RULES = Object.freeze([ maxSubscriptions: -1, maxPayload: -1, allowedConnectionTypes: ['LEAFNODE', 'WEBSOCKET'] + }), + Object.freeze({ + name: CONTROLLER_USER_RULE_NAME, + description: 'Controller WebSocket relay user with standard app limits, scoped to controller.relay.v1.>', + bearerToken: false, + allowedConnectionTypes: ['STANDARD'], + maxData: -1, + maxSubscriptions: -1, + maxPayload: -1, + pubAllow: [CONTROLLER_NATS_RELAY_SUBJECT_ALLOW], + subAllow: [CONTROLLER_NATS_RELAY_SUBJECT_ALLOW] }) ]) @@ -125,6 +158,9 @@ module.exports = { MICROSERVICE_USER_RULE_NAME, MQTT_BEARER_USER_RULE_NAME, DEFAULT_LEAF_USER_RULE_NAME, + CONTROLLER_ACCOUNT_RULE_NAME, + CONTROLLER_USER_RULE_NAME, + CONTROLLER_NATS_RELAY_SUBJECT_ALLOW, isReservedRuleName, getSystemAccountRuleDefinitions, getSystemUserRuleDefinitions, diff --git a/src/config/telemetry.js b/src/config/telemetry.js index 8c2c51f8..646e8ea7 100644 --- a/src/config/telemetry.js +++ b/src/config/telemetry.js @@ -44,9 +44,9 @@ async function startTelemetry () { try { await sdk.start() - const RouterConnectionService = require('../services/router-connection-service') + const RouterConnectionManager = require('../services/router-connection-manager') const { initWsMetrics } = require('../websocket/ws-metrics') - initWsMetrics(RouterConnectionService) + initWsMetrics(RouterConnectionManager) logger.info('OpenTelemetry initialized successfully') } catch (error) { logger.error('Error initializing OpenTelemetry:', error) diff --git a/src/helpers/connect-endpoint-utils.js b/src/helpers/connect-endpoint-utils.js new file mode 100644 index 00000000..f3d93602 --- /dev/null +++ b/src/helpers/connect-endpoint-utils.js @@ -0,0 +1,32 @@ +function createDedupeHostList () { + const hosts = [] + const seen = new Set() + const addHost = (candidate) => { + if (!candidate) return + const trimmed = String(candidate).trim() + if (trimmed.length === 0 || seen.has(trimmed)) return + seen.add(trimmed) + hosts.push(trimmed) + } + return { hosts, addHost } +} + +function formatConnectAttempts (attempts, port) { + if (!attempts || attempts.length === 0) { + return 'no hosts configured' + } + return attempts.map((entry) => `${entry.host}:${port} (${entry.error})`).join('; ') +} + +function aggregateConnectError (message, attempts, port) { + const detail = formatConnectAttempts(attempts, port) + const error = new Error(`${message}: ${detail}`) + error.connectAttempts = attempts + return error +} + +module.exports = { + createDedupeHostList, + formatConnectAttempts, + aggregateConnectError +} diff --git a/src/helpers/constants.js b/src/helpers/constants.js index 7abfe6b7..fb4f06b1 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -67,6 +67,7 @@ module.exports = { ROUTER_BRIDGE_DNS_SAN: 'router.default.svc.bridge.local', NATS_BRIDGE_DNS_SAN: 'nats.default.svc.bridge.local', DEFAULT_ROUTER_K8S_SERVICE: 'router', + DEFAULT_NATS_K8S_SERVICE: 'nats-server', RESERVED_PORTS: [54321, 54322, 53], diff --git a/src/services/amqp-relay-transport.js b/src/services/amqp-relay-transport.js new file mode 100644 index 00000000..0bfb95a3 --- /dev/null +++ b/src/services/amqp-relay-transport.js @@ -0,0 +1,77 @@ +const WsRelayTransport = require('./ws-relay-transport') +const WebSocketQueueService = require('./websocket-queue-service') +const RouterConnectionManager = require('./router-connection-manager') + +class AmqpRelayTransport extends WsRelayTransport { + constructor (queueService = WebSocketQueueService) { + super() + this._queueService = queueService + this._recoveryCallbacks = [] + + this._queueService.onRecovery((sessionId, meta) => { + for (const cb of this._recoveryCallbacks) { + try { + cb(sessionId, meta) + } catch (error) { + // Queue service already logs; swallow to protect other callbacks. + } + } + }) + } + + getTransport () { + return 'amqp' + } + + async isAvailable () { + return RouterConnectionManager.isRouterAvailable() + } + + async enableForSession (session, cleanupCb) { + return this._queueService.enableForSession(session, cleanupCb) + } + + async enableForLogSession (session, cleanupCb) { + return this._queueService.enableForLogSession(session, cleanupCb) + } + + async publishToAgent (execId, buffer, opts) { + return this._queueService.publishToAgent(execId, buffer, opts) + } + + async publishToUser (execId, buffer, opts) { + return this._queueService.publishToUser(execId, buffer, opts) + } + + async publishLogToUser (sessionId, buffer) { + return this._queueService.publishLogToUser(sessionId, buffer) + } + + shouldUseRelay (execId) { + return this._queueService.shouldUseQueue(execId) + } + + shouldUseRelayForLogs (sessionId) { + return this._queueService.shouldUseQueueForLogs(sessionId) + } + + async cleanup (execId) { + return this._queueService.cleanup(execId) + } + + async cleanupLogSession (sessionId) { + return this._queueService.cleanupLogSession(sessionId) + } + + async shutdown () { + return this._queueService.shutdown() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this._recoveryCallbacks.push(cb) + } + } +} + +module.exports = AmqpRelayTransport diff --git a/src/services/nats-api-service.js b/src/services/nats-api-service.js index 4d9dbfbf..8d07cb98 100644 --- a/src/services/nats-api-service.js +++ b/src/services/nats-api-service.js @@ -341,24 +341,31 @@ async function createUser (appName, payload, transaction) { } } -async function getUserCreds (appName, userName, transaction) { - const application = await ApplicationManager.findOne({ name: appName }, transaction) +async function _resolveAccountIdForCredsApi (appName, transaction) { const sysAccount = await NatsAccountManager.findOne({ name: appName }, transaction) - if (!application && (!sysAccount || (!sysAccount.isSystem && !sysAccount.isLeafSystem))) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_NAME, appName)) + if (NatsAuthService.isPlatformControllerNatsAccount(sysAccount)) { + return sysAccount.id } - let accountId = null + + const application = await ApplicationManager.findOne({ name: appName }, transaction) if (application) { const account = await NatsAccountManager.findOne({ applicationId: application.id }, transaction) if (!account) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_ID, application.id)) } - accountId = account.id + return account.id } + if (sysAccount && (sysAccount.isSystem || sysAccount.isLeafSystem)) { - accountId = sysAccount.id + return sysAccount.id } + + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_APPLICATION_NAME, appName)) +} + +async function getUserCreds (appName, userName, transaction) { + const accountId = await _resolveAccountIdForCredsApi(appName, transaction) const user = await NatsUserManager.findOne({ accountId, name: userName }, transaction) if (!user) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_NAME, userName)) diff --git a/src/services/nats-auth-service.js b/src/services/nats-auth-service.js index 623bd1eb..30892991 100644 --- a/src/services/nats-auth-service.js +++ b/src/services/nats-auth-service.js @@ -28,6 +28,17 @@ const leafSystemAccountUserName = (fog) => `admin-leaf-${slugifyName(fog.name)}` const accountSeedSecretName = (applicationName) => `nats-account-seed-${slugifyName(applicationName)}` const mqttBearerSecretName = (applicationName, userName) => `nats-mqtt-creds-${slugifyName(applicationName)}-${slugifyName(userName)}` const microserviceCredsSecretName = (applicationName, microserviceName) => `nats-creds-${slugifyName(applicationName)}-${slugifyName(microserviceName)}` +const CONTROLLER_NATS_ACCOUNT_NAME = 'controller' +const CONTROLLER_NATS_USER_NAME = 'controller' +const controllerNatsCredsSecretName = () => `nats-creds-${slugifyName(CONTROLLER_NATS_ACCOUNT_NAME)}-${slugifyName(CONTROLLER_NATS_USER_NAME)}` + +function isPlatformControllerNatsAccount (account) { + return account && + account.name === CONTROLLER_NATS_ACCOUNT_NAME && + account.applicationId == null && + !account.isSystem && + !account.isLeafSystem +} function _parseJsonText (value, fallback) { if (!value) { @@ -174,6 +185,19 @@ async function _resolveNatsUserRule (ruleName, defaultRuleName, transaction) { return NatsUserRuleManager.findOne({ name: defaultRuleName }, transaction) } +async function _resolveAccountRuleForAccount (account, app, transaction) { + if (account.isSystem) { + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) + } + if (account.name === CONTROLLER_NATS_ACCOUNT_NAME && account.applicationId == null) { + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME }, transaction) + } + if (app && app.natsRuleId) { + return NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) + } + return NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction) +} + async function _encodeAccountJwtWithRuleAndRevocations (accountName, accountKp, operatorKp, rule, existingAccountJwt) { const existingRevocations = _extractAccountRevocations(existingAccountJwt) return encodeAccount( @@ -188,10 +212,16 @@ async function _encodeAccountJwtWithRuleAndRevocations (accountName, accountKp, } function _normalizeSystemUserRuleForPersistence (rule) { - return { - ...rule, - allowedConnectionTypes: rule.allowedConnectionTypes ? JSON.stringify(rule.allowedConnectionTypes) : undefined + const out = { ...rule } + if (out.allowedConnectionTypes) { + out.allowedConnectionTypes = JSON.stringify(out.allowedConnectionTypes) } + for (const field of ['pubAllow', 'pubDeny', 'subAllow', 'subDeny', 'src', 'tags']) { + if (Array.isArray(out[field])) { + out[field] = JSON.stringify(out[field]) + } + } + return out } /** @@ -316,11 +346,7 @@ async function rotateOperator (transaction) { const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const newAccountJwt = await _encodeAccountJwtWithRuleAndRevocations( account.name, accountKp, @@ -507,6 +533,78 @@ async function deleteServerSysUserForFog (fog, isHub, transaction) { _triggerResolverArtifactsReconcile({ fogUuids: [fog.uuid] }) } +async function ensureControllerNatsAccount (transaction, ...rest) { + const options = _triggerOptionsFromArgs(rest) + await ensureDefaultRules(transaction) + + const existingAccount = await NatsAccountManager.findOne({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (existingAccount) { + const existingUser = await NatsUserManager.findOne({ + accountId: existingAccount.id, + name: CONTROLLER_NATS_USER_NAME + }, transaction) + if (existingUser) { + return { account: existingAccount, user: existingUser } + } + const result = await createUserForAccount( + existingAccount.id, + CONTROLLER_NATS_USER_NAME, + null, + NatsSystemRules.CONTROLLER_USER_RULE_NAME, + null, + transaction + ) + _triggerResolverArtifactsReconcile(options) + return result + } + + const operator = await ensureOperator(transaction, options) + const operatorSeed = await _loadSeedFromSecret(operator.seedSecretName, transaction) + const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) + + const accountKp = createAccount() + const accountRule = await NatsAccountRuleManager.findOne({ + name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME + }, transaction) + const accountJwt = await encodeAccount( + CONTROLLER_NATS_ACCOUNT_NAME, + accountKp, + _buildAccountRuleClaims(accountRule), + { signer: operatorKp } + ) + const accountSeed = new TextDecoder().decode(accountKp.getSeed()) + + const seedSecretName = accountSeedSecretName(CONTROLLER_NATS_ACCOUNT_NAME) + await _upsertOpaqueSecret(seedSecretName, { seed: accountSeed }, transaction) + + const account = await NatsAccountManager.create({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + publicKey: accountKp.getPublicKey(), + jwt: accountJwt, + seedSecretName, + operatorId: operator.id, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + + const result = await createUserForAccount( + account.id, + CONTROLLER_NATS_USER_NAME, + null, + NatsSystemRules.CONTROLLER_USER_RULE_NAME, + null, + transaction + ) + _triggerResolverArtifactsReconcile(options) + return result +} + async function ensureAccountForApplication (applicationId, transaction) { await ensureDefaultRules(transaction) const existing = await NatsAccountManager.findOne({ applicationId }, transaction) @@ -891,6 +989,7 @@ async function ensureLeafUserForAccount (accountId, fogName, transaction, natsIn } async function reissueForAccountRule (accountRuleId, transaction) { + const rule = await NatsAccountRuleManager.findOne({ id: accountRuleId }, transaction) const applications = await ApplicationManager.findAll({ natsRuleId: accountRuleId }, transaction) logger.info(`Reissuing account JWTs for rule ${accountRuleId}`) for (const app of applications) { @@ -903,7 +1002,6 @@ async function reissueForAccountRule (accountRuleId, transaction) { const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) - const rule = await NatsAccountRuleManager.findOne({ id: accountRuleId }, transaction) const accountJwt = await _encodeAccountJwtWithRuleAndRevocations( app.name, accountKp, @@ -913,6 +1011,29 @@ async function reissueForAccountRule (accountRuleId, transaction) { ) await NatsAccountManager.update({ id: account.id }, { jwt: accountJwt }, transaction) } + if (rule && rule.name === NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME) { + const relayAccount = await NatsAccountManager.findOne({ + name: CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (relayAccount) { + const operator = await ensureOperator(transaction) + const operatorSeed = await _loadSeedFromSecret(operator.seedSecretName, transaction) + const operatorKp = fromSeed(new TextEncoder().encode(operatorSeed)) + const accountSeed = await _loadSeedFromSecret(relayAccount.seedSecretName, transaction) + const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) + const accountJwt = await _encodeAccountJwtWithRuleAndRevocations( + CONTROLLER_NATS_ACCOUNT_NAME, + accountKp, + operatorKp, + rule, + relayAccount.jwt + ) + await NatsAccountManager.update({ id: relayAccount.id }, { jwt: accountJwt }, transaction) + } + } _triggerResolverArtifactsReconcile({ reason: 'account-rule-updated', accountRuleId }) } @@ -926,11 +1047,7 @@ async function _addRevocationToAccount (account, publicKey, transaction) { const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -949,11 +1066,7 @@ async function _reissueOneUserForRule (user, userRuleId, operatorKp, transaction const accountSeed = await _loadSeedFromSecret(account.seedSecretName, transaction) const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1018,11 +1131,7 @@ async function revokeMicroserviceUser (microserviceUuid, transaction) { const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1104,11 +1213,7 @@ async function revokeUserByAccountAndName (accountId, userName, transaction) { const accountKp = fromSeed(new TextEncoder().encode(accountSeed)) const app = account.applicationId ? await ApplicationManager.findOne({ id: account.applicationId }, transaction) : null - const accountRule = account.isSystem - ? await NatsAccountRuleManager.findOne({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }, transaction) - : (app && app.natsRuleId - ? await NatsAccountRuleManager.findOne({ id: app.natsRuleId }, transaction) - : await NatsAccountRuleManager.findOne({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }, transaction)) + const accountRule = await _resolveAccountRuleForAccount(account, app, transaction) const revocations = _extractAccountRevocations(account.jwt) revocations[user.publicKey] = Math.floor(Date.now() / 1000) const accountJwt = await encodeAccount( @@ -1177,12 +1282,17 @@ function scheduleReissueUsersForMicroservices (microserviceUuids = []) { module.exports = { SYSTEM_ACCOUNT_NAME, + CONTROLLER_NATS_ACCOUNT_NAME, + CONTROLLER_NATS_USER_NAME, + controllerNatsCredsSecretName, + isPlatformControllerNatsAccount, sysUserNameForServer, leafSystemAccountName, leafSystemAccountUserName, ensureOperator: TransactionDecorator.generateTransaction(ensureOperator), rotateOperator: TransactionDecorator.generateTransaction(rotateOperator), ensureSystemAccount: TransactionDecorator.generateTransaction(ensureSystemAccount), + ensureControllerNatsAccount: TransactionDecorator.generateTransaction(ensureControllerNatsAccount), ensureSysUserForServer: TransactionDecorator.generateTransaction(ensureSysUserForServer), ensureLeafSystemAccount: TransactionDecorator.generateTransaction(ensureLeafSystemAccount), ensureLeafSystemAccountUser: TransactionDecorator.generateTransaction(ensureLeafSystemAccountUser), diff --git a/src/services/nats-relay-connection-manager.js b/src/services/nats-relay-connection-manager.js new file mode 100644 index 00000000..a85a0d25 --- /dev/null +++ b/src/services/nats-relay-connection-manager.js @@ -0,0 +1,336 @@ +const { connect, credsAuthenticator } = require('@nats-io/transport-node') +const config = require('../config') +const logger = require('../logger') +const Constants = require('../helpers/constants') +const { createDedupeHostList, aggregateConnectError } = require('../helpers/connect-endpoint-utils') +const NatsInstanceManager = require('../data/managers/nats-instance-manager') +const NatsAccountManager = require('../data/managers/nats-account-manager') +const NatsUserManager = require('../data/managers/nats-user-manager') +const NatsAuthService = require('./nats-auth-service') +const SecretService = require('./secret-service') + +const NATS_DEFAULT_PORT = 4222 + +class NatsRelayConnectionManager { + constructor (deps = {}) { + this._connectFn = deps.connectFn || connect + this._config = deps.config || config + this.maxReconnectAttempts = deps.maxReconnectAttempts ?? -1 + this.fakeTransaction = { fakeTransaction: true } + this.connection = null + this.connectionPromise = null + this.cachedHubRecord = null + this.cachedCreds = null + this.credsPromise = null + this.recoveryListeners = [] + this.statusTask = null + this.shuttingDown = false + } + + onReconnect (cb) { + if (typeof cb === 'function') { + this.recoveryListeners.push(cb) + } + } + + _notifyReconnect () { + for (const listener of this.recoveryListeners) { + try { + listener() + } catch (error) { + logger.error({ error: error.message }, '[NATS][RELAY] Reconnect listener failed') + } + } + } + + isConnected () { + return !!(this.connection && !this.connection.isClosed()) + } + + async isAvailable () { + if (this.isConnected()) { + return true + } + try { + await this.getConnection() + return this.isConnected() + } catch (error) { + logger.debug({ error: error.message }, '[NATS][RELAY] Hub unavailable for relay') + return false + } + } + + async getConnection () { + if (this.shuttingDown) { + throw new Error('NATS relay connection manager is shutting down') + } + if (this.connection && !this.connection.isClosed()) { + return this.connection + } + if (this.connectionPromise) { + return this.connectionPromise + } + + this.connectionPromise = this._createConnection() + return this.connectionPromise + } + + async _createConnection () { + try { + const { hosts, port } = await this._resolveHubEndpoint() + const creds = await this._loadControllerRelayCreds() + const connectAttempts = [] + + for (let attempt = 0; attempt < hosts.length; attempt++) { + const host = hosts[attempt] + const servers = `nats://${host}:${port}` + try { + const nc = await this._connectFn({ + servers, + authenticator: credsAuthenticator(creds), + maxReconnectAttempts: this.maxReconnectAttempts, + reconnect: true, + reconnectTimeWait: 1000, + name: 'controller-ws-relay' + }) + + this.connection = nc + this.connectionPromise = null + this._watchConnectionStatus(nc) + + logger.info({ + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' + }, '[NATS][RELAY] Hub connection established') + return nc + } catch (error) { + const errorMessage = error.message || String(error) + connectAttempts.push({ host, error: errorMessage }) + logger.warn({ + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + error: errorMessage + }, '[NATS][RELAY] Hub connect attempt failed') + } + } + + const aggregateError = aggregateConnectError( + 'Unable to connect NATS hub after all fallback hosts', + connectAttempts, + port + ) + logger.error({ + hosts, + port, + connectAttempts, + error: aggregateError.message + }, '[NATS][RELAY] Unable to connect hub after all fallback hosts') + throw aggregateError + } catch (error) { + this.connectionPromise = null + throw error + } + } + + _watchConnectionStatus (nc) { + if (this.statusTask) { + return + } + + this.statusTask = (async () => { + try { + for await (const status of nc.status()) { + if (status.type === 'disconnect') { + logger.warn({ + error: status.data ? status.data.message : undefined + }, '[NATS][RELAY] Hub connection disconnected') + } else if (status.type === 'reconnect') { + logger.info('[NATS][RELAY] Hub connection reconnected') + this._notifyReconnect() + } + } + } catch (error) { + if (!this.shuttingDown) { + logger.debug({ error: error.message }, '[NATS][RELAY] Connection status watcher ended') + } + } finally { + this.statusTask = null + } + })() + } + + async _resolveHubEndpoint () { + const hub = await this._getHubRecord() + const port = hub.serverPort || NATS_DEFAULT_PORT + const hosts = this._buildHubHostList(hub) + return { hosts, port, hubUuid: hub.iofogUuid } + } + + _buildHubHostList (hub) { + if (this._isKubernetes()) { + return this._kubernetesHubHosts(hub) + } + return this._remoteHubHosts(hub) + } + + _kubernetesHubHosts (hub) { + const { hosts, addHost } = createDedupeHostList() + + const namespace = process.env.CONTROLLER_NAMESPACE || this._config.get('app.namespace') + if (namespace && namespace.trim().length > 0) { + addHost(`${Constants.DEFAULT_NATS_K8S_SERVICE}.${namespace.trim()}.svc.cluster.local`) + } + if (hub.host) { + addHost(hub.host) + } + if (hosts.length === 0) { + addHost(Constants.DEFAULT_NATS_K8S_SERVICE) + } + return hosts + } + + _remoteHubHosts (hub) { + const { hosts, addHost } = createDedupeHostList() + + addHost(Constants.NATS_BRIDGE_DNS_SAN) + + if (hub.host) { + addHost(hub.host) + } + + return hosts + } + + async _getHubRecord () { + if (this.cachedHubRecord) { + return this.cachedHubRecord + } + const hub = await NatsInstanceManager.findOne({ isHub: true }, this.fakeTransaction) + if (!hub) { + throw new Error('NATS hub not found. Ensure a hub NatsInstances row with isHub=true exists.') + } + this.cachedHubRecord = hub + return hub + } + + async _loadControllerRelayCreds () { + if (this.cachedCreds) { + return this.cachedCreds + } + if (this.credsPromise) { + return this.credsPromise + } + + this.credsPromise = (async () => { + try { + await this._ensureControllerNatsAccount() + const creds = await this._fetchControllerRelayCreds() + this.cachedCreds = creds + return creds + } finally { + this.credsPromise = null + } + })() + return this.credsPromise + } + + async _ensureControllerNatsAccount () { + const hub = await NatsInstanceManager.findOne({ isHub: true }, this.fakeTransaction) + if (!hub) { + return + } + await NatsAuthService.ensureControllerNatsAccount() + } + + async _fetchControllerRelayCreds () { + const account = await NatsAccountManager.findOne({ + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, this.fakeTransaction) + + let credsSecretName = NatsAuthService.controllerNatsCredsSecretName() + if (account) { + const user = await NatsUserManager.findOne({ + accountId: account.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME + }, this.fakeTransaction) + if (user && user.credsSecretName) { + credsSecretName = user.credsSecretName + } + } + + const secret = await this._safeGetSecret(credsSecretName) + if (!secret || !secret.data) { + throw new Error(`Controller relay NATS creds secret not found: ${credsSecretName}`) + } + + const credsKey = Object.keys(secret.data).find((key) => key.endsWith('.creds')) || 'creds' + const raw = secret.data[credsKey] + if (!raw) { + throw new Error(`Missing creds payload in secret ${credsSecretName}`) + } + + const credsText = typeof raw === 'string' + ? raw + : Buffer.from(raw, 'base64').toString('utf8') + + return new TextEncoder().encode(credsText) + } + + async _safeGetSecret (name) { + try { + return await SecretService.getSecretEndpoint(name) + } catch (error) { + if (error.name === 'NotFoundError') { + logger.debug({ secret: name }, '[NATS][RELAY] Secret not found') + return null + } + throw error + } + } + + _isKubernetes () { + const controlPlane = process.env.CONTROL_PLANE || this._config.get('app.ControlPlane') + return controlPlane && controlPlane.toLowerCase() === 'kubernetes' + } + + async shutdown () { + this.shuttingDown = true + if (this.connection && !this.connection.isClosed()) { + try { + await this.connection.drain() + } catch (error) { + logger.debug({ error: error.message }, '[NATS][RELAY] Drain failed during shutdown') + try { + await this.connection.close() + } catch (closeError) { + logger.debug({ error: closeError.message }, '[NATS][RELAY] Close failed during shutdown') + } + } + } + this.connection = null + this.connectionPromise = null + logger.info('[NATS][RELAY] Connection manager shut down') + } + + resetForTests () { + this.connection = null + this.connectionPromise = null + this.cachedHubRecord = null + this.cachedCreds = null + this.credsPromise = null + this.statusTask = null + this.shuttingDown = false + this.recoveryListeners = [] + } +} + +module.exports = new NatsRelayConnectionManager() +module.exports.NatsRelayConnectionManager = NatsRelayConnectionManager diff --git a/src/services/nats-relay-transport-impl.js b/src/services/nats-relay-transport-impl.js new file mode 100644 index 00000000..3929b488 --- /dev/null +++ b/src/services/nats-relay-transport-impl.js @@ -0,0 +1,627 @@ +const WebSocket = require('ws') +const { headers: natsHeaders } = require('@nats-io/transport-node') +const msgpack = require('@msgpack/msgpack') +const config = require('../config') +const logger = require('../logger') +const NatsRelayConnectionManager = require('./nats-relay-connection-manager') + +const SUBJECT_PREFIX = 'controller.relay.v1' +const LOG_BACKPRESSURE_BUFFER_BYTES = 256 * 1024 +const LOG_MESSAGE_TYPES = { LOG_ERROR: 9, LOG_LINE: 6 } + +const MESSAGE_TYPES = { + CLOSE: 4 +} + +const HEADER_MESSAGE_TYPE = 'messageType' +const HEADER_CLOSE_ACK = 'closeAck' + +function execAgentSubject (sessionId) { + return `${SUBJECT_PREFIX}.exec.${sessionId}.agent` +} + +function execUserSubject (sessionId) { + return `${SUBJECT_PREFIX}.exec.${sessionId}.user` +} + +function logUserSubject (sessionId) { + return `${SUBJECT_PREFIX}.log.${sessionId}.user` +} + +function decodeLogMessageType (buffer) { + try { + const msg = msgpack.decode(buffer) + return typeof msg.type === 'number' ? msg.type : null + } catch (error) { + return null + } +} + +function getBufferFromData (data) { + if (!data) return Buffer.alloc(0) + if (Buffer.isBuffer(data)) return data + if (data instanceof Uint8Array) return Buffer.from(data) + return Buffer.from(data) +} + +class NatsRelayTransportImpl { + constructor (connectionManager = NatsRelayConnectionManager, configOverride = null) { + this._connectionManager = connectionManager + this._config = configOverride || config + this.maxPendingBytes = this._config.get('server.webSocket.relay.nats.maxPendingBytes', 33554432) + this.maxPendingMessages = this._config.get('server.webSocket.relay.nats.maxPendingMessages', 8192) + this.publishTimeoutMs = this._config.get('server.webSocket.relay.nats.publishTimeoutMs', 5000) + this.execBridges = new Map() + this.logBridges = new Map() + this.recoveryCallbacks = [] + this.rebinding = new Set() + + this._connectionManager.onReconnect(() => { + return this._handleReconnect().catch((error) => { + logger.error('[NATS][RELAY] Reconnect handling failed', { error: error.message }) + }) + }) + } + + async isAvailable () { + return this._connectionManager.isAvailable() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryCallbacks.push(cb) + } + } + + async enableForSession (session, cleanupCallback) { + const execId = session.execId + if (!execId) { + logger.warn('[NATS][RELAY] Missing execId for session, skipping bridge enablement') + return false + } + + const bridge = this.execBridges.get(execId) || { + execId, + session: null, + subscriptions: {}, + sockets: {}, + cleanupCallback: null + } + + bridge.session = session + if (cleanupCallback) { + bridge.cleanupCallback = cleanupCallback + } + + const nc = await this._connectionManager.getConnection() + + if (session.user) { + await this._ensureExecSubscription(bridge, 'user', session.user, nc) + } + if (session.agent) { + await this._ensureExecSubscription(bridge, 'agent', session.agent, nc) + } + + this.execBridges.set(execId, bridge) + return true + } + + shouldUseRelay (execId) { + return this.execBridges.has(execId) + } + + async publishToAgent (execId, buffer, options = {}) { + await this._publishExec(execId, execAgentSubject(execId), buffer, options) + } + + async publishToUser (execId, buffer, options = {}) { + await this._publishExec(execId, execUserSubject(execId), buffer, options) + } + + async cleanup (execId) { + const bridge = this.execBridges.get(execId) + if (!bridge) return + this._closeExecBridge(bridge) + this.execBridges.delete(execId) + } + + async enableForLogSession (session, cleanupCallback) { + const sessionId = session.sessionId + if (!sessionId) { + logger.warn('[NATS][RELAY] Missing sessionId for log session, skipping bridge enablement') + return false + } + + const bridge = this.logBridges.get(sessionId) || { + sessionId, + session: null, + subscription: null, + cleanupCallback: null, + backpressureNotified: false, + pendingBytes: 0, + pendingMessages: 0 + } + + bridge.session = session + if (cleanupCallback) { + bridge.cleanupCallback = cleanupCallback + } + + if (session.user) { + const nc = await this._connectionManager.getConnection() + await this._ensureLogUserSubscription(bridge, session.user, nc) + } + + this.logBridges.set(sessionId, bridge) + return true + } + + shouldUseRelayForLogs (sessionId) { + return this.logBridges.has(sessionId) + } + + async publishLogToUser (sessionId, buffer) { + const bridge = this.logBridges.get(sessionId) + if (!bridge) { + throw new Error(`Log bridge missing for sessionId=${sessionId}`) + } + + const messageType = decodeLogMessageType(buffer) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + + try { + await this._publish(logUserSubject(sessionId), buffer, {}, sessionId) + } catch (error) { + if (isLogLine) { + this._dropLogLineForPublishBackpressure(bridge, sessionId) + return + } + logger.error('[NATS][RELAY] Failed to publish log message', { + sessionId, + error: error.message + }) + throw error + } + } + + async cleanupLogSession (sessionId) { + const bridge = this.logBridges.get(sessionId) + if (!bridge) return + + if (bridge.subscription) { + try { + await bridge.subscription.drain() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to drain log subscription during cleanup', { + sessionId, + error: error.message + }) + } + bridge.subscription = null + } + + if (bridge.cleanupCallback) { + bridge.cleanupCallback = null + } + + this.logBridges.delete(sessionId) + } + + async shutdown () { + for (const execId of this.execBridges.keys()) { + await this.cleanup(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.cleanupLogSession(sessionId) + } + await this._connectionManager.shutdown() + } + + async _handleReconnect () { + for (const execId of this.execBridges.keys()) { + await this.rebindSessionBridges(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.rebindLogSessionBridges(sessionId) + } + } + + async rebindSessionBridges (execId) { + if (this.rebinding.has(execId)) return + this.rebinding.add(execId) + + try { + const bridge = this.execBridges.get(execId) + if (!bridge || !bridge.session) return + + logger.info('[NATS][RELAY] Rebinding exec session subscriptions after reconnect', { execId }) + + this._closeExecBridge(bridge) + bridge.subscriptions = {} + bridge.sockets = {} + + const nc = await this._connectionManager.getConnection() + const session = bridge.session + if (session.user) { + await this._ensureExecSubscription(bridge, 'user', session.user, nc) + } + if (session.agent) { + await this._ensureExecSubscription(bridge, 'agent', session.agent, nc) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(execId, { kind: 'exec' }) + } catch (error) { + logger.error('[NATS][RELAY] Exec recovery callback failed', { + execId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(execId) + } + } + + async rebindLogSessionBridges (sessionId) { + const key = `log:${sessionId}` + if (this.rebinding.has(key)) return + this.rebinding.add(key) + + try { + const bridge = this.logBridges.get(sessionId) + if (!bridge || !bridge.session) return + + logger.info('[NATS][RELAY] Rebinding log session subscription after reconnect', { sessionId }) + + if (bridge.subscription) { + try { + await bridge.subscription.drain() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to drain log subscription during rebind', { + sessionId, + error: error.message + }) + } + bridge.subscription = null + } + + const session = bridge.session + if (session.user) { + const nc = await this._connectionManager.getConnection() + await this._ensureLogUserSubscription(bridge, session.user, nc) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(sessionId, { kind: 'log' }) + } catch (error) { + logger.error('[NATS][RELAY] Log recovery callback failed', { + sessionId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(key) + } + } + + _closeExecBridge (bridge) { + for (const side of Object.keys(bridge.subscriptions || {})) { + const sub = bridge.subscriptions[side] + if (!sub) continue + try { + sub.unsubscribe() + } catch (error) { + logger.debug('[NATS][RELAY] Failed to unsubscribe exec side during cleanup', { + execId: bridge.execId, + side, + error: error.message + }) + } + } + bridge.subscriptions = {} + } + + async _ensureExecSubscription (bridge, side, socket, nc) { + if (!socket) return + + bridge.sockets[side] = socket + if (bridge.subscriptions[side]) { + return + } + + const subject = side === 'agent' ? execAgentSubject(bridge.execId) : execUserSubject(bridge.execId) + logger.info('[NATS][RELAY] Subscribing exec relay subject', { + execId: bridge.execId, + side, + subject + }) + + const sub = nc.subscribe(subject, { + callback: (err, msg) => { + if (err) { + logger.error('[NATS][RELAY] Exec subscription error', { + execId: bridge.execId, + side, + error: err.message + }) + return + } + this._handleExecMessage(bridge, side, msg).catch((error) => { + logger.error('[NATS][RELAY] Failed to handle exec relay message', { + execId: bridge.execId, + side, + error: error.message + }) + }) + } + }) + + bridge.subscriptions[side] = sub + } + + async _handleExecMessage (bridge, side, msg) { + const currentBridge = this.execBridges.get(bridge.execId) + if (!currentBridge) return + + const ws = currentBridge.sockets[side] + const body = getBufferFromData(msg.data) + const msgTypeHeader = msg.headers ? msg.headers.get(HEADER_MESSAGE_TYPE) : null + const msgType = msgTypeHeader != null && msgTypeHeader !== '' ? Number(msgTypeHeader) : null + + if (msgType === MESSAGE_TYPES.CLOSE) { + await this._handleCloseMessage({ + bridge: currentBridge, + session: currentBridge.session, + side, + ws, + msg, + body + }) + return + } + + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(body, { binary: true, compress: false, mask: false, fin: true }) + logger.debug('[NATS][RELAY] Delivered exec message to socket', { + execId: bridge.execId, + side, + messageSize: body.length + }) + } else { + logger.debug('[NATS][RELAY] No socket available for exec delivery', { + execId: bridge.execId, + side, + hasSocket: !!ws, + socketState: ws ? ws.readyState : 'N/A' + }) + } + } + + async _handleCloseMessage ({ bridge, session, side, ws, msg, body }) { + const execId = session.execId + const closeInitiator = side === 'user' ? 'agent' : 'user' + const closeAck = Boolean(msg.headers && msg.headers.get(HEADER_CLOSE_ACK) === 'true') + + logger.info('[NATS][RELAY] Received CLOSE message via relay', { + execId, + side, + closeInitiator, + closeAck + }) + + if (ws && ws.readyState === WebSocket.OPEN) { + try { + const reason = closeInitiator === 'agent' ? 'Agent closed connection' : 'User closed connection' + ws.close(1000, reason) + } catch (error) { + logger.warn('[NATS][RELAY] Failed to close WebSocket after CLOSE message', { + execId, + side, + error: error.message + }) + } + } + + if (!closeAck && this.execBridges.has(execId)) { + const ackSide = side === 'user' ? 'agent' : 'user' + try { + const hdrs = natsHeaders() + hdrs.set(HEADER_MESSAGE_TYPE, String(MESSAGE_TYPES.CLOSE)) + hdrs.set(HEADER_CLOSE_ACK, 'true') + const subject = ackSide === 'agent' ? execAgentSubject(execId) : execUserSubject(execId) + await this._publish(subject, body, { headers: hdrs }, execId) + } catch (error) { + logger.warn('[NATS][RELAY] Failed to send CLOSE acknowledgement', { + execId, + ackSide, + error: error.message + }) + } + } + + if (bridge && bridge.cleanupCallback) { + try { + await bridge.cleanupCallback(execId) + } catch (error) { + logger.error('[NATS][RELAY] Error in cleanup callback during CLOSE handling', { + execId, + error: error.message + }) + } + } + } + + async _ensureLogUserSubscription (bridge, userWs, nc) { + bridge.session.user = userWs + if (bridge.subscription) { + return + } + + const subject = logUserSubject(bridge.sessionId) + logger.info('[NATS][RELAY] Subscribing log relay subject', { + sessionId: bridge.sessionId, + subject + }) + + bridge.pendingBytes = 0 + bridge.pendingMessages = 0 + + const sub = nc.subscribe(subject, { + callback: (err, msg) => { + if (err) { + logger.error('[NATS][RELAY] Log subscription error', { + sessionId: bridge.sessionId, + error: err.message + }) + return + } + this._handleLogMessage(bridge, msg).catch((error) => { + logger.error('[NATS][RELAY] Failed to handle log relay message', { + sessionId: bridge.sessionId, + error: error.message + }) + }) + } + }) + + bridge.subscription = sub + } + + async _handleLogMessage (bridge, msg) { + const currentBridge = this.logBridges.get(bridge.sessionId) + if (!currentBridge) return + + const ws = currentBridge.session && currentBridge.session.user + const body = getBufferFromData(msg.data) + const messageType = decodeLogMessageType(body) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + + currentBridge.pendingBytes += body.length + currentBridge.pendingMessages += 1 + + const overPendingLimits = + currentBridge.pendingBytes > this.maxPendingBytes || + currentBridge.pendingMessages > this.maxPendingMessages + + if (isLogLine && overPendingLimits) { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + this._dropLogLineForReceiveBackpressure(currentBridge, bridge.sessionId) + return + } + + if (ws && ws.readyState === WebSocket.OPEN) { + if (isLogLine && ws.bufferedAmount > LOG_BACKPRESSURE_BUFFER_BYTES) { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + this._dropLogLineForReceiveBackpressure(currentBridge, bridge.sessionId) + return + } + + ws.send(body, { binary: true }) + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + } else { + currentBridge.pendingBytes = Math.max(0, currentBridge.pendingBytes - body.length) + currentBridge.pendingMessages = Math.max(0, currentBridge.pendingMessages - 1) + } + } + + _dropLogLineForReceiveBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + logger.warn('[NATS][RELAY] Dropping log line due to subscription backpressure', { sessionId }) + this._notifyLogBackpressure(bridge, sessionId) + } + } + + _dropLogLineForPublishBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + logger.warn('[NATS][RELAY] Dropping log line due to publish-side backpressure', { sessionId }) + this._notifyLogBackpressure(bridge, sessionId) + } + } + + _notifyLogBackpressure (bridge, sessionId) { + const user = bridge.session && bridge.session.user + if (user && user.readyState === WebSocket.OPEN) { + try { + const errorBody = msgpack.encode({ + type: LOG_MESSAGE_TYPES.LOG_ERROR, + data: Buffer.from('Log stream backpressure: dropping lines until client catches up\n'), + sessionId, + timestamp: Date.now() + }) + user.send(errorBody, { binary: true }) + } catch (error) { + logger.debug('[NATS][RELAY] Failed to notify user of log backpressure', { + sessionId, + error: error.message + }) + } + } + } + + async _publishExec (execId, subject, buffer, options = {}) { + const hdrs = natsHeaders() + const hasMessageType = Object.prototype.hasOwnProperty.call(options, 'messageType') + const messageType = hasMessageType ? options.messageType : null + if (messageType !== null && messageType !== undefined) { + hdrs.set(HEADER_MESSAGE_TYPE, String(messageType)) + } + + const applicationProperties = (options && options.applicationProperties) || {} + if (applicationProperties.closeAck) { + hdrs.set(HEADER_CLOSE_ACK, 'true') + } + + await this._publish(subject, buffer, { headers: hdrs }, execId) + } + + async _publish (subject, buffer, opts = {}, sessionKey = null) { + const nc = await this._connectionManager.getConnection() + nc.publish(subject, buffer, opts) + + const flushPromise = nc.flush() + const timeoutMs = this.publishTimeoutMs + let timer = null + try { + await Promise.race([ + flushPromise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new Error(`NATS publish not flushed within ${timeoutMs}ms`)) + }, timeoutMs) + }) + ]) + } catch (error) { + logger.error('[NATS][RELAY] Failed to publish message', { + subject, + sessionKey, + error: error.message + }) + throw error + } finally { + if (timer) clearTimeout(timer) + } + + logger.debug('[NATS][RELAY] Published relay message', { + subject, + sessionKey, + messageSize: buffer.length + }) + } +} + +const singleton = new NatsRelayTransportImpl() + +module.exports = singleton +module.exports.NatsRelayTransportImpl = NatsRelayTransportImpl +module.exports.execAgentSubject = execAgentSubject +module.exports.execUserSubject = execUserSubject +module.exports.logUserSubject = logUserSubject diff --git a/src/services/nats-relay-transport.js b/src/services/nats-relay-transport.js new file mode 100644 index 00000000..e7d47e51 --- /dev/null +++ b/src/services/nats-relay-transport.js @@ -0,0 +1,76 @@ +const WsRelayTransport = require('./ws-relay-transport') +const NatsRelayTransportImpl = require('./nats-relay-transport-impl') + +class NatsRelayTransport extends WsRelayTransport { + constructor (transportImpl = NatsRelayTransportImpl) { + super() + this._impl = transportImpl + this._recoveryCallbacks = [] + + this._impl.onRecovery((sessionId, meta) => { + for (const cb of this._recoveryCallbacks) { + try { + cb(sessionId, meta) + } catch (error) { + // Impl already logs; protect other callbacks. + } + } + }) + } + + getTransport () { + return 'nats' + } + + async isAvailable () { + return this._impl.isAvailable() + } + + async enableForSession (session, cleanupCb) { + return this._impl.enableForSession(session, cleanupCb) + } + + async enableForLogSession (session, cleanupCb) { + return this._impl.enableForLogSession(session, cleanupCb) + } + + async publishToAgent (execId, buffer, opts) { + return this._impl.publishToAgent(execId, buffer, opts) + } + + async publishToUser (execId, buffer, opts) { + return this._impl.publishToUser(execId, buffer, opts) + } + + async publishLogToUser (sessionId, buffer) { + return this._impl.publishLogToUser(sessionId, buffer) + } + + shouldUseRelay (execId) { + return this._impl.shouldUseRelay(execId) + } + + shouldUseRelayForLogs (sessionId) { + return this._impl.shouldUseRelayForLogs(sessionId) + } + + async cleanup (execId) { + return this._impl.cleanup(execId) + } + + async cleanupLogSession (sessionId) { + return this._impl.cleanupLogSession(sessionId) + } + + async shutdown () { + return this._impl.shutdown() + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this._recoveryCallbacks.push(cb) + } + } +} + +module.exports = NatsRelayTransport diff --git a/src/services/nats-service.js b/src/services/nats-service.js index c372c5a5..c33d88dd 100644 --- a/src/services/nats-service.js +++ b/src/services/nats-service.js @@ -336,6 +336,17 @@ async function _buildHubJwtBundle (transaction) { jwtBundle[`${account.publicKey}.jwt`] = account.jwt } } + if (config.getBoolean('nats.enabled', false)) { + const relayAccount = await NatsAccountManager.findOne({ + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + }, transaction) + if (relayAccount) { + jwtBundle[`${relayAccount.publicKey}.jwt`] = relayAccount.jwt + } + } return jwtBundle } @@ -1367,6 +1378,12 @@ async function _reconcileResolverArtifactsOnce (options = {}, transaction) { const reconcileTriggerOptions = { triggerReconcile: false } await NatsAuthServiceRuntime.ensureSystemAccount(transaction, reconcileTriggerOptions) + if (config.getBoolean('nats.enabled', false)) { + const hubInstance = (natsInstances || []).find((ni) => ni.isHub) + if (hubInstance) { + await NatsAuthServiceRuntime.ensureControllerNatsAccount(transaction, reconcileTriggerOptions) + } + } const systemAccount = await NatsAccountManager.findOne({ isSystem: true }, transaction) const systemNatsMicroservices = candidateFogUuids.length > 0 diff --git a/src/services/router-connection-manager.js b/src/services/router-connection-manager.js new file mode 100644 index 00000000..22d7aaf6 --- /dev/null +++ b/src/services/router-connection-manager.js @@ -0,0 +1,625 @@ +const rhea = require('rhea') +const config = require('../config') +const logger = require('../logger') +const Constants = require('../helpers/constants') +const { createDedupeHostList, aggregateConnectError } = require('../helpers/connect-endpoint-utils') +const RouterManager = require('../data/managers/router-manager') +const CertificateService = require('./certificate-service') +const SecretService = require('./secret-service') +const os = require('os') + +const CONTROLLER_CERT_PREFIX = 'controller-exec-session-client' +const hostname = process.env.HOSTNAME || os.hostname() +const CONTROLLER_CERT_NAME = hostname ? `${CONTROLLER_CERT_PREFIX}-${hostname}` : CONTROLLER_CERT_PREFIX + +const AMQP_DEFAULT_PORT = 5671 +const RELAY_CONTAINER_PREFIX = 'controller-relay' + +function hashSessionId (sessionId) { + let hash = 5381 + const value = String(sessionId) + for (let i = 0; i < value.length; i++) { + hash = ((hash << 5) + hash) ^ value.charCodeAt(i) + } + return Math.abs(hash >>> 0) +} + +function isCircularBufferOverflow (error) { + const msg = error && error.message ? error.message : String(error) + return /circular buffer overflow/i.test(msg) +} + +function isTlsError (error) { + if (!error) return false + const msg = error.message ? error.message : String(error) + return /tls|certificate|cert|ssl|handshake/i.test(msg) +} + +class PoolSlot { + constructor (manager, slotId) { + this.manager = manager + this.slotId = slotId + this.containerId = `${RELAY_CONTAINER_PREFIX}-${hostname || 'local'}-${slotId}` + this.container = rhea.create_container({ + id: this.containerId, + enable_sasl_external: true + }) + this.connection = null + this.connectionPromise = null + this.healthy = true + this.unsettledCount = 0 + } +} + +class RouterConnectionManager { + constructor () { + this.poolSize = config.get('server.webSocket.relay.amqp.poolSize', 8) + this.sendTimeoutMs = config.get('server.webSocket.relay.amqp.sendTimeoutMs', 5000) + this.unsettledWarnThreshold = config.get('server.webSocket.relay.amqp.unsettledWarnThreshold', 1800) + this.certificatePromise = null + this.cachedCertificate = null + this.cachedRouterRecord = null + this.fakeTransaction = { fakeTransaction: true } + this.slots = Array.from({ length: this.poolSize }, (_, slotId) => new PoolSlot(this, slotId)) + this.recoveryListeners = [] + this.saturationCount = 0 + this.shuttingDown = false + } + + slotIdForSession (sessionId) { + return hashSessionId(sessionId) % this.poolSize + } + + async acquire (sessionId) { + if (this.shuttingDown) { + throw new Error('Router connection pool is shutting down') + } + const slotId = this.slotIdForSession(sessionId) + return this._getSlotConnection(slotId) + } + + async getConnection () { + return this.acquire('__legacy__') + } + + getSlot (slotId) { + return this.slots[slotId] + } + + isConnected () { + return this.slots.some((slot) => slot.connection && slot.connection.is_open && slot.connection.is_open()) + } + + getHealthyPoolCount () { + return this.slots.filter((slot) => + slot.healthy && + slot.connection && + slot.connection.is_open && + slot.connection.is_open() + ).length + } + + getTotalUnsettled () { + return this.slots.reduce((sum, slot) => sum + (slot.unsettledCount || 0), 0) + } + + recordSessionSaturation () { + this.saturationCount += 1 + } + + getSaturationCount () { + return this.saturationCount + } + + onSlotRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryListeners.push(cb) + } + } + + _notifySlotRecovery (slotId) { + for (const listener of this.recoveryListeners) { + try { + listener(slotId) + } catch (error) { + logger.error('[AMQP] Slot recovery listener failed', { + slotId, + error: error.message + }) + } + } + } + + async isRouterAvailable () { + if (this.isConnected()) { + return true + } + try { + await this.acquire('__healthcheck__') + return this.isConnected() + } catch (error) { + return false + } + } + + async waitForSendable (sender, timeoutMs = this.sendTimeoutMs) { + if (!sender) { + throw new Error('AMQP sender is missing') + } + if (typeof sender.sendable === 'function' && sender.sendable()) { + return + } + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup() + reject(new Error(`AMQP sender not sendable within ${timeoutMs}ms`)) + }, timeoutMs) + + const onSendable = () => { + cleanup() + resolve() + } + + const onError = (context) => { + cleanup() + reject(context.error || new Error('AMQP sender error while waiting for sendable')) + } + + const cleanup = () => { + clearTimeout(timer) + sender.removeListener('sendable', onSendable) + sender.removeListener('error', onError) + } + + sender.once('sendable', onSendable) + sender.once('error', onError) + }) + } + + async markSlotUnhealthy (slotId, reason) { + const slot = this.slots[slotId] + if (!slot) return + + slot.healthy = false + this.recordSessionSaturation() + logger.warn('[AMQP] Marking router pool slot unhealthy', { + slotId, + containerId: slot.containerId, + reason: reason || 'unknown' + }) + + await this.reconnectSlot(slotId) + } + + async reconnectSlot (slotId) { + const slot = this.slots[slotId] + if (!slot || this.shuttingDown) return + + this._closeSlotConnection(slot) + + try { + await this._createSlotConnection(slot) + slot.healthy = true + logger.info('[AMQP] Router pool slot reconnected', { + slotId, + containerId: slot.containerId + }) + this._notifySlotRecovery(slotId) + } catch (error) { + slot.healthy = false + logger.error('[AMQP] Router pool slot reconnect failed', { + slotId, + containerId: slot.containerId, + error: error.message + }) + throw error + } + } + + handleSendError (sessionId, error) { + if (isCircularBufferOverflow(error)) { + const slotId = this.slotIdForSession(sessionId) + this.markSlotUnhealthy(slotId, error.message).catch((reconnectError) => { + logger.error('[AMQP] Failed to recover slot after overflow', { + slotId, + error: reconnectError.message + }) + }) + return true + } + return false + } + + updateUnsettledCount (slotId, count) { + const slot = this.slots[slotId] + if (!slot) return + slot.unsettledCount = count + if (count >= this.unsettledWarnThreshold) { + logger.warn('[AMQP] Router pool slot unsettled deliveries high', { + slotId, + unsettled: count, + threshold: this.unsettledWarnThreshold + }) + } + } + + async _getSlotConnection (slotId) { + const slot = this.slots[slotId] + if (!slot) { + throw new Error(`Invalid router pool slot: ${slotId}`) + } + + if (slot.connection && slot.connection.is_open && slot.connection.is_open() && slot.healthy) { + return slot.connection + } + + if (slot.connectionPromise) { + return slot.connectionPromise + } + + slot.connectionPromise = this._createSlotConnection(slot) + return slot.connectionPromise + } + + async _createSlotConnection (slot) { + try { + const { hosts, port } = await this._resolveRouterEndpoint() + const certBundle = await this._ensureControllerCertificate() + + const connectAttempts = [] + for (let attempt = 0; attempt < hosts.length; attempt++) { + const host = hosts[attempt] + const options = this._buildConnectOptions(host, port, certBundle, slot.containerId) + try { + const connection = await this._connectToHost(slot, host, port, options) + this.cachedCertificate = certBundle + logger.info({ + slotId: slot.slotId, + containerId: slot.containerId, + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length + }, '[AMQP] Router pool slot connection established') + return connection + } catch (error) { + const errorMessage = error.message || String(error) + connectAttempts.push({ host, error: errorMessage }) + logger.warn({ + slotId: slot.slotId, + host, + port, + attempt: attempt + 1, + totalAttempts: hosts.length, + error: errorMessage + }, '[AMQP] Router pool slot connect attempt failed') + } + } + + const aggregateError = aggregateConnectError( + 'Unable to connect router pool slot after all fallback hosts', + connectAttempts, + port + ) + logger.error({ + transport: 'amqp', + slotId: slot.slotId, + hosts, + port, + connectAttempts, + error: aggregateError.message + }, '[AMQP] Unable to connect router pool slot after all fallback hosts') + throw aggregateError + } catch (error) { + slot.connectionPromise = null + throw error + } + } + + _buildConnectOptions (host, port, certBundle, containerId) { + return { + transport: 'tls', + host, + hostname: host, + port, + rejectUnauthorized: true, + idle_time_out: 300000, + reconnect: false, + username: '', + password: '', + container_id: containerId, + cert: certBundle.cert, + key: certBundle.key, + ca: [certBundle.ca] + } + } + + _connectToHost (slot, host, port, options) { + return new Promise((resolve, reject) => { + const connection = slot.container.connect(options) + let settled = false + + const settle = (handler) => (context) => { + if (settled) return + settled = true + handler(context) + } + + const cleanupPromise = () => { + slot.connection = null + slot.connectionPromise = null + slot.healthy = false + } + + connection.once('connection_open', settle(() => { + slot.connection = connection + slot.connectionPromise = null + slot.healthy = true + + connection.on('connection_error', (context) => { + logger.error({ + err: context.error, + transport: 'amqp', + msg: '[AMQP] Pool slot connection error event', + slotId: slot.slotId, + host, + port + }) + if (isTlsError(context.error)) { + this.cachedCertificate = null + this.reconnectSlot(slot.slotId).catch((error) => { + logger.error('[AMQP] TLS reconnect failed for pool slot', { + slotId: slot.slotId, + error: error.message + }) + }) + } + }) + + connection.on('connection_close', () => { + logger.warn('[AMQP] Router pool slot connection closed', { + slotId: slot.slotId, + host, + port + }) + cleanupPromise() + }) + + connection.on('disconnected', (context) => { + logger.warn('[AMQP] Router pool slot disconnected', { + slotId: slot.slotId, + host, + port, + error: context.error ? context.error.message : 'unknown' + }) + cleanupPromise() + if (context.error && isTlsError(context.error)) { + this.cachedCertificate = null + } + }) + + resolve(connection) + })) + + connection.once('connection_close', settle((context) => { + reject(context.error || new Error('Router connection closed before opening')) + })) + + connection.once('disconnected', settle((context) => { + reject(context.error || new Error('Router disconnected during connect')) + })) + }) + } + + _closeSlotConnection (slot) { + if (!slot.connection) return + try { + slot.connection.removeAllListeners() + slot.connection.close() + } catch (error) { + logger.debug('[AMQP] Failed to close pool slot connection during reconnect', { + slotId: slot.slotId, + error: error.message + }) + } + slot.connection = null + slot.connectionPromise = null + } + + async shutdown () { + this.shuttingDown = true + for (const slot of this.slots) { + this._closeSlotConnection(slot) + } + logger.info('[AMQP] Router connection pool shut down', { poolSize: this.poolSize }) + } + + async _resolveRouterEndpoint () { + logger.debug({ msg: '[AMQP] Resolving default router endpoint' }) + try { + const router = await this._getDefaultRouterRecord() + const port = router.messagingPort || AMQP_DEFAULT_PORT + const hosts = this._buildRouterHostList(router) + const host = hosts[0] + logger.debug({ + msg: '[AMQP] Default router resolved', + routerHost: router.host, + hosts, + host, + port, + routerUuid: router.iofogUuid, + controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' + }) + return { + host, + hosts, + port, + routerUuid: router.iofogUuid + } + } catch (error) { + logger.error({ err: error, msg: '[AMQP] Failed while resolving router endpoint' }) + throw error + } + } + + _buildRouterHostList (router) { + if (this._isKubernetes()) { + return this._kubernetesRouterHosts(router) + } + return this._remoteRouterHosts(router) + } + + _kubernetesRouterHosts (router) { + const { hosts, addHost } = createDedupeHostList() + + const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') + if (namespace && namespace.trim().length > 0) { + addHost(`${Constants.DEFAULT_ROUTER_K8S_SERVICE}.${namespace.trim()}.svc.cluster.local`) + } + if (router.host) { + addHost(router.host) + } + if (hosts.length === 0) { + addHost(Constants.DEFAULT_ROUTER_K8S_SERVICE) + } + return hosts + } + + _remoteRouterHosts (router) { + const { hosts, addHost } = createDedupeHostList() + + addHost(Constants.ROUTER_BRIDGE_DNS_SAN) + + if (router.host) { + addHost(router.host) + } + + return hosts + } + + async _getDefaultRouterRecord () { + if (this.cachedRouterRecord) { + return this.cachedRouterRecord + } + const router = await RouterManager.findOne({ isDefault: true }, this.fakeTransaction) + if (!router) { + throw new Error('Default router not found. Please ensure default router is provisioned.') + } + this.cachedRouterRecord = router + return router + } + + _isKubernetes () { + const controlPlane = process.env.CONTROL_PLANE || config.get('app.ControlPlane') + return controlPlane && controlPlane.toLowerCase() === 'kubernetes' + } + + async _ensureControllerCertificate () { + if (this.cachedCertificate) { + return this.cachedCertificate + } + if (this.certificatePromise) { + return this.certificatePromise + } + this.certificatePromise = (async () => { + try { + const bundle = await this._createControllerCertificate() + this.cachedCertificate = bundle + return bundle + } finally { + this.certificatePromise = null + } + })() + return this.certificatePromise + } + + async _createControllerCertificate () { + logger.debug('[AMQP] Ensuring controller certificate secret exists', { name: CONTROLLER_CERT_NAME }) + await CertificateService.ensureRouterLocalCA(this.fakeTransaction) + const existingSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) + const caName = Constants.DEFAULT_ROUTER_LOCAL_CA + if (existingSecret) { + const caSecret = await this._safeGetSecret(caName) + const bundle = this._decodeCertificate(existingSecret, caSecret) + logger.debug({ msg: '[AMQP] Using existing controller-exec-session-client certificate', ca: caName }) + return bundle + } + + const hosts = this._buildControllerHosts() + logger.debug({ msg: '[AMQP] Generating controller-exec-session-client certificate', hosts, ca: caName }) + + try { + await CertificateService.createCertificateEndpoint({ + name: CONTROLLER_CERT_NAME, + subject: CONTROLLER_CERT_NAME, + hosts: hosts.join(','), + ca: { + type: 'direct', + secretName: caName + }, + expiration: 36 + }) + } catch (error) { + logger.error({ err: error, ca: caName, msg: '[AMQP] Failed to create controller certificate' }) + throw error + } + + const certSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) + const caSecret = await this._safeGetSecret(caName) + if (!certSecret || !caSecret) { + throw new Error('Controller certificate creation succeeded but secret not found') + } + logger.debug({ msg: '[AMQP] controller-exec-session-client certificate generated successfully', ca: caName }) + return this._decodeCertificate(certSecret, caSecret) + } + + _buildControllerHosts () { + const hosts = new Set(['localhost', '127.0.0.1']) + if (hostname) hosts.add(hostname) + if (this._isKubernetes() && (process.env.CONTROLLER_NAMESPACE != null && process.env.CONTROLLER_NAMESPACE !== '')) { + hosts.add(`controller.${process.env.CONTROLLER_NAMESPACE}.svc.cluster.local`) + } + if (process.env.CONTROLLER_HOST) hosts.add(process.env.CONTROLLER_HOST) + return Array.from(hosts) + } + + _decodeCertificate (certSecret, caSecret) { + if (!certSecret || !certSecret.data) { + throw new Error(`Secret ${CONTROLLER_CERT_NAME} is empty or missing.`) + } + if (!caSecret || !caSecret.data) { + throw new Error('CA secret not found for router connection.') + } + const decode = (value, label) => { + if (!value) { + throw new Error(`Missing ${label} in certificate secret`) + } + return Buffer.from(value, 'base64') + } + return { + cert: decode(certSecret.data['tls.crt'], 'tls.crt'), + key: decode(certSecret.data['tls.key'], 'tls.key'), + ca: decode(caSecret.data['tls.crt'], 'ca.crt') + } + } + + async _safeGetSecret (name) { + try { + return await SecretService.getSecretEndpoint(name) + } catch (error) { + if (error.name === 'NotFoundError') { + logger.debug('[AMQP] Secret not found', { secret: name }) + return null + } + logger.error('[AMQP] Unexpected error while fetching secret', { + secret: name, + error: error.message, + stack: error.stack + }) + throw error + } + } +} + +module.exports = new RouterConnectionManager() diff --git a/src/services/router-connection-service.js b/src/services/router-connection-service.js index 77f70412..d9b7376d 100644 --- a/src/services/router-connection-service.js +++ b/src/services/router-connection-service.js @@ -1,392 +1,4 @@ -const rhea = require('rhea') -const config = require('../config') -const logger = require('../logger') -const Constants = require('../helpers/constants') -const RouterManager = require('../data/managers/router-manager') -const CertificateService = require('./certificate-service') -const SecretService = require('./secret-service') -const os = require('os') - -const CONTROLLER_CERT_PREFIX = 'controller-exec-session-client' -const hostname = process.env.HOSTNAME || os.hostname() -const CONTROLLER_CERT_NAME = hostname ? `${CONTROLLER_CERT_PREFIX}-${hostname}` : CONTROLLER_CERT_PREFIX - -const DEFAULT_ROUTER_SERVICE = 'router' -const AMQP_DEFAULT_PORT = 5671 - -class RouterConnectionService { - constructor () { - this.connection = null - this.connectionPromise = null - this.certificatePromise = null - this.cachedCertificate = null - this.connectionOptions = null - this.cachedRouterRecord = null - this.fakeTransaction = { fakeTransaction: true } - this.container = rhea.create_container({ - id: 'controller-exec-session-client', - enable_sasl_external: true - }) - } - - async getConnection () { - if (this.connection && this.connection.is_open && this.connection.is_open()) { - return this.connection - } - if (this.connectionPromise) { - return this.connectionPromise - } - this.connectionPromise = this._createConnection() - return this.connectionPromise - } - - isConnected () { - return !!(this.connection && this.connection.is_open && this.connection.is_open()) - } - - async isRouterAvailable () { - if (this.isConnected()) { - return true - } - try { - await this.getConnection() - return this.isConnected() - } catch (error) { - return false - } - } - - async _createConnection () { - try { - logger.debug({ msg: '[AMQP] Preparing router connection options' }) - - const { hosts, port } = await this._resolveRouterEndpoint() - logger.debug({ msg: '[AMQP] Router endpoint resolved', hosts, port }) - const certBundle = await this._ensureControllerCertificate() - - let lastError = null - for (let attempt = 0; attempt < hosts.length; attempt++) { - const host = hosts[attempt] - const options = this._buildConnectOptions(host, port, certBundle) - try { - const connection = await this._connectToHost(host, port, options) - this.connectionOptions = { - transport: 'tls', - host, - hostname: host, - port, - rejectUnauthorized: true, - idle_time_out: 300000, - reconnect: true, - reconnect_limit: 100, - username: '', - password: '', - container_id: 'controller-exec-session-client' - } - this.cachedCertificate = certBundle - logger.info({ - msg: '[AMQP] Router connection established', - host, - port, - attempt: attempt + 1, - totalAttempts: hosts.length - }) - return connection - } catch (error) { - lastError = error - logger.warn({ - msg: '[AMQP] Router connect attempt failed', - host, - port, - attempt: attempt + 1, - totalAttempts: hosts.length, - err: error.message || String(error) - }) - } - } - - const aggregateError = lastError || new Error('No router hosts available for connection') - logger.error({ - err: aggregateError, - transport: 'amqp', - msg: '[AMQP] Unable to connect to router after all fallback hosts', - hosts, - port - }) - throw aggregateError - } catch (error) { - this.connectionPromise = null - logger.error('[AMQP] Failed to create router connection', { - error: error.message, - stack: error.stack - }) - throw error - } - } - - _buildConnectOptions (host, port, certBundle) { - logger.debug({ msg: '[AMQP] Router connection options built', host, port }) - return { - transport: 'tls', - host, - hostname: host, - port, - rejectUnauthorized: true, - idle_time_out: 300000, - reconnect: false, - username: '', - password: '', - container_id: 'controller-exec-session-client', - cert: certBundle.cert, - key: certBundle.key, - ca: [certBundle.ca] - } - } - - _connectToHost (host, port, options) { - return new Promise((resolve, reject) => { - const connection = this.container.connect(options) - let settled = false - - const settle = (handler) => (context) => { - if (settled) return - settled = true - handler(context) - } - - const cleanupPromise = () => { - this.connection = null - this.connectionPromise = null - } - - connection.once('connection_open', settle(() => { - this.connection = connection - this.connectionPromise = null - connection.on('connection_error', (context) => { - logger.error({ - err: context.error, - transport: 'amqp', - msg: '[AMQP] Connection error event', - host, - port - }) - }) - connection.on('connection_close', () => { - logger.warn('[AMQP] Router connection closed', { host, port }) - cleanupPromise() - }) - connection.on('disconnected', (context) => { - logger.warn('[AMQP] Router connection disconnected', { - host, - port, - error: context.error ? context.error.message : 'unknown' - }) - cleanupPromise() - }) - resolve(connection) - })) - - connection.once('connection_close', settle((context) => { - reject(context.error || new Error('Router connection closed before opening')) - })) - - connection.once('disconnected', settle((context) => { - reject(context.error || new Error('Router disconnected during connect')) - })) - }) - } - - async _resolveRouterEndpoint () { - logger.debug({ msg: '[AMQP] Resolving default router endpoint' }) - try { - const router = await this._getDefaultRouterRecord() - const port = router.messagingPort || AMQP_DEFAULT_PORT - const hosts = this._buildRouterHostList(router) - const host = hosts[0] - logger.debug({ - msg: '[AMQP] Default router resolved', - routerHost: router.host, - hosts, - host, - port, - routerUuid: router.iofogUuid, - controlPlane: this._isKubernetes() ? 'kubernetes' : 'remote' - }) - return { - host, - hosts, - port, - routerUuid: router.iofogUuid - } - } catch (error) { - logger.error({ err: error, msg: '[AMQP] Failed while resolving router endpoint' }) - throw error - } - } - - _buildRouterHostList (router) { - if (this._isKubernetes()) { - return [this._kubernetesRouterHost(router)] - } - return this._remoteRouterHosts(router) - } - - _kubernetesRouterHost (router) { - const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') - if (namespace && namespace.trim().length > 0) { - return `${DEFAULT_ROUTER_SERVICE}.${namespace.trim()}.svc.cluster.local` - } - const dbHost = router.host && router.host.trim().length > 0 ? router.host.trim() : '' - return dbHost || DEFAULT_ROUTER_SERVICE - } - - _remoteRouterHosts (router) { - const hosts = [] - const seen = new Set() - const addHost = (candidate) => { - if (!candidate) return - const trimmed = String(candidate).trim() - if (trimmed.length === 0 || seen.has(trimmed)) return - seen.add(trimmed) - hosts.push(trimmed) - } - - addHost(Constants.ROUTER_BRIDGE_DNS_SAN) - - if (router.host) { - addHost(router.host) - } - - const namespace = process.env.CONTROLLER_NAMESPACE || config.get('app.namespace') - if (namespace && namespace.trim().length > 0) { - addHost(`${DEFAULT_ROUTER_SERVICE}.${namespace.trim()}.svc.cluster.local`) - } - - return hosts - } - - async _getDefaultRouterRecord () { - if (this.cachedRouterRecord) { - return this.cachedRouterRecord - } - const router = await RouterManager.findOne({ isDefault: true }, this.fakeTransaction) - if (!router) { - throw new Error('Default router not found. Please ensure default router is provisioned.') - } - this.cachedRouterRecord = router - return router - } - - _isKubernetes () { - const controlPlane = process.env.CONTROL_PLANE || config.get('app.ControlPlane') - return controlPlane && controlPlane.toLowerCase() === 'kubernetes' - } - - async _ensureControllerCertificate () { - if (this.cachedCertificate) { - return this.cachedCertificate - } - if (this.certificatePromise) { - return this.certificatePromise - } - this.certificatePromise = (async () => { - try { - const bundle = await this._createControllerCertificate() - this.cachedCertificate = bundle - return bundle - } finally { - this.certificatePromise = null - } - })() - return this.certificatePromise - } - - async _createControllerCertificate () { - logger.debug('[AMQP] Ensuring controller certificate secret exists', { name: CONTROLLER_CERT_NAME }) - await CertificateService.ensureRouterLocalCA(this.fakeTransaction) - const existingSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) - const caName = Constants.DEFAULT_ROUTER_LOCAL_CA - if (existingSecret) { - const caSecret = await this._safeGetSecret(caName) - const bundle = this._decodeCertificate(existingSecret, caSecret) - logger.debug({ msg: '[AMQP] Using existing controller-exec-session-client certificate', ca: caName }) - return bundle - } - - const hosts = this._buildControllerHosts() - logger.debug({ msg: '[AMQP] Generating controller-exec-session-client certificate', hosts, ca: caName }) - - try { - await CertificateService.createCertificateEndpoint({ - name: CONTROLLER_CERT_NAME, - subject: CONTROLLER_CERT_NAME, - hosts: hosts.join(','), - ca: { - type: 'direct', - secretName: caName - }, - expiration: 36 // months - }) - } catch (error) { - logger.error({ err: error, ca: caName, msg: '[AMQP] Failed to create controller certificate' }) - throw error - } - - const certSecret = await this._safeGetSecret(CONTROLLER_CERT_NAME) - const caSecret = await this._safeGetSecret(caName) - if (!certSecret || !caSecret) { - throw new Error('Controller certificate creation succeeded but secret not found') - } - logger.debug({ msg: '[AMQP] controller-exec-session-client certificate generated successfully', ca: caName }) - return this._decodeCertificate(certSecret, caSecret) - } - - _buildControllerHosts () { - const hosts = new Set(['localhost', '127.0.0.1']) - if (hostname) hosts.add(hostname) - if (this._isKubernetes() && (process.env.CONTROLLER_NAMESPACE != null && process.env.CONTROLLER_NAMESPACE !== '')) { - hosts.add(`controller.${process.env.CONTROLLER_NAMESPACE}.svc.cluster.local`) - } - if (process.env.CONTROLLER_HOST) hosts.add(process.env.CONTROLLER_HOST) - return Array.from(hosts) - } - - _decodeCertificate (certSecret, caSecret) { - if (!certSecret || !certSecret.data) { - throw new Error(`Secret ${CONTROLLER_CERT_NAME} is empty or missing.`) - } - if (!caSecret || !caSecret.data) { - throw new Error('CA secret not found for router connection.') - } - const decode = (value, label) => { - if (!value) { - throw new Error(`Missing ${label} in certificate secret`) - } - return Buffer.from(value, 'base64') - } - return { - cert: decode(certSecret.data['tls.crt'], 'tls.crt'), - key: decode(certSecret.data['tls.key'], 'tls.key'), - ca: decode(caSecret.data['tls.crt'], 'ca.crt') - } - } - - async _safeGetSecret (name) { - try { - return await SecretService.getSecretEndpoint(name) - } catch (error) { - if (error.name === 'NotFoundError') { - logger.debug('[AMQP] Secret not found', { secret: name }) - return null - } - logger.error('[AMQP] Unexpected error while fetching secret', { - secret: name, - error: error.message, - stack: error.stack - }) - throw error - } - } -} - -module.exports = new RouterConnectionService() +/** + * Backward-compatible export — implementation is RouterConnectionManager (Plan 18-B pool). + */ +module.exports = require('./router-connection-manager') diff --git a/src/services/websocket-queue-service.js b/src/services/websocket-queue-service.js index 1052e31c..595c9124 100644 --- a/src/services/websocket-queue-service.js +++ b/src/services/websocket-queue-service.js @@ -1,12 +1,14 @@ const WebSocket = require('ws') const logger = require('../logger') -const RouterConnectionService = require('./router-connection-service') -const { recordAmqpPublishError } = require('../websocket/ws-metrics') +const RouterConnectionManager = require('./router-connection-manager') +const { + recordAmqpPublishError, + recordAmqpSessionSaturation +} = require('../websocket/ws-metrics') const msgpack = require('@msgpack/msgpack') -// Plan 16-C: drop LOG_LINE when user WS buffer exceeds threshold (see forwardLogToUser). const LOG_BACKPRESSURE_BUFFER_BYTES = 256 * 1024 -const LOG_MESSAGE_TYPES = { LOG_ERROR: 9 } +const LOG_MESSAGE_TYPES = { LOG_ERROR: 9, LOG_LINE: 6 } const MESSAGE_TYPES = { STDIN: 0, @@ -38,10 +40,49 @@ function getBufferFromBody (body) { return Buffer.from(body) } +function decodeLogMessageType (buffer) { + try { + const msg = msgpack.decode(buffer) + return typeof msg.type === 'number' ? msg.type : null + } catch (error) { + return null + } +} + class WebSocketQueueService { constructor () { this.execBridges = new Map() - this.logBridges = new Map() // New: for log sessions + this.logBridges = new Map() + this.recoveryCallbacks = [] + this.rebinding = new Set() + + RouterConnectionManager.onSlotRecovery((slotId) => { + this._handleSlotRecovery(slotId).catch((error) => { + logger.error('[AMQP][QUEUE] Slot recovery handling failed', { + slotId, + error: error.message + }) + }) + }) + } + + onRecovery (cb) { + if (typeof cb === 'function') { + this.recoveryCallbacks.push(cb) + } + } + + async _handleSlotRecovery (slotId) { + for (const [execId, bridge] of this.execBridges.entries()) { + if (bridge.slotId === slotId) { + await this.rebindSessionBridges(execId) + } + } + for (const [sessionId, bridge] of this.logBridges.entries()) { + if (bridge.slotId === slotId) { + await this.rebindLogSessionBridges(sessionId) + } + } } async enableForSession (session, cleanupCallback) { @@ -51,14 +92,19 @@ class WebSocketQueueService { return false } + const slotId = RouterConnectionManager.slotIdForSession(execId) const bridge = this.execBridges.get(execId) || { execId, + slotId, + session: null, senders: {}, receivers: {}, + linkRefs: {}, cleanupCallback: null } - // Store cleanup callback for CLOSE message handling + bridge.slotId = slotId + bridge.session = session if (cleanupCallback) { bridge.cleanupCallback = cleanupCallback } @@ -93,6 +139,47 @@ class WebSocketQueueService { this.execBridges.delete(execId) } + async rebindSessionBridges (execId) { + if (this.rebinding.has(execId)) return + this.rebinding.add(execId) + + try { + const bridge = this.execBridges.get(execId) + if (!bridge || !bridge.session) return + + logger.info('[AMQP][QUEUE] Rebinding exec session bridges after slot recovery', { + execId, + slotId: bridge.slotId + }) + + this._closeBridgeLinks(bridge, execId) + bridge.senders = {} + bridge.receivers = {} + bridge.linkRefs = {} + + const session = bridge.session + if (session.user) { + await this._ensureReceiver(bridge, 'user', session.user, session) + } + if (session.agent) { + await this._ensureReceiver(bridge, 'agent', session.agent, session) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(execId, { kind: 'exec', slotId: bridge.slotId }) + } catch (error) { + logger.error('[AMQP][QUEUE] Exec recovery callback failed', { + execId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(execId) + } + } + _closeBridgeLinks (bridge, sessionKey) { const closeLink = (linkWrapper) => { if (!linkWrapper) return @@ -116,18 +203,25 @@ class WebSocketQueueService { closeLink(bridge.receivers?.user) closeLink(bridge.senders?.agent) closeLink(bridge.senders?.user) + bridge.linkRefs = {} } _invalidateExecSender (bridge, side) { if (bridge.senders[side]) { bridge.senders[side] = null } + if (bridge.linkRefs) { + bridge.linkRefs[`sender:${side}`] = null + } } _invalidateExecReceiver (bridge, side) { if (bridge.receivers[side]) { bridge.receivers[side] = null } + if (bridge.linkRefs) { + bridge.linkRefs[`receiver:${side}`] = null + } } _attachSenderLifecycle (bridge, side, sender, execId) { @@ -169,11 +263,15 @@ class WebSocketQueueService { } async _send (execId, side, buffer, options = {}) { - const bridge = await this._ensureSender(execId, side) - if (!bridge) { - throw new Error('Queue bridge missing for execId=' + execId) - } + let bridge try { + bridge = await this._ensureSender(execId, side) + if (!bridge) { + throw new Error('Queue bridge missing for execId=' + execId) + } + + await RouterConnectionManager.waitForSendable(bridge.sender) + const message = { body: buffer, content_type: 'application/octet-stream' @@ -199,6 +297,7 @@ class WebSocketQueueService { } catch (error) { recordAmqpPublishError({ sessionType: 'exec', side }) logger.error('[AMQP][QUEUE] Failed to publish message', { execId, side, error: error.message }) + RouterConnectionManager.handleSendError(execId, error) const execBridge = this.execBridges.get(execId) if (execBridge) { this._invalidateExecSender(execBridge, side) @@ -219,7 +318,7 @@ class WebSocketQueueService { side === 'agent' ? MESSAGE_QUEUE_PREFIX.agent : MESSAGE_QUEUE_PREFIX.user, execId ) - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(execId) const sender = await new Promise((resolve, reject) => { const link = connection.open_sender({ target: { @@ -238,13 +337,13 @@ class WebSocketQueueService { this._attachSenderLifecycle(bridge, side, sender, execId) bridge.senders[side] = { sender } + bridge.linkRefs[`sender:${side}`] = sender return bridge.senders[side] } async _ensureReceiver (bridge, side, socket, session) { if (!socket) return if (bridge.receivers[side]) { - // Update socket reference if receiver already exists bridge.receivers[side].socket = socket logger.debug('[AMQP][QUEUE] Updated socket reference for existing receiver', { execId: session.execId, @@ -263,7 +362,7 @@ class WebSocketQueueService { side, queueName }) - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.execId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -290,7 +389,6 @@ class WebSocketQueueService { receiver.on('message', async (context) => { try { - // Always get the latest socket reference from the bridge const currentBridge = this.execBridges.get(session.execId) const ws = currentBridge && currentBridge.receivers[side] ? currentBridge.receivers[side].socket : null const body = getBufferFromBody(context.message.body) @@ -298,7 +396,6 @@ class WebSocketQueueService { ? context.message.application_properties.messageType : null - // Handle CLOSE messages (works for both user and agent sides) if (msgType === MESSAGE_TYPES.CLOSE) { await this._handleCloseMessage({ bridge: currentBridge, @@ -311,7 +408,6 @@ class WebSocketQueueService { return } - // Forward message to socket (normal message or non-CLOSE message) if (ws && ws.readyState === WebSocket.OPEN) { try { ws.send(body, { @@ -363,6 +459,7 @@ class WebSocketQueueService { }) bridge.receivers[side] = { receiver, socket } + bridge.linkRefs[`receiver:${side}`] = receiver logger.info('[AMQP][QUEUE] Receiver setup complete', { execId: session.execId, side, @@ -385,7 +482,6 @@ class WebSocketQueueService { closeAck }) - // Attempt to close the socket gracefully (if present) if (ws && ws.readyState === WebSocket.OPEN) { try { const reason = closeInitiator === 'agent' ? 'Agent closed connection' : 'User closed connection' @@ -432,7 +528,6 @@ class WebSocketQueueService { } } - // Invoke cleanup callback to remove session/queue resources if (bridge && bridge.cleanupCallback) { try { await bridge.cleanupCallback(execId) @@ -445,11 +540,6 @@ class WebSocketQueueService { } } - // ========== Log Session Queue Methods ========== - - // Enable queue bridge for log session (unidirectional: agent → user, one-to-one) - // Following exec session pattern: Use queues for ALL scenarios - // Each sessionId has its own queues (one-to-one, like exec sessions) async enableForLogSession (session, cleanupCallback) { const sessionId = session.sessionId if (!sessionId) { @@ -457,25 +547,30 @@ class WebSocketQueueService { return false } + const slotId = RouterConnectionManager.slotIdForSession(sessionId) const bridge = this.logBridges.get(sessionId) || { sessionId, + slotId, + session: null, agentSender: null, agentReceiver: null, - userReceiver: null, // Single user receiver (one-to-one) - userSender: null, // User queue sender - cleanupCallback: null + userReceiver: null, + userSender: null, + linkRefs: {}, + cleanupCallback: null, + backpressureNotified: false } + bridge.slotId = slotId + bridge.session = session if (cleanupCallback) { bridge.cleanupCallback = cleanupCallback } - // Agent side: single receiver (agent receives from queue) if (session.agent) { await this._ensureLogAgentReceiver(bridge, session.agent, session) } - // User side: single receiver (one-to-one, like exec sessions) if (session.user) { await this._ensureLogUserReceiver(bridge, session.user, session) } @@ -488,51 +583,46 @@ class WebSocketQueueService { return this.logBridges.has(sessionId) } - // Forward to user via queue (one-to-one, like exec sessions) - // Note: Exec sessions use queues for BOTH single and multi-replica - // We follow the same pattern for consistency - // Buffer is already MessagePack encoded from agent async publishLogToUser (sessionId, buffer, options = {}) { const bridge = this.logBridges.get(sessionId) if (!bridge) { throw new Error(`Log bridge missing for sessionId=${sessionId}`) } - // Ensure user queue sender exists + const messageType = decodeLogMessageType(buffer) + const isLogLine = messageType === LOG_MESSAGE_TYPES.LOG_LINE + if (!bridge.userSender) { - const userQueueName = `logs-user-${sessionId}` - const connection = await RouterConnectionService.getConnection() - const sender = await new Promise((resolve, reject) => { - const link = connection.open_sender({ - target: { - address: userQueueName, - durable: 0, - expiry_policy: 'link-detach' - }, - autosettle: true - }) + await this._ensureLogUserSender(sessionId) + } - link.once('sender_open', () => resolve(link)) - link.once('sender_close', reject) - link.once('error', reject) - }) - bridge.userSender = { sender } + if (isLogLine && bridge.userSender && bridge.userSender.sender) { + const sender = bridge.userSender.sender + if (typeof sender.sendable === 'function' && !sender.sendable()) { + return this._dropLogLineForPublishBackpressure(bridge, sessionId) + } } - // Buffer is already MessagePack encoded, send as binary const message = { - body: buffer, // MessagePack encoded buffer + body: buffer, content_type: 'application/octet-stream', application_properties: options.applicationProperties || {} } try { + if (bridge.userSender && bridge.userSender.sender) { + await RouterConnectionManager.waitForSendable(bridge.userSender.sender) + } bridge.userSender.sender.send(message) logger.debug('[AMQP][QUEUE] Published log message to user queue', { sessionId, messageSize: buffer.length }) } catch (error) { + if (isLogLine) { + RouterConnectionManager.handleSendError(sessionId, error) + return this._dropLogLineForPublishBackpressure(bridge, sessionId) + } recordAmqpPublishError({ sessionType: 'log', side: 'user' }) logger.error('[AMQP][QUEUE] Failed to publish log message to user queue', { sessionId, @@ -543,16 +633,64 @@ class WebSocketQueueService { } } - // Setup receiver for agent queue (one-to-one per sessionId) + _dropLogLineForPublishBackpressure (bridge, sessionId) { + if (!bridge.backpressureNotified) { + bridge.backpressureNotified = true + recordAmqpSessionSaturation() + logger.warn('[AMQP][QUEUE] Dropping log line due to publish-side backpressure', { sessionId }) + const session = bridge.session + const user = session && session.user + if (user && user.readyState === WebSocket.OPEN) { + try { + const errorBody = msgpack.encode({ + type: LOG_MESSAGE_TYPES.LOG_ERROR, + data: Buffer.from('Log stream backpressure: dropping lines until client catches up\n'), + sessionId, + timestamp: Date.now() + }) + user.send(errorBody, { binary: true }) + } catch (sendError) { + logger.debug('[AMQP][QUEUE] Failed to notify user of log publish backpressure', { + sessionId, + error: sendError.message + }) + } + } + } + } + + async _ensureLogUserSender (sessionId) { + const bridge = this.logBridges.get(sessionId) + if (!bridge || bridge.userSender) return + + const userQueueName = `logs-user-${sessionId}` + const connection = await RouterConnectionManager.acquire(sessionId) + const sender = await new Promise((resolve, reject) => { + const link = connection.open_sender({ + target: { + address: userQueueName, + durable: 0, + expiry_policy: 'link-detach' + }, + autosettle: true + }) + + link.once('sender_open', () => resolve(link)) + link.once('sender_close', reject) + link.once('error', reject) + }) + bridge.userSender = { sender } + bridge.linkRefs.userSender = sender + } + async _ensureLogAgentReceiver (bridge, agentWs, session) { if (bridge.agentReceiver) { bridge.agentReceiver.socket = agentWs return } - // Queue name per sessionId (one-to-one) const queueName = `logs-agent-${session.sessionId}` - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.sessionId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -574,8 +712,6 @@ class WebSocketQueueService { const ws = currentBridge && currentBridge.agentReceiver ? currentBridge.agentReceiver.socket : null const body = getBufferFromBody(context.message.body) - // Body is already MessagePack encoded from agent - // Forward directly to agent WebSocket (binary) if (ws && ws.readyState === WebSocket.OPEN) { ws.send(body, { binary: true }) context.delivery.accept() @@ -585,47 +721,17 @@ class WebSocketQueueService { }) bridge.agentReceiver = { receiver, socket: agentWs } + bridge.linkRefs.agentReceiver = receiver } - // Setup sender for agent queue (one-to-one per sessionId) - async _ensureLogAgentSender (sessionId) { - const bridge = this.logBridges.get(sessionId) - if (!bridge) return null - if (bridge.agentSender) return bridge.agentSender - - // Queue name per sessionId (one-to-one) - const queueName = `logs-agent-${sessionId}` - const connection = await RouterConnectionService.getConnection() - - const sender = await new Promise((resolve, reject) => { - const link = connection.open_sender({ - target: { - address: queueName, - durable: 0, - expiry_policy: 'link-detach' - }, - autosettle: true - }) - - link.once('sender_open', () => resolve(link)) - link.once('sender_close', reject) - link.once('error', reject) - }) - - bridge.agentSender = { sender } - return bridge.agentSender - } - - // Setup receiver for user queue (one-to-one, like exec session pattern) async _ensureLogUserReceiver (bridge, userWs, session) { if (bridge.userReceiver) { bridge.userReceiver.socket = userWs return } - // Queue name per sessionId (one-to-one) const queueName = `logs-user-${session.sessionId}` - const connection = await RouterConnectionService.getConnection() + const connection = await RouterConnectionManager.acquire(session.sessionId) const receiver = await new Promise((resolve, reject) => { const link = connection.open_receiver({ @@ -677,9 +783,71 @@ class WebSocketQueueService { }) bridge.userReceiver = { receiver, socket: userWs } + bridge.linkRefs.userReceiver = receiver + } + + async rebindLogSessionBridges (sessionId) { + if (this.rebinding.has(`log:${sessionId}`)) return + this.rebinding.add(`log:${sessionId}`) + + try { + const bridge = this.logBridges.get(sessionId) + if (!bridge || !bridge.session) return + + logger.info('[AMQP][QUEUE] Rebinding log session bridges after slot recovery', { + sessionId, + slotId: bridge.slotId + }) + + const closeLink = (linkWrapper) => { + if (!linkWrapper) return + try { + if (linkWrapper.receiver) { + linkWrapper.receiver.removeAllListeners() + linkWrapper.receiver.close() + } else if (linkWrapper.sender) { + linkWrapper.sender.removeAllListeners() + linkWrapper.sender.close() + } + } catch (error) { + logger.debug('[AMQP][QUEUE] Failed to close log link during rebind', { + sessionId, + error: error.message + }) + } + } + + closeLink(bridge.agentReceiver) + closeLink(bridge.userReceiver) + closeLink(bridge.userSender) + bridge.agentReceiver = null + bridge.userReceiver = null + bridge.userSender = null + bridge.linkRefs = {} + + const session = bridge.session + if (session.agent) { + await this._ensureLogAgentReceiver(bridge, session.agent, session) + } + if (session.user) { + await this._ensureLogUserReceiver(bridge, session.user, session) + } + + for (const cb of this.recoveryCallbacks) { + try { + cb(sessionId, { kind: 'log', slotId: bridge.slotId }) + } catch (error) { + logger.error('[AMQP][QUEUE] Log recovery callback failed', { + sessionId, + error: error.message + }) + } + } + } finally { + this.rebinding.delete(`log:${sessionId}`) + } } - // Cleanup log session (one-to-one) async cleanupLogSession (sessionId) { const bridge = this.logBridges.get(sessionId) if (!bridge) return @@ -710,6 +878,16 @@ class WebSocketQueueService { this.logBridges.delete(sessionId) } + + async shutdown () { + for (const execId of this.execBridges.keys()) { + await this.cleanup(execId) + } + for (const sessionId of this.logBridges.keys()) { + await this.cleanupLogSession(sessionId) + } + await RouterConnectionManager.shutdown() + } } module.exports = new WebSocketQueueService() diff --git a/src/services/ws-relay-transport-factory.js b/src/services/ws-relay-transport-factory.js new file mode 100644 index 00000000..578889af --- /dev/null +++ b/src/services/ws-relay-transport-factory.js @@ -0,0 +1,38 @@ +const config = require('../config') +const logger = require('../logger') +const AmqpRelayTransport = require('./amqp-relay-transport') +const NatsRelayTransport = require('./nats-relay-transport') + +let transportInstance = null +let resolvedTransportName = null + +/** + * Resolve singleton WsRelayTransport from nats.enabled (fixed at first call). + * @param {Object} [configOverride] - Optional config for tests. + * @returns {import('./ws-relay-transport')} + */ +function resolveTransport (configOverride) { + const cfg = configOverride || config + const useNats = cfg.getBoolean('nats.enabled', false) + const transportName = useNats ? 'nats' : 'amqp' + + if (!transportInstance || resolvedTransportName !== transportName) { + logger.info(`[RELAY] transport=${transportName}`) + transportInstance = useNats + ? new NatsRelayTransport() + : new AmqpRelayTransport() + resolvedTransportName = transportName + } + + return transportInstance +} + +function resetTransportForTests () { + transportInstance = null + resolvedTransportName = null +} + +module.exports = { + resolveTransport, + resetTransportForTests +} diff --git a/src/services/ws-relay-transport.js b/src/services/ws-relay-transport.js new file mode 100644 index 00000000..0c84aa15 --- /dev/null +++ b/src/services/ws-relay-transport.js @@ -0,0 +1,111 @@ +/** + * Cross-replica exec/log relay transport interface (Plan 18). + * Implementations: AmqpRelayTransport, NatsRelayTransport (18-D). + */ +class WsRelayTransport { + /** + * @returns {'amqp' | 'nats'} + */ + getTransport () { + throw new Error('WsRelayTransport.getTransport() not implemented') + } + + /** + * Lazy health check for session fail-fast. + * @returns {Promise} + */ + async isAvailable () { + throw new Error('WsRelayTransport.isAvailable() not implemented') + } + + /** + * @param {Object} session + * @param {Function} [cleanupCb] + * @returns {Promise} + */ + async enableForSession (session, cleanupCb) { + throw new Error('WsRelayTransport.enableForSession() not implemented') + } + + /** + * @param {Object} session + * @param {Function} [cleanupCb] + * @returns {Promise} + */ + async enableForLogSession (session, cleanupCb) { + throw new Error('WsRelayTransport.enableForLogSession() not implemented') + } + + /** + * @param {string} execId + * @param {Buffer} buffer + * @param {Object} [opts] + */ + async publishToAgent (execId, buffer, opts) { + throw new Error('WsRelayTransport.publishToAgent() not implemented') + } + + /** + * @param {string} execId + * @param {Buffer} buffer + * @param {Object} [opts] + */ + async publishToUser (execId, buffer, opts) { + throw new Error('WsRelayTransport.publishToUser() not implemented') + } + + /** + * @param {string} sessionId + * @param {Buffer} buffer + */ + async publishLogToUser (sessionId, buffer) { + throw new Error('WsRelayTransport.publishLogToUser() not implemented') + } + + /** + * @param {string} execId + * @returns {boolean} + */ + shouldUseRelay (execId) { + throw new Error('WsRelayTransport.shouldUseRelay() not implemented') + } + + /** + * @param {string} sessionId + * @returns {boolean} + */ + shouldUseRelayForLogs (sessionId) { + throw new Error('WsRelayTransport.shouldUseRelayForLogs() not implemented') + } + + /** + * @param {string} execId + */ + async cleanup (execId) { + throw new Error('WsRelayTransport.cleanup() not implemented') + } + + /** + * @param {string} sessionId + */ + async cleanupLogSession (sessionId) { + throw new Error('WsRelayTransport.cleanupLogSession() not implemented') + } + + /** + * Drain on process shutdown. + */ + async shutdown () { + // Default no-op; implementations may override. + } + + /** + * Optional callback registration for bridge rebind after reconnect (18-B / 18-D). + * @param {Function} _cb + */ + onRecovery (_cb) { + // Default no-op. + } +} + +module.exports = WsRelayTransport diff --git a/src/websocket/server.js b/src/websocket/server.js index 8dcd0477..88754bf2 100644 --- a/src/websocket/server.js +++ b/src/websocket/server.js @@ -12,8 +12,7 @@ const { microserviceState } = require('../enums/microservice-state') const AuthDecorator = require('../decorators/authorization-decorator') const TransactionDecorator = require('../decorators/transaction-decorator') const msgpack = require('@msgpack/msgpack') -const WebSocketQueueService = require('../services/websocket-queue-service') -const RouterConnectionService = require('../services/router-connection-service') +const { resolveTransport } = require('../services/ws-relay-transport-factory') const { recordExecSessionActive, recordLogSessionActive @@ -39,8 +38,8 @@ const MESSAGE_TYPES = { LOG_ERROR: 9 // Log streaming error } -const ROUTER_UNAVAILABLE_CLOSE_CODE = 1013 -const ROUTER_UNAVAILABLE_CLOSE_REASON = 'Router unavailable for cross-replica session' +const RELAY_UNAVAILABLE_CLOSE_CODE = 1013 +const RELAY_UNAVAILABLE_CLOSE_REASON = 'Relay unavailable for cross-replica session' const DRAIN_CLOSE_CODE = 1001 const DRAIN_CLOSE_REASON = 'Server draining' // when user WS bufferedAmount exceeds this, drop LOG_LINE silently and emit LOG_ERROR once. @@ -168,7 +167,23 @@ class WebSocketServer { this.logSessionManager = new LogSessionManager(config.get('server.webSocket')) this.execSessionManager = new ExecSessionManager(config.get('server.webSocket')) this.sessionConfig = config.get('server.webSocket.session') - this.queueService = WebSocketQueueService + this.relayTransport = resolveTransport() + this.relayTransport.onRecovery(async (sessionId, meta) => { + if (!meta || meta.kind !== 'exec') return + const session = this.execSessionManager.getExecSession(sessionId) + if (!session || !session.user || !session.agent) return + session.activationSent = false + try { + await TransactionDecorator.generateTransaction(async (tx) => { + await this.sendExecActivationToExecSession(session, sessionId, tx) + })() + } catch (error) { + logger.error('[RELAY] Failed to resend exec activation after relay recovery', { + sessionId, + error: error.message + }) + } + }) this.pendingCloseTimeouts = new Map() // Track pending CLOSE messages in cross-replica scenarios this.haConfig = config.get('server.webSocket.ha') || {} this.isDraining = false @@ -376,15 +391,17 @@ class WebSocketServer { return !!(session && (!session.agent || !session.user)) } - async requireRouterForCrossReplica (ws) { + async requireRelayForCrossReplica (ws) { if (this.haConfig.failFastOnRouterUnavailable === false) { return true } - const available = await RouterConnectionService.isRouterAvailable() + const available = await this.relayTransport.isAvailable() if (!available) { - logger.warn('[WS-HA] Router unavailable for cross-replica session') + logger.warn('[RELAY] Relay backend unavailable for cross-replica session', { + transport: this.relayTransport.getTransport() + }) if (ws && ws.readyState === WebSocket.OPEN) { - ws.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + ws.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } return false } @@ -1020,7 +1037,7 @@ class WebSocketServer { let session = this.execSessionManager.getExecSession(sessionId) if (!session) { - if (!(await this.requireRouterForCrossReplica(ws))) { + if (!(await this.requireRelayForCrossReplica(ws))) { return } session = this.execSessionManager.createExecSession( @@ -1106,12 +1123,12 @@ class WebSocketServer { closeTransaction ) - const queueEnabled = this.queueService.shouldUseQueue(sessionId) + const relayEnabled = this.relayTransport.shouldUseRelay(sessionId) if (!currentSession.user) { await this.cleanupExecSession(sessionId, closeTransaction) } else { - if (queueEnabled) { + if (relayEnabled) { try { const closeMsg = { type: MESSAGE_TYPES.CLOSE, @@ -1122,7 +1139,7 @@ class WebSocketServer { data: Buffer.from('Agent closed connection') } const encoded = this.encodeMessage(closeMsg) - await this.queueService.publishToUser(sessionId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) + await this.relayTransport.publishToUser(sessionId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) } catch (error) { logger.error('[WS-CLOSE] Failed to send CLOSE to user via queue after agent exec disconnect', { sessionId, @@ -1236,7 +1253,7 @@ class WebSocketServer { logger.info('[RELAY] Exec session activation sent to agent:' + JSON.stringify({ sessionId, microserviceUuid: session.microserviceUuid, - queueEnabled: this.queueService.shouldUseQueue(sessionId) + relayEnabled: this.relayTransport.shouldUseRelay(sessionId) })) } else { logger.error('[RELAY] Exec session activation to agent failed:' + JSON.stringify({ @@ -1465,6 +1482,10 @@ class WebSocketServer { execSessions: execSessionIds.length, logSessions: logSessionIds.length }) + + await this.relayTransport.shutdown().catch((error) => { + logger.warn('[WS-DRAIN] Relay transport shutdown failed', { error: error.message }) + }) })() return this.drainPromise @@ -1582,11 +1603,11 @@ class WebSocketServer { async sendMessageToAgent (agent, message, execId, microserviceUuid) { try { const encoded = this.encodeMessage(message) - const isQueueEnabled = this.queueService.shouldUseQueue(execId) + const relayEnabled = this.relayTransport.shouldUseRelay(execId) const messageType = typeof message.type === 'number' ? message.type : null - if (isQueueEnabled) { - await this.queueService.publishToAgent(execId, encoded, { messageType }) + if (relayEnabled) { + await this.relayTransport.publishToAgent(execId, encoded, { messageType }) logger.debug('[RELAY] Queued message for agent via AMQP:' + JSON.stringify({ execId, microserviceUuid, @@ -2103,7 +2124,7 @@ class WebSocketServer { let session = this.logSessionManager.getLogSession(sessionId) if (!session) { // Session might be on different replica, create it - if (!(await this.requireRouterForCrossReplica(ws))) { + if (!(await this.requireRelayForCrossReplica(ws))) { return } session = this.logSessionManager.createLogSession( @@ -2339,7 +2360,7 @@ class WebSocketServer { } // Enable queue bridge for cross-replica support (one-to-one, like exec sessions) - await this.queueService.enableForLogSession(session, (sessionId) => { + await this.relayTransport.enableForLogSession(session, (sessionId) => { this.cleanupLogSession(sessionId, transaction) }) @@ -2445,18 +2466,18 @@ class WebSocketServer { // Buffer is already MessagePack encoded from agent // Following exec session pattern: Use queue for ALL scenarios (single and multi-replica) // One-to-one forwarding (agent → user) via queue - const useQueue = this.queueService.shouldUseQueueForLogs(sessionId) + const useRelay = this.relayTransport.shouldUseRelayForLogs(sessionId) logger.debug('forwardLogToUser:' + JSON.stringify({ sessionId, - useQueue, + useRelay, hasUser: !!session.user, userState: session.user ? session.user.readyState : 'N/A', bufferLength: buffer.length })) - if (useQueue) { - // Publish MessagePack encoded buffer to user queue - await this.queueService.publishLogToUser(sessionId, buffer) + if (useRelay) { + // Publish MessagePack encoded buffer to user relay + await this.relayTransport.publishLogToUser(sessionId, buffer) } else { // Fallback: Direct WebSocket (only if queue not enabled) if (this._shouldDropLogLineForBackpressure(session, sessionId)) { @@ -2499,7 +2520,7 @@ class WebSocketServer { } this.logBackpressureNotified.delete(sessionId) await this.logSessionManager.removeLogSession(sessionId, transaction) - await this.queueService.cleanupLogSession(sessionId) + await this.relayTransport.cleanupLogSession(sessionId) } async setupExecMessageForwarding (sessionId, transaction) { @@ -2514,7 +2535,7 @@ class WebSocketServer { const wasQueueBridgeEnabled = session.queueBridgeEnabled try { - await this.queueService.enableForSession(session, async (closeExecId) => { + await this.relayTransport.enableForSession(session, async (closeExecId) => { const timeout = this.pendingCloseTimeouts.get(closeExecId) if (timeout) { clearTimeout(timeout) @@ -2524,9 +2545,10 @@ class WebSocketServer { }) session.queueBridgeEnabled = true if (!wasQueueBridgeEnabled) { - logger.info('[RELAY] AMQP queue bridge enabled for exec session', { + logger.info('[RELAY] Relay bridge enabled for exec session', { sessionId, - microserviceUuid: session.microserviceUuid + microserviceUuid: session.microserviceUuid, + transport: this.relayTransport.getTransport() }) } } catch (error) { @@ -2534,27 +2556,36 @@ class WebSocketServer { const requireQueue = this.isCrossReplicaSession(session) && this.haConfig.failFastOnRouterUnavailable !== false if (requireQueue && !wasQueueBridgeEnabled) { - logger.error('[RELAY] AMQP required for cross-replica exec session but router bridge failed', { + logger.error('[RELAY] Relay required for cross-replica exec session but bridge failed', { sessionId, + transport: this.relayTransport.getTransport(), error: error.message }) if (session.user && session.user.readyState === WebSocket.OPEN) { - session.user.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + session.user.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } if (session.agent && session.agent.readyState === WebSocket.OPEN) { - session.agent.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + session.agent.close(RELAY_UNAVAILABLE_CLOSE_CODE, RELAY_UNAVAILABLE_CLOSE_REASON) } await this.cleanupExecSession(sessionId, transaction) return } - logger.warn('[RELAY] Failed to enable AMQP queue bridge for exec session', { + logger.warn('[RELAY] Failed to enable relay bridge for exec session', { sessionId, + transport: this.relayTransport.getTransport(), error: error.message }) } if (user && agent) { - await this.sendExecActivationToExecSession(session, sessionId, transaction) + const activated = await this.sendExecActivationToExecSession(session, sessionId, transaction) + if (!activated) { + logger.error('[RELAY] Exec session activation failed; aborting message forwarding setup', { + sessionId, + microserviceUuid: session.microserviceUuid + }) + return + } } if (user) { @@ -2576,7 +2607,11 @@ class WebSocketServer { sessionId, timestamp: Date.now() } - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + const sent = await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + if (!sent && this.relayTransport.shouldUseRelay(execId)) { + logger.error('[RELAY] Exec relay publish failed; closing session', { sessionId: execId }) + await this.cleanupExecSession(execId, transaction) + } return } @@ -2591,8 +2626,8 @@ class WebSocketServer { if (msg.type === MESSAGE_TYPES.CLOSE) { await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { const timeout = setTimeout(async () => { const currentSession = this.execSessionManager.getExecSession(execId) if (currentSession && currentSession.user && currentSession.user.readyState === WebSocket.OPEN) { @@ -2635,7 +2670,11 @@ class WebSocketServer { } } - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + const sent = await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + if (!sent && this.relayTransport.shouldUseRelay(execId)) { + logger.error('[RELAY] Exec relay publish failed; closing session', { sessionId: execId }) + await this.cleanupExecSession(execId, transaction) + } } catch (error) { logger.error('[RELAY] Failed to process exec user message:' + JSON.stringify({ sessionId: execId, @@ -2657,10 +2696,10 @@ class WebSocketServer { const msg = this.decodeMessage(buffer) if (msg.type === MESSAGE_TYPES.CLOSE) { - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { try { - await this.queueService.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) + await this.relayTransport.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) } catch (error) { logger.error('[RELAY] Failed to enqueue exec CLOSE for user', { sessionId: execId, @@ -2674,9 +2713,17 @@ class WebSocketServer { return } - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { - await this.queueService.publishToUser(execId, buffer) + const relayEnabled = this.relayTransport.shouldUseRelay(execId) + if (relayEnabled) { + try { + await this.relayTransport.publishToUser(execId, buffer) + } catch (error) { + logger.error('[RELAY] Exec relay publish to user failed; closing session', { + sessionId: execId, + error: error.message + }) + await this.cleanupExecSession(execId, transaction) + } } else if (session.user && session.user.readyState === WebSocket.OPEN) { if (msg.type === MESSAGE_TYPES.STDOUT || msg.type === MESSAGE_TYPES.STDERR) { if (msg.data && msg.data.length > 0) { @@ -2744,7 +2791,7 @@ class WebSocketServer { } await this.execSessionManager.removeExecSession(sessionId, transaction) - await this.queueService.cleanup(sessionId) + await this.relayTransport.cleanup(sessionId) .catch(error => { logger.warn('[RELAY] Failed to cleanup exec queue bridge during session cleanup', { sessionId, diff --git a/src/websocket/ws-metrics.js b/src/websocket/ws-metrics.js index 0aa9f5f6..d1aa5822 100644 --- a/src/websocket/ws-metrics.js +++ b/src/websocket/ws-metrics.js @@ -9,7 +9,9 @@ let logSessionsActive = null let pendingPairings = null let pairingDurationMs = null let amqpPublishErrors = null -let routerConnected = null +let routerPoolConnections = null +let routerPoolUnsettled = null +let amqpSessionSaturated = null function getMeter () { if (!meter) { @@ -18,7 +20,7 @@ function getMeter () { return meter } -function initWsMetrics (routerConnectionService) { +function initWsMetrics (routerConnectionManager) { const m = getMeter() execSessionsActive = m.createUpDownCounter('ws_exec_sessions_active', { @@ -37,14 +39,29 @@ function initWsMetrics (routerConnectionService) { amqpPublishErrors = m.createCounter('ws_amqp_publish_errors', { description: 'AMQP publish failures for exec/log relay' }) + amqpSessionSaturated = m.createCounter('ws_amqp_session_saturated', { + description: 'AMQP relay saturation events (overflow / publish backpressure)' + }) + + if (routerConnectionManager) { + routerPoolConnections = m.createObservableGauge('ws_router_pool_connections', { + description: 'Healthy router AMQP pool connections on this replica' + }) + routerPoolConnections.addCallback((result) => { + const healthy = typeof routerConnectionManager.getHealthyPoolCount === 'function' + ? routerConnectionManager.getHealthyPoolCount() + : (routerConnectionManager.isConnected() ? 1 : 0) + result.observe(healthy) + }) - if (routerConnectionService) { - routerConnected = m.createObservableGauge('ws_router_connected', { - description: 'Router AMQP connection availability (1=connected, 0=disconnected)' + routerPoolUnsettled = m.createObservableGauge('ws_router_pool_unsettled', { + description: 'Total unsettled AMQP deliveries across router pool slots' }) - routerConnected.addCallback((result) => { - const connected = routerConnectionService.isConnected() ? 1 : 0 - result.observe(connected) + routerPoolUnsettled.addCallback((result) => { + const unsettled = typeof routerConnectionManager.getTotalUnsettled === 'function' + ? routerConnectionManager.getTotalUnsettled() + : 0 + result.observe(unsettled) }) } } @@ -71,11 +88,16 @@ function recordAmqpPublishError (attributes = {}) { amqpPublishErrors?.add(1, attributes) } +function recordAmqpSessionSaturation () { + amqpSessionSaturated?.add(1) +} + module.exports = { initWsMetrics, recordExecSessionActive, recordLogSessionActive, recordPendingPairing, recordPairingDurationMs, - recordAmqpPublishError + recordAmqpPublishError, + recordAmqpSessionSaturation } diff --git a/test/src/services/nats-api-service.test.js b/test/src/services/nats-api-service.test.js new file mode 100644 index 00000000..d56da614 --- /dev/null +++ b/test/src/services/nats-api-service.test.js @@ -0,0 +1,101 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const NatsApiService = require('../../../src/services/nats-api-service') +const NatsAuthService = require('../../../src/services/nats-auth-service') +const ApplicationManager = require('../../../src/data/managers/application-manager') +const NatsAccountManager = require('../../../src/data/managers/nats-account-manager') +const NatsUserManager = require('../../../src/data/managers/nats-user-manager') +const SecretService = require('../../../src/services/secret-service') + +describe('NatsApiService', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + const sampleCredsText = [ + '-----BEGIN NATS USER JWT-----', + 'test-jwt', + '------END NATS USER JWT------', + '' + ].join('\n') + + afterEach(() => $sandbox.restore()) + + describe('getUserCreds()', () => { + it('returns creds for platform controller NATS account', async () => { + const platformAccount = { + id: 42, + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false + } + const platformUser = { + id: 7, + accountId: platformAccount.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME, + credsSecretName: NatsAuthService.controllerNatsCredsSecretName() + } + + $sandbox.stub(NatsAccountManager, 'findOne').resolves(platformAccount) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + $sandbox.stub(NatsUserManager, 'findOne').resolves(platformUser) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: { + 'controller/controller.creds': sampleCredsText + } + }) + + const result = await NatsApiService.getUserCreds( + NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + NatsAuthService.CONTROLLER_NATS_USER_NAME, + transaction + ) + + expect(result.credsBase64).to.equal(Buffer.from(sampleCredsText, 'utf8').toString('base64')) + expect(ApplicationManager.findOne).to.not.have.been.called + }) + + it('returns creds for SYS account by account name', async () => { + const sysAccount = { + id: 1, + name: 'SYS', + applicationId: null, + isSystem: true, + isLeafSystem: false + } + const sysUser = { + id: 2, + accountId: sysAccount.id, + name: 'admin-hub', + credsSecretName: 'nats-creds-sys-admin-hub' + } + + $sandbox.stub(NatsAccountManager, 'findOne').resolves(sysAccount) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + $sandbox.stub(NatsUserManager, 'findOne').resolves(sysUser) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: { + 'sys/admin-hub.creds': sampleCredsText + } + }) + + const result = await NatsApiService.getUserCreds('SYS', 'admin-hub', transaction) + + expect(result.credsBase64).to.equal(Buffer.from(sampleCredsText, 'utf8').toString('base64')) + }) + + it('rejects unknown account names that are not applications or system accounts', async () => { + $sandbox.stub(NatsAccountManager, 'findOne').resolves(null) + $sandbox.stub(ApplicationManager, 'findOne').resolves(null) + + try { + await NatsApiService.getUserCreds('missing-account', 'some-user', transaction) + expect.fail('expected getUserCreds to fail') + } catch (error) { + expect(error.name).to.equal('NotFoundError') + expect(error.message).to.include('missing-account') + } + }) + }) +}) diff --git a/test/src/services/nats-auth-service.test.js b/test/src/services/nats-auth-service.test.js index 2642dee0..a665254f 100644 --- a/test/src/services/nats-auth-service.test.js +++ b/test/src/services/nats-auth-service.test.js @@ -7,11 +7,18 @@ const NatsUserManager = require('../../../src/data/managers/nats-user-manager') const NatsUserRuleManager = require('../../../src/data/managers/nats-user-rule-manager') const NatsAccountRuleManager = require('../../../src/data/managers/nats-account-rule-manager') const ApplicationManager = require('../../../src/data/managers/application-manager') +const NatsOperatorManager = require('../../../src/data/managers/nats-operator-manager') const SecretService = require('../../../src/services/secret-service') const NatsService = require('../../../src/services/nats-service') const NatsAuthService = require('../../../src/services/nats-auth-service') +const NatsSystemRules = require('../../../src/config/nats-system-rules') const { createOperator, createAccount } = require('@nats-io/nkeys') +function decodeJwtPayload (jwt) { + const parts = jwt.split('.') + return JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) +} + describe('NATS Auth Service', () => { def('sandbox', () => sinon.createSandbox()) const transaction = {} @@ -135,4 +142,144 @@ describe('NATS Auth Service', () => { }) }) }) + + describe('ensureControllerNatsAccount', () => { + const operatorKp = createOperator() + const operatorSeed = new TextDecoder().decode(operatorKp.getSeed()) + const operator = { + id: 1, + name: 'test-operator', + seedSecretName: 'nats-operator-seed', + publicKey: operatorKp.getPublicKey() + } + const relayAccountRule = { + id: 50, + name: NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME, + maxConnections: -1, + maxLeafNodeConnections: -1, + maxData: -1, + maxExports: -1, + maxImports: -1, + maxMsgPayload: -1, + maxSubscriptions: -1, + exportsAllowWildcards: true + } + const relayUserRule = { + id: 51, + name: NatsSystemRules.CONTROLLER_USER_RULE_NAME, + bearerToken: false, + allowedConnectionTypes: JSON.stringify(['STANDARD']), + maxData: -1, + maxSubscriptions: -1, + maxPayload: -1, + pubAllow: JSON.stringify([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]), + subAllow: JSON.stringify([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + } + let createdAccount + let createdUser + + beforeEach(() => { + createdAccount = null + createdUser = null + $sandbox.stub(NatsAccountRuleManager, 'updateOrCreate').resolves() + $sandbox.stub(NatsUserRuleManager, 'updateOrCreate').resolves() + $sandbox.stub(NatsService, 'enqueueReconcileTask').resolves() + $sandbox.stub(NatsOperatorManager, 'findOne').resolves(operator) + $sandbox.stub(SecretService, 'getSecretEndpoint').callsFake((secretName) => { + if (secretName === operator.seedSecretName) { + return Promise.resolve({ data: { seed: operatorSeed } }) + } + if (secretName && secretName.startsWith('nats-account-seed-')) { + const accountKp = createAccount() + return Promise.resolve({ data: { seed: new TextDecoder().decode(accountKp.getSeed()) } }) + } + return Promise.resolve(null) + }) + $sandbox.stub(SecretService, 'createSecretEndpoint').resolves() + $sandbox.stub(SecretService, 'updateSecretEndpointIfChanged').resolves() + $sandbox.stub(NatsAccountRuleManager, 'findOne').callsFake(({ name }) => { + if (name === NatsSystemRules.CONTROLLER_ACCOUNT_RULE_NAME) { + return Promise.resolve(relayAccountRule) + } + if (name === NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.SYSTEM_ACCOUNT_RULE_NAME }) + } + if (name === NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.APPLICATION_ACCOUNT_RULE_NAME }) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsUserRuleManager, 'findOne').callsFake(({ name }) => { + if (name === NatsSystemRules.CONTROLLER_USER_RULE_NAME) { + return Promise.resolve(relayUserRule) + } + if (name === NatsSystemRules.MICROSERVICE_USER_RULE_NAME) { + return Promise.resolve({ name: NatsSystemRules.MICROSERVICE_USER_RULE_NAME }) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsAccountManager, 'findOne').callsFake((query) => { + if (createdAccount && query.name === NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME) { + return Promise.resolve(createdAccount) + } + if (createdAccount && query.id === createdAccount.id) { + return Promise.resolve(createdAccount) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsAccountManager, 'create').callsFake((data) => { + createdAccount = { id: 100, ...data } + return Promise.resolve(createdAccount) + }) + $sandbox.stub(NatsUserManager, 'findOne').callsFake(({ accountId, name }) => { + if (createdUser && accountId === createdAccount.id && name === NatsAuthService.CONTROLLER_NATS_USER_NAME) { + return Promise.resolve(createdUser) + } + return Promise.resolve(null) + }) + $sandbox.stub(NatsUserManager, 'create').callsFake((data) => { + createdUser = { id: 200, ...data } + return Promise.resolve(createdUser) + }) + }) + + it('creates controller NATS account and user with scoped pub/sub JWT claims', async () => { + const result = await NatsAuthService.ensureControllerNatsAccount(transaction, { triggerReconcile: false }) + + expect(result.account.name).to.equal(NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME) + expect(result.user.name).to.equal(NatsAuthService.CONTROLLER_NATS_USER_NAME) + expect(result.user.credsSecretName).to.equal(NatsAuthService.controllerNatsCredsSecretName()) + + const payload = decodeJwtPayload(result.user.jwt) + expect(payload.nats.pub.allow).to.deep.equal([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + expect(payload.nats.sub.allow).to.deep.equal([NatsSystemRules.CONTROLLER_NATS_RELAY_SUBJECT_ALLOW]) + expect(NatsAccountManager.create).to.have.been.calledOnce + expect(NatsUserManager.create).to.have.been.calledOnce + }) + + it('is idempotent when account and user already exist', async () => { + createdAccount = { + id: 100, + name: NatsAuthService.CONTROLLER_NATS_ACCOUNT_NAME, + applicationId: null, + isSystem: false, + isLeafSystem: false, + seedSecretName: 'nats-account-seed-controller' + } + createdUser = { + id: 200, + accountId: createdAccount.id, + name: NatsAuthService.CONTROLLER_NATS_USER_NAME, + credsSecretName: NatsAuthService.controllerNatsCredsSecretName(), + jwt: 'existing.jwt.token' + } + + const result = await NatsAuthService.ensureControllerNatsAccount(transaction, { triggerReconcile: false }) + + expect(result.account).to.equal(createdAccount) + expect(result.user).to.equal(createdUser) + expect(NatsAccountManager.create).to.not.have.been.called + expect(NatsUserManager.create).to.not.have.been.called + }) + }) }) diff --git a/test/src/services/nats-relay-connection-manager.test.js b/test/src/services/nats-relay-connection-manager.test.js new file mode 100644 index 00000000..49913ba8 --- /dev/null +++ b/test/src/services/nats-relay-connection-manager.test.js @@ -0,0 +1,198 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const { + NatsRelayConnectionManager +} = require('../../../src/services/nats-relay-connection-manager') +const NatsInstanceManager = require('../../../src/data/managers/nats-instance-manager') +const NatsAccountManager = require('../../../src/data/managers/nats-account-manager') +const NatsUserManager = require('../../../src/data/managers/nats-user-manager') +const SecretService = require('../../../src/services/secret-service') +const NatsAuthService = require('../../../src/services/nats-auth-service') +const Constants = require('../../../src/helpers/constants') + +describe('NatsRelayConnectionManager', () => { + def('sandbox', () => sinon.createSandbox()) + + const hubRecord = { + host: 'hub.example.com', + serverPort: 4222 + } + + const sampleCredsText = [ + '-----BEGIN NATS USER JWT-----', + 'test-jwt', + '------END NATS USER JWT------', + '', + '-----BEGIN USER NKEY SEED-----', + 'SUATESTSEED', + '------END USER NKEY SEED------', + '' + ].join('\n') + + function relayCredsSecretData (credsText = sampleCredsText) { + return { + 'controller/controller.creds': credsText + } + } + + function createManager (overrides = {}) { + const cfg = { + get: sinon.stub().callsFake((key, defaultValue) => { + if (key === 'app.namespace') return 'pot-ns' + if (key === 'app.ControlPlane') return 'Remote' + return defaultValue + }), + getBoolean: sinon.stub().returns(false) + } + + return new NatsRelayConnectionManager({ + connectFn: overrides.connectFn || sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }), + config: cfg, + maxReconnectAttempts: 0 + }) + } + + afterEach(() => { + $sandbox.restore() + }) + + it('resolves remote hub hosts in fallback order', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + manager._connectFn = connectFn + + await manager.getConnection() + + expect(connectFn).to.have.been.calledOnce + const options = connectFn.firstCall.args[0] + expect(options.servers).to.equal(`nats://${Constants.NATS_BRIDGE_DNS_SAN}:4222`) + expect(NatsAuthService.ensureControllerNatsAccount).to.have.been.calledOnce + }) + + it('ensures controller NATS account before loading creds when hub exists', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + const ensureStub = $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + manager._connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + await manager.getConnection() + + expect(ensureStub).to.have.been.calledBefore(SecretService.getSecretEndpoint) + }) + + it('skips ensure when no hub record exists', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(null) + const ensureStub = $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + + await manager._ensureControllerNatsAccount() + + expect(ensureStub).to.not.have.been.called + }) + + it('resolves kubernetes hub service host first', async () => { + const manager = createManager() + manager._config.get = sinon.stub().callsFake((key, defaultValue) => { + if (key === 'app.namespace') return 'prod' + if (key === 'app.ControlPlane') return 'kubernetes' + return defaultValue + }) + + const hosts = manager._buildHubHostList(hubRecord) + expect(hosts[0]).to.equal('nats-server.prod.svc.cluster.local') + expect(hosts).to.deep.equal([ + 'nats-server.prod.svc.cluster.local', + 'hub.example.com' + ]) + }) + + it('resolves remote hub hosts without cluster.local fallback', () => { + const manager = createManager() + const hosts = manager._remoteHubHosts(hubRecord) + + expect(hosts).to.deep.equal([ + Constants.NATS_BRIDGE_DNS_SAN, + hubRecord.host + ]) + }) + + it('uses hub serverPort for all remote connect attempts', async () => { + const manager = createManager() + const customPort = 14222 + $sandbox.stub(NatsInstanceManager, 'findOne').resolves({ + ...hubRecord, + serverPort: customPort + }) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const connectFn = sinon.stub().resolves({ isClosed: () => false, status: async function * () {} }) + manager._connectFn = connectFn + + await manager.getConnection() + + expect(connectFn.firstCall.args[0].servers).to.equal(`nats://${Constants.NATS_BRIDGE_DNS_SAN}:${customPort}`) + }) + + it('throws aggregate error listing all connect attempts', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + manager._connectFn = sinon.stub().rejects(new Error('ECONNREFUSED')) + + try { + await manager.getConnection() + expect.fail('expected connect to fail') + } catch (error) { + expect(error.message).to.include(Constants.NATS_BRIDGE_DNS_SAN) + expect(error.message).to.include(hubRecord.host) + expect(error.message).to.include('ECONNREFUSED') + } + }) + + it('loads opaque creds secret as plain UTF-8 text', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(hubRecord) + $sandbox.stub(NatsAuthService, 'ensureControllerNatsAccount').resolves() + $sandbox.stub(NatsAccountManager, 'findOne').resolves({ id: 1 }) + $sandbox.stub(NatsUserManager, 'findOne').resolves({ credsSecretName: 'nats-creds-controller-controller' }) + $sandbox.stub(SecretService, 'getSecretEndpoint').resolves({ + data: relayCredsSecretData() + }) + + const creds = await manager._loadControllerRelayCreds() + expect(Buffer.from(creds).toString('utf8')).to.equal(sampleCredsText) + }) + + it('reports unavailable when hub record is missing', async () => { + const manager = createManager() + $sandbox.stub(NatsInstanceManager, 'findOne').resolves(null) + + const available = await manager.isAvailable() + expect(available).to.equal(false) + }) +}) diff --git a/test/src/services/nats-relay-transport.test.js b/test/src/services/nats-relay-transport.test.js new file mode 100644 index 00000000..2c872ad1 --- /dev/null +++ b/test/src/services/nats-relay-transport.test.js @@ -0,0 +1,217 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const msgpack = require('@msgpack/msgpack') +const WebSocket = require('ws') + +const { + NatsRelayTransportImpl, + execAgentSubject, + execUserSubject, + logUserSubject +} = require('../../../src/services/nats-relay-transport-impl') + +function createMockNatsConnection () { + const subscriptions = new Map() + + return { + subscriptions, + publish (subject, data, opts = {}) { + const handlers = subscriptions.get(subject) || [] + for (const handler of handlers) { + handler(null, { + subject, + data, + headers: opts.headers + }) + } + }, + subscribe (subject, opts = {}) { + if (!subscriptions.has(subject)) { + subscriptions.set(subject, []) + } + subscriptions.get(subject).push(opts.callback) + return { + unsubscribe () { + const list = subscriptions.get(subject) || [] + subscriptions.set(subject, list.filter((fn) => fn !== opts.callback)) + }, + drain: async () => { + const list = subscriptions.get(subject) || [] + subscriptions.set(subject, list.filter((fn) => fn !== opts.callback)) + } + } + }, + flush: async () => {}, + drain: async () => {}, + close: async () => {}, + isClosed: () => false + } +} + +function createMockConnectionManager (nc) { + const listeners = [] + return { + onReconnect (cb) { + listeners.push(cb) + }, + async getConnection () { + return nc + }, + async isAvailable () { + return true + }, + async shutdown () {}, + triggerReconnect () { + return Promise.all(listeners.map((cb) => Promise.resolve(cb()))) + } + } +} + +function createMockWebSocket () { + const ws = { + readyState: WebSocket.OPEN, + bufferedAmount: 0, + _sent: [], + send (data) { + ws._sent.push(Buffer.from(data)) + }, + close: sinon.stub() + } + return ws +} + +describe('NatsRelayTransportImpl', () => { + def('sandbox', () => sinon.createSandbox()) + + let nc + let connectionManager + let transport + const execId = 'exec-session-1' + const logSessionId = 'log-session-1' + + beforeEach(() => { + nc = createMockNatsConnection() + connectionManager = createMockConnectionManager(nc) + transport = new NatsRelayTransportImpl(connectionManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.maxPendingBytes') return 33554432 + if (key === 'server.webSocket.relay.nats.maxPendingMessages') return 8192 + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 50 + return defaultValue + } + }) + }) + + afterEach(() => { + $sandbox.restore() + }) + + it('relays exec stdin from user publish to agent subscription', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + + await transport.enableForSession({ execId, user: userWs }) + await transport.enableForSession({ execId, agent: agentWs }) + + const frame = Buffer.from('hello-from-user') + await transport.publishToAgent(execId, frame) + + expect(agentWs._sent).to.have.length(1) + expect(agentWs._sent[0].toString()).to.equal('hello-from-user') + }) + + it('relays exec stdout from agent publish to user subscription', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + + await transport.enableForSession({ execId, user: userWs }) + await transport.enableForSession({ execId, agent: agentWs }) + + const frame = Buffer.from('hello-from-agent') + await transport.publishToUser(execId, frame) + + expect(userWs._sent).to.have.length(1) + expect(userWs._sent[0].toString()).to.equal('hello-from-agent') + }) + + it('forwards log lines to user websocket by sessionId', async () => { + const userWs = createMockWebSocket() + + await transport.enableForLogSession({ sessionId: logSessionId, user: userWs }) + + const line = msgpack.encode({ + type: 6, + data: Buffer.from('line-1\n'), + sessionId: logSessionId + }) + + nc.publish(logUserSubject(logSessionId), line) + + expect(userWs._sent).to.have.length(1) + expect(msgpack.decode(userWs._sent[0]).data.toString()).to.equal('line-1\n') + }) + + it('drops log lines and emits LOG_ERROR when pending limits exceeded', async () => { + const userWs = createMockWebSocket() + const limitedTransport = new NatsRelayTransportImpl(connectionManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.maxPendingBytes') return 32 + if (key === 'server.webSocket.relay.nats.maxPendingMessages') return 1 + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 50 + return defaultValue + } + }) + + await limitedTransport.enableForLogSession({ sessionId: logSessionId, user: userWs }) + const bridge = limitedTransport.logBridges.get(logSessionId) + bridge.pendingMessages = 1 + bridge.pendingBytes = 32 + + const line = msgpack.encode({ + type: 6, + data: Buffer.from('overflow-line\n'), + sessionId: logSessionId + }) + + nc.publish(logUserSubject(logSessionId), line) + + expect(userWs._sent).to.have.length(1) + expect(msgpack.decode(userWs._sent[0]).type).to.equal(9) + }) + + it('re-subscribes active exec sessions after reconnect', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const recovery = sinon.stub() + + transport.onRecovery(recovery) + await transport.enableForSession({ execId, user: userWs, agent: agentWs }) + + expect(nc.subscriptions.get(execUserSubject(execId))).to.have.length(1) + expect(nc.subscriptions.get(execAgentSubject(execId))).to.have.length(1) + + nc.subscriptions.clear() + await connectionManager.triggerReconnect() + + expect(nc.subscriptions.get(execUserSubject(execId))).to.have.length(1) + expect(nc.subscriptions.get(execAgentSubject(execId))).to.have.length(1) + expect(recovery).to.have.been.calledOnceWith(execId, { kind: 'exec' }) + }) + + it('fails exec publish when flush exceeds timeout', async () => { + const slowNc = createMockNatsConnection() + slowNc.flush = () => new Promise(() => {}) + const slowManager = createMockConnectionManager(slowNc) + const slowTransport = new NatsRelayTransportImpl(slowManager, { + get: (key, defaultValue) => { + if (key === 'server.webSocket.relay.nats.publishTimeoutMs') return 20 + return defaultValue + } + }) + + await slowTransport.enableForSession({ execId, user: createMockWebSocket() }) + + await expect(slowTransport.publishToAgent(execId, Buffer.from('x'))) + .to.be.rejectedWith(/not flushed within 20ms/) + }) +}) diff --git a/test/src/services/router-connection-service.test.js b/test/src/services/router-connection-service.test.js index 24b64fc3..bdaf9b66 100644 --- a/test/src/services/router-connection-service.test.js +++ b/test/src/services/router-connection-service.test.js @@ -6,9 +6,9 @@ const config = require('../../../src/config') const RouterManager = require('../../../src/data/managers/router-manager') const CertificateService = require('../../../src/services/certificate-service') const SecretService = require('../../../src/services/secret-service') -const RouterConnectionService = require('../../../src/services/router-connection-service') +const RouterConnectionManager = require('../../../src/services/router-connection-manager') -describe('Router Connection Service', () => { +describe('Router Connection Manager', () => { def('sandbox', () => sinon.createSandbox()) const defaultRouter = { @@ -21,12 +21,17 @@ describe('Router Connection Service', () => { const originalNamespace = process.env.CONTROLLER_NAMESPACE beforeEach(() => { - RouterConnectionService.connection = null - RouterConnectionService.connectionPromise = null - RouterConnectionService.cachedRouterRecord = null - RouterConnectionService.cachedCertificate = null - RouterConnectionService.connectionOptions = null - RouterConnectionService.certificatePromise = null + for (const slot of RouterConnectionManager.slots) { + slot.connection = null + slot.connectionPromise = null + slot.healthy = true + slot.unsettledCount = 0 + } + RouterConnectionManager.cachedRouterRecord = null + RouterConnectionManager.cachedCertificate = null + RouterConnectionManager.certificatePromise = null + RouterConnectionManager.shuttingDown = false + RouterConnectionManager.saturationCount = 0 $sandbox.stub(RouterManager, 'findOne').resolves(defaultRouter) }) @@ -47,33 +52,53 @@ describe('Router Connection Service', () => { } }) + describe('pool acquire — sticky assignment', () => { + it('maps the same sessionId to the same slot', () => { + const sessionId = 'exec-session-sticky-abc' + const first = RouterConnectionManager.slotIdForSession(sessionId) + const second = RouterConnectionManager.slotIdForSession(sessionId) + expect(first).to.equal(second) + expect(first).to.be.at.least(0).and.below(RouterConnectionManager.poolSize) + }) + + it('distributes different sessionIds across the pool range', () => { + const slots = new Set() + for (let i = 0; i < 64; i++) { + slots.add(RouterConnectionManager.slotIdForSession(`session-${i}`)) + } + expect(slots.size).to.be.at.least(2) + }) + }) + describe('_resolveRouterEndpoint() — Kubernetes control plane', () => { beforeEach(() => { process.env.CONTROL_PLANE = 'kubernetes' process.env.CONTROLLER_NAMESPACE = 'iofog' }) - it('returns a single cluster.local service host', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('returns cluster service host then DB host fallback', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() - expect(result.hosts).to.deep.equal(['router.iofog.svc.cluster.local']) + expect(result.hosts).to.deep.equal([ + 'router.iofog.svc.cluster.local', + defaultRouter.host + ]) expect(result.host).to.equal('router.iofog.svc.cluster.local') expect(result.port).to.equal(5671) expect(result.routerUuid).to.equal(defaultRouter.iofogUuid) }) - it('does not include bridge DNS or DB host in the fallback list', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('does not include bridge DNS in the fallback list', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.not.include(Constants.ROUTER_BRIDGE_DNS_SAN) - expect(result.hosts).to.not.include(defaultRouter.host) }) it('falls back to DB host when namespace is unset', async () => { delete process.env.CONTROLLER_NAMESPACE $sandbox.stub(config, 'get').withArgs('app.namespace').returns('') - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([defaultRouter.host]) }) @@ -86,7 +111,7 @@ describe('Router Connection Service', () => { host: '' }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal(['router']) }) @@ -98,13 +123,12 @@ describe('Router Connection Service', () => { process.env.CONTROLLER_NAMESPACE = 'edge-ns' }) - it('returns ordered fallback hosts: bridge DNS, DB host, cluster.local', async () => { - const result = await RouterConnectionService._resolveRouterEndpoint() + it('returns ordered fallback hosts: bridge DNS then DB host', async () => { + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ Constants.ROUTER_BRIDGE_DNS_SAN, - defaultRouter.host, - 'router.edge-ns.svc.cluster.local' + defaultRouter.host ]) expect(result.host).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) expect(result.port).to.equal(5671) @@ -116,11 +140,10 @@ describe('Router Connection Service', () => { host: Constants.ROUTER_BRIDGE_DNS_SAN }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ - Constants.ROUTER_BRIDGE_DNS_SAN, - 'router.edge-ns.svc.cluster.local' + Constants.ROUTER_BRIDGE_DNS_SAN ]) }) @@ -128,7 +151,7 @@ describe('Router Connection Service', () => { delete process.env.CONTROLLER_NAMESPACE $sandbox.stub(config, 'get').withArgs('app.namespace').returns('') - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.hosts).to.deep.equal([ Constants.ROUTER_BRIDGE_DNS_SAN, @@ -142,7 +165,7 @@ describe('Router Connection Service', () => { messagingPort: null }) - const result = await RouterConnectionService._resolveRouterEndpoint() + const result = await RouterConnectionManager._resolveRouterEndpoint() expect(result.port).to.equal(5671) }) @@ -172,7 +195,7 @@ describe('Router Connection Service', () => { }) it('ensures default-router-local-ca before creating the exec client certificate', async () => { - await RouterConnectionService._createControllerCertificate() + await RouterConnectionManager._createControllerCertificate() expect(CertificateService.ensureRouterLocalCA).to.have.been.calledOnce expect(CertificateService.createCertificateEndpoint).to.have.been.calledOnce @@ -183,7 +206,7 @@ describe('Router Connection Service', () => { }) }) - describe('_createConnection() — Remote connect fallback', () => { + describe('_createSlotConnection() — Remote connect fallback', () => { const certBundle = { cert: Buffer.from('cert'), key: Buffer.from('key'), @@ -193,44 +216,105 @@ describe('Router Connection Service', () => { def('mockConnection', () => ({ is_open: () => true })) beforeEach(() => { - $sandbox.stub(RouterConnectionService, '_ensureControllerCertificate').resolves(certBundle) + $sandbox.stub(RouterConnectionManager, '_ensureControllerCertificate').resolves(certBundle) }) it('tries hosts in order until one connects', async () => { - const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host, 'router.edge-ns.svc.cluster.local'] - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - const connectStub = $sandbox.stub(RouterConnectionService, '_connectToHost') + const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host] + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + const connectStub = $sandbox.stub(RouterConnectionManager, '_connectToHost') .onCall(0).rejects(new Error('ECONNREFUSED')) .onCall(1).resolves($mockConnection) - const connection = await RouterConnectionService._createConnection() + const slot = RouterConnectionManager.slots[0] + const connection = await RouterConnectionManager._createSlotConnection(slot) expect(connection).to.equal($mockConnection) expect(connectStub).to.have.been.calledTwice - expect(connectStub.firstCall.args[0]).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) - expect(connectStub.secondCall.args[0]).to.equal(defaultRouter.host) + expect(connectStub.firstCall.args[1]).to.equal(Constants.ROUTER_BRIDGE_DNS_SAN) + expect(connectStub.secondCall.args[1]).to.equal(defaultRouter.host) }) - it('throws after all hosts fail', async () => { + it('throws aggregate error after all hosts fail', async () => { const hosts = [Constants.ROUTER_BRIDGE_DNS_SAN, defaultRouter.host] - const lastError = new Error('all hosts down') - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - $sandbox.stub(RouterConnectionService, '_connectToHost').rejects(lastError) - - await expect(RouterConnectionService._createConnection()).to.be.rejectedWith('all hosts down') - expect(RouterConnectionService._connectToHost).to.have.been.calledTwice + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + $sandbox.stub(RouterConnectionManager, '_connectToHost') + .onCall(0).rejects(new Error('bridge down')) + .onCall(1).rejects(new Error('db host down')) + + const slot = RouterConnectionManager.slots[0] + try { + await RouterConnectionManager._createSlotConnection(slot) + expect.fail('expected connect to fail') + } catch (error) { + expect(error.message).to.include(Constants.ROUTER_BRIDGE_DNS_SAN) + expect(error.message).to.include(defaultRouter.host) + expect(error.message).to.include('bridge down') + expect(error.message).to.include('db host down') + } + expect(RouterConnectionManager._connectToHost).to.have.been.calledTwice }) - it('connects on first host for Kubernetes single-host list', async () => { - const hosts = ['router.iofog.svc.cluster.local'] - $sandbox.stub(RouterConnectionService, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) - const connectStub = $sandbox.stub(RouterConnectionService, '_connectToHost').resolves($mockConnection) + it('connects on second host when first fails for Kubernetes host list', async () => { + const hosts = ['router.iofog.svc.cluster.local', defaultRouter.host] + $sandbox.stub(RouterConnectionManager, '_resolveRouterEndpoint').resolves({ hosts, port: 5671 }) + const connectStub = $sandbox.stub(RouterConnectionManager, '_connectToHost') + .onCall(0).rejects(new Error('ECONNREFUSED')) + .onCall(1).resolves($mockConnection) - const connection = await RouterConnectionService._createConnection() + const slot = RouterConnectionManager.slots[0] + const connection = await RouterConnectionManager._createSlotConnection(slot) expect(connection).to.equal($mockConnection) - expect(connectStub).to.have.been.calledOnce - expect(connectStub.firstCall.args[0]).to.equal('router.iofog.svc.cluster.local') + expect(connectStub).to.have.been.calledTwice + expect(connectStub.firstCall.args[1]).to.equal('router.iofog.svc.cluster.local') + expect(connectStub.secondCall.args[1]).to.equal(defaultRouter.host) + }) + }) + + describe('waitForSendable()', () => { + it('resolves immediately when sender is already sendable', async () => { + const sender = { + sendable: () => true, + once: sinon.stub(), + removeListener: sinon.stub() + } + + await RouterConnectionManager.waitForSendable(sender, 100) + expect(sender.once).to.not.have.been.called + }) + + it('waits for sendable event when sender is not ready', async () => { + const sender = new (require('events').EventEmitter)() + sender.sendable = () => false + + const pending = RouterConnectionManager.waitForSendable(sender, 500) + setImmediate(() => sender.emit('sendable')) + await pending + }) + + it('rejects on timeout when sender never becomes sendable', async () => { + const sender = new (require('events').EventEmitter)() + sender.sendable = () => false + + await expect(RouterConnectionManager.waitForSendable(sender, 20)) + .to.be.rejectedWith(/not sendable/) + }) + }) + + describe('handleSendError()', () => { + it('marks slot unhealthy on circular buffer overflow', async () => { + const sessionId = 'overflow-session' + const slotId = RouterConnectionManager.slotIdForSession(sessionId) + const reconnectStub = $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + + const handled = RouterConnectionManager.handleSendError( + sessionId, + new Error('circular buffer overflow') + ) + + expect(handled).to.equal(true) + expect(reconnectStub).to.have.been.calledOnceWith(slotId, 'circular buffer overflow') }) }) }) diff --git a/test/src/services/websocket-queue-service.test.js b/test/src/services/websocket-queue-service.test.js new file mode 100644 index 00000000..cd80007f --- /dev/null +++ b/test/src/services/websocket-queue-service.test.js @@ -0,0 +1,140 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const msgpack = require('@msgpack/msgpack') + +const WebSocketQueueService = require('../../../src/services/websocket-queue-service') +const RouterConnectionManager = require('../../../src/services/router-connection-manager') + +describe('WebSocketQueueService — AMQP hardening', () => { + def('sandbox', () => sinon.createSandbox()) + + const sessionId = 'log-session-backpressure' + const execId = 'exec-session-overflow' + + afterEach(() => { + $sandbox.restore() + WebSocketQueueService.execBridges.clear() + WebSocketQueueService.logBridges.clear() + }) + + describe('publishLogToUser() publish-side backpressure', () => { + it('drops LOG_LINE when sender is not sendable', async () => { + const userWs = { + readyState: 1, + send: sinon.stub() + } + const sender = { + sendable: () => false, + send: sinon.stub() + } + + WebSocketQueueService.logBridges.set(sessionId, { + sessionId, + slotId: 0, + session: { sessionId, user: userWs }, + userSender: { sender }, + backpressureNotified: false + }) + + const buffer = msgpack.encode({ + type: 6, + data: Buffer.from('line\n'), + sessionId + }) + + await WebSocketQueueService.publishLogToUser(sessionId, buffer) + + expect(sender.send).to.not.have.been.called + expect(userWs.send).to.have.been.calledOnce + }) + }) + + describe('overflow recovery', () => { + it('recovers publish after slot reconnect without poisoning other sessions', async () => { + const bridge = { + execId, + slotId: RouterConnectionManager.slotIdForSession(execId), + session: { execId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(execId, bridge) + + const sender = { + sendable: () => true, + send: sinon.stub() + } + let ensureCount = 0 + $sandbox.stub(WebSocketQueueService, '_ensureSender').callsFake(async () => { + ensureCount += 1 + if (ensureCount === 1) { + throw new Error('circular buffer overflow') + } + return { sender } + }) + + const markStub = $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + $sandbox.stub(RouterConnectionManager, 'waitForSendable').resolves() + + await expect(WebSocketQueueService.publishToAgent(execId, Buffer.from('fail'))) + .to.be.rejectedWith(/circular buffer overflow/) + + expect(markStub).to.have.been.calledOnce + + await WebSocketQueueService.publishToAgent(execId, Buffer.from('ok')) + + expect(sender.send).to.have.been.calledOnceWith(sinon.match({ body: Buffer.from('ok') })) + }) + + it('does not poison unrelated exec sessions when one slot overflows', async () => { + const otherExecId = 'exec-session-healthy' + const healthyBridge = { + execId: otherExecId, + slotId: RouterConnectionManager.slotIdForSession(otherExecId), + session: { execId: otherExecId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(otherExecId, healthyBridge) + + const overflowBridge = { + execId, + slotId: RouterConnectionManager.slotIdForSession(execId), + session: { execId }, + senders: {}, + receivers: {}, + linkRefs: {} + } + WebSocketQueueService.execBridges.set(execId, overflowBridge) + + const healthySender = { + sendable: () => true, + send: sinon.stub() + } + const overflowSender = { + sendable: () => true, + send: sinon.stub().throws(new Error('circular buffer overflow')) + } + + $sandbox.stub(WebSocketQueueService, '_ensureSender').callsFake(async (id) => { + if (id === execId) { + return { sender: overflowSender } + } + return { sender: healthySender } + }) + + $sandbox.stub(RouterConnectionManager, 'markSlotUnhealthy').resolves() + $sandbox.stub(RouterConnectionManager, 'waitForSendable').resolves() + + await expect(WebSocketQueueService.publishToAgent(execId, Buffer.from('fail'))) + .to.be.rejectedWith(/circular buffer overflow/) + + await WebSocketQueueService.publishToAgent(otherExecId, Buffer.from('ok')) + + expect(healthySender.send).to.have.been.calledOnceWith(sinon.match({ body: Buffer.from('ok') })) + expect(overflowSender.send).to.have.been.calledOnce + }) + }) +}) diff --git a/test/src/services/ws-relay-transport-factory.test.js b/test/src/services/ws-relay-transport-factory.test.js new file mode 100644 index 00000000..8fd94961 --- /dev/null +++ b/test/src/services/ws-relay-transport-factory.test.js @@ -0,0 +1,52 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const config = require('../../../src/config') +const AmqpRelayTransport = require('../../../src/services/amqp-relay-transport') +const NatsRelayTransport = require('../../../src/services/nats-relay-transport') +const { + resolveTransport, + resetTransportForTests +} = require('../../../src/services/ws-relay-transport-factory') + +describe('ws-relay-transport-factory', () => { + def('sandbox', () => sinon.createSandbox()) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + }) + + it('selects amqp when nats.enabled is false', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return false + return defaultValue + }) + + const transport = resolveTransport() + expect(transport).to.be.instanceOf(AmqpRelayTransport) + expect(transport.getTransport()).to.equal('amqp') + }) + + it('selects nats transport when nats.enabled is true', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return true + return defaultValue + }) + + const transport = resolveTransport() + expect(transport).to.be.instanceOf(NatsRelayTransport) + expect(transport.getTransport()).to.equal('nats') + }) + + it('returns the same singleton instance on repeated calls', () => { + $sandbox.stub(config, 'getBoolean').callsFake((key, defaultValue = false) => { + if (key === 'nats.enabled') return false + return defaultValue + }) + + const first = resolveTransport() + const second = resolveTransport() + expect(first).to.equal(second) + }) +}) diff --git a/test/src/websocket/ws-cross-replica-nats.test.js b/test/src/websocket/ws-cross-replica-nats.test.js new file mode 100644 index 00000000..277177ac --- /dev/null +++ b/test/src/websocket/ws-cross-replica-nats.test.js @@ -0,0 +1,211 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const WebSocketServerClass = require('../../../src/websocket/server') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const AppHelper = require('../../../src/helpers/app-helper') +const EventService = require('../../../src/services/event-service') +const { + MESSAGE_TYPES, + createMockWebSocket, + createMockRequest, + buildExecFrame, + decodeExecMessage, + createMockNatsRelayTransport, + resetWebSocketServerSingleton, + newTestIds, + waitForSent, + delay +} = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') + +describe('WebSocket exec/log — cross-replica mock NATS', () => { + def('sandbox', () => sinon.createSandbox()) + def('ids', () => newTestIds()) + + let wsServer + let mockRelay + let transaction + + beforeEach(() => { + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + wsServer = new WebSocketServerClass() + mockRelay = createMockNatsRelayTransport() + wsServer.relayTransport = mockRelay + transaction = { fakeTransaction: true } + + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(EventService, 'createWsConnectEvent').resolves() + $sandbox.stub(EventService, 'createWsDisconnectEvent').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'create').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').callsFake(async () => ({ + sessionId: $ids.sessionId, + microserviceUuid: $ids.microserviceUuid, + status: 'PENDING', + userConnected: true, + agentConnected: false + })) + $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) + $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(AppHelper, 'generateUUID').returns($ids.sessionId) + $sandbox.stub(wsServer, 'countExecSessionsInDb').resolves(0) + }) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + }) + + it('uses nats transport for cross-replica exec relay', async () => { + expect(mockRelay.getTransport()).to.equal('nats') + + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + null, + userWs, + transaction + ) + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(50) + + const session = wsServer.execSessionManager.getExecSession(sessionId) + session.agent = agentWs + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(50) + + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) + + const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, sessionId, $ids.microserviceUuid, 'echo hi\n') + userWs.emit('message', stdinFrame, true) + await waitForSent(agentWs, 1) + + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) + }) + + it('delivers agent STDOUT to user through mock NATS publishToUser', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + transaction + ) + await mockRelay.enableForSession( + wsServer.execSessionManager.getExecSession(sessionId), + () => {} + ) + + const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, sessionId, $ids.microserviceUuid, 'line\n') + await mockRelay.publishToUser(sessionId, stdoutFrame) + await waitForSent(userWs, 1) + + const userReceived = decodeExecMessage(lastSent(userWs)) + expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) + }) + + it('routes log lines through mock NATS bridge', async () => { + const userWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.logSessionManager.createLogSession( + sessionId, + $ids.microserviceUuid, + null, + userWs, + { lines: 100, follow: true, since: null, until: null }, + transaction + ) + + await mockRelay.enableForLogSession( + { sessionId, microserviceUuid: $ids.microserviceUuid, user: userWs, agent: null }, + () => {} + ) + + const logLine = buildExecFrame(MESSAGE_TYPES.LOG_LINE, sessionId, $ids.microserviceUuid, 'log entry\n') + await mockRelay.publishLogToUser(sessionId, logLine) + await delay(20) + + expect(mockRelay.logBridges.has(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelayForLogs(sessionId)).to.equal(true) + }) + + it('user-first then agent-second delivers ACTIVATION and STDIN via NATS bridge', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const userReq = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + userReq.headers.authorization = 'Bearer user-jwt' + + await wsServer.handleUserExecConnection( + userWs, + userReq, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + + expect(mockRelay.shouldUseRelay($ids.sessionId)).to.equal(true) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) + await delay(50) + + const activationFrames = agentWs._sentMessages.filter((entry) => { + try { + return decodeExecMessage(entry.data).type === MESSAGE_TYPES.ACTIVATION + } catch (e) { + return false + } + }) + expect(activationFrames.length).to.be.at.least(1) + + const stdinFrame = buildExecFrame( + MESSAGE_TYPES.STDIN, + $ids.sessionId, + $ids.microserviceUuid, + 'echo hi\n' + ) + userWs.emit('message', stdinFrame, true) + await waitForSent(agentWs, activationFrames.length + 1) + + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) + }) +}) + +function lastSent (ws) { + const entry = ws._sentMessages[ws._sentMessages.length - 1] + return entry.data +} diff --git a/test/src/websocket/ws-cross-replica.test.js b/test/src/websocket/ws-cross-replica.test.js index 689414ae..0712ffac 100644 --- a/test/src/websocket/ws-cross-replica.test.js +++ b/test/src/websocket/ws-cross-replica.test.js @@ -7,7 +7,6 @@ const MicroserviceManager = require('../../../src/data/managers/microservice-man const FogManager = require('../../../src/data/managers/iofog-manager') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const AppHelper = require('../../../src/helpers/app-helper') -const RouterConnectionService = require('../../../src/services/router-connection-service') const EventService = require('../../../src/services/event-service') const { MESSAGE_TYPES, @@ -15,29 +14,30 @@ const { createMockRequest, buildExecFrame, decodeExecMessage, - createMockQueueService, + createMockRelayTransport, resetWebSocketServerSingleton, newTestIds, waitForSent, delay } = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') describe('WebSocket exec/log — cross-replica mock AMQP', () => { def('sandbox', () => sinon.createSandbox()) def('ids', () => newTestIds()) let wsServer - let mockQueue + let mockRelay let transaction beforeEach(() => { + resetTransportForTests() resetWebSocketServerSingleton(WebSocketServerClass) wsServer = new WebSocketServerClass() - mockQueue = createMockQueueService() - wsServer.queueService = mockQueue + mockRelay = createMockRelayTransport() + wsServer.relayTransport = mockRelay transaction = { fakeTransaction: true } - $sandbox.stub(RouterConnectionService, 'isRouterAvailable').resolves(true) $sandbox.stub(MicroserviceManager, 'update').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) @@ -63,6 +63,7 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { afterEach(() => { $sandbox.restore() + resetTransportForTests() resetWebSocketServerSingleton(WebSocketServerClass) }) @@ -86,13 +87,13 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { await wsServer.setupExecMessageForwarding(sessionId, transaction) await delay(50) - expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, sessionId, $ids.microserviceUuid, 'echo hi\n') userWs.emit('message', stdinFrame, true) await waitForSent(agentWs, 1) - expect(mockQueue.execBridges.has(sessionId)).to.equal(true) + expect(mockRelay.execBridges.has(sessionId)).to.equal(true) const agentReceived = decodeExecMessage(lastSent(agentWs)) expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) }) @@ -109,16 +110,16 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { userWs, transaction ) - await mockQueue.enableForSession( + await mockRelay.enableForSession( wsServer.execSessionManager.getExecSession(sessionId), () => {} ) const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, sessionId, $ids.microserviceUuid, 'line\n') - await mockQueue.publishToUser(sessionId, stdoutFrame) + await mockRelay.publishToUser(sessionId, stdoutFrame) await waitForSent(userWs, 1) - expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay(sessionId)).to.equal(true) const userReceived = decodeExecMessage(lastSent(userWs)) expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) }) @@ -136,17 +137,17 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { transaction ) - await mockQueue.enableForLogSession( + await mockRelay.enableForLogSession( { sessionId, microserviceUuid: $ids.microserviceUuid, user: userWs, agent: null }, () => {} ) const logLine = buildExecFrame(MESSAGE_TYPES.LOG_LINE, sessionId, $ids.microserviceUuid, 'log entry\n') - await mockQueue.publishLogToUser(sessionId, logLine) + await mockRelay.publishLogToUser(sessionId, logLine) await delay(20) - expect(mockQueue.logBridges.has(sessionId)).to.equal(true) - expect(mockQueue.shouldUseLogQueue(sessionId)).to.equal(true) + expect(mockRelay.logBridges.has(sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelayForLogs(sessionId)).to.equal(true) }) it('user-first then agent-second delivers ACTIVATION and STDIN via AMQP bridge', async () => { @@ -164,7 +165,7 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { transaction ) - expect(mockQueue.shouldUseQueue($ids.sessionId)).to.equal(true) + expect(mockRelay.shouldUseRelay($ids.sessionId)).to.equal(true) expect(wsServer.execSessionManager.getExecSession($ids.sessionId).agent).to.equal(null) const agentReq = createMockRequest( diff --git a/test/src/websocket/ws-drain.test.js b/test/src/websocket/ws-drain.test.js index ed94815c..85ca9992 100644 --- a/test/src/websocket/ws-drain.test.js +++ b/test/src/websocket/ws-drain.test.js @@ -28,8 +28,8 @@ describe('WebSocket graceful drain', () => { drainTimeoutMs: 500 } - $sandbox.stub(wsServer.queueService, 'cleanup').resolves() - $sandbox.stub(wsServer.queueService, 'cleanupLogSession').resolves() + $sandbox.stub(wsServer.relayTransport, 'cleanup').resolves() + $sandbox.stub(wsServer.relayTransport, 'cleanupLogSession').resolves() $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) diff --git a/test/src/websocket/ws-exec-activation-failfast.test.js b/test/src/websocket/ws-exec-activation-failfast.test.js new file mode 100644 index 00000000..4052a708 --- /dev/null +++ b/test/src/websocket/ws-exec-activation-failfast.test.js @@ -0,0 +1,75 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const logger = require('../../../src/logger') +const WebSocketServerClass = require('../../../src/websocket/server') +const { + createMockWebSocket, + resetWebSocketServerSingleton, + newTestIds, + delay +} = require('../../support/ws-session-harness') +const { resetTransportForTests } = require('../../../src/services/ws-relay-transport-factory') + +describe('WebSocket exec activation fail-fast', () => { + def('sandbox', () => sinon.createSandbox()) + def('ids', () => newTestIds()) + + let wsServer + let transaction + + beforeEach(() => { + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + wsServer = new WebSocketServerClass() + wsServer.relayTransport = { + getTransport: () => 'amqp', + isAvailable: async () => true, + enableForSession: async () => true, + shouldUseRelay: () => true, + publishToAgent: async () => {}, + publishToUser: async () => {}, + cleanup: async () => {}, + onRecovery: () => {}, + shutdown: async () => {} + } + transaction = { fakeTransaction: true } + + $sandbox.stub(logger, 'info') + $sandbox.stub(logger, 'error') + $sandbox.stub(logger, 'warn') + $sandbox.stub(logger, 'debug') + $sandbox.stub(wsServer, 'sendMessageToAgent').resolves(false) + $sandbox.stub(wsServer, 'cleanupExecSession').resolves() + }) + + afterEach(() => { + $sandbox.restore() + resetTransportForTests() + resetWebSocketServerSingleton(WebSocketServerClass) + }) + + it('does not log setup complete when activation fails', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId + + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + transaction + ) + + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(20) + + const setupCompleteCalls = logger.info.getCalls().filter((call) => { + const first = call.args[0] + return typeof first === 'string' && first.includes('Exec message forwarding setup complete') + }) + + expect(setupCompleteCalls).to.have.length(0) + }) +}) diff --git a/test/src/websocket/ws-exec-same-replica.test.js b/test/src/websocket/ws-exec-same-replica.test.js index 6f19b788..aea148b5 100644 --- a/test/src/websocket/ws-exec-same-replica.test.js +++ b/test/src/websocket/ws-exec-same-replica.test.js @@ -37,9 +37,9 @@ describe('WebSocket exec — same-replica integration (Plan 17)', () => { agentWs = createMockWebSocket() transaction = { fakeTransaction: true } - $sandbox.stub(wsServer.queueService, 'enableForSession').resolves(true) - $sandbox.stub(wsServer.queueService, 'shouldUseQueue').returns(false) - $sandbox.stub(wsServer.queueService, 'cleanup').resolves() + $sandbox.stub(wsServer.relayTransport, 'enableForSession').resolves(true) + $sandbox.stub(wsServer.relayTransport, 'shouldUseRelay').returns(false) + $sandbox.stub(wsServer.relayTransport, 'cleanup').resolves() $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) diff --git a/test/support/ws-session-harness.js b/test/support/ws-session-harness.js index 5eaec0a7..b2e0e26e 100644 --- a/test/support/ws-session-harness.js +++ b/test/support/ws-session-harness.js @@ -86,9 +86,10 @@ function buildExecFrame (type, execId, microserviceUuid, data) { } /** - * In-memory AMQP stub for cross-replica exec/log relay tests. + * In-memory relay stub for cross-replica exec/log tests. + * @param {'amqp'|'nats'} [transport='amqp'] */ -function createMockQueueService () { +function createMockRelayTransport (transport = 'amqp') { const execBridges = new Map() const logBridges = new Map() @@ -96,6 +97,14 @@ function createMockQueueService () { execBridges, logBridges, + getTransport () { + return transport + }, + + async isAvailable () { + return true + }, + async enableForSession (session, cleanupCallback) { const execId = session.execId if (!execId) return false @@ -111,7 +120,7 @@ function createMockQueueService () { return true }, - shouldUseQueue (execId) { + shouldUseRelay (execId) { return execBridges.has(execId) }, @@ -139,7 +148,7 @@ function createMockQueueService () { return true }, - shouldUseLogQueue (sessionId) { + shouldUseRelayForLogs (sessionId) { return logBridges.has(sessionId) }, @@ -152,10 +161,23 @@ function createMockQueueService () { async cleanupLogSession (sessionId) { logBridges.delete(sessionId) - } + }, + + async shutdown () {}, + + onRecovery () {} } } +function createMockNatsRelayTransport () { + return createMockRelayTransport('nats') +} + +/** @deprecated use createMockRelayTransport */ +function createMockQueueService () { + return createMockRelayTransport() +} + function resetWebSocketServerSingleton (WebSocketServerClass) { if (WebSocketServerClass.instance) { const instance = WebSocketServerClass.instance @@ -220,6 +242,8 @@ module.exports = { decodeExecMessage, buildAgentInitialMessage, buildExecFrame, + createMockRelayTransport, + createMockNatsRelayTransport, createMockQueueService, resetWebSocketServerSingleton, buildFakeJwt, From 239e98ded1154cb2a6b6e317bc076e73531b92b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Mon, 29 Jun 2026 00:51:05 +0300 Subject: [PATCH 3/4] Document WebSocket relay HA, NATS vs AMQP backend selection, and operator troubleshooting. Update architecture, ws-sessions guide, swagger WS/NATS creds paths, and changelog for relay production and NATS platform account naming. --- docs/architecture.md | 56 +++++++++++++++----- docs/operations/ws-sessions.md | 52 +++++++++++++++---- docs/swagger.yaml | 94 ++++++++++++++++++++++++++++++---- 3 files changed, 171 insertions(+), 31 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 786e92f8..e19adfd3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -206,7 +206,37 @@ Full spec: [`.cursor/controllerv3.8/docs/15-fog-platform-reconcile.md`](../.curs ## WebSocket exec & log sessions -Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens log sessions and shared WS infra (HA, drain, OTEL). **Plan 17** redesigns **microservice exec** to log-style multi-session flow (3 concurrent per MS, agent poll + session-scoped WS) — **Edgelet agent wire change required** for exec (see [edgelet-invariants.md §10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md)). +Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens log sessions and shared WS infra (HA, drain, OTEL). **Plan 17** redesigns **microservice exec** to log-style multi-session flow (3 concurrent per MS, agent poll + session-scoped WS). **Plan 18** production-hardens cross-replica relay via **`WsRelayTransport`** — AMQP pool + recovery when `nats.enabled=false`, NATS Core when `nats.enabled=true` (R102–R113). **Edgelet agent wire change required** for exec only (see [edgelet-invariants.md §10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md)). + +```mermaid +flowchart TB + subgraph ws [WebSocketServer] + U[User WS] + A[Agent WS] + end + + subgraph factory [WsRelayTransportFactory] + SEL{nats.enabled?} + end + + subgraph amqp [AmqpRelayTransport] + POOL[RouterConnectionManager pool x8] + Q[agent/user queues per sessionId] + end + + subgraph nats [NatsRelayTransport] + NC[NatsRelayConnectionManager] + SUB[Core pub/sub per sessionId] + end + + U --> ws + A --> ws + ws --> factory + SEL -->|false| amqp + SEL -->|true| nats + POOL --> Q + NC --> SUB +``` ```mermaid sequenceDiagram @@ -214,7 +244,7 @@ sequenceDiagram participant C as Controller participant DB as MicroserviceExecSessions participant CT as Change tracking - participant Q as AMQP + participant R as WsRelayTransport participant A as Edgelet agent Note over U,A: MS exec (R92–R98) — log-style @@ -228,11 +258,11 @@ sequenceDiagram A->>C: GET /agent/exec/sessions A->>C: WS /agent/exec/microservice/:uuid/:sessionId C->>DB: ACTIVE agentConnected - C->>Q: agent-{sessionId} user-{sessionId} + C->>R: enable bridge if cross-replica U->>C: STDIN - C->>Q->>A: relay + C->>R->>A: relay A->>C: STDOUT - C->>Q->>U: relay + C->>R->>U: relay U-->>C: close C->>DB: DELETE row C->>CT: execSessions if needed @@ -242,12 +272,12 @@ sequenceDiagram C->>C: provision debug system MS U->>C: WS /microservices/system/exec/:debugMsUuid - Note over U,A: Logs (R82–R83, R84) — unchanged Plan 16 + Note over U,A: Logs (R82–R83, R84/R112) U->>C: WS logs + tail params C->>DB: PENDING sessionId A->>C: WS agent/logs/:sessionId A->>C: LOG_LINE - C->>Q->>U: logs-user-{sessionId} + C->>R->>U: relay ``` | Topic | Normative value | @@ -263,7 +293,7 @@ sequenceDiagram | Log concurrency | **3** user log WS per microservice (or per fog for node logs) | | Log limits | Tail max **5,000** lines; **120s** pending; **2h** idle | | Log content | Live relay only — no log line persistence; audit connect/disconnect | -| HA relay | Cross-replica sessions **require** AMQP (`WebSocketQueueService`); same-replica may use direct WS; **fail fast** when router down | +| HA relay | Cross-replica sessions **require** a **relay backend** (R112): **AMQP** router queues when `nats.enabled=false`; **NATS Core** subjects on hub when `nats.enabled=true`. Same-replica may use direct WS; **fail fast** close **1013** when active backend unavailable | | Graceful drain | **30s** on SIGTERM / k8s `preStop` — CLOSE frames, queue cleanup, session row delete | | Security | Agent handlers validate fog token **before** message processing; **50** upgrades/min/IP; **100** active WS/IP; JWT in `?token=` (ingress log redaction required) | | Scale SLO | **500** concurrent WS per replica; **p99 pairing < 5s** | @@ -271,13 +301,15 @@ sequenceDiagram **OTEL metric names (R87):** `ws_exec_sessions_active`, `ws_log_sessions_active`, `ws_pending_pairings`, `ws_pairing_duration_ms` (histogram), `ws_amqp_publish_errors`, `ws_router_connected` (gauge). Emitted when `ENABLE_TELEMETRY=true`; see `src/websocket/ws-metrics.js`. -**HA config (`server.webSocket.ha`):** `crossReplicaRequiresAmqp` (default `true`), `failFastOnRouterUnavailable` (default `true`). Env: `WS_HA_CROSS_REPLICA_REQUIRES_AMQP`, `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE`. Graceful drain timeout: `server.webSocket.session.drainTimeoutMs` (default **30s**, env `WS_DRAIN_TIMEOUT_MS`). +**Relay transport (Plan 18, R102):** Selected once at startup from existing `nats.enabled` / `NATS_ENABLED` — **no new relay env var**. `false` → AMQP pool (8 connections, sticky by `sessionId`); `true` → NATS Core on hub with **`controller`** NATS account. Relay connect is **lazy** — does not block Controller startup. + +**HA config (`server.webSocket.ha`):** `crossReplicaRequiresAmqp` (default `true`; semantics: cross-replica requires **active relay backend** per R112), `failFastOnRouterUnavailable` (default `true`; applies to selected backend). Env: `WS_HA_CROSS_REPLICA_REQUIRES_AMQP`, `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE`. Graceful drain timeout: `server.webSocket.session.drainTimeoutMs` (default **30s**, env `WS_DRAIN_TIMEOUT_MS`). -**Core modules:** `src/websocket/server.js`, `exec-session-manager.js` (Plan 17), `log-session-manager.js`, `src/services/websocket-queue-service.js`, `src/services/router-connection-service.js`. +**Core modules:** `src/websocket/server.js`, `exec-session-manager.js` (Plan 17), `log-session-manager.js`, `ws-relay-transport-factory.js` / `amqp-relay-transport.js` / `nats-relay-transport.js` (Plan 18), `src/services/websocket-queue-service.js`, `src/services/router-connection-manager.js` (Plan 18), `src/services/nats-relay-connection-manager.js` (Plan 18). -**Operator guide:** [operations/ws-sessions.md](operations/ws-sessions.md) — ingress `?token=` log redaction, HTTPS/WSS, multi-replica AMQP requirement, k8s preStop drain, load SLO probe. +**Operator guide:** [operations/ws-sessions.md](operations/ws-sessions.md) — ingress `?token=` log redaction, HTTPS/WSS, multi-replica relay backend (`nats.enabled`), k8s preStop drain, load SLO probe. -Full spec: Plan 16 [logs + shared infra](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · Plan 17 [MS exec](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md) · RFC R80–R91, R92–R101 · Edgelet contract: [edgelet-invariants.md §10–§10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md). +Full spec: Plan 16 [logs + shared infra](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · Plan 17 [MS exec](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md) · Plan 18 [WS relay production](../.cursor/controllerv3.8/docs/18-ws-relay-production.md) · RFC R80–R91, R92–R101, R102–R113 · Edgelet contract: [edgelet-invariants.md §10–§10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md). --- diff --git a/docs/operations/ws-sessions.md b/docs/operations/ws-sessions.md index 29c95802..34e268a1 100644 --- a/docs/operations/ws-sessions.md +++ b/docs/operations/ws-sessions.md @@ -6,7 +6,7 @@ ## Overview -Controller exposes **interactive exec** and **log streaming** over WebSocket on the API port (default **51121**). Sessions pair an operator browser/CLI client (Bearer JWT) with an Edgelet agent (fog token). In multi-replica deployments, cross-replica relay requires the **Skupper-style AMQP router** microservice. +Controller exposes **interactive exec** and **log streaming** over WebSocket on the API port (default **51121**). Sessions pair an operator browser/CLI client (Bearer JWT) with an Edgelet agent (fog token). In multi-replica deployments, cross-replica relay uses a **relay backend** selected at startup by **`nats.enabled`** (Plan 18, R102): **AMQP** router queues when `false`, **NATS Core** on the platform hub when `true`. --- @@ -38,18 +38,41 @@ Without redaction, long-lived bearer tokens may appear in load balancer logs. ## Multi-replica HA +Relay transport is selected **once at startup** from existing platform config — **no separate relay env var** (R102): + +| `nats.enabled` | Cross-replica relay backend | +|----------------|----------------------------| +| `false` (default) | **AMQP** — Skupper-style router queues via `WebSocketQueueService` | +| `true` | **NATS Core** — hub pub/sub subjects `controller.relay.v1.*` via `NatsRelayTransport` | + +Set `NATS_ENABLED=true` only when the platform NATS hub is deployed and all Controller replicas share the same value. + | Setting | Default | Env | |---------|---------|-----| -| Cross-replica requires AMQP | `true` | `WS_HA_CROSS_REPLICA_REQUIRES_AMQP` | -| Fail fast when router down | `true` | `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE` | +| Cross-replica requires relay backend | `true` | `WS_HA_CROSS_REPLICA_REQUIRES_AMQP` | +| Fail fast when relay backend down | `true` | `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE | -**Requirements:** +> Env names retain `AMQP`/`ROUTER` for backward compatibility; semantics apply to the **active relay backend** (AMQP or NATS) per R112. -1. Deploy the **router** system microservice and ensure Controller can reach AMQP (`RouterConnectionService`). +### AMQP relay (`nats.enabled=false`) + +1. Deploy the **router** system microservice and ensure Controller can reach AMQP (`RouterConnectionManager` pool). 2. Run **2+ Controller replicas** behind a load balancer with **sticky sessions optional** — cross-replica exec/log uses AMQP queues (`agent-{sessionId}`, `user-{sessionId}`, `logs-user-{sessionId}`). -3. When the router is unavailable, new cross-replica sessions close with WebSocket code **1013** (`Router unavailable for cross-replica session`). +3. When the router/AMQP backend is unavailable, new cross-replica sessions close with WebSocket code **1013** (`Router unavailable for cross-replica session`). + +Plan 18 adds an **8-connection AMQP pool** per replica with overflow recovery — intense log streams must not poison other sessions (no router restart required). **Remote CP** resolves **`router.default.svc.bridge.local`** then default router `host`; **Kubernetes CP** resolves **`router.{namespace}.svc.cluster.local`** then default router `host`. Port from `Routers.messagingPort` (default **5671**). -Same-replica sessions may relay directly without AMQP when both user and agent land on the same pod. +### NATS relay (`nats.enabled=true`) + +1. Platform NATS hub must be running with `NatsInstances.isHub=true`. +2. Controller provisions dedicated **`controller`** NATS account/user (not SYS / `admin-hub`) via NATS auth reconcile. +3. Cross-replica exec uses subjects `controller.relay.v1.exec.{sessionId}.agent` / `.user`; logs use `controller.relay.v1.log.{sessionId}.user`. Plain TCP to hub — port from `NatsInstances.serverPort` (default **4222**) for every host in the resolver list. +4. **Remote CP:** Controller resolves **`nats.default.svc.bridge.local`** (Edgelet internal DNS) then hub `host`; both use hub `serverPort`. +5. **Kubernetes CP:** Controller resolves **`nats-server.{namespace}.svc.cluster.local`** then hub `host`. +6. Remote ControlPlane replicas connect to the **hub** NATS only — not local fog NATS leaf. +7. When NATS relay is unavailable, fail-fast semantics match AMQP (close **1013** when configured). + +Same-replica sessions may relay directly without AMQP or NATS when both user and agent land on the same pod. --- @@ -59,7 +82,7 @@ On shutdown, Controller drains WebSocket sessions for up to **`WS_DRAIN_TIMEOUT_ 1. Reject new upgrades (`verifyClient` → draining). 2. Close pending users with code **1001** (`Server draining`). -3. Send CLOSE frames, clean exec/log session DB rows, tear down AMQP bridges. +3. Send CLOSE frames, clean exec/log session DB rows, tear down relay bridges (AMQP or NATS). ### Kubernetes manifest example @@ -113,6 +136,15 @@ node test/load/ws-pairing-load.js --multi-ms 100 The `--multi-ms` mode creates **3 exec sessions per microservice** (100 MS × 3 = 300 pairs) to validate multi-session pairing latency under the same p99 SLO. +**AMQP profile** (`nats.enabled=false`): run the probe above on a dev machine — it exercises in-process `ExecSessionManager` pairing only (no router required). Record p99 from stdout; target **< 5000 ms**. + +**NATS profile** (`nats.enabled=true`): the same probe validates session-manager pairing latency (transport-agnostic SLO). For end-to-end NATS relay validation in staging: + +1. Deploy Controller with **`NATS_ENABLED=true`** on **2+ replicas** and a platform NATS hub (`NatsInstances.isHub=true`). +2. Confirm **`controller`** NATS account reconcile succeeded (NATS auth logs). +3. Run cross-replica exec/log sessions (user on replica A, agent on replica B) while recording OTEL **`ws_pairing_duration_ms`** p99. +4. Optionally repeat `node test/load/ws-pairing-load.js --pairs 500` against staging API with agent simulators — same **p99 < 5s** SLO applies. + For production validation, repeat against a staging cluster with real agent simulators and record p99 from Controller OTEL histogram `ws_pairing_duration_ms`. --- @@ -128,7 +160,9 @@ Enable `ENABLE_TELEMETRY=true`. Key metrics (`src/websocket/ws-metrics.js`): | `ws_pending_pairings` | gauge | | `ws_pairing_duration_ms` | histogram | | `ws_amqp_publish_errors` | counter | -| `ws_router_connected` | gauge | +| `ws_amqp_session_saturated` | counter (Plan 18 overflow/backpressure) | +| `ws_router_pool_connections` | gauge (Plan 18 AMQP pool health) | +| `ws_router_pool_unsettled` | gauge (Plan 18 AMQP unsettled deliveries) | --- diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c0ff8311..abd65f4d 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2774,8 +2774,7 @@ paths: `{ sessionId, microserviceUuid }` in the `data` field and top-level `sessionId`, followed by STDERR "waiting for agent…". Relay frames use `execId` equal to `sessionId`. - **HA:** Multi-replica deployments require AMQP router (`WebSocketQueueService`). - Cross-replica sessions fail fast with close code **1013** when router is unavailable. + **HA (R112):** Multi-replica deployments require a **relay backend** selected at startup by **`nats.enabled`**: AMQP router queues when `false` (default), NATS Core pub/sub on the platform hub when `true`. Cross-replica sessions fail fast with close code **1013** when the active relay backend is unavailable. See `docs/operations/ws-sessions.md`. See `#/components/schemas/WsExecMessageTypes` and `#/components/schemas/WsCloseCodes`. operationId: userMicroserviceExecWebSocket @@ -6593,11 +6592,16 @@ paths: tags: - NATS summary: Gets NATS creds for specific account user + description: | + Returns base64-encoded `.creds` file content for the user. + + Path `{appName}` is an **application name** for app-linked accounts, or a **NATS account name** for platform/system accounts (`SYS`, leaf accounts, **`controller`**). operationId: getNatsUserCreds parameters: - in: path name: appName required: true + description: Application name or NATS account name (e.g. `SYS`, `controller`) schema: type: string - in: path @@ -7033,7 +7037,7 @@ tags: - name: WebSocketSessions description: | Interactive exec and log streaming WebSocket endpoints (MessagePack binary). - Multi-replica HA requires AMQP router — see operations/ws-sessions.md. + Multi-replica HA requires a relay backend per **`nats.enabled`** (AMQP when `false`, NATS Core when `true`) — see operations/ws-sessions.md. - name: User description: Manage your users - name: Secrets @@ -8276,7 +8280,11 @@ components: description: True when microservice is the ControlPlane controller workload (DB column) ControllerRegisterRequest: type: object - description: Slim register body for Edgelet ControlPlane controller workload + description: >- + Edgelet ControlPlane controller workload register body. Accepts the same + container/workload fields as user microservice deploy (excluding application, + serviceAccount, and natsConfig). Ownership fields (iofogUuid, application) + are derived from the authenticated system fog. required: - uuid - images @@ -8298,6 +8306,31 @@ components: $ref: "#/components/schemas/MicroserviceContainerImage" registryId: type: integer + config: + type: string + annotations: + type: string + hostNetworkMode: + type: boolean + isPrivileged: + type: boolean + logSize: + type: integer + minimum: 0 + runtime: + type: string + pidMode: + type: string + ipcMode: + type: string + runAsUser: + type: string + platform: + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer ports: type: array items: @@ -8306,16 +8339,57 @@ components: type: array items: $ref: "#/components/schemas/VolumeMappingRequest" + extraHosts: + type: array + items: + type: object + required: + - name + - address + properties: + name: + type: string + address: + type: string env: type: array items: $ref: "#/components/schemas/AgentEnvRequest" - config: - type: string - hostNetworkMode: - type: boolean - runtime: - type: string + cmd: + type: array + items: + type: string + cdiDevices: + type: array + items: + type: string + capAdd: + type: array + items: + type: string + capDrop: + type: array + items: + type: string + healthCheck: + type: object + required: + - test + properties: + test: + type: array + items: + type: string + interval: + type: integer + timeout: + type: integer + startPeriod: + type: integer + startInterval: + type: integer + retries: + type: integer schedule: type: integer enum: From 1c1f23c80fee6162c442e24b7f630f2c42dfc3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Mon, 29 Jun 2026 15:09:33 +0300 Subject: [PATCH 4/4] bump console version and Dockerfile base images versions --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- Dockerfile | 10 +++++----- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 153d3d3e..8898b64e 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.0.2 +EDGEOPS_CONSOLE_VERSION=v1.0.3 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index 71381fa0..7c831f55 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.0.2"; fi + if [ -z "$VERSION" ]; then VERSION="1.0.3"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/Dockerfile b/Dockerfile index e6c9e7d7..3cfe9399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # ioFog overrides: EDGEOPS_CONSOLE_REPO=https://github.com/eclipse-iofog/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=iofog # node:24-bookworm — pin manifest list digest for reproducible multi-arch builds -FROM node:24-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS console-builder +FROM node:24-bookworm@sha256:fdddfb3e688158251943d52eba361de991548f6814007acba4917ae6b512d6be AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.3 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -28,7 +28,7 @@ RUN test -f build/index.html \ && cp -a build /tmp/console/build -FROM node:24-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS builder +FROM node:24-bookworm@sha256:fdddfb3e688158251943d52eba361de991548f6814007acba4917ae6b512d6be AS builder ARG PKG_VERSION @@ -48,9 +48,9 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e76548c58c4a29907cef1e01cb6a4cab426eb071ffe28ac1f25dd58d14a89569 +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:cc7648f8e1c7d628e4334328a712f30ea0820787bb92836cc93e349674c689bf -ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.3 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 0757d507..49685283 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.2 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.3 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.0.2 +EDGEOPS_CONSOLE_VERSION ?= v1.0.3 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index 46ab99f2..118b572f 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.2' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.3' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) {