From 9cec90a60fb788659f630e53abab21e0d68e3710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:09:51 +0300 Subject: [PATCH 1/7] Bump embedded EdgeOps Console default version to v1.0.2. --- .env.example | 2 +- .github/actions/set-build-env/action.yml | 2 +- Dockerfile | 4 ++-- Makefile | 4 ++-- scripts/build-console-dev.js | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 4455cebf..153d3d3e 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.0.1 +EDGEOPS_CONSOLE_VERSION=v1.0.2 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index 80c5bb69..71381fa0 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.0.1"; fi + if [ -z "$VERSION" ]; then VERSION="1.0.2"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/Dockerfile b/Dockerfile index b2df0e76..e6c9e7d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ FROM node:24-bookworm@sha256:40ad9f3064e67d6860b4bc3fe1880b2953934fd6320ada990e45fe0efa6badd7 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.0.1 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -50,7 +50,7 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e76548c58c4a29907cef1e01cb6a4cab426eb071ffe28ac1f25dd58d14a89569 -ARG EDGEOPS_CONSOLE_VERSION=v1.0.1 +ARG EDGEOPS_CONSOLE_VERSION=v1.0.2 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index 9b8ed316..0757d507 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.1 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.2 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.0.1 +EDGEOPS_CONSOLE_VERSION ?= v1.0.2 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index db7d41b5..46ab99f2 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.1' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.2' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { From 9519622e84efd228fd765357fcb04e2ab2900839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:10:01 +0300 Subject: [PATCH 2/7] Use hub router port fields when creating non-edge router microservice ports. --- src/services/router-service.js | 4 ++-- test/src/services/router-service.test.js | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/services/router-service.js b/src/services/router-service.js index d0ebd60b..7663eb45 100644 --- a/src/services/router-service.js +++ b/src/services/router-service.js @@ -120,8 +120,8 @@ async function createRouterForFog (fogData, uuid, upstreamRouters, transaction) const routerMicroservice = await _createRouterMicroservice(isEdge, uuid, microserviceConfig, transaction) await _createRouterPorts(routerMicroservice.uuid, messagingPort, transaction) if (!isEdge) { - await _createRouterPorts(routerMicroservice.uuid, fogData.edgeRouterPort, transaction) - await _createRouterPorts(routerMicroservice.uuid, fogData.interRouterPort, transaction) + await _createRouterPorts(routerMicroservice.uuid, router.edgeRouterPort, transaction) + await _createRouterPorts(routerMicroservice.uuid, router.interRouterPort, transaction) } await _ensureRouterTlsVolumeMountsAndMappings(uuid, routerMicroservice.uuid, transaction, false) diff --git a/test/src/services/router-service.test.js b/test/src/services/router-service.test.js index 46bb6b7f..a385075a 100644 --- a/test/src/services/router-service.test.js +++ b/test/src/services/router-service.test.js @@ -227,6 +227,26 @@ describe('Router Service', () => { expect(JSON.parse(createArgs.config).metadata.mode).to.equal('interior') }) }) + + context('Interior router with default ports', () => { + def('fogData', () => ({ ...fogData, routerMode: 'interior' })) + def('createRouterResponse', () => Promise.resolve({ + ...router, + edgeRouterPort: 45671, + interRouterPort: 55671, + isEdge: false + })) + + it('Should open messaging, edge and inter port using router defaults', async () => { + await $subject + const mappingData = { + microserviceUuid: routerMsvc.uuid + } + expect(MicroservicePortManager.create).to.have.been.calledWith({ ...mappingData, portExternal: 55671, portInternal: 55671 }, transaction) + expect(MicroservicePortManager.create).to.have.been.calledWith({ ...mappingData, portExternal: 45671, portInternal: 45671 }, transaction) + return expect(MicroservicePortManager.create).to.have.been.calledThrice + }) + }) }) describe('.updateConfig', () => { From fe6ae78f428fc4150eaa3f7775cea840a444af8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:10:15 +0300 Subject: [PATCH 3/7] Handle concurrent ConfigMap creation during NAT reconcile. Retry update when another replica creates the ConfigMap first instead of failing with ConflictError. --- src/services/nats-service.js | 30 +++++++--- test/src/services/nats-service.test.js | 77 ++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 7 deletions(-) diff --git a/src/services/nats-service.js b/src/services/nats-service.js index 3b1fd079..c372c5a5 100644 --- a/src/services/nats-service.js +++ b/src/services/nats-service.js @@ -140,17 +140,32 @@ function _configMapDataHash (data) { return crypto.createHash('sha256').update(canonical).digest('hex') } +async function _updateConfigMapIfChanged (name, data, existing, transaction) { + const existingHash = _configMapDataHash(existing.data) + const newHash = _configMapDataHash(data) + if (existingHash === newHash) { + return + } + return ConfigMapService.updateConfigMapEndpoint(name, { data, immutable: false }, transaction) +} + async function _ensureConfigMap (name, data, transaction) { const existing = await ConfigMapManager.getConfigMap(name, transaction) if (existing) { - const existingHash = _configMapDataHash(existing.data) - const newHash = _configMapDataHash(data) - if (existingHash === newHash) { - return + return _updateConfigMapIfChanged(name, data, existing, transaction) + } + try { + return await ConfigMapService.createConfigMapEndpoint({ name, data, immutable: false, useVault: true }, transaction) + } catch (error) { + if (error.name !== 'ConflictError') { + throw error + } + const createdConcurrently = await ConfigMapManager.getConfigMap(name, transaction) + if (!createdConcurrently) { + throw error } - return ConfigMapService.updateConfigMapEndpoint(name, { data, immutable: false }, transaction) + return _updateConfigMapIfChanged(name, data, createdConcurrently, transaction) } - return ConfigMapService.createConfigMapEndpoint({ name, data, immutable: false, useVault: true }, transaction) } async function _ensureVolumeMount (name, opts, transaction) { @@ -1626,5 +1641,6 @@ module.exports = { isReconcileRunning, setReconcilePending, normalizeJetstreamSize, - mergeK8sHubClusterRoutes + mergeK8sHubClusterRoutes, + _ensureConfigMap } diff --git a/test/src/services/nats-service.test.js b/test/src/services/nats-service.test.js index a353c4de..36b0a93b 100644 --- a/test/src/services/nats-service.test.js +++ b/test/src/services/nats-service.test.js @@ -10,8 +10,10 @@ const MicroserviceManager = require('../../../src/data/managers/microservice-man const VolumeMappingManager = require('../../../src/data/managers/volume-mapping-manager') const VolumeMountService = require('../../../src/services/volume-mount-service') const ConfigMapService = require('../../../src/services/config-map-service') +const ConfigMapManager = require('../../../src/data/managers/config-map-manager') const NatsAuthService = require('../../../src/services/nats-auth-service') const SecretService = require('../../../src/services/secret-service') +const Errors = require('../../../src/helpers/errors') describe('NATS Service', () => { def('sandbox', () => sinon.createSandbox()) @@ -77,6 +79,81 @@ describe('NATS Service', () => { }) }) + describe('._ensureConfigMap()', () => { + const transaction = {} + const name = 'iofog-nats-jwt-bundle' + const data = { 'bundle.jwt': 'jwt-content' } + + def('subject', () => NatsService._ensureConfigMap(name, data, transaction)) + + beforeEach(() => { + $sandbox.stub(ConfigMapService, 'createConfigMapEndpoint') + $sandbox.stub(ConfigMapService, 'updateConfigMapEndpoint') + }) + + it('creates the ConfigMap when it does not exist', async () => { + $sandbox.stub(ConfigMapManager, 'getConfigMap').resolves(null) + ConfigMapService.createConfigMapEndpoint.resolves({ name }) + + await $subject + + expect(ConfigMapService.createConfigMapEndpoint).to.have.been.calledOnceWith({ + name, + data, + immutable: false, + useVault: true + }, transaction) + expect(ConfigMapService.updateConfigMapEndpoint).to.not.have.been.called + }) + + it('updates the ConfigMap when content changed', async () => { + $sandbox.stub(ConfigMapManager, 'getConfigMap').resolves({ name, data: { 'bundle.jwt': 'old' } }) + ConfigMapService.updateConfigMapEndpoint.resolves({ name }) + + await $subject + + expect(ConfigMapService.createConfigMapEndpoint).to.not.have.been.called + expect(ConfigMapService.updateConfigMapEndpoint).to.have.been.calledOnceWith(name, { data, immutable: false }, transaction) + }) + + it('skips update when content is unchanged', async () => { + $sandbox.stub(ConfigMapManager, 'getConfigMap').resolves({ name, data }) + + await $subject + + expect(ConfigMapService.createConfigMapEndpoint).to.not.have.been.called + expect(ConfigMapService.updateConfigMapEndpoint).to.not.have.been.called + }) + + it('reconciles after concurrent create ConflictError', async () => { + const conflict = new Errors.ConflictError('ConfigMap already exists') + $sandbox.stub(ConfigMapManager, 'getConfigMap') + .onFirstCall().resolves(null) + .onSecondCall().resolves({ name, data }) + ConfigMapService.createConfigMapEndpoint.rejects(conflict) + + await $subject + + expect(ConfigMapService.createConfigMapEndpoint).to.have.been.calledOnce + expect(ConfigMapManager.getConfigMap).to.have.been.calledTwice + expect(ConfigMapService.updateConfigMapEndpoint).to.not.have.been.called + }) + + it('updates after concurrent create when desired content differs', async () => { + const conflict = new Errors.ConflictError('ConfigMap already exists') + $sandbox.stub(ConfigMapManager, 'getConfigMap') + .onFirstCall().resolves(null) + .onSecondCall().resolves({ name, data: { 'bundle.jwt': 'old' } }) + ConfigMapService.createConfigMapEndpoint.rejects(conflict) + ConfigMapService.updateConfigMapEndpoint.resolves({ name }) + + await $subject + + expect(ConfigMapService.createConfigMapEndpoint).to.have.been.calledOnce + expect(ConfigMapService.updateConfigMapEndpoint).to.have.been.calledOnceWith(name, { data, immutable: false }, transaction) + }) + }) + describe('mergeK8sHubClusterRoutes', () => { it('preserves operator routes (nats-headless) and appends controller-managed routes', () => { const currentRoutes = [ From 0a4795e00b9a0652f2d1f91aa19ed7aef5230b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:10:22 +0300 Subject: [PATCH 4/7] Add MicroserviceExecSessions table and drop exec_enabled from microservice model. Introduce per-session exec rows keyed by sessionId and remove the microservice-level exec flag from the Sequelize model. --- .../microservice-exec-session-manager.js | 20 +++++++ .../mysql/db_migration_mysql_v3.8.0.sql | 16 +++++- .../postgres/db_migration_pg_v3.8.0.sql | 16 +++++- .../sqlite/db_migration_sqlite_v3.8.0.sql | 16 +++++- src/data/models/microservice.js | 10 ++-- src/data/models/microserviceExecSession.js | 53 +++++++++++++++++++ 6 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 src/data/managers/microservice-exec-session-manager.js create mode 100644 src/data/models/microserviceExecSession.js diff --git a/src/data/managers/microservice-exec-session-manager.js b/src/data/managers/microservice-exec-session-manager.js new file mode 100644 index 00000000..259bfcf7 --- /dev/null +++ b/src/data/managers/microservice-exec-session-manager.js @@ -0,0 +1,20 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceExecSession = models.MicroserviceExecSession + +class MicroserviceExecSessionManager extends BaseManager { + getEntity () { + return MicroserviceExecSession + } + + findBySessionId (sessionId, transaction) { + return this.findOne({ sessionId }, transaction) + } + + deleteBySessionId (sessionId, transaction) { + return this.delete({ sessionId }, transaction) + } +} + +const instance = new MicroserviceExecSessionManager() +module.exports = instance diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.8.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.8.0.sql index 5884bdbf..23a5f2ea 100644 --- a/src/data/migrations/mysql/db_migration_mysql_v3.8.0.sql +++ b/src/data/migrations/mysql/db_migration_mysql_v3.8.0.sql @@ -233,7 +233,6 @@ CREATE TABLE IF NOT EXISTS Microservices ( annotations TEXT, pid_mode VARCHAR(36), ipc_mode VARCHAR(36), - exec_enabled BOOLEAN DEFAULT false, schedule INT DEFAULT 50, cpu_set_cpus TEXT, memory_limit FLOAT, @@ -705,6 +704,21 @@ CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + CREATE TABLE IF NOT EXISTS FogLogStatuses ( id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, iofog_uuid VARCHAR(36), diff --git a/src/data/migrations/postgres/db_migration_pg_v3.8.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.8.0.sql index 17bb8664..19714e68 100644 --- a/src/data/migrations/postgres/db_migration_pg_v3.8.0.sql +++ b/src/data/migrations/postgres/db_migration_pg_v3.8.0.sql @@ -231,7 +231,6 @@ CREATE TABLE IF NOT EXISTS "Microservices" ( annotations TEXT, pid_mode VARCHAR(36), ipc_mode VARCHAR(36), - exec_enabled BOOLEAN DEFAULT false, schedule INT DEFAULT 50, cpu_set_cpus TEXT, memory_limit DOUBLE PRECISION, @@ -703,6 +702,21 @@ CREATE TABLE IF NOT EXISTS "MicroserviceLogStatuses" ( CREATE INDEX idx_microservice_log_status_microservice_uuid ON "MicroserviceLogStatuses" (microservice_uuid); CREATE INDEX idx_microservice_log_status_session_id ON "MicroserviceLogStatuses" (session_id); +CREATE TABLE IF NOT EXISTS "MicroserviceExecSessions" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at TIMESTAMP(0), + updated_at TIMESTAMP(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON "MicroserviceExecSessions" (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON "MicroserviceExecSessions" (session_id); + CREATE TABLE IF NOT EXISTS "FogLogStatuses" ( id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, iofog_uuid VARCHAR(36), diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.8.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.8.0.sql index b85c1c3d..9792e096 100644 --- a/src/data/migrations/sqlite/db_migration_sqlite_v3.8.0.sql +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.8.0.sql @@ -231,7 +231,6 @@ CREATE TABLE IF NOT EXISTS Microservices ( annotations TEXT, pid_mode VARCHAR(36), ipc_mode VARCHAR(36), - exec_enabled BOOLEAN DEFAULT false, schedule INT DEFAULT 50, cpu_set_cpus TEXT, memory_limit FLOAT, @@ -703,6 +702,21 @@ CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + CREATE TABLE IF NOT EXISTS FogLogStatuses ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, iofog_uuid VARCHAR(36), diff --git a/src/data/models/microservice.js b/src/data/models/microservice.js index 27a121f9..017a3821 100644 --- a/src/data/models/microservice.js +++ b/src/data/models/microservice.js @@ -94,11 +94,6 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.FLOAT, field: 'memory_limit' }, - execEnabled: { - type: DataTypes.BOOLEAN, - field: 'exec_enabled', - defaultValue: false - }, delete: { type: DataTypes.BOOLEAN, field: 'delete', @@ -212,6 +207,11 @@ module.exports = (sequelize, DataTypes) => { as: 'microserviceExecStatus' }) + Microservice.hasMany(models.MicroserviceExecSession, { + foreignKey: 'microservice_uuid', + as: 'microserviceExecSessions' + }) + Microservice.hasOne(models.MicroserviceHealthCheck, { foreignKey: 'microservice_uuid', as: 'healthCheck' diff --git a/src/data/models/microserviceExecSession.js b/src/data/models/microserviceExecSession.js new file mode 100644 index 00000000..5dfb01da --- /dev/null +++ b/src/data/models/microserviceExecSession.js @@ -0,0 +1,53 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceExecSession = sequelize.define('MicroserviceExecSession', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + microserviceUuid: { + type: DataTypes.STRING(36), + field: 'microservice_uuid', + allowNull: false + }, + sessionId: { + type: DataTypes.TEXT, + field: 'session_id', + allowNull: false, + unique: true + }, + status: { + type: DataTypes.TEXT, + field: 'status', + allowNull: true + }, + userConnected: { + type: DataTypes.BOOLEAN, + field: 'user_connected', + defaultValue: false + }, + agentConnected: { + type: DataTypes.BOOLEAN, + field: 'agent_connected', + defaultValue: false + } + }, { + tableName: 'MicroserviceExecSessions', + timestamps: true, + underscored: true + }) + MicroserviceExecSession.associate = function (models) { + MicroserviceExecSession.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceExecSession +} From c813ffba338e62e33a086b805aa3ad29ad74bd2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:10:32 +0300 Subject: [PATCH 5/7] Replace single-session exec with multi-session WebSocket flow. Remove microservice exec REST enable/disable endpoints. Users connect directly via WebSocket; agents poll GET /agent/exec/sessions and attach per session at WS /agent/exec/microservice/:uuid/:sessionId. Support three concurrent exec sessions per microservice with per-session lifecycle and ACTIVATION announce to the user. --- src/config/config.yaml | 1 + src/config/env-mapping.js | 1 + src/config/rbac-resources.yaml | 15 +- src/controllers/agent-controller.js | 5 + src/controllers/microservices-controller.js | 24 - src/jobs/ws-session-reconcile-job.js | 40 +- src/routes/agent.js | 39 +- src/routes/microservices.js | 156 +- src/services/agent-service.js | 38 +- src/services/iofog-service.js | 4 +- src/services/microservices-service.js | 83 - src/websocket/exec-session-manager.js | 196 ++ src/websocket/server.js | 1996 ++++++------------- src/websocket/session-manager.js | 658 ------ 14 files changed, 907 insertions(+), 2349 deletions(-) create mode 100644 src/websocket/exec-session-manager.js delete mode 100644 src/websocket/session-manager.js diff --git a/src/config/config.yaml b/src/config/config.yaml index 942cd893..0e39643f 100644 --- a/src/config/config.yaml +++ b/src/config/config.yaml @@ -25,6 +25,7 @@ server: cleanupInterval: 30000 # Session cleanup interval (30 seconds) execPendingTimeoutMs: 60000 # Exec: user wait for agent (R81) execMaxDurationMs: 28800000 # Exec: max active session 8h (R81) + execMaxConcurrentPerResource: 3 # Exec: max user WS per MS (R92) logPendingTimeoutMs: 120000 # Log: user wait for agent (R82) logIdleTimeoutMs: 7200000 # Log: idle session 2h (R82) logMaxConcurrentPerResource: 3 # Log: max user WS per MS or fog (R82) diff --git a/src/config/env-mapping.js b/src/config/env-mapping.js index b9ac0689..31844774 100644 --- a/src/config/env-mapping.js +++ b/src/config/env-mapping.js @@ -23,6 +23,7 @@ module.exports = { WS_CLEANUP_INTERVAL: 'server.webSocket.session.cleanupInterval', WS_EXEC_PENDING_TIMEOUT_MS: 'server.webSocket.session.execPendingTimeoutMs', WS_EXEC_MAX_DURATION_MS: 'server.webSocket.session.execMaxDurationMs', + WS_EXEC_MAX_CONCURRENT_PER_RESOURCE: 'server.webSocket.session.execMaxConcurrentPerResource', WS_LOG_PENDING_TIMEOUT_MS: 'server.webSocket.session.logPendingTimeoutMs', WS_LOG_IDLE_TIMEOUT_MS: 'server.webSocket.session.logIdleTimeoutMs', WS_LOG_MAX_CONCURRENT_PER_RESOURCE: 'server.webSocket.session.logMaxConcurrentPerResource', diff --git a/src/config/rbac-resources.yaml b/src/config/rbac-resources.yaml index ee700ffb..6c98ab84 100644 --- a/src/config/rbac-resources.yaml +++ b/src/config/rbac-resources.yaml @@ -154,11 +154,6 @@ resources: # Exec Sessions execSessions: routes: - - path: /api/v3/microservices/:uuid/exec - methods: - POST: [patch] - DELETE: [patch] - resourceNameParam: uuid - path: /api/v3/microservices/exec/:microserviceUuid methods: WS: [get] @@ -172,11 +167,6 @@ resources: POST: [patch] DELETE: [patch] resourceNameParam: uuid - - path: /api/v3/microservices/system/:uuid/exec - methods: - POST: [patch] - DELETE: [patch] - resourceNameParam: uuid - path: /api/v3/microservices/system/exec/:microserviceUuid methods: WS: [get] @@ -641,7 +631,10 @@ resources: - path: /api/v3/agent/logs/sessions methods: GET: [list] - - path: /api/v3/agent/exec/:microserviceUuid + - path: /api/v3/agent/exec/sessions + methods: + GET: [list] + - path: /api/v3/agent/exec/microservice/:microserviceUuid/:sessionId methods: WS: [get] resourceNameParam: microserviceUuid diff --git a/src/controllers/agent-controller.js b/src/controllers/agent-controller.js index 5822f4c7..1c6419f9 100644 --- a/src/controllers/agent-controller.js +++ b/src/controllers/agent-controller.js @@ -56,6 +56,10 @@ const getAgentLogSessionsEndPoint = async function (req, fog) { return AgentService.getAgentLogSessions(fog) } +const getAgentExecSessionsEndPoint = async function (req, fog) { + return AgentService.getAgentExecSessions(fog) +} + const getAgentMicroserviceEndPoint = async function (req, fog) { const microserviceUuid = req.params.microserviceUuid @@ -118,5 +122,6 @@ module.exports = { getAgentLinkedVolumeMountsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedVolumeMountsEndpoint), getControllerCAEndPoint: AuthDecorator.checkFogToken(getControllerCAEndPoint), getAgentLogSessionsEndPoint: AuthDecorator.checkFogToken(getAgentLogSessionsEndPoint), + getAgentExecSessionsEndPoint: AuthDecorator.checkFogToken(getAgentExecSessionsEndPoint), registerControllerMicroserviceEndPoint: AuthDecorator.checkFogToken(registerControllerMicroserviceEndPoint) } diff --git a/src/controllers/microservices-controller.js b/src/controllers/microservices-controller.js index 770e54f1..ab737c29 100644 --- a/src/controllers/microservices-controller.js +++ b/src/controllers/microservices-controller.js @@ -180,26 +180,6 @@ const deleteSystemMicroserviceVolumeMappingEndPoint = async function (req) { return MicroservicesService.deleteSystemVolumeMappingEndPoint(uuid, id, false) } -const createMicroserviceExecEndPoint = async function (req) { - const uuid = req.params.uuid - return MicroservicesService.createExecEndPoint(uuid, false) -} - -const deleteMicroserviceExecEndPoint = async function (req) { - const uuid = req.params.uuid - return MicroservicesService.deleteExecEndPoint(uuid, false) -} - -const createSystemMicroserviceExecEndPoint = async function (req) { - const uuid = req.params.uuid - return MicroservicesService.createSystemExecEndPoint(uuid, false) -} - -const deleteSystemMicroserviceExecEndPoint = async function (req) { - const uuid = req.params.uuid - return MicroservicesService.deleteSystemExecEndPoint(uuid, false) -} - const startMicroserviceEndPoint = async function (req) { const uuid = req.params.uuid return MicroservicesService.startMicroserviceEndPoint(uuid, false) @@ -240,10 +220,6 @@ module.exports = { getSystemMicroserviceConfigEndPoint, deleteMicroserviceConfigEndPoint, deleteSystemMicroserviceConfigEndPoint, - createMicroserviceExecEndPoint, - deleteMicroserviceExecEndPoint, - createSystemMicroserviceExecEndPoint, - deleteSystemMicroserviceExecEndPoint, startMicroserviceEndPoint, stopMicroserviceEndPoint } diff --git a/src/jobs/ws-session-reconcile-job.js b/src/jobs/ws-session-reconcile-job.js index 846e7c7f..b019d6a7 100644 --- a/src/jobs/ws-session-reconcile-job.js +++ b/src/jobs/ws-session-reconcile-job.js @@ -3,13 +3,12 @@ const logger = require('../logger') const Sequelize = require('sequelize') const Op = Sequelize.Op const WebSocketServer = require('../websocket/server') -const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') +const MicroserviceExecSessionManager = require('../data/managers/microservice-exec-session-manager') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const ChangeTrackingService = require('../services/change-tracking-service') const FogManager = require('../data/managers/iofog-manager') -const { microserviceExecState } = require('../enums/microservice-state') const TransactionDecorator = require('../decorators/transaction-decorator') function getIntervalMs () { @@ -34,7 +33,7 @@ async function run () { async function reconcileStaleSessions () { const wsServer = WebSocketServer.getInstance() - const sessionManager = wsServer.sessionManager + const execSessionManager = wsServer.execSessionManager const logSessionManager = wsServer.logSessionManager const sessionConfig = getSessionConfig() const execPendingTimeout = sessionConfig.execPendingTimeoutMs || 60000 @@ -47,35 +46,23 @@ async function reconcileStaleSessions () { let logCleaned = 0 await TransactionDecorator.generateTransaction(async (transaction) => { - const execStatuses = await MicroserviceExecStatusManager.findAll({ - status: { [Op.in]: [microserviceExecState.PENDING, microserviceExecState.ACTIVE] } + const execRows = await MicroserviceExecSessionManager.findAll({ + status: { [Op.in]: ['PENDING', 'ACTIVE'] } }, transaction) - for (const row of execStatuses) { + for (const row of execRows) { + const sessionId = row.sessionId const microserviceUuid = row.microserviceUuid - const execId = row.execSessionId - if (!microserviceUuid) continue - - const inMemory = execId && sessionManager.getSession(execId) - const hasPending = sessionManager.getPendingUserCount(microserviceUuid) > 0 || - (sessionManager.pendingAgents.has(microserviceUuid) && - sessionManager.pendingAgents.get(microserviceUuid).size > 0) + if (!sessionId || !microserviceUuid) continue - if (inMemory || hasPending) continue + if (execSessionManager.getExecSession(sessionId)) continue const age = now - new Date(row.updatedAt).getTime() - const threshold = row.status === microserviceExecState.PENDING - ? execPendingTimeout - : execMaxDuration - + const threshold = row.status === 'PENDING' ? execPendingTimeout : execMaxDuration if (age < threshold) continue - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: '', status: microserviceExecState.INACTIVE }, - transaction - ) - await MicroserviceManager.update({ uuid: microserviceUuid }, { execEnabled: false }, transaction) + await MicroserviceExecSessionManager.deleteBySessionId(sessionId, transaction) + const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) if (microservice) { await ChangeTrackingService.update( @@ -84,10 +71,11 @@ async function reconcileStaleSessions () { transaction ) } + execCleaned++ - logger.info('Reconciled stale exec status row:' + JSON.stringify({ + logger.info('Reconciled stale exec session row:' + JSON.stringify({ + sessionId, microserviceUuid, - execId, status: row.status, ageMs: age })) diff --git a/src/routes/agent.js b/src/routes/agent.js index b25ca68a..750993b0 100644 --- a/src/routes/agent.js +++ b/src/routes/agent.js @@ -571,9 +571,33 @@ module.exports = [ logger.apiRes({ req, res, responseObject }) } }, + { + method: 'get', + path: '/api/v3/agent/exec/sessions', + middleware: async (req, res) => { + logger.apiReq(req) + const successCode = constants.HTTP_CODE_SUCCESS + const errorCodes = [ + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] + } + ] + + const getAgentExecSessionsEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentExecSessionsEndPoint, + successCode, errorCodes) + const responseObject = await getAgentExecSessionsEndPoint(req) + + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, res, responseObject }) + } + }, { method: 'ws', - path: '/api/v3/agent/exec/:microserviceUuid', + path: '/api/v3/agent/exec/microservice/:microserviceUuid/:sessionId', middleware: async (ws, req) => { logger.apiReq(req) try { @@ -591,29 +615,28 @@ module.exports = [ return } - // Set flag to bypass route matching - // Token validation will be done by validateAgentConnection in handleAgentConnection req._rbacAuthorized = true - // Call handler directly (it will validate the token) const wsServer = WebSocketServer.getInstance() const microserviceUuid = req.params.microserviceUuid + const sessionId = req.params.sessionId await TransactionDecorator.generateTransaction(async (transaction) => { - await wsServer.handleAgentConnection(ws, req, token, microserviceUuid, transaction) + await wsServer.handleAgentExecConnection(ws, req, token, microserviceUuid, sessionId, transaction) })() } catch (error) { - logger.error('Error in agent WebSocket connection:' + JSON.stringify({ + logger.error('Error in agent exec WebSocket connection:' + JSON.stringify({ error: error.message, stack: error.stack, url: req.url, - microserviceUuid: req.params.microserviceUuid + microserviceUuid: req.params.microserviceUuid, + sessionId: req.params.sessionId })) try { if (ws.readyState === ws.OPEN) { ws.close(1008, error.message || 'Authentication failed') } } catch (closeError) { - logger.error('Error closing agent WebSocket:' + JSON.stringify({ + logger.error('Error closing agent exec WebSocket:' + JSON.stringify({ error: closeError.message, originalError: error.message })) diff --git a/src/routes/microservices.js b/src/routes/microservices.js index b3de3939..c9aa3172 100644 --- a/src/routes/microservices.js +++ b/src/routes/microservices.js @@ -995,158 +995,6 @@ module.exports = [ }) } }, - { - method: 'post', - path: '/api/v3/microservices/:uuid/exec', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_CREATED - const errorCodes = [ - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] - }, - { - code: constants.HTTP_CODE_UNAUTHORIZED, - errors: [Errors.AuthenticationError] - }, - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const createMicroserviceExecEndPoint = ResponseDecorator.handleErrors( - MicroservicesController.createMicroserviceExecEndPoint, - successCode, - errorCodes - ) - const responseObject = await createMicroserviceExecEndPoint(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, - { - method: 'post', - path: '/api/v3/microservices/system/:uuid/exec', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_CREATED - const errorCodes = [ - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] - }, - { - code: constants.HTTP_CODE_UNAUTHORIZED, - errors: [Errors.AuthenticationError] - }, - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const createSystemMicroserviceExecEndPoint = ResponseDecorator.handleErrors( - MicroservicesController.createSystemMicroserviceExecEndPoint, - successCode, - errorCodes - ) - const responseObject = await createSystemMicroserviceExecEndPoint(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, - { - method: 'delete', - path: '/api/v3/microservices/:uuid/exec', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_CREATED - const errorCodes = [ - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] - }, - { - code: constants.HTTP_CODE_UNAUTHORIZED, - errors: [Errors.AuthenticationError] - }, - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const deleteMicroserviceExecEndPoint = ResponseDecorator.handleErrors( - MicroservicesController.deleteMicroserviceExecEndPoint, - successCode, - errorCodes - ) - const responseObject = await deleteMicroserviceExecEndPoint(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, - { - method: 'delete', - path: '/api/v3/microservices/system/:uuid/exec', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_CREATED - const errorCodes = [ - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] - }, - { - code: constants.HTTP_CODE_UNAUTHORIZED, - errors: [Errors.AuthenticationError] - }, - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const deleteSystemMicroserviceExecEndPoint = ResponseDecorator.handleErrors( - MicroservicesController.deleteSystemMicroserviceExecEndPoint, - successCode, - errorCodes - ) - const responseObject = await deleteSystemMicroserviceExecEndPoint(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, { method: 'patch', path: '/api/v3/microservices/:uuid/start', @@ -1243,7 +1091,7 @@ module.exports = [ const wsServer = WebSocketServer.getInstance() const microserviceUuid = req.params.microserviceUuid await TransactionDecorator.generateTransaction(async (transaction) => { - await wsServer.handleUserConnection(ws, req, token, microserviceUuid, false, transaction) + await wsServer.handleUserExecConnection(ws, req, token, microserviceUuid, false, transaction) })() } catch (error) { logger.error('Error in microservice WebSocket connection:' + JSON.stringify({ @@ -1291,7 +1139,7 @@ module.exports = [ const wsServer = WebSocketServer.getInstance() const microserviceUuid = req.params.microserviceUuid await TransactionDecorator.generateTransaction(async (transaction) => { - await wsServer.handleUserConnection(ws, req, token, microserviceUuid, true, transaction) // true = expectSystem + await wsServer.handleUserExecConnection(ws, req, token, microserviceUuid, true, transaction) // true = expectSystem })() } catch (error) { logger.error('Error in system microservice WebSocket connection:' + JSON.stringify({ diff --git a/src/services/agent-service.js b/src/services/agent-service.js index 6278ed0f..3a8ee3f0 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -31,6 +31,7 @@ const constants = require('../helpers/constants') const SecretManager = require('../data/managers/secret-manager') const ConfigMapManager = require('../data/managers/config-map-manager') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') +const MicroserviceExecSessionManager = require('../data/managers/microservice-exec-session-manager') const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') @@ -466,7 +467,6 @@ const getAgentMicroservices = async function (fog, transaction) { isRouter, isNats, isController: microservice.isController, - execEnabled: microservice.execEnabled, schedule: microservice.schedule } @@ -682,6 +682,39 @@ const getAgentLogSessions = async function (fog, transaction) { return { logSessions: allSessions } } +const getAgentExecSessions = async function (fog, transaction) { + const Op = require('sequelize').Op + + const microservices = await MicroserviceManager.findAll( + { iofogUuid: fog.uuid }, + transaction + ) + + const allSessions = [] + const microserviceUuids = microservices.map(ms => ms.uuid) + + if (microserviceUuids.length > 0) { + const msSessions = await MicroserviceExecSessionManager.findAll( + { + microserviceUuid: { [Op.in]: microserviceUuids }, + status: { [Op.in]: ['PENDING', 'ACTIVE'] } + }, + transaction + ) + + for (const session of msSessions) { + allSessions.push({ + microserviceUuid: session.microserviceUuid, + sessionId: session.sessionId, + status: session.status, + agentConnected: session.agentConnected + }) + } + } + + return { execSessions: allSessions } +} + const getAgentLinkedVolumeMounts = async function (fog, transaction) { const volumeMounts = [] const resourceAttributes = [ @@ -758,5 +791,6 @@ module.exports = { deleteNode: TransactionDecorator.generateTransaction(deleteNode), getAgentLinkedVolumeMounts: TransactionDecorator.generateTransaction(getAgentLinkedVolumeMounts), getControllerCA: TransactionDecorator.generateTransaction(getControllerCA), - getAgentLogSessions: TransactionDecorator.generateTransaction(getAgentLogSessions) + getAgentLogSessions: TransactionDecorator.generateTransaction(getAgentLogSessions), + getAgentExecSessions: TransactionDecorator.generateTransaction(getAgentExecSessions) } diff --git a/src/services/iofog-service.js b/src/services/iofog-service.js index 4c17ca9a..875d5c22 100644 --- a/src/services/iofog-service.js +++ b/src/services/iofog-service.js @@ -1262,7 +1262,6 @@ async function enableNodeExecEndPoint (execData, isCLI, transaction) { isPrivileged: true, logSize: Constants.MICROSERVICE_DEFAULT_LOG_SIZE, schedule: 0, - execEnabled: true, configLastUpdated: Date.now() } @@ -1296,8 +1295,7 @@ async function enableNodeExecEndPoint (execData, isCLI, transaction) { isPrivileged: debugMicroserviceData.isPrivileged, logSize: debugMicroserviceData.logSize, schedule: debugMicroserviceData.schedule, - configLastUpdated: debugMicroserviceData.configLastUpdated, - execEnabled: debugMicroserviceData.execEnabled + configLastUpdated: debugMicroserviceData.configLastUpdated } if (execData.image) { diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index 6ed9b85b..db10325f 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -2665,85 +2665,6 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { return res } -async function createExecEndPoint (microserviceUuid, isCLI, transaction) { - const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) - if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { - throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) - } - if (!microservice) { - throw new Errors.NotFoundError(ErrorMessages.INVALID_MICROSERVICE_USER) - } - - await MicroserviceManager.update({ uuid: microservice.uuid }, { execEnabled: true }, transaction) - await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceExecSessions, transaction) - - const updatedMicroservice = await MicroserviceManager.findOneWithCategory({ uuid: microservice.uuid }, transaction) - - return { - uuid: microservice.uuid, - execEnabled: updatedMicroservice.execEnabled - } -} - -async function deleteExecEndPoint (microserviceUuid, isCLI, transaction) { - const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) - if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { - throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) - } - if (!microservice) { - throw new Errors.NotFoundError(ErrorMessages.INVALID_MICROSERVICE_USER) - } - - await MicroserviceManager.update({ uuid: microservice.uuid }, { execEnabled: false }, transaction) - await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceExecSessions, transaction) - - const updatedMicroservice = await MicroserviceManager.findOneWithCategory({ uuid: microservice.uuid }, transaction) - - return { - uuid: microservice.uuid, - execEnabled: updatedMicroservice.execEnabled - } -} - -async function createSystemExecEndPoint (microserviceUuid, isCLI, transaction) { - const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) - // if (microservice.catalogItem && microservice.catalogItem.category !== 'SYSTEM') { - // throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) - // } - if (!microservice) { - throw new Errors.NotFoundError(ErrorMessages.INVALID_MICROSERVICE_USER) - } - await MicroserviceManager.update({ uuid: microservice.uuid }, { execEnabled: true }, transaction) - await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceExecSessions, transaction) - - const updatedMicroservice = await MicroserviceManager.findOneWithCategory({ uuid: microservice.uuid }, transaction) - - return { - uuid: microservice.uuid, - execEnabled: updatedMicroservice.execEnabled - } -} - -async function deleteSystemExecEndPoint (microserviceUuid, isCLI, transaction) { - const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) - // if (microservice.catalogItem && microservice.catalogItem.category !== 'SYSTEM') { - // throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) - // } - if (!microservice) { - throw new Errors.NotFoundError(ErrorMessages.INVALID_MICROSERVICE_USER) - } - - await MicroserviceManager.update({ uuid: microservice.uuid }, { execEnabled: false }, transaction) - await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceExecSessions, transaction) - - const updatedMicroservice = await MicroserviceManager.findOneWithCategory({ uuid: microservice.uuid }, transaction) - - return { - uuid: microservice.uuid, - execEnabled: updatedMicroservice.execEnabled - } -} - async function startMicroserviceEndPoint (microserviceUuid, isCLI, transaction) { const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) if (!microservice) { @@ -2842,10 +2763,6 @@ module.exports = { rebuildSystemMicroserviceEndPoint: TransactionDecorator.generateTransaction(rebuildSystemMicroserviceEndPoint), buildGetMicroserviceResponse: _buildGetMicroserviceResponse, updateChangeTracking: _updateChangeTracking, - createExecEndPoint: TransactionDecorator.generateTransaction(createExecEndPoint), - deleteExecEndPoint: TransactionDecorator.generateTransaction(deleteExecEndPoint), - createSystemExecEndPoint: TransactionDecorator.generateTransaction(createSystemExecEndPoint), - deleteSystemExecEndPoint: TransactionDecorator.generateTransaction(deleteSystemExecEndPoint), startMicroserviceEndPoint: TransactionDecorator.generateTransaction(startMicroserviceEndPoint), stopMicroserviceEndPoint: TransactionDecorator.generateTransaction(stopMicroserviceEndPoint), reconcileNatsForApplication: TransactionDecorator.generateTransaction(reconcileNatsForApplication, bypassOptions), diff --git a/src/websocket/exec-session-manager.js b/src/websocket/exec-session-manager.js new file mode 100644 index 00000000..94d68dd0 --- /dev/null +++ b/src/websocket/exec-session-manager.js @@ -0,0 +1,196 @@ +const WebSocket = require('ws') +const logger = require('../logger') +const MicroserviceExecSessionManager = require('../data/managers/microservice-exec-session-manager') +const ChangeTrackingService = require('../services/change-tracking-service') +const FogManager = require('../data/managers/iofog-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') + +class ExecSessionManager { + constructor (config) { + if (!config || !config.session) { + const error = new Error('Invalid session manager configuration') + logger.error('Failed to initialize ExecSessionManager:' + error) + throw error + } + this.execSessions = new Map() + this.config = config + this.cleanupInterval = null + this.startCleanupInterval() + logger.info('ExecSessionManager initialized with config:' + JSON.stringify({ + execPendingTimeoutMs: config.session.execPendingTimeoutMs, + execMaxDurationMs: config.session.execMaxDurationMs, + cleanupInterval: config.session.cleanupInterval + })) + } + + countSessionsForResource (microserviceUuid) { + return this.getAllSessionsForMicroservice(microserviceUuid).length + } + + getActiveExecSessionCount () { + return this.execSessions.size + } + + getAllExecSessionIds () { + return Array.from(this.execSessions.keys()) + } + + createExecSession (sessionId, microserviceUuid, agentWs, userWs, transaction) { + const session = { + sessionId, + execId: sessionId, + microserviceUuid, + agent: agentWs, + user: userWs, + lastActivity: Date.now(), + createdAt: Date.now(), + transaction, + queueBridgeEnabled: false, + metricsActive: false + } + this.execSessions.set(sessionId, session) + return session + } + + getExecSession (sessionId) { + return this.execSessions.get(sessionId) || null + } + + getAllSessionsForMicroservice (microserviceUuid) { + const sessions = [] + for (const [, session] of this.execSessions) { + if (session.microserviceUuid === microserviceUuid) { + sessions.push(session) + } + } + return sessions + } + + updateLastActivity (sessionId) { + const session = this.execSessions.get(sessionId) + if (session) { + session.lastActivity = Date.now() + } + } + + async removeExecSession (sessionId, transaction) { + const session = this.execSessions.get(sessionId) + if (!session) return + + if (session.agent && session.agent.readyState === WebSocket.OPEN) { + session.agent.close() + } + if (session.user && session.user.readyState === WebSocket.OPEN) { + session.user.close() + } + this.execSessions.delete(sessionId) + + try { + await MicroserviceExecSessionManager.deleteBySessionId(sessionId, transaction) + + const microservice = await MicroserviceManager.findOne( + { uuid: session.microserviceUuid }, + transaction + ) + if (microservice) { + const fog = await FogManager.findOne({ uuid: microservice.iofogUuid }, transaction) + if (fog) { + await ChangeTrackingService.update( + fog.uuid, + ChangeTrackingService.events.microserviceExecSessions, + transaction + ) + } + } + } catch (error) { + logger.error('Error removing exec session from database:' + JSON.stringify({ + error: error.message, + stack: error.stack, + sessionId, + microserviceUuid: session.microserviceUuid + })) + } + } + + async cleanupExpiredSessions (transaction) { + const now = Date.now() + const pendingTimeout = this.config.session.execPendingTimeoutMs || 60000 + const maxDuration = this.config.session.execMaxDurationMs || 28800000 + const expiredSessions = [] + + for (const [sessionId, session] of this.execSessions) { + const timeSinceLastActivity = now - session.lastActivity + const timeSinceCreation = now - session.createdAt + + let isExpired = false + + if (!session.agent && session.user) { + isExpired = timeSinceCreation > pendingTimeout + } else if (session.agent && !session.user) { + isExpired = timeSinceLastActivity > pendingTimeout + } else if (session.agent && session.user) { + isExpired = timeSinceLastActivity > maxDuration + } else { + isExpired = timeSinceCreation > pendingTimeout + } + + if (isExpired) { + expiredSessions.push(sessionId) + } + } + + for (const sessionId of expiredSessions) { + logger.info('Cleaning up expired exec session:' + JSON.stringify({ sessionId })) + const session = this.execSessions.get(sessionId) + if (session && session.user && session.user.readyState === WebSocket.OPEN) { + try { + session.user.close(1008, session.agent ? 'Exec session max duration exceeded' : 'Timeout waiting for agent connection') + } catch (error) { + logger.warn('Failed to close expired exec user connection:' + error.message) + } + } + if (session && session.agent && session.agent.readyState === WebSocket.OPEN) { + try { + session.agent.close(1000, 'Exec session expired') + } catch (error) { + logger.warn('Failed to close expired exec agent connection:' + error.message) + } + } + await this.removeExecSession(sessionId, transaction) + } + + return expiredSessions.length + } + + startCleanupInterval () { + const interval = this.config.session.cleanupInterval || 30000 + this.cleanupInterval = setInterval(async () => { + try { + const models = require('../data/models') + const sequelize = models.sequelize + if (!sequelize) { + logger.warn('Sequelize not available, skipping exec session cleanup') + return + } + + await sequelize.transaction(async (transaction) => { + await this.cleanupExpiredSessions(transaction) + }) + } catch (error) { + logger.error('Error during exec session cleanup:' + JSON.stringify({ + error: error.message, + stack: error.stack + })) + } + }, interval) + } + + stopCleanupInterval () { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval) + this.cleanupInterval = null + } + } +} + +module.exports = ExecSessionManager diff --git a/src/websocket/server.js b/src/websocket/server.js index 8ade3375..8dcd0477 100644 --- a/src/websocket/server.js +++ b/src/websocket/server.js @@ -2,14 +2,13 @@ const WebSocket = require('ws') const config = require('../config') const baseLogger = require('../logger') const Errors = require('../helpers/errors') -const SessionManager = require('./session-manager') const LogSessionManager = require('./log-session-manager') +const ExecSessionManager = require('./exec-session-manager') const { WebSocketError } = require('./error-handler') const MicroserviceManager = require('../data/managers/microservice-manager') const ApplicationManager = require('../data/managers/application-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') -const { microserviceState, microserviceExecState } = require('../enums/microservice-state') -const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') +const { microserviceState } = require('../enums/microservice-state') const AuthDecorator = require('../decorators/authorization-decorator') const TransactionDecorator = require('../decorators/transaction-decorator') const msgpack = require('@msgpack/msgpack') @@ -17,11 +16,11 @@ const WebSocketQueueService = require('../services/websocket-queue-service') const RouterConnectionService = require('../services/router-connection-service') const { recordExecSessionActive, - recordLogSessionActive, - recordPairingDurationMs + recordLogSessionActive } = require('./ws-metrics') const AppHelper = require('../helpers/app-helper') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') +const MicroserviceExecSessionManager = require('../data/managers/microservice-exec-session-manager') const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const ChangeTrackingService = require('../services/change-tracking-service') const FogManager = require('../data/managers/iofog-manager') @@ -166,8 +165,8 @@ class WebSocketServer { this.userSessions = new Map() this.connectionLimits = new Map() this.rateLimits = new Map() - this.sessionManager = new SessionManager(config.get('server.webSocket')) this.logSessionManager = new LogSessionManager(config.get('server.webSocket')) + this.execSessionManager = new ExecSessionManager(config.get('server.webSocket')) this.sessionConfig = config.get('server.webSocket.session') this.queueService = WebSocketQueueService this.pendingCloseTimeouts = new Map() // Track pending CLOSE messages in cross-replica scenarios @@ -252,7 +251,12 @@ class WebSocketServer { } } - logger.info('Initializing WebSocket server with strict options:' + JSON.stringify(options)) + logger.info('Initializing WebSocket server with strict options', { + maxPayload: options.maxPayload, + perMessageDeflate: options.perMessageDeflate, + clientTracking: options.clientTracking, + tls: Boolean(server && (server.key || server.cert)) + }) this.wss = new WebSocket.Server(options) // Handle WebSocket server errors @@ -342,81 +346,16 @@ class WebSocketServer { processErrorHandlersRegistered = true } - - this.sessionManager.startCleanup() - this.sessionManager.setSessionExpiredHandler(async (microserviceUuid, execId) => { - await TransactionDecorator.generateTransaction(async (tx) => { - if (execId) { - await this.cleanupSession(execId, tx) - } else { - await this.disableExecForMicroservice(microserviceUuid, tx) - } - })() - }) - } - - async disableExecForMicroservice (microserviceUuid, transaction) { - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: '', status: microserviceExecState.INACTIVE }, - transaction - ) - await MicroserviceManager.update({ uuid: microserviceUuid }, { execEnabled: false }, transaction) - const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) - if (microservice) { - await ChangeTrackingService.update( - microservice.iofogUuid, - ChangeTrackingService.events.microserviceExecSessions, - transaction - ) - } - } - - /** - * Tear down all in-memory exec sessions and agent sockets for a microservice. - * Ensures Edgelet must open a fresh agent WS on the next exec attempt (exec_b / timeout). - */ - async cleanupExecSessionsForMicroservice (microserviceUuid, transaction) { - const execIdsToCleanup = [] - for (const [execId, session] of this.sessionManager.sessions) { - if (session.microserviceUuid === microserviceUuid) { - execIdsToCleanup.push(execId) - } - } - - for (const execId of execIdsToCleanup) { - await this.cleanupSession(execId, transaction) - } - - if (this.sessionManager.pendingAgents.has(microserviceUuid)) { - const agents = this.sessionManager.pendingAgents.get(microserviceUuid) - for (const [, agentInfo] of agents.entries()) { - if (agentInfo.ws && agentInfo.ws.readyState === WebSocket.OPEN) { - try { - agentInfo.ws.close(1000, 'Exec session ended') - } catch (error) { - logger.debug('[WS-CLEANUP] Failed to close pending agent socket', { - microserviceUuid, - error: error.message - }) - } - } - } - this.sessionManager.pendingAgents.delete(microserviceUuid) - } - - if (execIdsToCleanup.length > 0) { - logger.info('[WS-CLEANUP] Cleaned exec sessions for microservice:' + JSON.stringify({ - microserviceUuid, - sessionCount: execIdsToCleanup.length - })) - } } getLogConcurrencyLimit () { return this.sessionConfig.logMaxConcurrentPerResource || 3 } + getExecConcurrencyLimit () { + return this.sessionConfig.execMaxConcurrentPerResource || 3 + } + getLogTailMaxLines () { return this.sessionConfig.logTailMaxLines || 5000 } @@ -452,12 +391,6 @@ class WebSocketServer { return true } - recordPairingDuration (startedAt) { - if (startedAt) { - recordPairingDurationMs(Date.now() - startedAt) - } - } - async countLogSessionsInDb (microserviceUuid, fogUuid, transaction) { if (microserviceUuid) { const rows = await MicroserviceLogStatusManager.findAll({ microserviceUuid }, transaction) @@ -470,6 +403,14 @@ class WebSocketServer { return 0 } + async countExecSessionsInDb (microserviceUuid, transaction) { + if (!microserviceUuid) { + return 0 + } + const rows = await MicroserviceExecSessionManager.findAll({ microserviceUuid }, transaction) + return rows.length + } + parseLogTailConfig (url, ws) { const tailMax = this.getLogTailMaxLines() const tailDefault = 100 @@ -687,10 +628,6 @@ class WebSocketServer { try { if (ws.readyState === ws.OPEN) { ws.close(1008, error.message || 'Internal server error') - const microserviceUuid = this.extractMicroserviceUuid(req.url) - if (microserviceUuid) { - await this.disableExecForMicroservice(microserviceUuid, transaction) - } } } catch (closeError) { logger.error('Error closing WebSocket connection:' + JSON.stringify({ @@ -727,11 +664,16 @@ class WebSocketServer { // Determine connection type and route to appropriate handler // IMPORTANT: Check more specific routes (system) BEFORE general routes - if (req.url.startsWith('/api/v3/agent/exec/')) { - // Agent exec connection (agent routes don't use RBAC, but may come through here) - const microserviceUuid = this.extractMicroserviceUuid(req.url) - if (!microserviceUuid) { - logger.error('WebSocket internal routing failed: Invalid endpoint - no UUID found') + if (req.url.startsWith('/api/v3/agent/exec/microservice/')) { + const url = new URL(req.url, `http://${req.headers.host || 'localhost'}`) + const pathParts = url.pathname.split('/').filter(p => p) + const microserviceIndex = pathParts.indexOf('microservice') + const microserviceUuid = req.params.microserviceUuid || + (microserviceIndex >= 0 ? pathParts[microserviceIndex + 1] : null) + const sessionId = req.params.sessionId || + (microserviceIndex >= 0 ? pathParts[microserviceIndex + 2] : null) + if (!microserviceUuid || !sessionId) { + logger.error('WebSocket internal routing failed: Invalid agent exec endpoint') try { ws.close(1008, 'Invalid endpoint') } catch (error) { @@ -739,7 +681,7 @@ class WebSocketServer { } return } - await this.handleAgentConnection(ws, req, token, microserviceUuid, transaction) + await this.handleAgentExecConnection(ws, req, token, microserviceUuid, sessionId, transaction) } else if (req.url.startsWith('/api/v3/microservices/system/exec/')) { // System microservice exec - check BEFORE regular microservice exec const microserviceUuid = req.params.microserviceUuid || this.extractMicroserviceUuid(req.url) @@ -752,7 +694,7 @@ class WebSocketServer { } return } - await this.handleUserConnection(ws, req, token, microserviceUuid, true, transaction) // true = expectSystem + await this.handleUserExecConnection(ws, req, token, microserviceUuid, true, transaction) // true = expectSystem } else if (req.url.startsWith('/api/v3/microservices/exec/')) { // Regular microservice exec const microserviceUuid = req.params.microserviceUuid || this.extractMicroserviceUuid(req.url) @@ -765,7 +707,7 @@ class WebSocketServer { } return } - await this.handleUserConnection(ws, req, token, microserviceUuid, false, transaction) // false = not system + await this.handleUserExecConnection(ws, req, token, microserviceUuid, false, transaction) // false = not system } else if (req.url.includes('/logs')) { // Handle log connections - extract parameters from URL/req.params const url = new URL(req.url, `http://${req.headers.host || 'localhost'}`) @@ -854,199 +796,177 @@ class WebSocketServer { }) } - async processAgentInitialMessage (ws, req, data, isBinary, microserviceUuid, transaction) { - logger.debug('[WS-INIT] Received initial message from agent:' + JSON.stringify({ - isBinary, - url: req.url, - microserviceUuid - })) - - if (!isBinary) { - logger.error('[WS-ERROR] Expected binary message from agent') - ws.close(1008, 'Expected binary message') - return - } + async handleUserExecConnection (ws, req, token, microserviceUuid, expectSystem, transaction) { + try { + this.ensureSocketPongHandler(ws) - const buffer = Buffer.from(data) - logger.debug('[WS-INIT] Processing initial message from agent', { - isBinary, - length: buffer.length - }) + await this.validateUserConnection(token, microserviceUuid, expectSystem, transaction) - let execMsg - try { - execMsg = this.decodeMessage(buffer) - logger.info('[WS-INIT] Decoded MessagePack from agent:' + JSON.stringify(execMsg)) - } catch (err) { - logger.error('[WS-ERROR] Failed to decode MessagePack from agent:' + JSON.stringify({ - error: err.message, - stack: err.stack - })) - ws.close(1008, 'Invalid MessagePack') - return - } + const execConcurrencyLimit = this.getExecConcurrencyLimit() + const existingExecCount = await this.countExecSessionsInDb(microserviceUuid, transaction) + if (existingExecCount >= execConcurrencyLimit) { + ws.close(1008, `Maximum of ${execConcurrencyLimit} concurrent exec sessions allowed for this microservice.`) + return + } - const execId = execMsg instanceof Map ? execMsg.get('execId') : execMsg.execId - const msgMicroserviceUuid = execMsg instanceof Map ? execMsg.get('microserviceUuid') : execMsg.microserviceUuid - if (!execId || !msgMicroserviceUuid) { - logger.error('[WS-ERROR] Agent message missing execId or microserviceUuid:' + JSON.stringify(execMsg)) - ws.close(1008, 'Missing required fields') - return - } + const sessionId = AppHelper.generateUUID() - const existingSession = this.sessionManager.getSession(execId) - if (existingSession && existingSession.agent === ws && existingSession.awaitingUser && !existingSession.user) { - logger.debug('[WS-INIT] Ignoring duplicate agent init for pending session', { - execId, - microserviceUuid: msgMicroserviceUuid - }) - return - } + await MicroserviceExecSessionManager.create({ + microserviceUuid, + sessionId, + status: 'PENDING', + userConnected: true, + agentConnected: false + }, transaction) - for (const [existingExecId, existingSessionEntry] of this.sessionManager.sessions) { - if (existingSessionEntry.agent === ws && existingExecId !== execId) { - await this.cleanupSession(existingExecId, transaction, { preserveAgentSocket: true }) + const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) + if (!microservice) { + throw new Error(`Microservice not found: ${microserviceUuid}`) } - } - const session = await this.sessionManager.tryActivateSession(msgMicroserviceUuid, execId, ws, true, transaction) - if (session) { - logger.info('[WS-SESSION] Session activated for agent:' + JSON.stringify({ - execId, - microserviceUuid: msgMicroserviceUuid - })) - logger.debug('[WS-FORWARD] Setting up message forwarding:' + JSON.stringify({ - execId, - microserviceUuid: msgMicroserviceUuid - })) - await this.setupMessageForwarding(execId, transaction) - this.scheduleAgentExecConnectEvent(req, msgMicroserviceUuid) - return - } + const fog = await FogManager.findOne({ uuid: microservice.iofogUuid }, transaction) + if (!fog) { + throw new Error(`Fog not found: ${microservice.iofogUuid}`) + } - this.attachPendingKeepAliveHandler(ws) - try { - await MicroserviceExecStatusManager.update( - { microserviceUuid: msgMicroserviceUuid }, - { execSessionId: execId, status: microserviceExecState.PENDING }, + await ChangeTrackingService.update( + fog.uuid, + ChangeTrackingService.events.microserviceExecSessions, transaction ) - logger.debug('[WS-SESSION] Updated microservice exec status to PENDING', { - execId, - microserviceUuid: msgMicroserviceUuid - }) - } catch (error) { - logger.error('[WS-SESSION] Failed to update microservice exec status to PENDING', { - execId, - microserviceUuid: msgMicroserviceUuid, - error: error.message, - stack: error.stack - }) - } - - const agentOnlySession = this.sessionManager.createSession(execId, msgMicroserviceUuid, ws, null, transaction) - agentOnlySession.awaitingUser = true - try { - await this.queueService.enableForSession(agentOnlySession, async (closeExecId) => { - const timeout = this.pendingCloseTimeouts.get(closeExecId) - if (timeout) { - clearTimeout(timeout) - this.pendingCloseTimeouts.delete(closeExecId) - logger.debug('[WS-SESSION] Cleared pending CLOSE timeout - agent responded', { execId: closeExecId }) - } - await TransactionDecorator.generateTransaction(async (failTx) => { - await this.cleanupSession(closeExecId, failTx) - })() - }) - agentOnlySession.queueBridgeEnabled = true - } catch (error) { - logger.warn('[WS-SESSION] Optional queue bridge for pending agent failed; direct relay when user connects on same replica:', { - execId, - microserviceUuid: msgMicroserviceUuid, - error: error.message - }) - agentOnlySession.queueBridgeEnabled = false - } + logger.debug('Change tracking updated for exec session:' + JSON.stringify({ + fogUuid: fog.uuid, + microserviceUuid, + sessionId + })) - logger.info('[WS-SESSION] Agent pending — awaiting user before ACTIVATION:' + JSON.stringify({ - execId, - microserviceUuid: msgMicroserviceUuid - })) - this.scheduleAgentExecConnectEvent(req, msgMicroserviceUuid) - } + const execSession = this.execSessionManager.createExecSession( + sessionId, + microserviceUuid, + null, + ws, + transaction + ) + execSession.metricsActive = true + recordExecSessionActive(1) - async handleAgentConnection (ws, req, token, microserviceUuid, transaction) { - try { - this.ensureSocketPongHandler(ws) - ws._agentExecHandshakeContext = { req, microserviceUuid } - logger.debug('[WS-CONN] Processing agent connection:' + JSON.stringify({ - url: req.url, + const activationMsg = { + type: MESSAGE_TYPES.ACTIVATION, + data: Buffer.from(JSON.stringify({ sessionId, microserviceUuid })), + sessionId, microserviceUuid, - remoteAddress: req.socket.remoteAddress - })) + execId: sessionId, + timestamp: Date.now() + } + ws.send(this.encodeMessage(activationMsg), { binary: true }) - // Capture the first frame during validation so Edgelet's immediate-on-open msgpack is not lost (Plan 16-A). - let capturedInitialFrame = null - const captureHandler = (data, isBinary) => { - if (capturedInitialFrame === null) { - capturedInitialFrame = { data, isBinary } + try { + const waitingMsg = { + type: MESSAGE_TYPES.STDERR, + data: Buffer.from('Waiting for agent connection. Interactive exec will begin once the agent connects.\n'), + sessionId, + microserviceUuid, + execId: sessionId, + timestamp: Date.now() } + ws.send(this.encodeMessage(waitingMsg), { binary: true }) + logger.info('Sent waiting status message to user for exec session:' + JSON.stringify({ + sessionId, + microserviceUuid + })) + } catch (error) { + logger.warn('Failed to send waiting status message to user:' + JSON.stringify({ + error: error.message, + sessionId + })) } - ws.on('message', captureHandler) - const fog = await this.validateAgentConnection(token, microserviceUuid, transaction) - logger.debug('[WS-VALIDATE] Agent connection validated:' + JSON.stringify({ - fogUuid: fog.uuid, - microserviceUuid, - url: req.url - })) + await this.setupExecMessageForwarding(sessionId, transaction) - if (capturedInitialFrame) { - ws.removeListener('message', captureHandler) - await this.processAgentInitialMessage( - ws, - req, - capturedInitialFrame.data, - capturedInitialFrame.isBinary, + const EXEC_PENDING_TIMEOUT = this.getExecPendingTimeoutMs() + const pendingTimer = setTimeout(async () => { + const session = this.execSessionManager.getExecSession(sessionId) + if (!session || session.agent) { + return + } + logger.warn('Exec session pending timeout:' + JSON.stringify({ + sessionId, microserviceUuid, - transaction - ) - } else { - const initialMessageHandler = async (data, isBinary) => { - ws.removeListener('message', initialMessageHandler) - ws.removeListener('message', captureHandler) - await this.processAgentInitialMessage(ws, req, data, isBinary, microserviceUuid, transaction) + timeout: EXEC_PENDING_TIMEOUT + })) + try { + if (ws.readyState === WebSocket.OPEN) { + const timeoutMsg = { + type: MESSAGE_TYPES.STDERR, + data: Buffer.from('Timeout waiting for agent connection.\n'), + sessionId, + microserviceUuid, + execId: sessionId, + timestamp: Date.now() + } + ws.send(this.encodeMessage(timeoutMsg), { binary: true }) + ws.close(1008, 'Timeout waiting for agent connection') + } + } catch (error) { + logger.warn('Failed to close exec session on pending timeout:' + error.message) } - // Register the follow-up handler before removing capture to avoid losing a fast first frame. - ws.on('message', initialMessageHandler) - ws.removeListener('message', captureHandler) - } + try { + await TransactionDecorator.generateTransaction(async (timeoutTransaction) => { + await this.cleanupExecSession(sessionId, timeoutTransaction) + })() + } catch (error) { + logger.error('Failed to remove exec session after pending timeout:' + error.message) + } + }, EXEC_PENDING_TIMEOUT) + + setImmediate(async () => { + try { + let actorId = null + if (req.headers && req.headers.authorization) { + actorId = EventService.extractUsernameFromToken(req.headers.authorization) + } + await EventService.createWsConnectEvent({ + timestamp: Date.now(), + endpointType: 'user', + actorId, + path: req.url, + resourceId: microserviceUuid, + ipAddress: EventService.extractIPv4Address(req) || null + }) + } catch (err) { + logger.error('Failed to create WS_CONNECT event for user exec session (non-blocking):', err) + } + }) - // Handle connection close ws.on('close', async (code, reason) => { - // Record WebSocket disconnection event (non-blocking) + clearTimeout(pendingTimer) + const session = this.execSessionManager.getExecSession(sessionId) + if (session) { + session.user = null + session.lastActivity = Date.now() + + try { + await TransactionDecorator.generateTransaction(async (closeTransaction) => { + await this.cleanupExecSession(sessionId, closeTransaction) + })() + } catch (err) { + logger.error('Failed to cleanup exec session on user disconnect:' + JSON.stringify({ + error: err.message, + sessionId + })) + } + } + setImmediate(async () => { try { - const authHeader = req.headers.authorization let actorId = null - if (authHeader) { - const [scheme, token] = authHeader.split(' ') - if (scheme.toLowerCase() === 'bearer' && token) { - try { - const tokenParts = token.split('.') - if (tokenParts.length === 3) { - const payload = JSON.parse(Buffer.from(tokenParts[1], 'base64').toString()) - actorId = payload.sub || null - } - } catch (err) { - // Ignore token parsing errors - } - } + if (req.headers && req.headers.authorization) { + actorId = EventService.extractUsernameFromToken(req.headers.authorization) } await EventService.createWsDisconnectEvent({ timestamp: Date.now(), - endpointType: 'agent', + endpointType: 'user', actorId, path: req.url, resourceId: microserviceUuid, @@ -1054,537 +974,201 @@ class WebSocketServer { closeCode: code }) } catch (err) { - logger.error('Failed to create WS_DISCONNECT event (non-blocking):', err) + logger.error('Failed to create WS_DISCONNECT event for user exec session (non-blocking):', err) } }) - - await TransactionDecorator.generateTransaction(async (closeTransaction) => { - for (const [execId, session] of this.sessionManager.sessions) { - if (session.agent === ws) { - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { - try { - const closeMsg = { - type: MESSAGE_TYPES.CLOSE, - execId, - microserviceUuid: session.microserviceUuid, - timestamp: Date.now(), - data: Buffer.from('Agent closed connection') - } - const encoded = this.encodeMessage(closeMsg) - await this.queueService.publishToUser(execId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) - logger.info('[WS-CLOSE] Sent CLOSE message to user via queue after agent disconnect', { - execId, - microserviceUuid: session.microserviceUuid - }) - } catch (error) { - logger.error('[WS-CLOSE] Failed to send CLOSE message to user via queue', { - execId, - error: error.message - }) - } - } - await this.cleanupSession(execId, closeTransaction) - } - } - })() - this.sessionManager.removePendingAgent(microserviceUuid, ws) - logger.debug('[WS-CLOSE] Agent connection closed:' + JSON.stringify({ - url: req.url, - microserviceUuid - })) - }) - - // Handle errors - ws.on('error', (error) => { - logger.error('[WS-ERROR] Agent connection error:' + JSON.stringify({ - error: error.message, - url: req.url, - microserviceUuid - })) }) } catch (error) { - logger.error('[WS-ERROR] Error in handleAgentConnection:' + JSON.stringify({ - error: error.message, - stack: error.stack, - url: req.url, - microserviceUuid - })) - if (ws.readyState === ws.OPEN) { - ws.close(1008, error.message || 'Connection error') + logger.error('User exec connection error:', error) + if (ws.readyState === WebSocket.OPEN) { + ws.close(1008, error.message) } } } - async getPendingAgentExecIdsFromDB (microserviceUuid, transaction) { + async handleAgentExecConnection (ws, req, token, microserviceUuid, sessionId, transaction) { try { - const pendingExecStatus = await MicroserviceExecStatusManager.findAllExcludeFields( - { - microserviceUuid, - status: microserviceExecState.PENDING - }, - transaction - ) + this.ensureSocketPongHandler(ws) - const execIds = pendingExecStatus.map(status => status.execSessionId) - logger.debug('Database query for pending agents:' + JSON.stringify({ - microserviceUuid, - foundExecIds: execIds, - count: execIds.length - })) + await this.validateAgentExecConnection(token, microserviceUuid, sessionId, transaction) - return execIds - } catch (error) { - logger.error('Failed to query database for pending agents:' + JSON.stringify({ - error: error.message, - microserviceUuid - })) - return [] - } - } - - async handleUserConnection (ws, req, token, microserviceUuid, expectSystem, transaction) { - try { - this.ensureSocketPongHandler(ws) - await this.validateUserConnection(token, microserviceUuid, expectSystem, transaction) - logger.info('User connection validated successfully for microservice:' + microserviceUuid) + const execRow = await MicroserviceExecSessionManager.findBySessionId(sessionId, transaction) + if (!execRow) { + logger.error('Agent exec: session not found:' + JSON.stringify({ sessionId, microserviceUuid })) + ws.close(1008, 'Session not found') + return + } - // Check if there's already an active or pending exec session for this microservice - if (this.sessionManager.hasActiveOrPendingUser(microserviceUuid)) { - logger.debug('Microservice already has an exec session in progress:' + JSON.stringify({ - microserviceUuid + if (execRow.microserviceUuid !== microserviceUuid) { + logger.error('Agent exec: session microservice mismatch:' + JSON.stringify({ + sessionId, + microserviceUuid, + rowMicroserviceUuid: execRow.microserviceUuid })) - ws.close(1008, 'An exec session is already in progress for this microservice. Only one user exec WebSocket is allowed.') + ws.close(1008, 'Session mismatch') return } - // Get pending agent execIds from database (multi-replica compatible) - const pendingAgentExecIds = await this.getPendingAgentExecIdsFromDB(microserviceUuid, transaction) - logger.info('Pending agent execIds from database:' + JSON.stringify(pendingAgentExecIds)) - - // Simplified logic: find any available pending agent - const hasPendingAgents = pendingAgentExecIds.length > 0 - - if (hasPendingAgents) { - // Find any available pending agent - const availableExecId = pendingAgentExecIds[0] - const pendingAgent = this.sessionManager.findPendingAgentForExecId(microserviceUuid, availableExecId) - - if (pendingAgent) { - // Activate session using agent's execId (agent is on same replica) - const session = await this.sessionManager.tryActivateSession(microserviceUuid, availableExecId, ws, false, transaction) - if (session) { - logger.info('Session activated for user:', { - execId: availableExecId, - microserviceUuid, - userState: ws.readyState, - agentState: pendingAgent.readyState - }) - await this.setupMessageForwarding(availableExecId, transaction) + await MicroserviceExecSessionManager.update( + { sessionId }, + { agentConnected: true, status: 'ACTIVE' }, + transaction + ) - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { - try { - // Extract actorId from token (req.kauth not available for WebSocket connections) - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) - } - await EventService.createWsConnectEvent({ - timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null - }) - } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) - } - }) - return - } - } else { - // Agent is on a different replica - create session with just user and enable queue bridge - // The AMQP queues will handle message relay between replicas - logger.info('Found PENDING execId in DB but agent is on different replica, activating session with user only:', { - execId: availableExecId, - microserviceUuid - }) - if (!(await this.requireRouterForCrossReplica(ws))) { - return - } - this.sessionManager.createSession(availableExecId, microserviceUuid, null, ws, transaction) - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: availableExecId, status: microserviceExecState.ACTIVE }, - transaction - ) - await this.setupMessageForwarding(availableExecId, transaction) - logger.info('Cross-replica session activated with user only:', { - execId: availableExecId, - microserviceUuid, - userState: ws.readyState - }) + const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) + const fog = await FogManager.findOne({ uuid: microservice.iofogUuid }, transaction) - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { - try { - // Extract actorId from token (req.kauth not available for WebSocket connections) - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) - } - await EventService.createWsConnectEvent({ - timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null - }) - } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) - } - }) + let session = this.execSessionManager.getExecSession(sessionId) + if (!session) { + if (!(await this.requireRouterForCrossReplica(ws))) { return } + session = this.execSessionManager.createExecSession( + sessionId, + microserviceUuid, + ws, + null, + transaction + ) + session.metricsActive = true + recordExecSessionActive(1) + } else { + session.agent = ws + session.lastActivity = Date.now() + session.activationSent = false } - // If we reach here, either no pending agent or activation failed - // Add user to pending list to wait for agent - logger.info('No immediate agent available, adding user to pending list:' + JSON.stringify({ - microserviceUuid, - hasPendingAgents, - pendingAgentCount: pendingAgentExecIds.length - })) - this.sessionManager.addPendingUser(microserviceUuid, ws) - this.attachPendingKeepAliveHandler(ws) + await this.setupExecMessageForwarding(sessionId, transaction) - // IMMEDIATE RE-CHECK: Look for any newly available agents after adding user (database query) - const retryPendingAgents = await this.getPendingAgentExecIdsFromDB(microserviceUuid, transaction) - if (retryPendingAgents.length > 0) { - logger.info('Found available agent after adding user, attempting immediate activation:' + JSON.stringify({ - microserviceUuid, - availableExecIds: retryPendingAgents - })) - - // Try to activate session with first available agent - const availableExecId = retryPendingAgents[0] - const pendingAgent = this.sessionManager.findPendingAgentForExecId(microserviceUuid, availableExecId) - - if (pendingAgent) { - // Remove user from pending first since we're activating - this.sessionManager.removePendingUser(microserviceUuid, ws) - const session = await this.sessionManager.tryActivateSession(microserviceUuid, availableExecId, ws, false, transaction) - if (session) { - logger.info('Session activated immediately after re-check:' + JSON.stringify({ - execId: availableExecId, - microserviceUuid, - userState: ws.readyState, - agentState: pendingAgent.readyState - })) + if (session.user && session.user.readyState === WebSocket.OPEN) { + try { + const readyMsg = { + type: MESSAGE_TYPES.STDERR, + data: Buffer.from('Agent connected. Interactive exec is ready.\n'), + sessionId, + microserviceUuid, + execId: sessionId, + timestamp: Date.now() + } + session.user.send(this.encodeMessage(readyMsg), { binary: true }) + } catch (error) { + logger.warn('Failed to notify user that exec agent connected:' + JSON.stringify({ + sessionId, + error: error.message + })) + } + } - await this.setupMessageForwarding(availableExecId, transaction) + this.scheduleAgentExecConnectEvent(req, microserviceUuid) - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { + setImmediate(async () => { + try { + const authHeader = req.headers.authorization + let actorId = null + if (authHeader) { + const [scheme, authToken] = authHeader.split(' ') + if (scheme.toLowerCase() === 'bearer' && authToken) { try { - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) + const tokenParts = authToken.split('.') + if (tokenParts.length === 3) { + const payload = JSON.parse(Buffer.from(tokenParts[1], 'base64').toString()) + actorId = payload.sub || null } - await EventService.createWsConnectEvent({ - timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null - }) } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) - } - }) - return // Exit early, session activated successfully - } - } else { - // Agent is on different replica - activate with user only - logger.info('Found PENDING execId in retry but agent is on different replica, activating session with user only:', { - execId: availableExecId, - microserviceUuid - }) - if (!(await this.requireRouterForCrossReplica(ws))) { - this.sessionManager.removePendingUser(microserviceUuid, ws) - return - } - // Remove user from pending first - this.sessionManager.removePendingUser(microserviceUuid, ws) - this.sessionManager.createSession(availableExecId, microserviceUuid, null, ws, transaction) - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: availableExecId, status: microserviceExecState.ACTIVE }, - transaction - ) - await this.setupMessageForwarding(availableExecId, transaction) - logger.info('Cross-replica session activated with user only (retry):', { - execId: availableExecId, - microserviceUuid, - userState: ws.readyState - }) - - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { - try { - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) + // Ignore token parsing errors } - await EventService.createWsConnectEvent({ - timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null - }) - } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) } + } + await EventService.createWsConnectEvent({ + timestamp: Date.now(), + endpointType: 'agent', + actorId, + path: req.url, + resourceId: microserviceUuid, + ipAddress: EventService.extractIPv4Address(req) || null }) - return + } catch (err) { + logger.error('Failed to create WS_CONNECT event for agent exec session (non-blocking):', err) } - } + }) - // Only proceed with timeout mechanism if we still couldn't activate - logger.info('No immediate agent available after re-check, proceeding with timeout mechanism') + ws.on('close', async (code, reason) => { + const currentSession = this.execSessionManager.getExecSession(sessionId) + if (currentSession) { + currentSession.agent = null + currentSession.lastActivity = Date.now() - // Send status message to user when added to pending using STDERR - try { - const statusMsg = { - type: MESSAGE_TYPES.STDERR, - data: Buffer.from('Waiting for agent connection. Please ensure the microservice/agent is running.\n'), - microserviceUuid, - execId: 'pending', // Since we don't have execSessionId anymore - timestamp: Date.now() - } - const encoded = this.encodeMessage(statusMsg) - ws.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - logger.info('Sent waiting status message to user:' + JSON.stringify({ - microserviceUuid, - messageType: 'STDERR', - encodedLength: encoded.length - })) - } catch (error) { - logger.warn('Failed to send status message to user:' + JSON.stringify({ - error: error.message, - microserviceUuid - })) - } + try { + await TransactionDecorator.generateTransaction(async (closeTransaction) => { + await MicroserviceExecSessionManager.update( + { sessionId }, + { agentConnected: false }, + closeTransaction + ) - // Start periodic retry timer for pending users (every 10 seconds) - const RETRY_INTERVAL = 10000 - const startTime = Date.now() - const retryTimer = setInterval(async () => { - if (this.sessionManager.isUserStillPending(microserviceUuid, ws)) { - logger.debug('Periodic retry: checking for available agents:' + JSON.stringify({ - microserviceUuid, - retryCount: Math.floor((Date.now() - startTime) / RETRY_INTERVAL) - })) + const queueEnabled = this.queueService.shouldUseQueue(sessionId) - try { - const periodicRetryExecIds = await this.getPendingAgentExecIdsFromDB(microserviceUuid, transaction) - if (periodicRetryExecIds.length > 0) { - logger.info('Periodic retry found available agent:' + JSON.stringify({ - microserviceUuid, - availableExecIds: periodicRetryExecIds - })) - - // Attempt session activation with first available agent - const availableExecId = periodicRetryExecIds[0] - const pendingAgent = this.sessionManager.findPendingAgentForExecId(microserviceUuid, availableExecId) - - if (pendingAgent) { - // Remove user from pending first - this.sessionManager.removePendingUser(microserviceUuid, ws) - const session = await this.sessionManager.tryActivateSession(microserviceUuid, availableExecId, ws, false, transaction) - if (session) { - logger.info('Session activated via periodic retry:' + JSON.stringify({ - execId: availableExecId, - microserviceUuid, - userState: ws.readyState, - agentState: pendingAgent.readyState - })) - - await this.setupMessageForwarding(availableExecId, transaction) - - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { - try { - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) - } - await EventService.createWsConnectEvent({ - timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null - }) - } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) - } - }) - clearInterval(retryTimer) // Stop retry timer - // Exit early, session activated successfully - } + if (!currentSession.user) { + await this.cleanupExecSession(sessionId, closeTransaction) } else { - // Agent is on different replica - activate with user only - logger.info('Periodic retry found PENDING execId but agent is on different replica, activating session with user only:', { - execId: availableExecId, - microserviceUuid - }) - if (!(await this.requireRouterForCrossReplica(ws))) { - clearInterval(retryTimer) - this.sessionManager.removePendingUser(microserviceUuid, ws) - return - } - // Remove user from pending first - this.sessionManager.removePendingUser(microserviceUuid, ws) - this.sessionManager.createSession(availableExecId, microserviceUuid, null, ws, transaction) - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: availableExecId, status: microserviceExecState.ACTIVE }, - transaction - ) - await this.setupMessageForwarding(availableExecId, transaction) - logger.info('Cross-replica session activated with user only (periodic retry):', { - execId: availableExecId, - microserviceUuid, - userState: ws.readyState - }) - - // Record WebSocket connection event (non-blocking) - setImmediate(async () => { + if (queueEnabled) { try { - let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) - } - await EventService.createWsConnectEvent({ + const closeMsg = { + type: MESSAGE_TYPES.CLOSE, + execId: sessionId, + sessionId, + microserviceUuid: currentSession.microserviceUuid, timestamp: Date.now(), - endpointType: 'user', - actorId, - path: req.url, - resourceId: microserviceUuid, - ipAddress: EventService.extractIPv4Address(req) || null + data: Buffer.from('Agent closed connection') + } + const encoded = this.encodeMessage(closeMsg) + await this.queueService.publishToUser(sessionId, encoded, { messageType: MESSAGE_TYPES.CLOSE }) + } catch (error) { + logger.error('[WS-CLOSE] Failed to send CLOSE to user via queue after agent exec disconnect', { + sessionId, + error: error.message }) - } catch (err) { - logger.error('Failed to create WS_CONNECT event (non-blocking):', err) } - }) - clearInterval(retryTimer) // Stop retry timer - } - } - } catch (retryError) { - logger.warn('Periodic retry failed:' + JSON.stringify({ - error: retryError.message, - microserviceUuid - })) - } - } else { - // User no longer pending, clear retry timer - clearInterval(retryTimer) - } - }, RETRY_INTERVAL) - - // Store timer reference for cleanup - this.sessionManager.setUserRetryTimer(microserviceUuid, ws, retryTimer) - - // Add timeout mechanism for pending users - const PENDING_USER_TIMEOUT = this.getExecPendingTimeoutMs() - setTimeout(() => { - if (this.sessionManager.isUserStillPending(microserviceUuid, ws)) { - logger.warn('Pending user timeout, closing connection:' + JSON.stringify({ - microserviceUuid, - timeout: PENDING_USER_TIMEOUT - })) - - // Send timeout message before closing - try { - const timeoutMsg = { - type: MESSAGE_TYPES.STDERR, - data: Buffer.from('Timeout waiting for agent connection. Please try again.\n'), - microserviceUuid, - execId: 'pending', // Since we don't have execSessionId anymore - timestamp: Date.now() - } - const encoded = this.encodeMessage(timeoutMsg) - ws.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - logger.info('Sent timeout message to user:' + JSON.stringify({ - microserviceUuid, - messageType: 'STDERR', - encodedLength: encoded.length - })) - } catch (timeoutError) { - logger.warn('Failed to send timeout message to user:' + JSON.stringify({ - error: timeoutError.message, - microserviceUuid - })) - } + } else if (currentSession.user.readyState === WebSocket.OPEN) { + currentSession.user.close(1000, 'Agent closed connection') + } - try { - ws.close(1008, 'Timeout waiting for agent connection') - } catch (closeError) { - logger.error('Error closing timed out user connection:' + JSON.stringify({ - error: closeError.message, - microserviceUuid + await ChangeTrackingService.update( + fog.uuid, + ChangeTrackingService.events.microserviceExecSessions, + closeTransaction + ) + } + })() + } catch (err) { + logger.error('Failed to handle agent exec disconnect:' + JSON.stringify({ + sessionId, + error: err.message })) } - // Clear retry timer before removing user - const retryTimer = this.sessionManager.getUserRetryTimer(microserviceUuid, ws) - if (retryTimer) { - clearInterval(retryTimer) - this.sessionManager.clearUserRetryTimer(microserviceUuid, ws) - } - - this.sessionManager.removePendingUser(microserviceUuid, ws) - TransactionDecorator.generateTransaction(async (timeoutTransaction) => { - await this.cleanupExecSessionsForMicroservice(microserviceUuid, timeoutTransaction) - await this.disableExecForMicroservice(microserviceUuid, timeoutTransaction) - })().catch((err) => { - logger.error('Failed to disable exec after pending user timeout:' + JSON.stringify({ - error: err.message, - microserviceUuid - })) - }) } - }, PENDING_USER_TIMEOUT) - ws.on('close', (code, reason) => { - // Record WebSocket disconnection event (non-blocking) setImmediate(async () => { try { - // Extract actorId from token (req.kauth not available for WebSocket connections) + const authHeader = req.headers.authorization let actorId = null - if (req.headers && req.headers.authorization) { - actorId = EventService.extractUsernameFromToken(req.headers.authorization) + if (authHeader) { + const [scheme, authToken] = authHeader.split(' ') + if (scheme.toLowerCase() === 'bearer' && authToken) { + try { + const tokenParts = authToken.split('.') + if (tokenParts.length === 3) { + const payload = JSON.parse(Buffer.from(tokenParts[1], 'base64').toString()) + actorId = payload.sub || null + } + } catch (err) { + // Ignore token parsing errors + } + } } await EventService.createWsDisconnectEvent({ timestamp: Date.now(), - endpointType: 'user', + endpointType: 'agent', actorId, path: req.url, resourceId: microserviceUuid, @@ -1592,56 +1176,22 @@ class WebSocketServer { closeCode: code }) } catch (err) { - logger.error('Failed to create WS_DISCONNECT event (non-blocking):', err) - } - }) - - TransactionDecorator.generateTransaction(async (closeTransaction) => { - const wasPending = this.sessionManager.isUserStillPending(microserviceUuid, ws) - for (const [execId, session] of this.sessionManager.sessions) { - if (session.user === ws) { - await this.cleanupSession(execId, closeTransaction) - } - } - if (wasPending) { - await this.cleanupExecSessionsForMicroservice(microserviceUuid, closeTransaction) - await this.disableExecForMicroservice(microserviceUuid, closeTransaction) + logger.error('Failed to create WS_DISCONNECT event for agent exec session (non-blocking):', err) } - })().catch((err) => { - logger.error('Failed to cleanup exec session on user disconnect:' + JSON.stringify({ - error: err.message, - microserviceUuid - })) }) + }) - // Clear retry timer before removing user - const retryTimer = this.sessionManager.getUserRetryTimer(microserviceUuid, ws) - if (retryTimer) { - clearInterval(retryTimer) - this.sessionManager.clearUserRetryTimer(microserviceUuid, ws) - } - - this.sessionManager.removePendingUser(microserviceUuid, ws) - logger.info('User WebSocket disconnected:' + JSON.stringify({ - microserviceUuid, - userState: ws.readyState + ws.on('error', (error) => { + logger.error('[WS-ERROR] Agent exec connection error:' + JSON.stringify({ + error: error.message, + sessionId, + microserviceUuid })) }) } catch (error) { - logger.error('User connection validation failed:' + JSON.stringify({ - error: error.message, - stack: error.stack - })) - // Handle error gracefully instead of throwing + logger.error('Agent exec connection error:', error) if (ws.readyState === WebSocket.OPEN) { - try { - ws.close(1008, error.message || 'Authentication failed') - } catch (closeError) { - logger.error('Error closing WebSocket:' + JSON.stringify({ - error: closeError.message, - originalError: error.message - })) - } + ws.close(1008, error.message || 'Connection error') } } } @@ -1657,8 +1207,8 @@ class WebSocketServer { // return noisePatterns.some(pattern => pattern.test(output)) // } - async sendExecActivationToAgent (session, execId, transaction) { - if (!session.user) { + async sendExecActivationToExecSession (session, sessionId, transaction) { + if (!session.user || !session.agent) { return false } if (session.activationSent) { @@ -1668,462 +1218,48 @@ class WebSocketServer { const activationMsg = { type: MESSAGE_TYPES.ACTIVATION, data: Buffer.from(JSON.stringify({ - execId, + sessionId, + execId: sessionId, microserviceUuid: session.microserviceUuid, timestamp: Date.now() })), + sessionId, microserviceUuid: session.microserviceUuid, - execId, + execId: sessionId, timestamp: Date.now() } try { - const success = await this.sendMessageToAgent(session.agent, activationMsg, execId, session.microserviceUuid) + const success = await this.sendMessageToAgent(session.agent, activationMsg, sessionId, session.microserviceUuid) if (success) { session.activationSent = true - session.awaitingUser = false - logger.info('[RELAY] Session activation complete:' + JSON.stringify({ - execId, + logger.info('[RELAY] Exec session activation sent to agent:' + JSON.stringify({ + sessionId, microserviceUuid: session.microserviceUuid, - agentState: session.agent ? session.agent.readyState : 'N/A (cross-replica)', - queueEnabled: this.queueService.shouldUseQueue(execId) + queueEnabled: this.queueService.shouldUseQueue(sessionId) })) } else { - logger.error('[RELAY] Session activation failed:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, - agentState: session.agent ? session.agent.readyState : 'N/A', - queueEnabled: this.queueService.shouldUseQueue(execId) + logger.error('[RELAY] Exec session activation to agent failed:' + JSON.stringify({ + sessionId, + microserviceUuid: session.microserviceUuid })) if (session.agent) { - await this.cleanupSession(execId, transaction) + await this.cleanupExecSession(sessionId, transaction) } } return success } catch (error) { - logger.error('[RELAY] Session activation error:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, + logger.error('[RELAY] Exec session activation error:' + JSON.stringify({ + sessionId, error: error.message })) if (session.agent) { - await this.cleanupSession(execId, transaction) + await this.cleanupExecSession(sessionId, transaction) } return false } } - async setupMessageForwarding (execId, transaction) { - const session = this.sessionManager.getSession(execId) - if (!session) { - logger.error('[RELAY] Failed to setup message forwarding: No session found for execId=' + execId) - return - } - - const { agent, user } = session - logger.info('[RELAY] Setting up message forwarding for session:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, - agentConnected: !!agent, - userConnected: !!user, - agentState: agent ? agent.readyState : 'N/A', - userState: user ? user.readyState : 'N/A' - })) - this.detachPendingKeepAliveHandler(user) - this.detachPendingKeepAliveHandler(agent) - if (!session.queueBridgeEnabled) { - try { - // Pass cleanup callback so queue service can notify us when CLOSE is received - await this.queueService.enableForSession(session, async (execId) => { - // Clear timeout if it exists (agent responded to CLOSE) - const timeout = this.pendingCloseTimeouts.get(execId) - if (timeout) { - clearTimeout(timeout) - this.pendingCloseTimeouts.delete(execId) - logger.debug('[RELAY] Cleared pending CLOSE timeout - agent responded', { execId }) - } - const currentTransaction = session.transaction - await this.cleanupSession(execId, currentTransaction) - }) - session.queueBridgeEnabled = true - logger.info('[RELAY] AMQP queue bridge enabled for exec session', { - execId, - microserviceUuid: session.microserviceUuid - }) - } catch (error) { - session.queueBridgeEnabled = false - if (this.isCrossReplicaSession(session) && this.haConfig.failFastOnRouterUnavailable !== false) { - logger.error('[RELAY] AMQP required for cross-replica session but router bridge failed', { - execId, - error: error.message - }) - if (session.user && session.user.readyState === WebSocket.OPEN) { - session.user.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) - } - if (session.agent && session.agent.readyState === WebSocket.OPEN) { - session.agent.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) - } - await this.cleanupSession(execId, transaction) - return - } - logger.warn('[RELAY] Failed to enable AMQP queue bridge, falling back to direct WebSocket relay', { - execId, - error: error.message - }) - } - } - - if (user) { - if (!session.metricsActive) { - session.metricsActive = true - recordExecSessionActive(1) - this.recordPairingDuration(session.pairingStartedAt) - } - await this.sendExecActivationToAgent(session, execId, transaction) - } else { - logger.debug('[RELAY] Relay handlers deferred — user leg not connected; ACTIVATION withheld', { - execId, - microserviceUuid: session.microserviceUuid - }) - } - - // Remove any previous message handlers to avoid duplicates - if (user) { - logger.debug('[RELAY] Removing previous user message handlers for execId=' + execId) - user.removeAllListeners('message') - } - if (agent) { - logger.debug('[RELAY] Removing previous agent message handlers for execId=' + execId) - agent.removeAllListeners('message') - } - - // Forward user -> agent (works for both direct WebSocket and queue-based forwarding) - if (user) { - logger.debug('[RELAY] Setting up user->agent message forwarding for execId=' + execId) - user.on('message', async (data, isBinary) => { - logger.debug('[RELAY] User message received:' + JSON.stringify({ - execId, - isBinary, - dataType: typeof data, - dataLength: data.length, - userState: user.readyState, - agentState: agent ? agent.readyState : 'N/A (cross-replica)', - queueEnabled: this.queueService.shouldUseQueue(execId) - })) - - if (!isBinary) { - // Handle text messages from user - const text = data.toString() - logger.debug('[RELAY] Received text message from user:' + JSON.stringify({ - execId, - text, - length: text.length, - userState: user.readyState, - agentState: session.agent ? session.agent.readyState : 'N/A (cross-replica)', - queueEnabled: this.queueService.shouldUseQueue(execId) - })) - - // Convert text to binary message in agent's expected format - const msg = { - type: MESSAGE_TYPES.STDIN, - data: Buffer.from(text + '\n'), // Add newline for command execution - microserviceUuid: session.microserviceUuid, - execId, - timestamp: Date.now() - } - - // sendMessageToAgent handles queue-based forwarding when agent is null - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) - return - } - - const buffer = Buffer.from(data) - try { - const msg = this.decodeMessage(buffer) - // Ensure message has all required fields - if (!msg.microserviceUuid) msg.microserviceUuid = session.microserviceUuid - if (!msg.execId) msg.execId = execId - if (!msg.timestamp) msg.timestamp = Date.now() - - if (msg.type === MESSAGE_TYPES.CLOSE) { - logger.info(`[RELAY] User sent CLOSE for execId=${execId}`) - - const queueEnabled = this.queueService.shouldUseQueue(execId) - - // Forward CLOSE to agent first - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) - - if (queueEnabled) { - // Cross-replica scenario: Don't close socket immediately - // Wait for agent's CLOSE response via queue - // The queue service will handle closing the socket in _handleCloseMessage - // when it receives the agent's CLOSE response - logger.debug('[RELAY] Cross-replica CLOSE: waiting for agent response via queue', { - execId, - microserviceUuid: session.microserviceUuid - }) - - // Set timeout in case agent doesn't respond - const timeout = setTimeout(async () => { - const currentSession = this.sessionManager.getSession(execId) - if (currentSession && currentSession.user && currentSession.user.readyState === WebSocket.OPEN) { - logger.warn('[RELAY] Agent did not respond to CLOSE within timeout, closing user socket', { - execId, - microserviceUuid: session.microserviceUuid, - timeout: this.config.closeResponseTimeout - }) - try { - currentSession.user.close(1000, 'Session closed (timeout)') - const currentTransaction = currentSession.transaction - await this.cleanupSession(execId, currentTransaction) - } catch (error) { - logger.error('[RELAY] Failed to close user socket on timeout', { - execId, - error: error.message - }) - } - } - this.pendingCloseTimeouts.delete(execId) - }, this.config.closeResponseTimeout) - - this.pendingCloseTimeouts.set(execId, timeout) - // Don't cleanup yet - queue service will call cleanup callback when agent responds - return - } else { - // Same replica: Close immediately (existing behavior) - // Close user WebSocket with code 1000 so client's onclose handler shows "Successfully closed" - // The client expects code 1000 (normal closure) to display the success message - if (user && user.readyState === WebSocket.OPEN) { - try { - user.close(1000, 'Session closed') - logger.debug('[RELAY] Closed user WebSocket with code 1000:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid - })) - } catch (error) { - logger.warn('[RELAY] Failed to close user WebSocket:' + JSON.stringify({ - execId, - error: error.message - })) - } - } - - // Get current transaction from the session and cleanup - const currentTransaction = session.transaction - await this.cleanupSession(execId, currentTransaction) - return - } - } - - if (msg.type === MESSAGE_TYPES.CONTROL) { - // Handle keep-alive messages from user - const controlData = msg.data.toString() - if (controlData === 'keepalive') { - // Send keep-alive response back to user - const keepAliveResponse = { - type: MESSAGE_TYPES.CONTROL, - data: Buffer.from('keepalive'), - microserviceUuid: session.microserviceUuid, - execId, - timestamp: Date.now() - } - const encoded = this.encodeMessage(keepAliveResponse) - user.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - logger.debug('[RELAY] Sent keep-alive response to user:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid - })) - return // Don't forward keep-alive to agent - } - } - - // sendMessageToAgent handles queue-based forwarding when agent is null - await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) - } catch (error) { - logger.error('[RELAY] Failed to process binary message:' + JSON.stringify({ - execId, - error: error.message, - stack: error.stack, - bufferLength: buffer.length, - userState: user.readyState, - agentState: session.agent ? session.agent.readyState : 'N/A (cross-replica)' - })) - } - }) - } - - // Forward agent -> user (works for both direct WebSocket and queue-based forwarding) - if (agent) { - logger.debug('[RELAY] Setting up agent->user message forwarding for execId=' + execId) - agent.on('message', async (data, isBinary) => { - logger.debug('[RELAY] Agent message received:' + JSON.stringify({ - execId, - isBinary, - dataType: typeof data, - dataLength: data.length, - userState: session.user ? session.user.readyState : 'N/A (cross-replica)', - agentState: agent.readyState, - queueEnabled: this.queueService.shouldUseQueue(execId) - })) - - try { - const buffer = Buffer.from(data) - const msg = this.decodeMessage(buffer) - logger.debug('[RELAY] Decoded agent message:' + JSON.stringify({ - execId, - type: msg.type, - hasData: !!msg.data, - messageSize: buffer.length - })) - - if (msg.type === MESSAGE_TYPES.CLOSE) { - logger.info(`[RELAY] Agent sent CLOSE for execId=${execId}`) - - const queueEnabled = this.queueService.shouldUseQueue(execId) - - // In cross-replica scenarios, publish CLOSE to queue so user's replica can handle it - if (queueEnabled) { - try { - // Pass message type so queue receiver can detect CLOSE without decoding - await this.queueService.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) - logger.debug('[RELAY] Forwarded agent CLOSE message to user via queue:' + JSON.stringify({ - execId, - type: msg.type - })) - } catch (error) { - logger.error('[RELAY] Failed to enqueue CLOSE message for user', { - execId, - error: error.message - }) - } - } else if (session.user && session.user.readyState === WebSocket.OPEN) { - // Direct connection - close user WebSocket immediately - session.user.close(1000, 'Agent closed connection') - } - - // Get current transaction from the session - const currentTransaction = session.transaction - await this.cleanupSession(execId, currentTransaction) - return - } - - const queueEnabled = this.queueService.shouldUseQueue(execId) - if (queueEnabled) { - try { - await this.queueService.publishToUser(execId, buffer) - logger.debug('[RELAY] Forwarded agent message to user via queue:' + JSON.stringify({ - execId, - type: msg.type - })) - } catch (error) { - logger.error('[RELAY] Failed to enqueue message for user', { - execId, - error: error.message - }) - } - } else if (session.user && session.user.readyState === WebSocket.OPEN) { - if (msg.type === MESSAGE_TYPES.STDOUT || msg.type === MESSAGE_TYPES.STDERR) { - if (msg.data && msg.data.length > 0) { - // Create MessagePack message for user - const userMsg = { - type: msg.type, - data: msg.data, - microserviceUuid: session.microserviceUuid, - execId, - timestamp: Date.now() - } - // Encode and send as binary - const encoded = this.encodeMessage(userMsg) - session.user.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - - logger.debug('[RELAY] Forwarded agent message to user:' + JSON.stringify({ - execId, - type: msg.type, - encodedLength: encoded.length, - messageType: msg.type - })) - } - } else if (msg.type === MESSAGE_TYPES.CONTROL) { - session.user.send(data, { - binary: true, - compress: false, - mask: false, - fin: true - }) - } - } else { - logger.debug('[RELAY] User not available (cross-replica), message should be delivered via queue:' + JSON.stringify({ - execId, - userState: session.user ? session.user.readyState : 'N/A', - messageType: msg.type, - queueEnabled - })) - } - } catch (error) { - logger.error('[RELAY] Failed to process agent message:', error) - } - }) - } - - logger.info('[RELAY] Message forwarding setup complete for session:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, - agentConnected: !!agent, - userConnected: !!user, - agentState: agent ? agent.readyState : 'N/A', - userState: user ? user.readyState : 'N/A' - })) - } - - async validateAgentConnection (token, microserviceUuid, transaction) { - try { - // Use AuthDecorator to validate the token and get the fog - let fog = {} - const req = { headers: { authorization: token }, transaction } - const handler = AuthDecorator.checkFogToken(async (req, fogObj) => { - fog = fogObj - return fogObj - }) - await handler(req) - - if (!fog) { - logger.error('Agent validation failed: Invalid agent token') - throw new WebSocketError(1008, 'Invalid agent token') - } - - // Verify microservice exists and belongs to this fog - const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) - if (!microservice || microservice.iofogUuid !== fog.uuid) { - logger.error('Agent validation failed: Microservice not found or not associated with this agent' + JSON.stringify({ - microserviceUuid, - fogUuid: fog.uuid, - found: !!microservice, - microserviceFogUuid: microservice ? microservice.iofogUuid : null - })) - throw new WebSocketError(1008, 'Microservice not found or not associated with this agent') - } - - return fog - } catch (error) { - logger.error('Agent validation error:' + JSON.stringify({ - error: error.message, - stack: error.stack, - microserviceUuid - })) - throw error // Propagate the original error - } - } - async validateAgentLogsConnection (token, microserviceUuid, iofogUuid, sessionId, transaction) { try { // 1. Validate agent token and get fog @@ -2179,6 +1315,10 @@ class WebSocketServer { } } + async validateAgentExecConnection (token, microserviceUuid, sessionId, transaction) { + return this.validateAgentLogsConnection(token, microserviceUuid, null, sessionId, transaction) + } + async validateUserConnection (token, microserviceUuid, expectSystem, transaction) { try { // 1. Basic token validation @@ -2277,18 +1417,16 @@ class WebSocketServer { this.drainPromise = (async () => { const deadline = Date.now() + drainBudgetMs - this.sessionManager.closeAllPendingUsers(DRAIN_CLOSE_CODE, DRAIN_CLOSE_REASON) - - const execIds = this.sessionManager.getAllExecSessionIds() const logSessionIds = this.logSessionManager.getAllLogSessionIds() + const execSessionIds = this.execSessionManager.getAllExecSessionIds() const cleanupTasks = [] - for (const execId of execIds) { + for (const sessionId of execSessionIds) { cleanupTasks.push( TransactionDecorator.generateTransaction(async (tx) => { - await this.cleanupSession(execId, tx) + await this.cleanupExecSession(sessionId, tx) })().catch((error) => { - logger.warn('[WS-DRAIN] Exec session cleanup failed', { execId, error: error.message }) + logger.warn('[WS-DRAIN] Exec session cleanup failed', { sessionId, error: error.message }) }) ) } @@ -2324,88 +1462,22 @@ class WebSocketServer { } logger.info('[WS-DRAIN] Graceful drain complete', { - execSessions: execIds.length, + execSessions: execSessionIds.length, logSessions: logSessionIds.length }) - })() - - return this.drainPromise - } - - static getInstance () { - if (!WebSocketServer.instance) { - WebSocketServer.instance = new WebSocketServer() - } - return WebSocketServer.instance - } - - // Clean up session and close sockets - async cleanupSession (execId, transaction, options = {}) { - const preserveAgentSocket = options.preserveAgentSocket === true - const session = this.sessionManager.getSession(execId) - if (!session) return - - if (session.metricsActive) { - recordExecSessionActive(-1) - session.metricsActive = false - } - - // Clear any pending CLOSE timeout - const timeout = this.pendingCloseTimeouts.get(execId) - if (timeout) { - clearTimeout(timeout) - this.pendingCloseTimeouts.delete(execId) - logger.debug('[RELAY] Cleared pending CLOSE timeout during cleanup', { execId }) - } - - // Send CLOSE message to agent if it's still connected - if (!preserveAgentSocket && session.agent && session.agent.readyState === WebSocket.OPEN) { - const closeMsg = { - type: MESSAGE_TYPES.CLOSE, - execId, - microserviceUuid: session.microserviceUuid, - timestamp: Date.now(), - data: Buffer.from('Session closed') - } - - try { - const encoded = this.encodeMessage(closeMsg) - session.agent.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - logger.info('[RELAY] Sent CLOSE message to agent for execId=' + execId) - } catch (error) { - logger.error('[RELAY] Failed to send CLOSE message to agent:' + JSON.stringify({ - execId, - error: error.message, - stack: error.stack - })) - } - } + })() - // Close the connections (only if not already closed) - // Note: User connection may already be closed if user initiated the close - if (session.user && session.user.readyState === WebSocket.OPEN) { - session.user.close(1000, 'Session closed') - } - if (!preserveAgentSocket && session.agent && session.agent.readyState === WebSocket.OPEN) { - session.agent.close(1000, 'Session closed') - } + return this.drainPromise + } - await this.sessionManager.removeSession(execId, transaction) - logger.info('[RELAY] Session cleaned up for execId=' + execId) - this.queueService.cleanup(execId) - .catch(error => { - logger.warn('[RELAY] Failed to cleanup queue bridge during session cleanup', { - execId, - error: error.message - }) - }) + static getInstance () { + if (!WebSocketServer.instance) { + WebSocketServer.instance = new WebSocketServer() + } + return WebSocketServer.instance } + // Clean up session and close sockets // Utility to extract microserviceUuid from path extractUuidFromPath (path) { const match = path.match(/([a-f0-9-]{36})/i) @@ -2559,93 +1631,6 @@ class WebSocketServer { } } - attachPendingKeepAliveHandler (ws) { - if (!ws) { - return - } - - if (ws._pendingKeepAliveHandler) { - ws.removeListener('message', ws._pendingKeepAliveHandler) - } - - ws._pendingKeepAliveHandler = (data, isBinary) => { - if (!isBinary) return - let msg - try { - msg = this.decodeMessage(Buffer.from(data)) - } catch (error) { - return - } - - const msgType = msg instanceof Map ? msg.get('type') : msg.type - const execId = msg instanceof Map ? msg.get('execId') : msg.execId - const msgMicroserviceUuid = msg instanceof Map ? msg.get('microserviceUuid') : msg.microserviceUuid - - if (msgType === MESSAGE_TYPES.CONTROL) { - const controlData = msg.data ? msg.data.toString() : '' - if (controlData === 'keepalive') { - this._sendKeepAliveResponse(ws, execId || 'pending', msgMicroserviceUuid || null) - } - return - } - - // Edgelet may reuse an open agent socket and send a fresh init frame (execId + microserviceUuid, no type). - if (execId && msgMicroserviceUuid && (msgType === undefined || msgType === null) && ws._agentExecHandshakeContext) { - const ctx = ws._agentExecHandshakeContext - TransactionDecorator.generateTransaction(async (tx) => { - await this.processAgentInitialMessage(ws, ctx.req, data, isBinary, ctx.microserviceUuid, tx) - })().catch((err) => { - logger.error('[WS-INIT] Failed to process agent re-init on reused socket', { - error: err.message, - microserviceUuid: ctx.microserviceUuid - }) - }) - } - } - ws.on('message', ws._pendingKeepAliveHandler) - - if (!ws._pendingKeepAlivePingHandler) { - ws._pendingKeepAlivePingHandler = () => { - if (ws.readyState === WebSocket.OPEN) { - try { - ws.pong() - } catch (error) { - logger.debug('[RELAY] Failed to send pong on pending connection', { error: error.message }) - } - } - } - ws.on('ping', ws._pendingKeepAlivePingHandler) - } - } - - detachPendingKeepAliveHandler (ws) { - if (ws && ws._pendingKeepAliveHandler) { - ws.removeListener('message', ws._pendingKeepAliveHandler) - ws._pendingKeepAliveHandler = null - } - } - - _sendKeepAliveResponse (ws, execId, microserviceUuid) { - try { - const keepAliveResponse = { - type: MESSAGE_TYPES.CONTROL, - data: Buffer.from('keepalive'), - microserviceUuid, - execId, - timestamp: Date.now() - } - const encoded = this.encodeMessage(keepAliveResponse) - ws.send(encoded, { - binary: true, - compress: false, - mask: false, - fin: true - }) - } catch (error) { - logger.debug('[RELAY] Failed to send keepalive response', { error: error.message }) - } - } - // Helper method to check if auth is configured isAuthConfigured () { return isOidcAuthConfigured() @@ -3516,6 +2501,257 @@ class WebSocketServer { await this.logSessionManager.removeLogSession(sessionId, transaction) await this.queueService.cleanupLogSession(sessionId) } + + async setupExecMessageForwarding (sessionId, transaction) { + const session = this.execSessionManager.getExecSession(sessionId) + if (!session) { + logger.warn('setupExecMessageForwarding: Session not found:' + JSON.stringify({ sessionId })) + return + } + + const { agent, user } = session + const execId = sessionId + const wasQueueBridgeEnabled = session.queueBridgeEnabled + + try { + await this.queueService.enableForSession(session, async (closeExecId) => { + const timeout = this.pendingCloseTimeouts.get(closeExecId) + if (timeout) { + clearTimeout(timeout) + this.pendingCloseTimeouts.delete(closeExecId) + } + await this.cleanupExecSession(closeExecId, transaction) + }) + session.queueBridgeEnabled = true + if (!wasQueueBridgeEnabled) { + logger.info('[RELAY] AMQP queue bridge enabled for exec session', { + sessionId, + microserviceUuid: session.microserviceUuid + }) + } + } catch (error) { + session.queueBridgeEnabled = false + const requireQueue = this.isCrossReplicaSession(session) && + this.haConfig.failFastOnRouterUnavailable !== false + if (requireQueue && !wasQueueBridgeEnabled) { + logger.error('[RELAY] AMQP required for cross-replica exec session but router bridge failed', { + sessionId, + error: error.message + }) + if (session.user && session.user.readyState === WebSocket.OPEN) { + session.user.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + } + if (session.agent && session.agent.readyState === WebSocket.OPEN) { + session.agent.close(ROUTER_UNAVAILABLE_CLOSE_CODE, ROUTER_UNAVAILABLE_CLOSE_REASON) + } + await this.cleanupExecSession(sessionId, transaction) + return + } + logger.warn('[RELAY] Failed to enable AMQP queue bridge for exec session', { + sessionId, + error: error.message + }) + } + + if (user && agent) { + await this.sendExecActivationToExecSession(session, sessionId, transaction) + } + + if (user) { + user.removeAllListeners('message') + } + if (agent) { + agent.removeAllListeners('message') + } + + if (user) { + user.on('message', async (data, isBinary) => { + if (!isBinary) { + const text = data.toString() + const msg = { + type: MESSAGE_TYPES.STDIN, + data: Buffer.from(text + '\n'), + microserviceUuid: session.microserviceUuid, + execId, + sessionId, + timestamp: Date.now() + } + await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + return + } + + const buffer = Buffer.from(data) + try { + const msg = this.decodeMessage(buffer) + if (!msg.microserviceUuid) msg.microserviceUuid = session.microserviceUuid + if (!msg.execId) msg.execId = execId + if (!msg.sessionId) msg.sessionId = sessionId + if (!msg.timestamp) msg.timestamp = Date.now() + + if (msg.type === MESSAGE_TYPES.CLOSE) { + await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + + const queueEnabled = this.queueService.shouldUseQueue(execId) + if (queueEnabled) { + const timeout = setTimeout(async () => { + const currentSession = this.execSessionManager.getExecSession(execId) + if (currentSession && currentSession.user && currentSession.user.readyState === WebSocket.OPEN) { + try { + currentSession.user.close(1000, 'Session closed (timeout)') + await this.cleanupExecSession(execId, transaction) + } catch (error) { + logger.error('[RELAY] Failed to close exec user socket on CLOSE timeout', { + sessionId: execId, + error: error.message + }) + } + } + this.pendingCloseTimeouts.delete(execId) + }, this.config.closeResponseTimeout) + this.pendingCloseTimeouts.set(execId, timeout) + return + } + + if (user && user.readyState === WebSocket.OPEN) { + user.close(1000, 'Session closed') + } + await this.cleanupExecSession(execId, transaction) + return + } + + if (msg.type === MESSAGE_TYPES.CONTROL) { + const controlData = msg.data.toString() + if (controlData === 'keepalive') { + const keepAliveResponse = { + type: MESSAGE_TYPES.CONTROL, + data: Buffer.from('keepalive'), + microserviceUuid: session.microserviceUuid, + execId, + sessionId, + timestamp: Date.now() + } + user.send(this.encodeMessage(keepAliveResponse), { binary: true }) + return + } + } + + await this.sendMessageToAgent(session.agent, msg, execId, session.microserviceUuid) + } catch (error) { + logger.error('[RELAY] Failed to process exec user message:' + JSON.stringify({ + sessionId: execId, + error: error.message + })) + } + }) + } + + if (agent) { + agent.on('message', async (data, isBinary) => { + if (!isBinary) { + logger.warn('[RELAY] Received non-binary message from exec agent, expected MessagePack') + return + } + + try { + const buffer = Buffer.from(data) + const msg = this.decodeMessage(buffer) + + if (msg.type === MESSAGE_TYPES.CLOSE) { + const queueEnabled = this.queueService.shouldUseQueue(execId) + if (queueEnabled) { + try { + await this.queueService.publishToUser(execId, buffer, { messageType: MESSAGE_TYPES.CLOSE }) + } catch (error) { + logger.error('[RELAY] Failed to enqueue exec CLOSE for user', { + sessionId: execId, + error: error.message + }) + } + } else if (session.user && session.user.readyState === WebSocket.OPEN) { + session.user.close(1000, 'Agent closed connection') + } + await this.cleanupExecSession(execId, transaction) + return + } + + const queueEnabled = this.queueService.shouldUseQueue(execId) + if (queueEnabled) { + await this.queueService.publishToUser(execId, buffer) + } else if (session.user && session.user.readyState === WebSocket.OPEN) { + if (msg.type === MESSAGE_TYPES.STDOUT || msg.type === MESSAGE_TYPES.STDERR) { + if (msg.data && msg.data.length > 0) { + const userMsg = { + type: msg.type, + data: msg.data, + microserviceUuid: session.microserviceUuid, + execId, + sessionId, + timestamp: Date.now() + } + session.user.send(this.encodeMessage(userMsg), { binary: true }) + } + } else if (msg.type === MESSAGE_TYPES.CONTROL) { + session.user.send(data, { binary: true }) + } + } + } catch (error) { + logger.error('[RELAY] Failed to process exec agent message:' + JSON.stringify({ + sessionId: execId, + error: error.message + })) + } + }) + } + + logger.info('[RELAY] Exec message forwarding setup complete:' + JSON.stringify({ + sessionId, + microserviceUuid: session.microserviceUuid, + agentConnected: !!agent, + userConnected: !!user + })) + } + + async cleanupExecSession (sessionId, transaction) { + const session = this.execSessionManager.getExecSession(sessionId) + if (session && session.metricsActive) { + recordExecSessionActive(-1) + session.metricsActive = false + } + + const timeout = this.pendingCloseTimeouts.get(sessionId) + if (timeout) { + clearTimeout(timeout) + this.pendingCloseTimeouts.delete(sessionId) + } + + if (session && session.agent && session.agent.readyState === WebSocket.OPEN) { + const closeMsg = { + type: MESSAGE_TYPES.CLOSE, + execId: sessionId, + sessionId, + microserviceUuid: session.microserviceUuid, + timestamp: Date.now(), + data: Buffer.from('Session closed') + } + try { + session.agent.send(this.encodeMessage(closeMsg), { binary: true }) + } catch (error) { + logger.warn('[RELAY] Failed to send CLOSE to agent during exec session cleanup', { + sessionId, + error: error.message + }) + } + } + + await this.execSessionManager.removeExecSession(sessionId, transaction) + await this.queueService.cleanup(sessionId) + .catch(error => { + logger.warn('[RELAY] Failed to cleanup exec queue bridge during session cleanup', { + sessionId, + error: error.message + }) + }) + } } module.exports = WebSocketServer diff --git a/src/websocket/session-manager.js b/src/websocket/session-manager.js deleted file mode 100644 index ad80c574..00000000 --- a/src/websocket/session-manager.js +++ /dev/null @@ -1,658 +0,0 @@ -const WebSocket = require('ws') -const logger = require('../logger') -const Errors = require('../helpers/errors') -const { recordPendingPairing } = require('./ws-metrics') -const MicroserviceManager = require('../data/managers/microservice-manager') -const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') -const ChangeTrackingService = require('../services/change-tracking-service') -const { microserviceExecState } = require('../enums/microservice-state') - -class SessionManager { - constructor (config) { - if (!config || !config.session) { - const error = new Errors.ValidationError('Invalid session manager configuration') - logger.error('Failed to initialize SessionManager:' + error) - throw error - } - this.sessions = new Map() - this.pendingUsers = new Map() // Map> - this.pendingAgents = new Map() // Map> - this.userRetryTimers = new Map() // Map> - this.config = config - this.cleanupInterval = null - this.sessionExpiredHandler = null - logger.info('SessionManager initialized with config:' + JSON.stringify({ - execPendingTimeoutMs: config.session.execPendingTimeoutMs, - execMaxDurationMs: config.session.execMaxDurationMs, - cleanupInterval: config.session.cleanupInterval - })) - } - - setSessionExpiredHandler (handler) { - this.sessionExpiredHandler = handler - } - - getPendingUserCount (microserviceUuid) { - if (!this.pendingUsers.has(microserviceUuid)) { - return 0 - } - return this.pendingUsers.get(microserviceUuid).size - } - - hasActiveOrPendingUser (microserviceUuid) { - for (const session of this.sessions.values()) { - if (session.microserviceUuid === microserviceUuid && - session.user && - session.user.readyState === WebSocket.OPEN) { - return true - } - } - return this.getPendingUserCount(microserviceUuid) > 0 - } - - createSession (execId, microserviceUuid, agentWs, userWs, transaction) { - const session = { - execId, - microserviceUuid, - agent: agentWs, - user: userWs, - lastActivity: Date.now(), - pairingStartedAt: Date.now(), - transaction - } - this.sessions.set(execId, session) - logger.info('Session created:' + JSON.stringify({ - execId, - microserviceUuid, - agentConnected: !!agentWs, - userConnected: !!userWs - })) - return session - } - - getSession (execId) { - return this.sessions.get(execId) || null - } - - async removeSession (execId, transaction) { - const session = this.sessions.get(execId) - if (session) { - logger.info('Removing session:' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid - })) - this.sessions.delete(execId) - await MicroserviceExecStatusManager.update( - { microserviceUuid: session.microserviceUuid }, - { execSessionId: '', status: microserviceExecState.INACTIVE }, - transaction - ) - await MicroserviceManager.update({ uuid: session.microserviceUuid }, { execEnabled: false }, transaction) - const microservice = await MicroserviceManager.findOne({ uuid: session.microserviceUuid }, transaction) - if (microservice) { - await ChangeTrackingService.update( - microservice.iofogUuid, - ChangeTrackingService.events.microserviceExecSessions, - transaction - ) - } - } - } - - addPendingUser (microserviceUuid, userWs) { - if (!this.pendingUsers.has(microserviceUuid)) { - this.pendingUsers.set(microserviceUuid, new Map()) - } - const users = this.pendingUsers.get(microserviceUuid) - users.set(userWs, { timestamp: Date.now() }) - recordPendingPairing(1) - - logger.info('Added pending user:' + JSON.stringify({ - microserviceUuid, - pendingUserCount: users.size - })) - } - - async addPendingAgent (microserviceUuid, execId, agentWs, transaction) { - if (!this.pendingAgents.has(microserviceUuid)) { - this.pendingAgents.set(microserviceUuid, new Map()) - // await MicroserviceExecStatusManager.update( - // { microserviceUuid: microserviceUuid }, - // { execSessionId: execId, status: microserviceExecState.PENDING }, - // transaction - // ) - } - const agents = this.pendingAgents.get(microserviceUuid) - - // Check if agent with this execId already exists - if (agents.has(execId)) { - logger.warn('Agent with execId already exists in pending list:' + JSON.stringify({ - microserviceUuid, - execId, - existingAgentState: agents.get(execId).ws.readyState, - newAgentState: agentWs.readyState - })) - // Remove old agent if it's not in OPEN state - if (agents.get(execId).ws.readyState !== WebSocket.OPEN) { - agents.delete(execId) - } else { - return // Skip adding if we already have an active agent with this execId - } - } - - const agentInfo = { ws: agentWs, execId } - agents.set(execId, agentInfo) - logger.info('Added pending agent:' + JSON.stringify({ - microserviceUuid, - execId, - pendingAgentCount: agents.size, - agentState: agentWs.readyState - })) - } - - removePendingUser (microserviceUuid, userWs) { - if (this.pendingUsers.has(microserviceUuid)) { - const users = this.pendingUsers.get(microserviceUuid) - users.delete(userWs) - recordPendingPairing(-1) - if (users.size === 0) { - this.pendingUsers.delete(microserviceUuid) - } - - // Clear retry timer when removing user - this.clearUserRetryTimer(microserviceUuid, userWs) - - logger.info('Removed pending user:' + JSON.stringify({ - microserviceUuid, - remainingUsers: users.size - })) - } - } - - removePendingAgent (microserviceUuid, agentWs) { - if (this.pendingAgents.has(microserviceUuid)) { - const agents = this.pendingAgents.get(microserviceUuid) - // Find and remove agent by WebSocket instance - for (const [execId, agentInfo] of agents.entries()) { - if (agentInfo.ws === agentWs) { - agents.delete(execId) - logger.info('Removed pending agent:' + JSON.stringify({ - microserviceUuid, - execId, - remainingAgents: agents.size - })) - break - } - } - - if (agents.size === 0) { - this.pendingAgents.delete(microserviceUuid) - } - } - } - - findPendingUserForExecId (microserviceUuid, execId) { - if (this.pendingUsers.has(microserviceUuid)) { - const users = this.pendingUsers.get(microserviceUuid) - // Find any available user (no execId matching needed) - for (const [userWs] of users.entries()) { - if (userWs.readyState === WebSocket.OPEN) { - return userWs - } - } - } - return null - } - - findPendingAgentForExecId (microserviceUuid, execId) { - if (this.pendingAgents.has(microserviceUuid)) { - const agents = this.pendingAgents.get(microserviceUuid) - const agentInfo = agents.get(execId) - if (agentInfo && agentInfo.ws.readyState === WebSocket.OPEN) { - return agentInfo.ws - } - } - const session = this.sessions.get(execId) - if (session && - session.microserviceUuid === microserviceUuid && - session.agent && - session.agent.readyState === WebSocket.OPEN && - !session.user) { - return session.agent - } - return null - } - - async tryActivateSession (microserviceUuid, execId, newConnection, isAgent, transaction) { - let pendingUser = null - let pendingAgent = null - let session = null - - try { - if (isAgent) { - pendingUser = this.findPendingUserForExecId(microserviceUuid, execId) - if (pendingUser) { - // Atomic operation: remove user and create session - this.removePendingUser(microserviceUuid, pendingUser) - session = this.createSession(execId, microserviceUuid, newConnection, pendingUser, transaction) - logger.info('Session activated with agent first:' + JSON.stringify({ - execId, - microserviceUuid, - userConnected: !!pendingUser, - agentConnected: !!newConnection, - userState: pendingUser.readyState, - agentState: newConnection.readyState - })) - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: execId, status: microserviceExecState.ACTIVE }, - transaction - ) - } else { - // Agent-only pairing is handled by createSession in handleAgentConnection (A6) - logger.info('No pending user found for agent, awaiting user connection:' + JSON.stringify({ - execId, - microserviceUuid, - agentState: newConnection.readyState - })) - } - } else { - pendingAgent = this.findPendingAgentForExecId(microserviceUuid, execId) - if (pendingAgent) { - this.removePendingAgent(microserviceUuid, pendingAgent) - const existingSession = this.sessions.get(execId) - if (existingSession && existingSession.agent === pendingAgent && !existingSession.user) { - existingSession.user = newConnection - existingSession.lastActivity = Date.now() - existingSession.transaction = transaction - session = existingSession - } else { - session = this.createSession(execId, microserviceUuid, pendingAgent, newConnection, transaction) - } - logger.info('Session activated with user first:' + JSON.stringify({ - execId, - microserviceUuid, - userConnected: !!newConnection, - agentConnected: !!pendingAgent, - userState: newConnection.readyState, - agentState: pendingAgent.readyState - })) - await MicroserviceExecStatusManager.update( - { microserviceUuid }, - { execSessionId: execId, status: microserviceExecState.ACTIVE }, - transaction - ) - } else { - this.addPendingUser(microserviceUuid, newConnection) - logger.info('No pending agent found for user, added to pending list:' + JSON.stringify({ - execId, - microserviceUuid, - userState: newConnection.readyState - })) - } - } - } catch (error) { - logger.error('Failed to activate session:' + JSON.stringify({ - error: error.message, - execId, - microserviceUuid, - isAgent, - userState: newConnection.readyState - })) - // Cleanup any partial state - if (session) { - await this.removeSession(execId, transaction) - } - throw error - } - - return session - } - - logSessionState () { - logger.info('--- WebSocket SessionManager State ---') - logger.info('Active sessions:') - for (const [execId, session] of this.sessions) { - logger.info(JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, - agentConnected: !!session.agent, - userConnected: !!session.user, - lastActivity: new Date(session.lastActivity).toISOString(), - agentState: session.agent ? session.agent.readyState : 'N/A', - userState: session.user ? session.user.readyState : 'N/A' - })) - } - logger.info('Pending users:') - for (const [microserviceUuid, users] of this.pendingUsers) { - logger.info(JSON.stringify({ - microserviceUuid, - count: users.size, - users: Array.from(users.entries()).map(([ws, info]) => ({ - timestamp: new Date(info.timestamp).toISOString(), - readyState: ws.readyState - })) - })) - } - logger.info('Pending agents:') - for (const [microserviceUuid, agents] of this.pendingAgents) { - logger.info(JSON.stringify({ - microserviceUuid, - count: agents.size, - execIds: Array.from(agents.keys()) - })) - } - logger.info('--------------------------------------') - } - - assignAgentToSession (execId, agentWs) { - const session = this.getSession(execId) - if (session) { - session.agent = agentWs - session.lastActivity = Date.now() - } - } - - assignUserToSession (execId, userWs) { - const session = this.getSession(execId) - if (session) { - session.user = userWs - session.lastActivity = Date.now() - } - } - - addConnection (sessionId, ws) { - try { - const session = this.getSession(sessionId) - session.connections.add(ws) - session.lastActivity = Date.now() - logger.info('Connection added to session' + JSON.stringify({ - sessionId, - connectionCount: session.connections.size - })) - } catch (error) { - logger.error('Failed to add connection:' + error) - throw error - } - } - - removeConnection (sessionId, ws) { - try { - const session = this.getSession(sessionId) - session.connections.delete(ws) - if (session.connections.size === 0) { - session.lastActivity = Date.now() - logger.info('Last connection removed from session' + JSON.stringify({ sessionId })) - } else { - logger.debug('Connection removed from session' + JSON.stringify({ - sessionId, - remainingConnections: session.connections.size - })) - } - } catch (error) { - logger.error('Failed to remove connection:' + error) - throw error - } - } - - handleReconnection (sessionId, ws) { - try { - const session = this.getSession(sessionId) - if (session.reconnectAttempts < this.config.maxReconnectAttempts) { - session.reconnectAttempts++ - this.addConnection(sessionId, ws) - logger.info('Reconnection successful' + JSON.stringify({ - sessionId, - attempt: session.reconnectAttempts - })) - return true - } else { - const error = new Errors.ValidationError('Max reconnection attempts reached') - logger.warn('Max reconnection attempts reached' + JSON.stringify({ - sessionId, - maxAttempts: this.config.maxReconnectAttempts, - error: error.message - })) - throw error - } - } catch (error) { - logger.error('Reconnection failed:' + error) - throw error - } - } - - startCleanup () { - if (this.cleanupInterval) { - logger.debug('Cleanup interval already running') - return - } - logger.info('Starting session cleanup service with interval: ' + this.config.session.cleanupInterval + 'ms') - this.cleanupInterval = setInterval(async () => { - const now = Date.now() - let cleanedCount = 0 - const execMaxDuration = this.config.session.execMaxDurationMs || 28800000 - const execPendingTimeout = this.config.session.execPendingTimeoutMs || 60000 - logger.debug('Running session cleanup cycle') - for (const [execId, session] of this.sessions) { - if (now - session.lastActivity > execMaxDuration) { - await this.cleanupSession(execId) - cleanedCount++ - } - } - for (const [microserviceUuid, users] of this.pendingUsers) { - for (const [userWs, info] of users.entries()) { - if (now - info.timestamp > execPendingTimeout) { - if (userWs.readyState === WebSocket.OPEN) { - try { - userWs.close(1008, 'Timeout waiting for agent connection') - } catch (error) { - logger.error('Failed to close timed out pending user:' + error.message) - } - } - this.removePendingUser(microserviceUuid, userWs) - if (this.sessionExpiredHandler) { - await this.sessionExpiredHandler(microserviceUuid, null) - } - cleanedCount++ - } - } - } - if (cleanedCount > 0) { - logger.info('Session cleanup completed' + JSON.stringify({ cleanedCount })) - } - this.logSessionState() - }, this.config.session.cleanupInterval) - } - - async cleanupSession (execId) { - try { - const session = this.getSession(execId) - if (!session) { - return - } - logger.info('Cleaning up exec session' + JSON.stringify({ - execId, - microserviceUuid: session.microserviceUuid, - agentConnected: !!session.agent, - userConnected: !!session.user - })) - if (this.sessionExpiredHandler) { - await this.sessionExpiredHandler(session.microserviceUuid, execId) - return - } - if (session.agent && session.agent.readyState === WebSocket.OPEN) { - session.agent.close(1000, 'Session timeout') - } - if (session.user && session.user.readyState === WebSocket.OPEN) { - session.user.close(1000, 'Session timeout') - } - this.sessions.delete(execId) - logger.debug('Exec session cleanup completed' + JSON.stringify({ execId })) - } catch (error) { - logger.error('Failed to cleanup exec session:' + error) - } - } - - stopCleanup () { - if (this.cleanupInterval) { - clearInterval(this.cleanupInterval) - logger.info('Session cleanup service stopped') - } - } - - getActiveConnections (sessionId) { - try { - const session = this.getSession(sessionId) - const count = session.connections.size - logger.debug('Getting active connections' + JSON.stringify({ sessionId, count })) - return count - } catch (error) { - logger.error('Failed to get active connections:' + error) - throw error - } - } - - broadcastToSession (sessionId, message) { - try { - const session = this.getSession(sessionId) - const messageStr = JSON.stringify(message) - let sentCount = 0 - for (const ws of session.connections) { - if (ws.readyState === WebSocket.OPEN) { - ws.send(messageStr) - sentCount++ - } - } - logger.debug('Broadcast message to session' + JSON.stringify({ - sessionId, - recipients: sentCount, - totalConnections: session.connections.size - })) - } catch (error) { - logger.error('Failed to broadcast message:' + error) - throw error - } - } - - bufferMessage (sessionId, message) { - try { - const session = this.getSession(sessionId) - session.buffer.push(message) - if (session.buffer.length > this.config.maxBufferSize) { - session.buffer.shift() // Remove oldest message - logger.debug('Buffer size limit reached, removed oldest message' + JSON.stringify({ - sessionId, - bufferSize: session.buffer.length - })) - } - } catch (error) { - logger.error('Failed to buffer message:' + error) - throw error - } - } - - getBufferedMessages (sessionId) { - try { - const session = this.getSession(sessionId) - const messages = session.buffer - logger.debug('Retrieved buffered messages' + JSON.stringify({ - sessionId, - messageCount: messages.length - })) - return messages - } catch (error) { - logger.error('Failed to get buffered messages:' + error) - throw error - } - } - - clearBuffer (sessionId) { - try { - const session = this.getSession(sessionId) - const count = session.buffer.length - session.buffer = [] - logger.info('Cleared message buffer' + JSON.stringify({ sessionId, clearedCount: count })) - } catch (error) { - logger.error('Failed to clear buffer:' + error) - throw error - } - } - - getPendingAgentExecIds (microserviceUuid) { - if (this.pendingAgents.has(microserviceUuid)) { - const agents = this.pendingAgents.get(microserviceUuid) - return Array.from(agents.keys()) - } - return [] - } - - getActiveExecSessionCount () { - return this.sessions.size - } - - getPendingPairingCount () { - let count = 0 - for (const users of this.pendingUsers.values()) { - count += users.size - } - for (const agents of this.pendingAgents.values()) { - count += agents.size - } - return count - } - - getAllExecSessionIds () { - return Array.from(this.sessions.keys()) - } - - closeAllPendingUsers (code, reason) { - for (const users of this.pendingUsers.values()) { - for (const [userWs] of users) { - if (userWs.readyState === WebSocket.OPEN) { - try { - userWs.close(code, reason) - } catch (error) { - logger.debug('Failed to close pending user during drain', { error: error.message }) - } - } - } - } - this.pendingUsers.clear() - } - - isUserStillPending (microserviceUuid, userWs) { - if (this.pendingUsers.has(microserviceUuid)) { - const users = this.pendingUsers.get(microserviceUuid) - return users.has(userWs) - } - return false - } - - setUserRetryTimer (microserviceUuid, userWs, timer) { - if (!this.userRetryTimers.has(microserviceUuid)) { - this.userRetryTimers.set(microserviceUuid, new Map()) - } - const timers = this.userRetryTimers.get(microserviceUuid) - timers.set(userWs, timer) - } - - getUserRetryTimer (microserviceUuid, userWs) { - if (this.userRetryTimers.has(microserviceUuid)) { - const timers = this.userRetryTimers.get(microserviceUuid) - return timers.get(userWs) - } - return null - } - - clearUserRetryTimer (microserviceUuid, userWs) { - if (this.userRetryTimers.has(microserviceUuid)) { - const timers = this.userRetryTimers.get(microserviceUuid) - timers.delete(userWs) - if (timers.size === 0) { - this.userRetryTimers.delete(microserviceUuid) - } - } - } -} - -module.exports = SessionManager From 492bdd6e5c52a7c04918d19384146e8bcc298ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:11:09 +0300 Subject: [PATCH 6/7] Add WebSocket tests for multi-session exec pairing and quota. Cover three concurrent sessions per microservice, sessionId mismatch rejection, cross-replica relay, drain, security, and load pairing. --- test/load/ws-pairing-load.js | 107 +++++-- test/src/websocket/ws-cross-replica.test.js | 203 ++++++++++-- test/src/websocket/ws-drain.test.js | 23 +- .../websocket/ws-exec-same-replica.test.js | 301 +++++++++++------- test/src/websocket/ws-lifecycle.test.js | 165 ++++++---- test/src/websocket/ws-security.test.js | 11 +- test/support/ws-session-harness.js | 16 +- 7 files changed, 590 insertions(+), 236 deletions(-) diff --git a/test/load/ws-pairing-load.js b/test/load/ws-pairing-load.js index 8bc8da35..c883fc00 100644 --- a/test/load/ws-pairing-load.js +++ b/test/load/ws-pairing-load.js @@ -2,23 +2,34 @@ /** * WebSocket pairing load probe. * - * Measures SessionManager pairing latency for N concurrent user+agent pairs. - * Target SLO (R88): 500 concurrent WS/replica, p99 pairing < 5s. + * Measures ExecSessionManager pairing latency for N concurrent user+agent pairs. * * Usage: * nvm use 24 * node test/load/ws-pairing-load.js * node test/load/ws-pairing-load.js --pairs 500 + * node test/load/ws-pairing-load.js --multi-ms 100 + * + * --multi-ms N: create 3 exec sessions per microservice (Plan 17 quota) for N microservices. * * Exit 0 when p99 < 5000ms; exit 1 otherwise. */ -const SessionManager = require('../../src/websocket/session-manager') -const MicroserviceExecStatusManager = require('../../src/data/managers/microservice-exec-status-manager') +const ExecSessionManager = require('../../src/websocket/exec-session-manager') +const MicroserviceExecSessionManager = require('../../src/data/managers/microservice-exec-session-manager') const { createMockWebSocket, newTestIds, delay } = require('../support/ws-session-harness') -const PAIR_COUNT = parseInt(process.argv.find((a) => a.startsWith('--pairs='))?.split('=')[1] || - (process.argv.includes('--pairs') ? process.argv[process.argv.indexOf('--pairs') + 1] : '500'), 10) +function parseArg (name, fallback) { + const eq = process.argv.find((a) => a.startsWith(`--${name}=`)) + if (eq) return eq.split('=')[1] + const idx = process.argv.indexOf(`--${name}`) + if (idx !== -1 && process.argv[idx + 1]) return process.argv[idx + 1] + return fallback +} + +const PAIR_COUNT = parseInt(parseArg('pairs', '500'), 10) +const MULTI_MS_COUNT = parseInt(parseArg('multi-ms', '0'), 10) +const SESSIONS_PER_MS = 3 const FAST_CONFIG = { session: { @@ -33,18 +44,15 @@ function percentile (sorted, p) { return sorted[Math.max(0, idx)] } -async function main () { - MicroserviceExecStatusManager.update = async () => {} - - const sessionManager = new SessionManager(FAST_CONFIG) +async function runPairBenchmark (execSessionManager, pairCount, label) { const latencies = [] const batchSize = 50 - console.log(`WS pairing load probe — ${PAIR_COUNT} pairs (batch ${batchSize})`) + console.log(`WS pairing load probe — ${label} (${pairCount} pairs, batch ${batchSize})`) - for (let batch = 0; batch < PAIR_COUNT; batch += batchSize) { + for (let batch = 0; batch < pairCount; batch += batchSize) { const tasks = [] - const count = Math.min(batchSize, PAIR_COUNT - batch) + const count = Math.min(batchSize, pairCount - batch) for (let i = 0; i < count; i++) { tasks.push((async () => { @@ -53,19 +61,61 @@ async function main () { const agentWs = createMockWebSocket() const transaction = { fakeTransaction: true } - sessionManager.addPendingUser(ids.microserviceUuid, userWs) const start = Date.now() - await sessionManager.tryActivateSession(ids.microserviceUuid, ids.execId, agentWs, true, transaction) + execSessionManager.createExecSession( + ids.sessionId, + ids.microserviceUuid, + agentWs, + userWs, + transaction + ) latencies.push(Date.now() - start) - sessionManager.sessions.delete(ids.execId) - sessionManager.removePendingUser(ids.microserviceUuid, userWs) + execSessionManager.execSessions.delete(ids.sessionId) })()) } await Promise.all(tasks) } + return latencies +} + +async function runMultiMsBenchmark (execSessionManager, microserviceCount) { + const latencies = [] + const pairCount = microserviceCount * SESSIONS_PER_MS + + console.log(`WS multi-session load probe — ${microserviceCount} MS × ${SESSIONS_PER_MS} sessions (${pairCount} pairs)`) + + for (let ms = 0; ms < microserviceCount; ms++) { + const ids = newTestIds() + for (let slot = 0; slot < SESSIONS_PER_MS; slot++) { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const sessionId = `${ids.microserviceUuid}-session-${slot}` + const transaction = { fakeTransaction: true } + + const start = Date.now() + execSessionManager.createExecSession( + sessionId, + ids.microserviceUuid, + agentWs, + userWs, + transaction + ) + latencies.push(Date.now() - start) + + execSessionManager.execSessions.delete(sessionId) + } + if (ms > 0 && ms % 50 === 0) { + await delay(0) + } + } + + return latencies +} + +function reportResults (latencies, pairCount) { latencies.sort((a, b) => a - b) const p50 = percentile(latencies, 50) const p99 = percentile(latencies, 99) @@ -76,12 +126,33 @@ async function main () { console.log('') console.log('Results:') - console.log(` pairs: ${PAIR_COUNT}`) + console.log(` pairs: ${pairCount}`) console.log(` p50: ${p50} ms`) console.log(` p99: ${p99} ms (SLO < ${sloMs} ms)`) console.log(` max: ${max} ms`) console.log(` status: ${pass ? 'PASS' : 'FAIL'}`) + return pass +} + +async function main () { + MicroserviceExecSessionManager.deleteBySessionId = async () => {} + + const execSessionManager = new ExecSessionManager(FAST_CONFIG) + execSessionManager.stopCleanupInterval() + + let latencies + let pairCount + + if (MULTI_MS_COUNT > 0) { + latencies = await runMultiMsBenchmark(execSessionManager, MULTI_MS_COUNT) + pairCount = MULTI_MS_COUNT * SESSIONS_PER_MS + } else { + latencies = await runPairBenchmark(execSessionManager, PAIR_COUNT, `${PAIR_COUNT} pairs`) + pairCount = PAIR_COUNT + } + + const pass = reportResults(latencies, pairCount) process.exit(pass ? 0 : 1) } diff --git a/test/src/websocket/ws-cross-replica.test.js b/test/src/websocket/ws-cross-replica.test.js index b2bdecad..689414ae 100644 --- a/test/src/websocket/ws-cross-replica.test.js +++ b/test/src/websocket/ws-cross-replica.test.js @@ -2,8 +2,11 @@ const { expect } = require('chai') const sinon = require('sinon') const WebSocketServerClass = require('../../../src/websocket/server') -const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const AppHelper = require('../../../src/helpers/app-helper') const RouterConnectionService = require('../../../src/services/router-connection-service') const EventService = require('../../../src/services/event-service') const { @@ -15,6 +18,7 @@ const { createMockQueueService, resetWebSocketServerSingleton, newTestIds, + waitForSent, delay } = require('../../support/ws-session-harness') @@ -34,10 +38,27 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { transaction = { fakeTransaction: true } $sandbox.stub(RouterConnectionService, 'isRouterAvailable').resolves(true) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) $sandbox.stub(EventService, 'createWsConnectEvent').resolves() $sandbox.stub(EventService, 'createWsDisconnectEvent').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'create').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').callsFake(async () => ({ + sessionId: $ids.sessionId, + microserviceUuid: $ids.microserviceUuid, + status: 'PENDING', + userConnected: true, + agentConnected: false + })) + $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) + $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(AppHelper, 'generateUUID').returns($ids.sessionId) + $sandbox.stub(wsServer, 'countExecSessionsInDb').resolves(0) }) afterEach(() => { @@ -45,48 +66,61 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { resetWebSocketServerSingleton(WebSocketServerClass) }) - it('relays user STDIN to agent via mock AMQP bridge', async () => { + it('relays user STDIN to agent via mock AMQP bridge keyed by sessionId', async () => { const userWs = createMockWebSocket() const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId - wsServer.sessionManager.createSession($ids.execId, $ids.microserviceUuid, null, userWs, transaction) - await wsServer.setupMessageForwarding($ids.execId, transaction) + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + null, + userWs, + transaction + ) + await wsServer.setupExecMessageForwarding(sessionId, transaction) await delay(50) - expect(mockQueue.shouldUseQueue($ids.execId)).to.equal(true) - - mockQueue.execBridges.get($ids.execId).session.agent = agentWs - const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, $ids.execId, $ids.microserviceUuid, 'echo hi\n') - await mockQueue.publishToAgent($ids.execId, stdinFrame) + const session = wsServer.execSessionManager.getExecSession(sessionId) + session.agent = agentWs + await wsServer.setupExecMessageForwarding(sessionId, transaction) + await delay(50) - expect(agentWs._sentMessages.length).to.be.at.least(0) - const listeners = agentWs.listenerCount('message') - expect(listeners).to.be.at.least(0) + expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, sessionId, $ids.microserviceUuid, 'echo hi\n') userWs.emit('message', stdinFrame, true) - await delay(50) + await waitForSent(agentWs, 1) - expect(mockQueue.execBridges.has($ids.execId)).to.equal(true) - await mockQueue.publishToAgent($ids.execId, stdinFrame) - expect(agentWs.listenerCount('message')).to.be.at.least(0) + expect(mockQueue.execBridges.has(sessionId)).to.equal(true) + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) }) - it('delivers agent STDOUT to user through mock AMQP publishToUser', async () => { + it('delivers agent STDOUT to user through mock AMQP publishToUser keyed by sessionId', async () => { const userWs = createMockWebSocket() const agentWs = createMockWebSocket() + const sessionId = $ids.sessionId - wsServer.sessionManager.createSession($ids.execId, $ids.microserviceUuid, agentWs, userWs, transaction) + wsServer.execSessionManager.createExecSession( + sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + transaction + ) await mockQueue.enableForSession( - wsServer.sessionManager.getSession($ids.execId), + wsServer.execSessionManager.getExecSession(sessionId), () => {} ) - const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, $ids.execId, $ids.microserviceUuid, 'line\n') - await mockQueue.publishToUser($ids.execId, stdoutFrame) - await delay(20) + const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, sessionId, $ids.microserviceUuid, 'line\n') + await mockQueue.publishToUser(sessionId, stdoutFrame) + await waitForSent(userWs, 1) - expect(userWs._sentMessages.length).to.be.at.least(0) - expect(mockQueue.shouldUseQueue($ids.execId)).to.equal(true) + expect(mockQueue.shouldUseQueue(sessionId)).to.equal(true) + const userReceived = decodeExecMessage(lastSent(userWs)) + expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) }) it('routes log lines through mock AMQP bridge', async () => { @@ -114,4 +148,123 @@ describe('WebSocket exec/log — cross-replica mock AMQP', () => { expect(mockQueue.logBridges.has(sessionId)).to.equal(true) expect(mockQueue.shouldUseLogQueue(sessionId)).to.equal(true) }) + + it('user-first then agent-second delivers ACTIVATION and STDIN via AMQP bridge', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const userReq = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + userReq.headers.authorization = 'Bearer user-jwt' + + await wsServer.handleUserExecConnection( + userWs, + userReq, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + + expect(mockQueue.shouldUseQueue($ids.sessionId)).to.equal(true) + expect(wsServer.execSessionManager.getExecSession($ids.sessionId).agent).to.equal(null) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) + await delay(50) + + const activationFrames = agentWs._sentMessages.filter((entry) => { + try { + return decodeExecMessage(entry.data).type === MESSAGE_TYPES.ACTIVATION + } catch (e) { + return false + } + }) + expect(activationFrames.length).to.be.at.least(1) + + const stdinFrame = buildExecFrame( + MESSAGE_TYPES.STDIN, + $ids.sessionId, + $ids.microserviceUuid, + 'echo hi\n' + ) + userWs.emit('message', stdinFrame, true) + await waitForSent(agentWs, activationFrames.length + 1) + + const agentReceived = decodeExecMessage(lastSent(agentWs)) + expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) + expect(agentReceived.data.toString()).to.include('echo hi') + }) + + it('resends ACTIVATION when agent WS reconnects', async () => { + const userWs = createMockWebSocket() + const agentWs = createMockWebSocket() + const userReq = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + userReq.headers.authorization = 'Bearer user-jwt' + + await wsServer.handleUserExecConnection( + userWs, + userReq, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) + await delay(50) + + function countAgentActivations (ws) { + return ws._sentMessages.filter((entry) => { + try { + return decodeExecMessage(entry.data).type === MESSAGE_TYPES.ACTIVATION + } catch (e) { + return false + } + }).length + } + + expect(countAgentActivations(agentWs)).to.be.at.least(1) + + const reconnectedAgentWs = createMockWebSocket() + await wsServer.handleAgentExecConnection( + reconnectedAgentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) + await delay(50) + + expect(countAgentActivations(reconnectedAgentWs)).to.be.at.least(1) + expect(wsServer.execSessionManager.getExecSession($ids.sessionId).agent).to.equal(reconnectedAgentWs) + }) }) + +function lastSent (ws) { + const entry = ws._sentMessages[ws._sentMessages.length - 1] + return entry.data +} diff --git a/test/src/websocket/ws-drain.test.js b/test/src/websocket/ws-drain.test.js index 36e39107..ed94815c 100644 --- a/test/src/websocket/ws-drain.test.js +++ b/test/src/websocket/ws-drain.test.js @@ -1,10 +1,11 @@ const { expect } = require('chai') const sinon = require('sinon') -const WebSocket = require('ws') const WebSocketServerClass = require('../../../src/websocket/server') -const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') const { createMockWebSocket, resetWebSocketServerSingleton, @@ -29,8 +30,10 @@ describe('WebSocket graceful drain', () => { $sandbox.stub(wsServer.queueService, 'cleanup').resolves() $sandbox.stub(wsServer.queueService, 'cleanupLogSession').resolves() - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() }) afterEach(() => { @@ -41,8 +44,13 @@ describe('WebSocket graceful drain', () => { it('sets draining flag and closes active exec sessions within timeout budget', async () => { const userWs = createMockWebSocket() const agentWs = createMockWebSocket() - wsServer.sessionManager.createSession($ids.execId, $ids.microserviceUuid, agentWs, userWs, $transaction) - wsServer.sessionManager.addPendingUser($ids.microserviceUuid, createMockWebSocket()) + wsServer.execSessionManager.createExecSession( + $ids.execId, + $ids.microserviceUuid, + agentWs, + userWs, + $transaction + ) const started = Date.now() await wsServer.drain(500) @@ -50,8 +58,7 @@ describe('WebSocket graceful drain', () => { expect(wsServer.isDraining).to.equal(true) expect(elapsed).to.be.at.most(800) - expect(wsServer.sessionManager.getSession($ids.execId)).to.equal(null) - expect(wsServer.sessionManager.getPendingUserCount($ids.microserviceUuid)).to.equal(0) + expect(wsServer.execSessionManager.getExecSession($ids.execId)).to.equal(null) }) it('verifyClient rejects new upgrades while draining', (done) => { diff --git a/test/src/websocket/ws-exec-same-replica.test.js b/test/src/websocket/ws-exec-same-replica.test.js index 275e8e39..6f19b788 100644 --- a/test/src/websocket/ws-exec-same-replica.test.js +++ b/test/src/websocket/ws-exec-same-replica.test.js @@ -2,15 +2,16 @@ const { expect } = require('chai') const sinon = require('sinon') const WebSocketServerClass = require('../../../src/websocket/server') -const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const EventService = require('../../../src/services/event-service') +const AppHelper = require('../../../src/helpers/app-helper') const { MESSAGE_TYPES, createMockWebSocket, createMockRequest, - buildAgentInitialMessage, buildExecFrame, decodeExecMessage, resetWebSocketServerSingleton, @@ -20,7 +21,7 @@ const { } = require('../../support/ws-session-harness') const WebSocket = require('ws') -describe('WebSocket exec — same-replica integration', () => { +describe('WebSocket exec — same-replica integration (Plan 17)', () => { def('sandbox', () => sinon.createSandbox()) def('ids', () => newTestIds()) @@ -41,12 +42,23 @@ describe('WebSocket exec — same-replica integration', () => { $sandbox.stub(wsServer.queueService, 'cleanup').resolves() $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) - $sandbox.stub(wsServer, 'validateAgentConnection').resolves({ uuid: $ids.fogUuid }) - $sandbox.stub(wsServer, 'getPendingAgentExecIdsFromDB').resolves([]) + $sandbox.stub(wsServer, 'validateAgentExecConnection').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(AppHelper, 'generateUUID').returns($ids.sessionId) + + $sandbox.stub(MicroserviceExecSessionManager, 'create').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'update').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceExecSessionManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').callsFake(async () => ({ + sessionId: $ids.sessionId, + microserviceUuid: $ids.microserviceUuid, + status: 'PENDING', + userConnected: true, + agentConnected: false + })) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) $sandbox.stub(ChangeTrackingService, 'update').resolves() $sandbox.stub(EventService, 'createWsConnectEvent').resolves() $sandbox.stub(EventService, 'createWsDisconnectEvent').resolves() @@ -60,14 +72,23 @@ describe('WebSocket exec — same-replica integration', () => { async function connectUserFirst () { const req = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) req.headers.authorization = 'Bearer user-jwt' - await wsServer.handleUserConnection(userWs, req, 'Bearer user-jwt', $ids.microserviceUuid, false, transaction) + await wsServer.handleUserExecConnection(userWs, req, 'Bearer user-jwt', $ids.microserviceUuid, false, transaction) } - async function connectAgentWithExecId () { - const agentReq = createMockRequest(`/api/v3/agent/exec/${$ids.microserviceUuid}`, '127.0.0.2') + async function connectAgentWithSessionId () { + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) agentReq.headers.authorization = 'Bearer fog-token' - await wsServer.handleAgentConnection(agentWs, agentReq, 'Bearer fog-token', $ids.microserviceUuid, transaction) - agentWs.emit('message', buildAgentInitialMessage($ids.execId, $ids.microserviceUuid), true) + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) await delay(50) } @@ -82,86 +103,48 @@ describe('WebSocket exec — same-replica integration', () => { }) } - it('agent-first: defers ACTIVATION until user connects', async () => { - wsServer.getPendingAgentExecIdsFromDB.restore() - $sandbox.stub(wsServer, 'getPendingAgentExecIdsFromDB').callsFake(async () => { - const pending = wsServer.sessionManager.getSession($ids.execId) - return pending && pending.agent && !pending.user ? [$ids.execId] : [] - }) - - await connectAgentWithExecId() - - expect(activationFramesSent(agentWs)).to.have.length(0) - expect(wsServer.sessionManager.getSession($ids.execId)).to.exist - expect(wsServer.sessionManager.getSession($ids.execId).user).to.equal(null) - + it('user-first: sends ACTIVATION to user with sessionId, then pairs agent by sessionId', async () => { await connectUserFirst() - await delay(50) - expect(activationFramesSent(agentWs).length).to.be.at.least(1) - const session = wsServer.sessionManager.getSession($ids.execId) - expect(session.user).to.equal(userWs) - expect(session.agent).to.equal(agentWs) - }) - - it('captures initial msgpack sent during agent validation', async () => { - $sandbox.restore() - resetWebSocketServerSingleton(WebSocketServerClass) - wsServer = new WebSocketServerClass() - userWs = createMockWebSocket() - agentWs = createMockWebSocket() - transaction = { fakeTransaction: true } - - $sandbox.stub(wsServer.queueService, 'enableForSession').resolves(true) - $sandbox.stub(wsServer.queueService, 'shouldUseQueue').returns(false) - $sandbox.stub(wsServer.queueService, 'cleanup').resolves() - $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) - $sandbox.stub(wsServer, 'getPendingAgentExecIdsFromDB').resolves([]) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() - $sandbox.stub(EventService, 'createWsConnectEvent').resolves() - $sandbox.stub(EventService, 'createWsDisconnectEvent').resolves() - - $sandbox.stub(wsServer, 'validateAgentConnection').callsFake(async () => { - agentWs.emit('message', buildAgentInitialMessage($ids.execId, $ids.microserviceUuid), true) - await delay(20) - return { uuid: $ids.fogUuid } - }) - - await connectUserFirst() - const agentReq = createMockRequest(`/api/v3/agent/exec/${$ids.microserviceUuid}`, '127.0.0.2') - agentReq.headers.authorization = 'Bearer fog-token' - await wsServer.handleAgentConnection(agentWs, agentReq, 'Bearer fog-token', $ids.microserviceUuid, transaction) - await delay(50) + expect(activationFramesSent(userWs).length).to.be.at.least(1) + const activation = decodeExecMessage(activationFramesSent(userWs)[0].data) + expect(activation.type).to.equal(MESSAGE_TYPES.ACTIVATION) + expect(activation.sessionId).to.equal($ids.sessionId) + expect(activation.microserviceUuid).to.equal($ids.microserviceUuid) + const activationPayload = JSON.parse(activation.data.toString()) + expect(activationPayload.sessionId).to.equal($ids.sessionId) + expect(activationPayload.microserviceUuid).to.equal($ids.microserviceUuid) - const session = wsServer.sessionManager.getSession($ids.execId) + const session = wsServer.execSessionManager.getExecSession($ids.sessionId) expect(session).to.exist expect(session.user).to.equal(userWs) + expect(session.agent).to.equal(null) + + await connectAgentWithSessionId() + expect(session.agent).to.equal(agentWs) expect(activationFramesSent(agentWs).length).to.be.at.least(1) }) - it('pairs user and agent, relays STDIN/STDOUT, and exec_b disables exec on CLOSE', async () => { + it('pairs user and agent, relays STDIN/STDOUT, and cleans up on CLOSE', async () => { await connectUserFirst() - expect(wsServer.sessionManager.getPendingUserCount($ids.microserviceUuid)).to.equal(1) - - await connectAgentWithExecId() + await connectAgentWithSessionId() - const session = wsServer.sessionManager.getSession($ids.execId) + const session = wsServer.execSessionManager.getExecSession($ids.sessionId) expect(session).to.exist expect(session.user).to.equal(userWs) expect(session.agent).to.equal(agentWs) - const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, $ids.execId, $ids.microserviceUuid, 'ls\n') + const stdinFrame = buildExecFrame(MESSAGE_TYPES.STDIN, $ids.sessionId, $ids.microserviceUuid, 'ls\n') userWs.emit('message', stdinFrame, true) await waitForSent(agentWs, 1) const agentReceived = decodeExecMessage(lastSent(agentWs)) expect(agentReceived.type).to.equal(MESSAGE_TYPES.STDIN) - const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, $ids.execId, $ids.microserviceUuid, 'output\n') + const stdoutFrame = buildExecFrame(MESSAGE_TYPES.STDOUT, $ids.sessionId, $ids.microserviceUuid, 'output\n') agentWs.emit('message', stdoutFrame, true) - await waitForSent(userWs, 1) + await waitForSent(userWs, 2) const userReceived = decodeExecMessage(lastSent(userWs)) expect(userReceived.type).to.equal(MESSAGE_TYPES.STDOUT) @@ -170,23 +153,22 @@ describe('WebSocket exec — same-replica integration', () => { userWs.close(1000, 'done') await delay(300) - expect(MicroserviceManager.update).to.have.been.calledWith( - sinon.match({ uuid: $ids.microserviceUuid }), - sinon.match({ execEnabled: false }), + expect(MicroserviceExecSessionManager.deleteBySessionId).to.have.been.calledWith( + $ids.sessionId, sinon.match.any ) - expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( - sinon.match({ microserviceUuid: $ids.microserviceUuid }), - sinon.match({ status: sinon.match.string }), - transaction + expect(ChangeTrackingService.update).to.have.been.calledWith( + $ids.fogUuid, + ChangeTrackingService.events.microserviceExecSessions, + sinon.match.any ) }) it('relays CLOSE from user to agent', async () => { await connectUserFirst() - await connectAgentWithExecId() + await connectAgentWithSessionId() - const closeFrame = buildExecFrame(MESSAGE_TYPES.CLOSE, $ids.execId, $ids.microserviceUuid, 'bye') + const closeFrame = buildExecFrame(MESSAGE_TYPES.CLOSE, $ids.sessionId, $ids.microserviceUuid, 'bye') const sentBefore = agentWs._sentMessages.length userWs.emit('message', closeFrame, true) await delay(100) @@ -198,52 +180,149 @@ describe('WebSocket exec — same-replica integration', () => { expect(closeSent).to.equal(true) }) - it('pending user timeout cleans up orphaned agent-only session', async () => { + it('pending user timeout cleans up when agent never connects', async () => { wsServer.getExecPendingTimeoutMs = () => 50 - await connectAgentWithExecId() - expect(wsServer.sessionManager.getSession($ids.execId)).to.exist - expect(wsServer.sessionManager.getSession($ids.execId).user).to.equal(null) - - wsServer.getPendingAgentExecIdsFromDB.restore() - $sandbox.stub(wsServer, 'getPendingAgentExecIdsFromDB').resolves([]) - await connectUserFirst() - expect(wsServer.sessionManager.getPendingUserCount($ids.microserviceUuid)).to.equal(1) + expect(wsServer.execSessionManager.getExecSession($ids.sessionId)).to.exist await delay(120) - expect(wsServer.sessionManager.getSession($ids.execId)).to.equal(null) - expect(agentWs.readyState).to.equal(WebSocket.CLOSED) - expect(MicroserviceManager.update).to.have.been.calledWith( - sinon.match({ uuid: $ids.microserviceUuid }), - sinon.match({ execEnabled: false }), - sinon.match.any - ) - expect(ChangeTrackingService.update).to.have.been.calledWith( - $ids.fogUuid, - ChangeTrackingService.events.microserviceExecSessions, + expect(wsServer.execSessionManager.getExecSession($ids.sessionId)).to.equal(null) + expect(userWs.readyState).to.equal(WebSocket.CLOSED) + expect(MicroserviceExecSessionManager.deleteBySessionId).to.have.been.calledWith( + $ids.sessionId, sinon.match.any ) }) - it('re-handshakes agent init on reused open socket', async () => { - wsServer.getPendingAgentExecIdsFromDB.restore() - $sandbox.stub(wsServer, 'getPendingAgentExecIdsFromDB').callsFake(async () => { - const pending = wsServer.sessionManager.getSession($ids.execId) - return pending && pending.agent && !pending.user ? [$ids.execId] : [] + it('allows three concurrent exec sessions on same microservice', async () => { + let dbSessionCount = 0 + $sandbox.stub(wsServer, 'countExecSessionsInDb').callsFake(async () => dbSessionCount) + MicroserviceExecSessionManager.create.restore() + $sandbox.stub(MicroserviceExecSessionManager, 'create').callsFake(async () => { + dbSessionCount++ }) - await connectAgentWithExecId() - expect(wsServer.sessionManager.getSession($ids.execId)).to.exist + const sessionIds = [] + AppHelper.generateUUID.restore() + $sandbox.stub(AppHelper, 'generateUUID').callsFake(() => { + const id = `session-${sessionIds.length}` + sessionIds.push(id) + return id + }) - const newExecId = `${$ids.execId}-reused` - agentWs.emit('message', buildAgentInitialMessage(newExecId, $ids.microserviceUuid), true) - await delay(50) + const userSockets = [] + for (let i = 0; i < 3; i++) { + const ws = createMockWebSocket() + userSockets.push(ws) + const req = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + req.headers.authorization = 'Bearer user-jwt' + await wsServer.handleUserExecConnection( + ws, + req, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + expect(ws.readyState).to.equal(WebSocket.OPEN) + } + + expect(wsServer.execSessionManager.countSessionsForResource($ids.microserviceUuid)).to.equal(3) + + const rejectedWs = createMockWebSocket() + const rejectedReq = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + rejectedReq.headers.authorization = 'Bearer user-jwt' + await wsServer.handleUserExecConnection( + rejectedWs, + rejectedReq, + 'Bearer user-jwt', + $ids.microserviceUuid, + false, + transaction + ) + expect(rejectedWs.readyState).to.equal(WebSocket.CLOSED) + }) + + it('closing one exec session does not affect sibling sessions', async () => { + const sessionIds = ['session-a', 'session-b'] + let connectCount = 0 + AppHelper.generateUUID.restore() + $sandbox.stub(AppHelper, 'generateUUID').callsFake(() => sessionIds[connectCount++]) + + MicroserviceExecSessionManager.findAll.restore() + $sandbox.stub(MicroserviceExecSessionManager, 'findAll').callsFake(async () => + sessionIds.slice(0, connectCount - 1).map((sessionId) => ({ sessionId })) + ) + + const userA = createMockWebSocket() + const userB = createMockWebSocket() + const reqA = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + reqA.headers.authorization = 'Bearer user-jwt' + const reqB = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + reqB.headers.authorization = 'Bearer user-jwt' + + await wsServer.handleUserExecConnection(userA, reqA, 'Bearer user-jwt', $ids.microserviceUuid, false, transaction) + await wsServer.handleUserExecConnection(userB, reqB, 'Bearer user-jwt', $ids.microserviceUuid, false, transaction) + + expect(wsServer.execSessionManager.getExecSession('session-a')).to.exist + expect(wsServer.execSessionManager.getExecSession('session-b')).to.exist + + userA.close(1000, 'done') + await delay(300) + + expect(wsServer.execSessionManager.getExecSession('session-a')).to.equal(null) + expect(wsServer.execSessionManager.getExecSession('session-b')).to.exist + expect(userB.readyState).to.equal(WebSocket.OPEN) + }) + + it('rejects agent WS when sessionId is unknown', async () => { + MicroserviceExecSessionManager.findBySessionId.restore() + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').resolves(null) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/unknown-session`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + 'unknown-session', + transaction + ) + + expect(agentWs.readyState).to.equal(WebSocket.CLOSED) + }) + + it('rejects agent WS when sessionId does not match microservice', async () => { + MicroserviceExecSessionManager.findBySessionId.restore() + $sandbox.stub(MicroserviceExecSessionManager, 'findBySessionId').resolves({ + sessionId: $ids.sessionId, + microserviceUuid: 'other-ms-uuid', + status: 'PENDING', + userConnected: true, + agentConnected: false + }) + + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}`, + '127.0.0.2' + ) + agentReq.headers.authorization = 'Bearer fog-token' + await wsServer.handleAgentExecConnection( + agentWs, + agentReq, + 'Bearer fog-token', + $ids.microserviceUuid, + $ids.sessionId, + transaction + ) - expect(wsServer.sessionManager.getSession($ids.execId)).to.equal(null) - expect(wsServer.sessionManager.getSession(newExecId)).to.exist - expect(wsServer.sessionManager.getSession(newExecId).agent).to.equal(agentWs) + expect(agentWs.readyState).to.equal(WebSocket.CLOSED) }) }) diff --git a/test/src/websocket/ws-lifecycle.test.js b/test/src/websocket/ws-lifecycle.test.js index fc8ba960..2d598e0e 100644 --- a/test/src/websocket/ws-lifecycle.test.js +++ b/test/src/websocket/ws-lifecycle.test.js @@ -2,12 +2,12 @@ const { expect } = require('chai') const sinon = require('sinon') const WebSocket = require('ws') -const SessionManager = require('../../../src/websocket/session-manager') +const ExecSessionManager = require('../../../src/websocket/exec-session-manager') const LogSessionManager = require('../../../src/websocket/log-session-manager') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const FogManager = require('../../../src/data/managers/iofog-manager') const WebSocketServerClass = require('../../../src/websocket/server') -const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const MicroserviceExecSessionManager = require('../../../src/data/managers/microservice-exec-session-manager') const MicroserviceLogStatusManager = require('../../../src/data/managers/microservice-log-status-manager') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') const { @@ -39,6 +39,39 @@ describe('WebSocket session lifecycle', () => { $sandbox.restore() }) + describe('exec 3-session quota', () => { + let wsServer + + beforeEach(() => { + resetWebSocketServerSingleton(WebSocketServerClass) + wsServer = new WebSocketServerClass() + wsServer.sessionConfig = { ...wsServer.sessionConfig, execMaxConcurrentPerResource: 3 } + }) + + afterEach(() => { + resetWebSocketServerSingleton(WebSocketServerClass) + }) + + it('rejects fourth concurrent exec session for same microservice', async () => { + $sandbox.stub(wsServer, 'validateUserConnection').resolves({ uuid: $ids.microserviceUuid }) + $sandbox.stub(wsServer, 'countExecSessionsInDb').resolves(3) + + const ws = createMockWebSocket() + const req = createMockRequest(`/api/v3/microservices/exec/${$ids.microserviceUuid}`) + + await wsServer.handleUserExecConnection( + ws, + req, + 'Bearer token', + $ids.microserviceUuid, + false, + $transaction + ) + + expect(ws.readyState).to.equal(WebSocket.CLOSED) + }) + }) + describe('log 3-viewer quota', () => { let wsServer @@ -122,100 +155,100 @@ describe('WebSocket session lifecycle', () => { }) describe('exec pending timeout (60s normative, accelerated in test)', () => { - let sessionManager + let execSessionManager beforeEach(() => { - sessionManager = new SessionManager(FAST_CONFIG) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() + execSessionManager = new ExecSessionManager(FAST_CONFIG) + execSessionManager.stopCleanupInterval() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + afterEach(() => { + execSessionManager.stopCleanupInterval() }) - it('closes pending user after execPendingTimeoutMs via cleanup cycle', async () => { + it('expires user-only pending exec session after execPendingTimeoutMs', async () => { const userWs = createMockWebSocket() - sessionManager.addPendingUser($ids.microserviceUuid, userWs) + execSessionManager.createExecSession( + $ids.sessionId, + $ids.microserviceUuid, + null, + userWs, + $transaction + ) - const users = sessionManager.pendingUsers.get($ids.microserviceUuid) - for (const [, info] of users.entries()) { - info.timestamp = Date.now() - 200 - } + const session = execSessionManager.getExecSession($ids.sessionId) + session.createdAt = 0 + session.lastActivity = 0 - let expiredMicroservice = null - sessionManager.setSessionExpiredHandler(async (microserviceUuid) => { - expiredMicroservice = microserviceUuid - }) - - const now = Date.now() - const execPendingTimeout = FAST_CONFIG.session.execPendingTimeoutMs - for (const [microserviceUuid, usersMap] of sessionManager.pendingUsers) { - for (const [userWsEntry, info] of usersMap.entries()) { - if (now - info.timestamp > execPendingTimeout) { - if (userWsEntry.readyState === WebSocket.OPEN) { - userWsEntry.close(1008, 'Timeout waiting for agent connection') - } - sessionManager.removePendingUser(microserviceUuid, userWsEntry) - if (sessionManager.sessionExpiredHandler) { - await sessionManager.sessionExpiredHandler(microserviceUuid, null) - } - } - } - } + const cleaned = await execSessionManager.cleanupExpiredSessions($transaction) - expect(expiredMicroservice).to.equal($ids.microserviceUuid) + expect(cleaned).to.equal(1) expect(userWs.readyState).to.equal(WebSocket.CLOSED) - expect(sessionManager.getPendingUserCount($ids.microserviceUuid)).to.equal(0) + expect(execSessionManager.getExecSession($ids.sessionId)).to.equal(null) }) }) describe('exec max duration (8h normative, accelerated in test)', () => { - let sessionManager + let execSessionManager beforeEach(() => { - sessionManager = new SessionManager(FAST_CONFIG) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() + execSessionManager = new ExecSessionManager(FAST_CONFIG) + execSessionManager.stopCleanupInterval() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + afterEach(() => { + execSessionManager.stopCleanupInterval() }) - it('invokes sessionExpiredHandler when execMaxDurationMs exceeded', async () => { + it('removes paired exec session when execMaxDurationMs exceeded', async () => { const userWs = createMockWebSocket() const agentWs = createMockWebSocket() - sessionManager.createSession($ids.execId, $ids.microserviceUuid, agentWs, userWs, $transaction) - const session = sessionManager.getSession($ids.execId) + execSessionManager.createExecSession( + $ids.sessionId, + $ids.microserviceUuid, + agentWs, + userWs, + $transaction + ) + const session = execSessionManager.getExecSession($ids.sessionId) session.lastActivity = Date.now() - 300 - let expiredExecId = null - sessionManager.setSessionExpiredHandler(async (microserviceUuid, execId) => { - expiredExecId = execId - sessionManager.sessions.delete(execId) - }) - - const execMaxDuration = FAST_CONFIG.session.execMaxDurationMs - const now = Date.now() - for (const [execId, activeSession] of sessionManager.sessions) { - if (now - activeSession.lastActivity > execMaxDuration) { - await sessionManager.cleanupSession(execId) - } - } + const cleaned = await execSessionManager.cleanupExpiredSessions($transaction) - expect(expiredExecId).to.equal($ids.execId) - expect(sessionManager.getSession($ids.execId)).to.equal(null) + expect(cleaned).to.equal(1) + expect(execSessionManager.getExecSession($ids.sessionId)).to.equal(null) }) }) - describe('exec_b lifecycle', () => { - it('removeSession sets execEnabled=false and notifies execSessions change', async () => { - const sessionManager = new SessionManager(FAST_CONFIG) - $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() - $sandbox.stub(MicroserviceManager, 'update').resolves() + describe('per-session exec cleanup (no exec_b)', () => { + it('removeExecSession deletes DB row and updates execSessions change tracking', async () => { + const execSessionManager = new ExecSessionManager(FAST_CONFIG) + execSessionManager.stopCleanupInterval() + $sandbox.stub(MicroserviceExecSessionManager, 'deleteBySessionId').resolves() $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $ids.fogUuid }) + $sandbox.stub(FogManager, 'findOne').resolves({ uuid: $ids.fogUuid }) $sandbox.stub(ChangeTrackingService, 'update').resolves() const userWs = createMockWebSocket() - sessionManager.createSession($ids.execId, $ids.microserviceUuid, null, userWs, $transaction) - await sessionManager.removeSession($ids.execId, $transaction) + execSessionManager.createExecSession( + $ids.sessionId, + $ids.microserviceUuid, + null, + userWs, + $transaction + ) + await execSessionManager.removeExecSession($ids.sessionId, $transaction) - expect(MicroserviceManager.update).to.have.been.calledWith( - sinon.match({ uuid: $ids.microserviceUuid }), - sinon.match({ execEnabled: false }), + expect(MicroserviceExecSessionManager.deleteBySessionId).to.have.been.calledWith( + $ids.sessionId, $transaction ) expect(ChangeTrackingService.update).to.have.been.calledWith( diff --git a/test/src/websocket/ws-security.test.js b/test/src/websocket/ws-security.test.js index d93c35ab..42995c27 100644 --- a/test/src/websocket/ws-security.test.js +++ b/test/src/websocket/ws-security.test.js @@ -117,20 +117,23 @@ describe('WebSocket session security', () => { const transaction = { fakeTransaction: true } let validationResolved = false - $sandbox.stub(wsServer, 'validateAgentConnection').callsFake(async () => { + $sandbox.stub(wsServer, 'validateAgentExecConnection').callsFake(async () => { await delay(30) validationResolved = true throw new Error('Invalid agent token') }) - const agentReq = createMockRequest(`/api/v3/agent/exec/${$ids.microserviceUuid}`) + const agentReq = createMockRequest( + `/api/v3/agent/exec/microservice/${$ids.microserviceUuid}/${$ids.sessionId}` + ) agentReq.headers.authorization = 'Bearer bad-token' - const handlerPromise = wsServer.handleAgentConnection( + const handlerPromise = wsServer.handleAgentExecConnection( agentWs, agentReq, 'Bearer bad-token', $ids.microserviceUuid, + $ids.sessionId, transaction ) @@ -140,7 +143,7 @@ describe('WebSocket session security', () => { expect(validationResolved).to.equal(true) expect(agentWs.readyState).to.equal(WebSocket.CLOSED) - expect(wsServer.sessionManager.sessions.size).to.equal(0) + expect(wsServer.execSessionManager.execSessions.size).to.equal(0) }) }) }) diff --git a/test/support/ws-session-harness.js b/test/support/ws-session-harness.js index dde57f30..5eaec0a7 100644 --- a/test/support/ws-session-harness.js +++ b/test/support/ws-session-harness.js @@ -99,7 +99,15 @@ function createMockQueueService () { async enableForSession (session, cleanupCallback) { const execId = session.execId if (!execId) return false - execBridges.set(execId, { session, cleanupCallback }) + const existing = execBridges.get(execId) + if (existing) { + existing.session = session + if (cleanupCallback) { + existing.cleanupCallback = cleanupCallback + } + } else { + execBridges.set(execId, { session, cleanupCallback }) + } return true }, @@ -110,14 +118,14 @@ function createMockQueueService () { async publishToAgent (execId, buffer) { const bridge = execBridges.get(execId) if (bridge && bridge.session.agent && bridge.session.agent.readyState === WebSocket.OPEN) { - bridge.session.agent.emit('message', buffer, true) + bridge.session.agent.send(buffer, { binary: true }) } }, async publishToUser (execId, buffer) { const bridge = execBridges.get(execId) if (bridge && bridge.session.user && bridge.session.user.readyState === WebSocket.OPEN) { - bridge.session.user.emit('message', buffer, true) + bridge.session.user.send(buffer, { binary: true }) } }, @@ -151,7 +159,7 @@ function createMockQueueService () { function resetWebSocketServerSingleton (WebSocketServerClass) { if (WebSocketServerClass.instance) { const instance = WebSocketServerClass.instance - instance.sessionManager.stopCleanup() + instance.execSessionManager.stopCleanupInterval() instance.logSessionManager.stopCleanupInterval() } WebSocketServerClass.instance = null From dd1fdfd0c006450fbab1aba766d1387e6511f539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emirhan=20Durmu=C5=9F?= Date: Fri, 26 Jun 2026 18:11:23 +0300 Subject: [PATCH 7/7] Document multi-session exec in swagger, architecture, and operator guide. --- CHANGELOG.md | 15 ++- docs/architecture.md | 62 +++++++---- docs/operations/ws-sessions.md | 12 ++- docs/swagger.yaml | 192 ++++++++++++--------------------- 4 files changed, 131 insertions(+), 150 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b92c444a..c7de671f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,17 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - **Diagnostics**, **strace**, image **snapshot** / **download** APIs. - All **`/api/v3/flow`** routes. +#### WebSocket exec — multi-session + +- **Microservice exec REST removed** — `POST/DELETE /api/v3/microservices/:uuid/exec` and `…/system/:uuid/exec` no longer exist. Open exec with **direct WebSocket** only: `WS /api/v3/microservices/exec/:uuid` (or `…/system/exec/:uuid`). +- **3 concurrent exec sessions** per microservice (was 1 user exec WS per MS in Plan 16). +- **Per-session lifecycle** — closing one exec session deletes only that session row only (no microservice-level exec flag). +- **`execEnabled` removed** — dropped `microservices.exec_enabled` column and agent MS list field; exec attach is poll-driven only (`GET /agent/exec/sessions`). +- **Agent exec discovery** — new `GET /api/v3/agent/exec/sessions` when change tracking reports `execSessions: true`. +- **Agent exec WebSocket** — `WS /api/v3/agent/exec/microservice/:microserviceUuid/:sessionId` only; legacy `WS /api/v3/agent/exec/:microserviceUuid` with initial MessagePack pairing frame **removed**. +- **User session announce** — Controller sends **ACTIVATION** (type 5) to user with `{ sessionId, microserviceUuid }` on connect. +- **Fog debug** — `POST/DELETE /api/v3/iofog/:uuid/exec` unchanged (provisions debug system MS); interactive shell via `WS /api/v3/microservices/system/exec/:debugMsUuid` (system path — debug MS is a system microservice). + #### Authentication - **`keycloak-connect` removed** — generic OIDC (`openid-client` + JWKS discovery). @@ -97,10 +108,12 @@ Controller v3.8 is a **greenfield** release aligned with **Edgelet**. There is * - **K8s control plane:** hub **`iofog-router`** ConfigMap patches serialized via DB lock; K8s Service create/update/delete with LoadBalancer watch timeout. - **`service-bridge-config.js`** — full recompute of service-derived TCP bridge config per fog on reconcile (preserves router base config). - **SQLite single-node production hardening** — WAL + `busy_timeout` pragmas, reconcile task claim retry on `SQLITE_BUSY`, staggered startup for reconcile-heavy background jobs (`settings.jobStartupDelaySeconds`). -- **WebSocket exec & log session hardening** — quotas (1 exec / 3 log WS per resource), exec_b lifecycle, 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). +- **WebSocket exec & log session hardening** — quotas (**3 exec** / 3 log WS per resource), per-session exec lifecycle (Plan 17), 60s/120s pending timeouts, 8h exec max, 30s graceful drain, OTEL metrics, HA AMQP fail-fast, integration tests, swagger WS protocol docs, operator guide (`docs/operations/ws-sessions.md`). +- **Multi exec sessions (Plan 17)** — `GET /api/v3/agent/exec/sessions`; agent exec WS `…/agent/exec/microservice/:uuid/:sessionId`; user ACTIVATION with `sessionId`; `MicroserviceExecSessions` table; `execMaxConcurrentPerResource` config (default **3**). ### Fixed +- Exec AMQP relay re-attaches queue receivers whenever user or agent WebSocket connects (fixes user-first sessions where ACTIVATION and STDIN never reached Edgelet); ACTIVATION is resent on agent WS reconnect. - Controller register accepts optional **`schedule: 0`**; server always enforces schedule **0** on create, re-register, and **`PATCH /api/v3/microservices/system/:uuid`** for controller workloads. - Agent version command (**`GET /api/v3/agent/version`**) refreshes the provision key on each pull instead of returning a stale or deleted key. - Controller AMQP certificate provisioning uses shared **`default-router-local-ca`** instead of per-fog router local CA secret names. diff --git a/docs/architecture.md b/docs/architecture.md index 4df7b97f..786e92f8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -206,29 +206,43 @@ Full spec: [`.cursor/controllerv3.8/docs/15-fog-platform-reconcile.md`](../.curs ## WebSocket exec & log sessions -Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens lifecycle, quotas, multi-replica HA relay, and observability — **without changing the Edgelet wire protocol**. +Interactive **exec** and **log streaming** use paired WebSocket sessions between operators (Bearer JWT), Controller, and Edgelet agents (fog token). Plan 16 hardens log sessions and shared WS infra (HA, drain, OTEL). **Plan 17** redesigns **microservice exec** to log-style multi-session flow (3 concurrent per MS, agent poll + session-scoped WS) — **Edgelet agent wire change required** for exec (see [edgelet-invariants.md §10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md)). ```mermaid sequenceDiagram participant U as User WS - participant C as Controller replica - participant DB as Database - participant Q as AMQP Router - participant A as Agent WS - - Note over U,A: Exec (R80–R81, R84) - U->>C: WS exec (RBAC) - A->>C: WS agent/exec + execId - C->>C: Pair SessionManager - C->>Q: Enable agent-{execId} user-{execId} + participant C as Controller + participant DB as MicroserviceExecSessions + participant CT as Change tracking + participant Q as AMQP + participant A as Edgelet agent + + Note over U,A: MS exec (R92–R98) — log-style + U->>C: WS /microservices/exec/:uuid + C->>DB: INSERT sessionId PENDING + C->>CT: execSessions=true + C->>U: ACTIVATION { sessionId } + C->>U: STDERR waiting… + + A->>C: GET /changes execSessions + A->>C: GET /agent/exec/sessions + A->>C: WS /agent/exec/microservice/:uuid/:sessionId + C->>DB: ACTIVE agentConnected + C->>Q: agent-{sessionId} user-{sessionId} U->>C: STDIN - C->>Q->>A: or direct WS same replica + C->>Q->>A: relay A->>C: STDOUT C->>Q->>U: relay U-->>C: close - C->>DB: execEnabled=false INACTIVE + C->>DB: DELETE row + C->>CT: execSessions if needed - Note over U,A: Logs (R82–R83, R84) + Note over U,A: Fog debug (R99) + U->>C: POST /iofog/:uuid/exec + C->>C: provision debug system MS + U->>C: WS /microservices/system/exec/:debugMsUuid + + Note over U,A: Logs (R82–R83, R84) — unchanged Plan 16 U->>C: WS logs + tail params C->>DB: PENDING sessionId A->>C: WS agent/logs/:sessionId @@ -236,15 +250,21 @@ sequenceDiagram C->>Q->>U: logs-user-{sessionId} ``` -| Topic | Normative value (RFC R80–R91) | -|-------|-------------------------------| -| Exec lifecycle | **exec_b** — WS close sets `execEnabled=false`; **1** user exec WS per microservice | -| Exec timeouts | **60s** pending for agent; **8h** max active session | +| Topic | Normative value | +|-------|-----------------| +| MS exec entry | **Direct user WS** — no `POST …/microservices/…/exec` (R92, R94) | +| MS exec concurrency | **3** user exec WS per microservice (R93) | +| MS exec lifecycle | **Per-session** — close deletes session row only; **no** `execEnabled=false` (R98) | +| MS exec pending / max | **60s** pending for agent; **8h** max active session (Plan 16 carry-over) | +| Agent exec discovery | `GET /agent/exec/sessions` on `execSessions` change flag (R95, R100) | +| Agent exec WS | `/agent/exec/microservice/:uuid/:sessionId` only — legacy `/agent/exec/:uuid` removed (R96) | +| User session notify | **ACTIVATION** (type 5) with `{ sessionId, microserviceUuid }` (R97) | +| Fog debug provision | `POST/DELETE /iofog/:uuid/exec` unchanged; shell via `WS /microservices/system/exec/:debugMsUuid` (R99) | | Log concurrency | **3** user log WS per microservice (or per fog for node logs) | | Log limits | Tail max **5,000** lines; **120s** pending; **2h** idle | | Log content | Live relay only — no log line persistence; audit connect/disconnect | | HA relay | Cross-replica sessions **require** AMQP (`WebSocketQueueService`); same-replica may use direct WS; **fail fast** when router down | -| Graceful drain | **30s** on SIGTERM / k8s `preStop` — CLOSE frames, queue cleanup, DB status update | +| Graceful drain | **30s** on SIGTERM / k8s `preStop` — CLOSE frames, queue cleanup, session row delete | | Security | Agent handlers validate fog token **before** message processing; **50** upgrades/min/IP; **100** active WS/IP; JWT in `?token=` (ingress log redaction required) | | Scale SLO | **500** concurrent WS per replica; **p99 pairing < 5s** | | Observability | OpenTelemetry: active/pending sessions, pairing latency, AMQP failures, router connectivity | @@ -253,11 +273,11 @@ sequenceDiagram **HA config (`server.webSocket.ha`):** `crossReplicaRequiresAmqp` (default `true`), `failFastOnRouterUnavailable` (default `true`). Env: `WS_HA_CROSS_REPLICA_REQUIRES_AMQP`, `WS_HA_FAIL_FAST_ON_ROUTER_UNAVAILABLE`. Graceful drain timeout: `server.webSocket.session.drainTimeoutMs` (default **30s**, env `WS_DRAIN_TIMEOUT_MS`). -**Core modules:** `src/websocket/server.js`, `session-manager.js`, `log-session-manager.js`, `src/services/websocket-queue-service.js`, `src/services/router-connection-service.js`. +**Core modules:** `src/websocket/server.js`, `exec-session-manager.js` (Plan 17), `log-session-manager.js`, `src/services/websocket-queue-service.js`, `src/services/router-connection-service.js`. **Operator guide:** [operations/ws-sessions.md](operations/ws-sessions.md) — ingress `?token=` log redaction, HTTPS/WSS, multi-replica AMQP requirement, k8s preStop drain, load SLO probe. -Full spec: [`.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md`](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · RFC R80–R91 · Edgelet contract: [edgelet-invariants.md §10](../.cursor/controllerv3.8/docs/edgelet-invariants.md). +Full spec: Plan 16 [logs + shared infra](../.cursor/controllerv3.8/docs/16-ws-exec-log-hardening.md) · Plan 17 [MS exec](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md) · RFC R80–R91, R92–R101 · Edgelet contract: [edgelet-invariants.md §10–§10.1](../.cursor/controllerv3.8/docs/edgelet-invariants.md). --- diff --git a/docs/operations/ws-sessions.md b/docs/operations/ws-sessions.md index 06b0c228..29c95802 100644 --- a/docs/operations/ws-sessions.md +++ b/docs/operations/ws-sessions.md @@ -18,6 +18,8 @@ Controller exposes **interactive exec** and **log streaming** over WebSocket on | **User auth** | Bearer JWT via `Authorization` header or `?token=` query param (browser Console). RBAC: `execSessions`, `logs`, `systemExecSessions`, `systemLogs`. | | **Agent auth** | Fog token on `/api/v3/agent/exec/*` and `/api/v3/agent/logs/*` — OIDC does **not** apply to agent routes. | +> **Plan 17 (MS exec):** Open exec with **direct WebSocket** — `wss://…/api/v3/microservices/exec/:uuid` (app MS) or `…/system/exec/:uuid` (system MS). **No** `POST …/exec` before connect. Up to **3** concurrent exec sessions per microservice. Agent discovers sessions via `GET /api/v3/agent/exec/sessions` and connects `WS /api/v3/agent/exec/microservice/:uuid/:sessionId`. Fog node debug: `POST/DELETE /api/v3/iofog/:uuid/exec` provisions the debug system MS, then **`WS …/microservices/system/exec/:debugMsUuid`** (not the app exec path). Full spec: [17-multi-exec-sessions.md](../.cursor/controllerv3.8/docs/17-multi-exec-sessions.md). + ### Ingress log redaction (required) Browser clients pass JWT in the query string: `wss://controller.example.com/api/v3/microservices/{uuid}/logs?token=…` @@ -44,7 +46,7 @@ Without redaction, long-lived bearer tokens may appear in load balancer logs. **Requirements:** 1. Deploy the **router** system microservice and ensure Controller can reach AMQP (`RouterConnectionService`). -2. Run **2+ Controller replicas** behind a load balancer with **sticky sessions optional** — cross-replica exec/log uses AMQP queues (`agent-{execId}`, `user-{execId}`, `logs-user-{sessionId}`). +2. Run **2+ Controller replicas** behind a load balancer with **sticky sessions optional** — cross-replica exec/log uses AMQP queues (`agent-{sessionId}`, `user-{sessionId}`, `logs-user-{sessionId}`). 3. When the router is unavailable, new cross-replica sessions close with WebSocket code **1013** (`Router unavailable for cross-replica session`). Same-replica sessions may relay directly without AMQP when both user and agent land on the same pod. @@ -88,7 +90,7 @@ spec: 1. Open an exec or log session against a running pod. 2. `kubectl delete pod --grace-period=45` -3. Confirm the client receives close code **1001** within ~30s and exec is disabled (`execEnabled=false` for exec sessions). +3. Confirm the client receives close code **1001** within ~30s and the session row is cleaned up (Plan 17: per-session delete; no global `execEnabled=false` for MS exec). 4. Confirm replacement pod accepts new sessions. --- @@ -99,14 +101,18 @@ spec: |--------|--------| | Concurrent WS per replica | **500** (`WS_REPLICA_MAX_CONCURRENT_WS`) | | p99 exec pairing latency | **< 5s** | +| Exec sessions per microservice | **3** concurrent user WS (Plan 17) | Run the load probe locally: ```bash nvm use 24 node test/load/ws-pairing-load.js --pairs 500 +node test/load/ws-pairing-load.js --multi-ms 100 ``` +The `--multi-ms` mode creates **3 exec sessions per microservice** (100 MS × 3 = 300 pairs) to validate multi-session pairing latency under the same p99 SLO. + For production validation, repeat against a staging cluster with real agent simulators and record p99 from Controller OTEL histogram `ws_pairing_duration_ms`. --- @@ -130,7 +136,7 @@ Enable `ENABLE_TELEMETRY=true`. Key metrics (`src/websocket/ws-metrics.js`): | Session | Limit | |---------|-------| -| Exec user WS per microservice | **1** | +| Exec user WS per microservice | **3** (Plan 17 — direct WS; no POST/DELETE MS exec REST) | | Exec pending (user waits for agent) | **60s** | | Exec max duration | **8h** | | Log user WS per microservice/fog | **3** | diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6738dda8..c0ff8311 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2754,63 +2754,6 @@ paths: description: Not Found "500": description: Internal Server Error - "/microservices/{uuid}/exec": - post: - tags: - - Microservices - summary: Enables a exec for microservice - operationId: enableMicroserviceExec - parameters: - - in: path - name: uuid - description: Microservice UUID - required: true - schema: - type: string - security: - - authToken: [] - responses: - "201": - description: Created - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - "401": - description: Not Authorized - "404": - description: Invalid Microservice UUID - "500": - description: Internal Server Error - delete: - tags: - - Microservices - summary: Disables a exec for microservice - operationId: disableMicroserviceExec - parameters: - - in: path - name: uuid - description: Microservice UUID - required: true - schema: - type: string - security: - - authToken: [] - responses: - "204": - description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - "401": - description: Not Authorized - "404": - description: Invalid Microservice UUID - "500": - description: Internal Server Error "/microservices/exec/{microserviceUuid}": get: tags: @@ -2822,9 +2765,14 @@ paths: **Auth:** Bearer JWT in `Authorization` header or `?token=` query param. **RBAC:** `execSessions` on `microservices` resource. - Pairing: user connects first (or agent first on `/agent/exec/{uuid}`); Controller pairs - via `SessionManager` and relays STDIN/STDOUT/STDERR. On disconnect, **exec_b** sets - `execEnabled=false` (see REST `POST/DELETE …/exec`). + Pairing: user connects directly via WebSocket; Controller creates a per-session row + in `MicroserviceExecSessions` and relays STDIN/STDOUT/STDERR by `sessionId`. + No prior REST enable step is required (Plan 17). Max **3** concurrent exec sessions + per microservice (close code **1008** when quota exceeded). + + On connect, Controller sends **ACTIVATION** (type 5) with JSON + `{ sessionId, microserviceUuid }` in the `data` field and top-level `sessionId`, + followed by STDERR "waiting for agent…". Relay frames use `execId` equal to `sessionId`. **HA:** Multi-replica deployments require AMQP router (`WebSocketQueueService`). Cross-replica sessions fail fast with close code **1013** when router is unavailable. @@ -3013,17 +2961,62 @@ paths: responses: "101": description: Switching Protocols - "/agent/exec/{microserviceUuid}": + "/agent/exec/sessions": + get: + tags: + - Agent + summary: List pending and active exec sessions for this agent + description: | + Returns exec sessions for microservices on the authenticated fog node. + Poll when `GET /agent/changes` reports `execSessions: true` (Plan 17). + + Each entry includes `microserviceUuid`, `sessionId`, `status` (`PENDING` | `ACTIVE`), + and `agentConnected`. Agent connects with + `WS /agent/exec/microservice/{microserviceUuid}/{sessionId}`. + operationId: getAgentExecSessions + security: + - fogToken: [] + responses: + "200": + description: Exec session list + content: + application/json: + schema: + type: object + properties: + execSessions: + type: array + items: + type: object + properties: + microserviceUuid: + type: string + format: uuid + sessionId: + type: string + format: uuid + status: + type: string + enum: [PENDING, ACTIVE] + agentConnected: + type: boolean + "401": + description: Not Authorized + "/agent/exec/microservice/{microserviceUuid}/{sessionId}": get: tags: - WebSocketSessions - summary: Agent exec WebSocket + summary: Agent exec WebSocket (session-scoped) description: | - Agent-side exec pairing. **Auth:** fog token (not OIDC). + Agent-side exec pairing for a specific session. **Auth:** fog token (not OIDC). + + Path includes the `sessionId` from `GET /agent/exec/sessions` or from user ACTIVATION. + Controller validates the session row and strict `sessionId` / `microserviceUuid` match + before relay. Legacy `WS /agent/exec/{microserviceUuid}` with initial MessagePack frame + is **removed** (Plan 17, R96). - First binary frame must be MessagePack with `execId` and `microserviceUuid`. Agent handlers validate fog token **before** accepting messages (R86). - operationId: agentExecWebSocket + operationId: agentExecMicroserviceWebSocket parameters: - in: path name: microserviceUuid @@ -3031,6 +3024,12 @@ paths: schema: type: string format: uuid + - in: path + name: sessionId + required: true + schema: + type: string + format: uuid security: - fogToken: [] responses: @@ -3084,63 +3083,6 @@ paths: responses: "101": description: Switching Protocols - "/microservices/system/{uuid}/exec": - post: - tags: - - Microservices - summary: Enables a exec for system microservice - operationId: enableSystemMicroserviceExec - parameters: - - in: path - name: uuid - description: Microservice UUID - required: true - schema: - type: string - security: - - authToken: [] - responses: - "201": - description: Created - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - "401": - description: Not Authorized - "404": - description: Invalid Microservice UUID - "500": - description: Internal Server Error - delete: - tags: - - Microservices - summary: Disables a exec for system microservice - operationId: disableSystemMicroserviceExec - parameters: - - in: path - name: uuid - description: Microservice UUID - required: true - schema: - type: string - security: - - authToken: [] - responses: - "204": - description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - "401": - description: Not Authorized - "404": - description: Invalid Microservice UUID - "500": - description: Internal Server Error "/iofog/{uuid}/tunnel": patch: tags: @@ -7165,7 +7107,7 @@ components: schemas: WsExecMessageTypes: type: object - description: MessagePack frame types for exec sessions (R80) + description: MessagePack frame types for exec sessions properties: types: type: array @@ -7185,7 +7127,7 @@ components: - { name: STDERR, value: 2, direction: agent → user } - { name: CONTROL, value: 3, direction: both } - { name: CLOSE, value: 4, direction: both } - - { name: ACTIVATION, value: 5, direction: controller → agent } + - { name: ACTIVATION, value: 5, direction: controller → user (session announce); controller → agent (optional, shell live) } WsLogMessageTypes: type: object description: MessagePack frame types for log sessions (R82) @@ -7214,7 +7156,7 @@ components: codes: - { code: 1000, reason: Normal closure, when: Session ended cleanly } - { code: 1001, reason: Server draining, when: SIGTERM / k8s preStop (R85) } - - { code: 1008, reason: Policy violation, when: RBAC deny, quota, invalid tail params, pending timeout } + - { code: 1008, reason: Policy violation, when: RBAC deny, exec/log quota (3 per MS), invalid tail params, pending timeout, sessionId mismatch } - { code: 1013, reason: Router unavailable for cross-replica session, when: AMQP router down (R84) } EventRecord: type: object