diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..e873e66 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "tencentdb-agent-memory-local", + "interface": { + "displayName": "TencentDB Agent Memory Local" + }, + "plugins": [ + { + "name": "memory-tencentdb-codex", + "source": { + "source": "local", + "path": "./codex-plugin" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Productivity" + } + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2d346..268b7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ --- +## [Unreleased] + +### ✨ 新功能 + +- **Codex adapter**：新增独立 `codex-plugin/` 适配层，覆盖 Codex CLI 与 Codex App 的生命周期 hook、MCP 检索工具、历史 JSONL 导入、工具输出 offload 与本地 Gateway 自动启动，并包含 Codex App 的额外适配与验证。 + +### 🔒 安全增强 + +- Gateway 默认要求 tokenized POST；无 token 时仅保留 loopback GET 探活，loopback tokenless POST 必须显式启用开发开关。 +- Codex adapter 默认拒绝非 loopback Gateway URL，MCP 默认不暴露跨项目检索或完整 offload 内容。 +- Gateway token 文件改为私有权限、owner 校验、atomic create；并发 autostart 不再可能生成互相覆盖的 token。 +- Codex hook 诊断写入私有 `hook.log`，日志内容先经过敏感字段 redaction。 + +### 🐛 修复 + +- scoped memory/conversation search 会扩展候选窗口到 store 记录总数，避免当前项目结果被其他项目的前 500 个候选挤掉。 +- `prepack` 不再因为已缺失的历史可选 bin-script 源目录而失败；存在对应 `tsconfig.json` 时仍会构建这些脚本。 + ## [0.3.4] - 2026-05-12 ### 🐛 修复 diff --git a/codex-plugin/.codex-plugin/plugin.json b/codex-plugin/.codex-plugin/plugin.json new file mode 100644 index 0000000..799a3a6 --- /dev/null +++ b/codex-plugin/.codex-plugin/plugin.json @@ -0,0 +1,30 @@ +{ + "name": "memory-tencentdb-codex", + "version": "0.1.0", + "description": "Codex adapter for TencentDB Agent Memory: auto-recall, auto-capture, hook context injection, MCP search/offload tools, compaction flush, and seed through the TDAI Gateway.", + "author": { + "name": "TencentDB Agent Memory Codex adapter" + }, + "license": "MIT", + "homepage": "https://github.com/Tencent/TencentDB-Agent-Memory", + "repository": "https://github.com/Tencent/TencentDB-Agent-Memory", + "interface": { + "displayName": "TencentDB Agent Memory", + "shortDescription": "Automatic recall, capture, flush, offload, and MCP search for Codex sessions through TencentDB Agent Memory.", + "longDescription": "Adds Codex hooks that recall relevant memory before each prompt, inject model-visible memory hints, capture completed turns, track tool and permission activity, offload large tool results into JSONL/ref/Mermaid artifacts, flush session memory after compaction or every N turns, expose memory and offload lookup as MCP tools, seed historical conversations, and manage a local TencentDB Agent Memory Gateway. The adapter supports Codex CLI and Codex App, with additional Codex App adaptation and validation.", + "developerName": "TencentDB Agent Memory Codex adapter", + "category": "Coding", + "capabilities": [ + "Interactive", + "Read", + "Write" + ], + "defaultPrompt": [ + "Use TencentDB Agent Memory to recall prior context before answering and capture important session details after the turn." + ], + "brandColor": "#2563EB" + }, + "skills": "./skills/", + "mcpServers": "./.mcp.json", + "hooks": "./hooks/hooks.codex.json" +} diff --git a/codex-plugin/.mcp.json b/codex-plugin/.mcp.json new file mode 100644 index 0000000..d4ec3a9 --- /dev/null +++ b/codex-plugin/.mcp.json @@ -0,0 +1,15 @@ +{ + "mcpServers": { + "tdai-memory": { + "command": "node", + "args": [ + "${CLAUDE_PLUGIN_ROOT}/scripts/mcp-server.mjs" + ], + "env": { + "TDAI_CODEX_AUTOSTART": "true" + }, + "startup_timeout_sec": 20, + "tool_timeout_sec": 60 + } + } +} diff --git a/codex-plugin/README.md b/codex-plugin/README.md new file mode 100644 index 0000000..c87f69a --- /dev/null +++ b/codex-plugin/README.md @@ -0,0 +1,312 @@ +# TencentDB Agent Memory Codex Adapter + +Codex adapter for the **memory-tencentdb** four-layer memory system +(L0 conversation capture -> L1 episodic extraction -> L2 scene blocks -> L3 +persona synthesis). + +The heavy lifting runs in the Node.js **Gateway** sidecar used by the other +TencentDB Agent Memory integrations. This adapter is a thin Codex plugin layer: +it translates Codex hooks and MCP tool calls into the Gateway API, keeps +Codex-specific state under the adapter data directory, and leaves the OpenClaw +and Hermes paths unchanged. + +The adapter targets Codex as a host, including Codex CLI and Codex App. +It also includes extra Codex App adaptation and validation for App session +history, archived JSONL import, plugin-cache loading, and App-observed hook +behavior. + +## Architecture + +```text +Codex (CLI and App) + +-- Hooks + | +-- SessionStart -> scripts/session-start.mjs + | +-- UserPromptSubmit -> scripts/user-prompt-submit.mjs + | +-- PreToolUse -> scripts/pre-tool-use.mjs + | +-- PermissionRequest -> scripts/permission-request.mjs + | +-- PostToolUse -> scripts/post-tool-use.mjs + | +-- PreCompact -> scripts/pre-compact.mjs + | +-- PostCompact -> scripts/post-compact.mjs + | +-- Stop -> scripts/stop.mjs + +-- MCP + +-- scripts/mcp-server.mjs + +-- tdai_memory_search + +-- tdai_conversation_search + +-- tdai_offload_lookup + | + v HTTP (127.0.0.1:8420 by default) + memory-tencentdb Gateway + +-- POST /recall + +-- POST /capture + +-- POST /search/memories + +-- POST /search/conversations + +-- POST /session/end + +-- POST /seed +``` + +The Codex-specific integration lives in this directory. The shared changes +outside `codex-plugin/` are limited to host-neutral Gateway and seed support +used by sidecar clients: a lightweight root metadata endpoint, optional +`started_at` capture metadata, and opt-in full-pipeline waiting for `/seed`. + +## Lifecycle Mapping + +| Codex surface | Gateway or local path | Behavior | +| --- | --- | --- | +| `SessionStart` | `/recall`, `/search/memories`, selective `/search/conversations` | Restores project/session context and returns Codex `additionalContext` when useful context exists. | +| `UserPromptSubmit` | Local turn state, `/recall`, `/search/memories`, selective `/search/conversations`, local L0 JSONL fallback | Starts a pending turn, recalls relevant memory, and injects bounded context; if Gateway recall/search has no useful context, scans project-scoped local L0 JSONL as a last resort. | +| `PreToolUse` | Local turn state | Records tool intent and returns a compact memory/offload hint. | +| `PermissionRequest` | Local turn state | Records permission activity for the current turn. | +| `PostToolUse` | Local turn state, context-offload files | Records tool results and can replace large tool output with compact hook feedback plus a lookup reference. | +| `PreCompact` | `/capture` | Captures pending turn state before compaction. | +| `PostCompact` | `/session/end` | Flushes pending Gateway pipeline work after compaction. | +| `Stop` | `/capture`, periodic `/session/end` | Captures the completed Codex turn and flushes every `TDAI_CODEX_FLUSH_EVERY_N_TURNS` captured turns. | +| MCP `tdai_memory_search` | `/search/memories` | Searches L1 structured memory. | +| MCP `tdai_conversation_search` | `/search/conversations` | Searches L0 raw conversation history. | +| MCP `tdai_offload_lookup` | Local context-offload index | Retrieves exact redacted tool results by `node_id`, `tool_call_id`, or query. | + +## Reliability Features + +- **Gateway supervision** - the adapter can auto-discover and start the Gateway + from a local TencentDB Agent Memory checkout, then poll `/health` before use. +- **Circuit breaker** - repeated Gateway failures pause calls for a short + cooldown instead of slowing every hook invocation. +- **Bounded prompt injection** - empty Gateway search responses are not injected, + recall output is capped by `TDAI_CODEX_CONTEXT_MAX_CHARS`, and tool hints are + intentionally compact. +- **Injected-context cleanup** - adapter-controlled capture, import, transcript, + and Gateway L0/L1 write paths strip TencentDB/Codex injected blocks before + persistence to avoid recall feedback loops. +- **Local L0 fallback** - when Gateway recall/search is unavailable or empty, + automatic prompt recall can stream recent local L0 JSONL and filter by the + current Codex project session-key prefix. +- **Short-term offload lookup** - large `PostToolUse` output can be stored under + local JSONL/ref/Mermaid artifacts and retrieved later even if the Gateway is + temporarily unavailable. + +## Installation Location + +This directory (`codex-plugin/`) is the source of truth for the Codex adapter. +Codex loads it as a local plugin or from a local marketplace/cache copy. + +The plugin manifest is: + +```text +codex-plugin/.codex-plugin/plugin.json +``` + +It declares the Codex skill, bundled hook config, and bundled MCP server config: + +```text +codex-plugin/hooks/hooks.codex.json +codex-plugin/.mcp.json +``` + +Codex can load these hooks as plugin-bundled hooks when `plugin_hooks` is +enabled, or as user-level hooks from `~/.codex/hooks.json`. When mirroring the +hook file into a user-level config, replace the plugin-root variable with the +installed adapter path because user-level hooks do not receive plugin-specific +environment variables. The bundled MCP config exposes memory search and offload +lookup tools; the manual `codex mcp add` command below is a fallback for local +development or older Codex builds. + +## Setup + +From the TencentDB-Agent-Memory repository root: + +```bash +npm install +``` + +Optional Codex adapter environment: + +```bash +export TDAI_CODEX_TDAI_ROOT="/path/to/TencentDB-Agent-Memory" +export TDAI_CODEX_DATA_DIR="$HOME/.memory-tencentdb/codex-memory-tdai" +export TDAI_CODEX_GATEWAY_URL="http://127.0.0.1:8420" +export TDAI_CODEX_AUTOSTART=true +export TDAI_CODEX_FLUSH_EVERY_N_TURNS=5 +export TDAI_CODEX_TOOL_OFFLOAD=true +``` + +When the adapter autostarts the Gateway it keeps the service on loopback by +default, creates a private bearer token under +`$TDAI_CODEX_DATA_DIR/codex-adapter/gateway-token`, and sends that token on +Gateway requests. The token is passed to the daemon through `TDAI_TOKEN_PATH` +instead of a generated token environment variable. Set `TDAI_CODEX_GATEWAY_TOKEN` +or `TDAI_TOKEN_PATH` if you want to manage the token yourself. Autostart refuses +non-loopback hosts unless `TDAI_CODEX_ALLOW_NON_LOOPBACK=true` is set explicitly. + +By default autostart uses the package bin +(`npx --yes --ignore-scripts --package @tencentdb-agent-memory/memory-tencentdb tdai-memory-gateway`), +so the copied Codex plugin does not need to import package dependencies from the +plugin directory and daemon launch does not run npm lifecycle scripts. For +source-tree development, set `TDAI_CODEX_TDAI_ROOT` to use +`npx tsx src/gateway/server.ts` from a local checkout, or set +`TDAI_CODEX_GATEWAY_PACKAGE` to override the package spec, including a pinned +version or tarball during release validation. Package-bin launch does not +hydrate additional shell-only LLM secrets unless +`TDAI_CODEX_HYDRATE_ENV_FOR_PACKAGE_GATEWAY=true` is set explicitly. + +The Gateway also rejects non-loopback browser origins by default and blocks +credential-bearing `/seed config_override` keys, so imported Codex history cannot +redirect configured LLM, embedding, TCVDB, or backend credentials to a different +network endpoint. + +When no Gateway token is configured, unauthenticated loopback access is limited +to `GET` routes such as `/health`. Tokenless `POST` routes require the explicit +loopback-only development flag `TDAI_GATEWAY_AUTH_DISABLED=true`; non-loopback +tokenless access is always rejected. + +Adapter requests also refuse non-loopback `TDAI_CODEX_GATEWAY_URL` values unless +`TDAI_CODEX_ALLOW_NON_LOOPBACK=true` is set explicitly. This prevents hooks from +sending local bearer tokens or captured memory to an unexpected remote URL. + +For L1/L2/L3 extraction, configure an OpenAI-compatible LLM for the Gateway: + +```bash +export TDAI_LLM_BASE_URL="https://api.openai.com/v1" +export TDAI_LLM_API_KEY="..." +export TDAI_LLM_MODEL="gpt-4o-mini" +``` + +The example Gateway config is `tdai-gateway.example.json`. Copy it to: + +```bash +$TDAI_CODEX_DATA_DIR/tdai-gateway.json +``` + +or use environment variables only. During autostart the adapter sets +`TDAI_GATEWAY_CONFIG=$TDAI_CODEX_DATA_DIR/tdai-gateway.json`, because the Gateway +normally discovers config files from the current working directory or its +default data directory unless this variable is explicit. + +## Register MCP Tools + +The plugin bundles `codex-plugin/.mcp.json`, so normal Codex plugin installation +can register the MCP server from the plugin manifest. For local development, +or if a Codex build does not load plugin-bundled MCP config, register it +manually: + +```bash +codex mcp add tdai-memory \ + --env TDAI_CODEX_TDAI_ROOT="/path/to/TencentDB-Agent-Memory" \ + --env TDAI_CODEX_DATA_DIR="$HOME/.memory-tencentdb/codex-memory-tdai" \ + --env TDAI_CODEX_GATEWAY_URL="http://127.0.0.1:8420" \ + --env TDAI_CODEX_AUTOSTART="true" \ + -- node "/path/to/TencentDB-Agent-Memory/codex-plugin/scripts/mcp-server.mjs" +``` + +MCP search tools are scoped to the current Codex project path by default. Pass +`all_projects: true` only when you intentionally want cross-project memory or +offload lookup. + +For model-facing MCP safety, cross-project search and exact offload content are +not exposed by default. To opt in from outside the model context, set: + +```bash +export TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true +export TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT=true +``` + +## Diagnostics + +```bash +node codex-plugin/scripts/gateway.mjs status +node codex-plugin/scripts/gateway.mjs start +node codex-plugin/scripts/query.mjs status +node codex-plugin/scripts/query.mjs memory "previous decision" +node codex-plugin/scripts/query.mjs conversation "continue where we left off" +node codex-plugin/scripts/query.mjs remember "This project uses X as the source of truth." +node codex-plugin/scripts/query.mjs flush +node codex-plugin/scripts/query.mjs seed ./historical-conversations.json +node codex-plugin/scripts/query.mjs import-codex-history --dry-run --since 30d +node codex-plugin/scripts/query.mjs import-codex-history --yes --since 30d --cwd "/path/to/project" +node codex-plugin/scripts/query.mjs offload list --all --limit 10 +node codex-plugin/scripts/query.mjs offload node Cxxxxxx_N001 --content +node codex-plugin/scripts/query.mjs offload canvas +node codex-plugin/scripts/mcp-server.mjs +``` + +Logs: + +```text +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/gateway.stdout.log +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/gateway.stderr.log +$TDAI_CODEX_DATA_DIR/codex-adapter/logs/hook.log +``` + +## Import Existing Codex History + +The Gateway supports historical seeding through `POST /seed`. The Codex adapter +adds a host-specific importer that converts local Codex JSONL rollouts into +that seed format. + +By default it reads: + +```text +~/.codex/sessions/**/*.jsonl +~/.codex/archived_sessions/*.jsonl +``` + +The importer is opt-in and runs as a dry run unless `--yes` is provided: + +```bash +node codex-plugin/scripts/import-codex-history.mjs --dry-run --since 30d +node codex-plugin/scripts/import-codex-history.mjs --yes --since 30d --cwd "/path/to/project" +``` + +It skips Codex-generated context scaffolding such as `AGENTS.md` injections and +imports only paired user/assistant rounds. Use `--no-archived` to exclude +archived sessions, `--limit` for a small trial import, and `--out` to inspect +the generated `/seed` payload before writing. + +By default, a real import requests `wait_for_full_pipeline`, so Gateway `/seed` +records L0, waits for L1, flushes L2 scene extraction, and waits for L3 persona +generation before returning. Use `--no-full-pipeline` when the faster L0/L1-only +seed behavior is preferred. + +## Short-Term Context Offload + +Codex does not expose OpenClaw's `slots.contextEngine`, so the adapter uses the +official Codex hook surface as the equivalent control point: + +1. `PostToolUse` evaluates tool-result size against mild, aggressive, and + emergency thresholds. +2. When offload is triggered, the full redacted result is written under + `$TDAI_CODEX_DATA_DIR/codex-adapter/context-offload//refs/`. +3. A structured `offload-.jsonl` entry is appended with `node_id`, + `tool_call_id`, summary, score, policy, and `result_ref`. +4. The deterministic L2 canvas at `mmds/001-codex-tool-offload.mmd` is rebuilt + and injected on later `SessionStart` / `UserPromptSubmit` hooks. +5. The model can drill down by calling `tdai_offload_lookup`; humans can use + `query.mjs offload node ... --content`. + +Thresholds are configurable: + +```bash +export TDAI_CODEX_TOOL_OFFLOAD_MIN_CHARS=20000 +export TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS=80000 +export TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS=250000 +export TDAI_CODEX_TOOL_OFFLOAD_PREVIEW_CHARS=2000 +export TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS=800 +export TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS=240 +``` + +## Codex Host Notes + +OpenClaw- or Claude Code-only interfaces such as host-specific slot APIs are not +applicable to Codex; this adapter uses Codex hook, MCP, JSONL history, and +context-injection surfaces instead. Codex can gate plugin-scoped hooks or omit +optional transcript fields in some builds; the adapter provides Codex-native +fallbacks through user-level hooks, local session state, tool-event summaries, +and history import. + +## Security Notes + +- Adapter-owned session state, gateway tokens, and offloaded tool-result files + are written with private owner-only permissions on POSIX filesystems. +- Tokenized Gateways require `Authorization: Bearer ...` for all routes. A + tokenless Gateway exposes only loopback `GET` probes by default; loopback + tokenless `POST` routes require explicit development opt-in, and non-loopback + tokenless access is rejected. diff --git a/codex-plugin/hooks/hooks.codex.json b/codex-plugin/hooks/hooks.codex.json new file mode 100644 index 0000000..1ae77b4 --- /dev/null +++ b/codex-plugin/hooks/hooks.codex.json @@ -0,0 +1,90 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear", + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/session-start.mjs", + "statusMessage": "tdai-memory: loading Codex memory context" + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/user-prompt-submit.mjs", + "statusMessage": "tdai-memory: recalling relevant memory" + } + ] + } + ], + "PreToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/pre-tool-use.mjs" + } + ] + } + ], + "PermissionRequest": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/permission-request.mjs" + } + ] + } + ], + "PostToolUse": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/post-tool-use.mjs" + } + ] + } + ], + "PreCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/pre-compact.mjs", + "statusMessage": "tdai-memory: preserving turn before compaction" + } + ] + } + ], + "PostCompact": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/post-compact.mjs", + "statusMessage": "tdai-memory: flushing memory after compaction" + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "node ${PLUGIN_ROOT}/scripts/stop.mjs", + "statusMessage": "tdai-memory: capturing completed Codex turn" + } + ] + } + ] + } +} diff --git a/codex-plugin/scripts/codex-security.test.mjs b/codex-plugin/scripts/codex-security.test.mjs new file mode 100644 index 0000000..cdeb94d --- /dev/null +++ b/codex-plugin/scripts/codex-security.test.mjs @@ -0,0 +1,358 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + beginTurn, + configuredGatewayTokenPath, + captureCurrentTurn, + debug, + ensureGatewayAuthToken, + healthCheck, + hookLogPath, + httpPost, + loadSessionState, + recallForPrompt, + readGatewayAuthToken, + sanitizeMemoryText, + sessionKeyFromPayload, +} from "./lib.mjs"; +import { + lookupCodexOffload, + recordCodexToolOffload, +} from "./offload-store.mjs"; + +let tmpDir; +let originalDataDir; +let originalAutostart; +let originalGatewayUrl; +let originalAllowNonLoopback; +let originalCodexGatewayToken; +let originalGatewayToken; +let originalTokenPath; + +beforeEach(() => { + originalDataDir = process.env.TDAI_CODEX_DATA_DIR; + originalAutostart = process.env.TDAI_CODEX_AUTOSTART; + originalGatewayUrl = process.env.TDAI_CODEX_GATEWAY_URL; + originalAllowNonLoopback = process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + originalCodexGatewayToken = process.env.TDAI_CODEX_GATEWAY_TOKEN; + originalGatewayToken = process.env.TDAI_GATEWAY_TOKEN; + originalTokenPath = process.env.TDAI_TOKEN_PATH; + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-codex-security-")); + process.env.TDAI_CODEX_DATA_DIR = tmpDir; +}); + +afterEach(() => { + if (originalDataDir === undefined) { + delete process.env.TDAI_CODEX_DATA_DIR; + } else { + process.env.TDAI_CODEX_DATA_DIR = originalDataDir; + } + if (originalAutostart === undefined) { + delete process.env.TDAI_CODEX_AUTOSTART; + } else { + process.env.TDAI_CODEX_AUTOSTART = originalAutostart; + } + if (originalGatewayUrl === undefined) { + delete process.env.TDAI_CODEX_GATEWAY_URL; + } else { + process.env.TDAI_CODEX_GATEWAY_URL = originalGatewayUrl; + } + if (originalAllowNonLoopback === undefined) { + delete process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + } else { + process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK = originalAllowNonLoopback; + } + if (originalCodexGatewayToken === undefined) { + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + } else { + process.env.TDAI_CODEX_GATEWAY_TOKEN = originalCodexGatewayToken; + } + if (originalGatewayToken === undefined) { + delete process.env.TDAI_GATEWAY_TOKEN; + } else { + process.env.TDAI_GATEWAY_TOKEN = originalGatewayToken; + } + if (originalTokenPath === undefined) { + delete process.env.TDAI_TOKEN_PATH; + } else { + process.env.TDAI_TOKEN_PATH = originalTokenPath; + } + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe("Codex adapter security defaults", () => { + it("strips injected memory blocks and redacts common secrets", () => { + const githubToken = ["github", "pat", "1234567890abcdefghijklmnopqrstuvwxyz"].join("_"); + const awsAccessKey = `AKIA${"1234567890ABCDEF"}`; + const keyKind = "PRIVATE KEY"; + const privateKeyBlock = [ + `-----BEGIN ${keyKind}-----`, + "secret material", + `-----END ${keyKind}-----`, + ].join("\n"); + const cleaned = sanitizeMemoryText(` +keep this +private injected context +${githubToken} +${awsAccessKey} +${privateKeyBlock} +`); + + expect(cleaned).toContain("keep this"); + expect(cleaned).not.toContain("private injected context"); + expect(cleaned).not.toContain(githubToken); + expect(cleaned).not.toContain(awsAccessKey); + expect(cleaned).not.toContain("secret material"); + expect(cleaned).toContain("[REDACTED_GITHUB_TOKEN]"); + expect(cleaned).toContain("[REDACTED_AWS_ACCESS_KEY]"); + expect(cleaned).toContain("[REDACTED_PRIVATE_KEY]"); + }); + + it("redacts JSON-style credential fields", () => { + const cleaned = sanitizeMemoryText(JSON.stringify({ + apiKey: "plain-secret-123", + password: "hunter2", + token: "abc123xyz", + authorization: "Basic abc123", + nested: { + clientSecret: "client-secret-value", + accessToken: "access-token-value", + }, + })); + + expect(cleaned).not.toContain("plain-secret-123"); + expect(cleaned).not.toContain("hunter2"); + expect(cleaned).not.toContain("abc123xyz"); + expect(cleaned).not.toContain("Basic abc123"); + expect(cleaned).not.toContain("client-secret-value"); + expect(cleaned).not.toContain("access-token-value"); + expect(cleaned.match(/\[REDACTED\]/g)?.length).toBeGreaterThanOrEqual(6); + }); + + it("redacts env-style credential fields with prefixes", () => { + const cleaned = sanitizeMemoryText([ + "CLIENT_SECRET=client-secret-value", + "ACCESS_TOKEN=access-token-value", + "DB_PASSWORD=hunter2", + ].join("\n")); + + expect(cleaned).not.toContain("client-secret-value"); + expect(cleaned).not.toContain("access-token-value"); + expect(cleaned).not.toContain("hunter2"); + expect(cleaned).toContain("CLIENT_SECRET=[REDACTED]"); + expect(cleaned).toContain("ACCESS_TOKEN=[REDACTED]"); + expect(cleaned).toContain("DB_PASSWORD=[REDACTED]"); + }); + + it("redacts full Authorization and Proxy-Authorization header values", () => { + const cleaned = sanitizeMemoryText([ + "Authorization: Basic dXNlcjpwYXNz", + "Proxy-Authorization: Token proxy-secret-value", + ].join("\n")); + + expect(cleaned).not.toContain("dXNlcjpwYXNz"); + expect(cleaned).not.toContain("proxy-secret-value"); + expect(cleaned).toContain("Authorization=[REDACTED]"); + expect(cleaned).toContain("Proxy-Authorization=[REDACTED]"); + }); + + it("writes redacted diagnostics to hook.log without throwing", () => { + debug("Gateway failed with Authorization: Bearer diagnostic-secret-value"); + + const log = fs.readFileSync(hookLogPath(), "utf-8"); + expect(log).toContain("Gateway failed"); + expect(log).toContain("Authorization=[REDACTED]"); + expect(log).not.toContain("diagnostic-secret-value"); + }); + + it("writes Codex state and offload files with private permissions", async () => { + const payload = { cwd: process.cwd(), session_id: "perm-test", prompt: "hello" }; + await beginTurn(payload); + await recordCodexToolOffload({ + sessionKey: sessionKeyFromPayload(payload), + sessionId: "perm-test", + cwd: process.cwd(), + toolName: "test-tool", + toolUseId: "tool-1", + inputSummary: "input", + redactedOutput: "output".repeat(100), + storedText: "stored output", + policy: { name: "mild", score: 8 }, + }); + + const sessionDir = path.join(tmpDir, "codex-adapter", "sessions"); + const sessionFile = path.join(sessionDir, fs.readdirSync(sessionDir)[0]); + const offloadBase = path.join(tmpDir, "codex-adapter", "context-offload"); + const offloadRoot = path.join(offloadBase, fs.readdirSync(offloadBase)[0]); + const refFile = path.join(offloadRoot, "refs", fs.readdirSync(path.join(offloadRoot, "refs"))[0]); + + expect(mode(sessionDir)).toBe("700"); + expect(mode(sessionFile)).toBe("600"); + expect(mode(offloadRoot)).toBe("700"); + expect(mode(refFile)).toBe("600"); + }); + + it("scopes offload lookup by project cwd unless explicitly omitted", async () => { + const cwdA = path.join(tmpDir, "project-a"); + const cwdB = path.join(tmpDir, "project-b"); + fs.mkdirSync(cwdA); + fs.mkdirSync(cwdB); + + await recordCodexToolOffload(offloadParams(cwdA, "session-a", "tool-a")); + await recordCodexToolOffload(offloadParams(cwdB, "session-b", "tool-b")); + + const scoped = await lookupCodexOffload({ cwd: cwdA, limit: 10 }); + expect(scoped.matches).toHaveLength(1); + expect(scoped.matches[0].tool_call_id).toBe("tool-a"); + + const all = await lookupCodexOffload({ limit: 10 }); + expect(all.matches.map((entry) => entry.tool_call_id).sort()).toEqual(["tool-a", "tool-b"]); + }); + + it("falls back to project-scoped local L0 JSONL search when Gateway recall is unavailable", async () => { + process.env.TDAI_CODEX_AUTOSTART = "false"; + process.env.TDAI_CODEX_GATEWAY_URL = "http://127.0.0.1:9"; + const cwdA = path.join(tmpDir, "project-a"); + const cwdB = path.join(tmpDir, "project-b"); + fs.mkdirSync(cwdA); + fs.mkdirSync(cwdB); + + const conversationsDir = path.join(tmpDir, "conversations"); + fs.mkdirSync(conversationsDir); + fs.writeFileSync(path.join(conversationsDir, "2026-05-18.jsonl"), [ + JSON.stringify({ + sessionKey: sessionKeyFromPayload({ cwd: cwdA, session_id: "a" }), + sessionId: "a", + recordedAt: "2026-05-18T01:00:00.000Z", + role: "assistant", + content: "The project decision was to use SQLite for local recall.", + }), + JSON.stringify({ + sessionKey: sessionKeyFromPayload({ cwd: cwdB, session_id: "b" }), + sessionId: "b", + recordedAt: "2026-05-18T02:00:00.000Z", + role: "assistant", + content: "The project decision was to use a remote service.", + }), + ].join("\n") + "\n"); + + const context = await recallForPrompt( + { cwd: cwdA, session_id: "a" }, + "previous project decision SQLite", + "prompt", + ); + + expect(context).toContain('source="local-jsonl-direct"'); + expect(context).toContain("SQLite for local recall"); + expect(context).not.toContain("remote service"); + }); + + it("keeps a pending turn when capture cannot reach the Gateway", async () => { + process.env.TDAI_CODEX_AUTOSTART = "false"; + process.env.TDAI_CODEX_GATEWAY_URL = "http://127.0.0.1:9"; + const payload = { cwd: process.cwd(), session_id: "capture-failure", prompt: "keep me pending" }; + const sessionKey = sessionKeyFromPayload(payload); + + await beginTurn(payload); + const result = await captureCurrentTurn(payload, "stop"); + const state = await loadSessionState(sessionKey); + + expect(result).toEqual({ captured: false, reason: "gateway_unavailable" }); + expect(state.currentTurn.userPrompt).toBe("keep me pending"); + expect(state.turns || []).toHaveLength(0); + }); + + it("writes explicit env token to the token file used by the spawned Gateway", async () => { + const customTokenPath = path.join(tmpDir, "custom-token"); + const defaultTokenPath = path.join(tmpDir, "codex-adapter", "gateway-token"); + fs.mkdirSync(path.dirname(defaultTokenPath), { recursive: true }); + fs.writeFileSync(defaultTokenPath, "stale-default-token\n", { mode: 0o600 }); + + process.env.TDAI_TOKEN_PATH = customTokenPath; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "explicit-env-token"; + delete process.env.TDAI_GATEWAY_TOKEN; + + await expect(ensureGatewayAuthToken()).resolves.toBe("explicit-env-token"); + expect(fs.readFileSync(customTokenPath, "utf-8").trim()).toBe("explicit-env-token"); + await expect(readGatewayAuthToken()).resolves.toBe("explicit-env-token"); + }); + + it("treats a custom token path as authoritative over a stale default token file", async () => { + const customTokenPath = path.join(tmpDir, "custom-token"); + const defaultTokenPath = path.join(tmpDir, "codex-adapter", "gateway-token"); + fs.mkdirSync(path.dirname(defaultTokenPath), { recursive: true }); + fs.writeFileSync(defaultTokenPath, "stale-default-token\n", { mode: 0o600 }); + + process.env.TDAI_TOKEN_PATH = customTokenPath; + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + delete process.env.TDAI_GATEWAY_TOKEN; + + const token = await ensureGatewayAuthToken(); + expect(token).not.toBe("stale-default-token"); + expect(fs.readFileSync(customTokenPath, "utf-8").trim()).toBe(token); + await expect(readGatewayAuthToken()).resolves.toBe(token); + }); + + it("expands tilde token paths consistently for adapter and spawned Gateway env", async () => { + const tokenFileName = `.tdai-codex-token-test-${process.pid}-${Date.now()}`; + const expandedTokenPath = path.join(os.homedir(), tokenFileName); + process.env.TDAI_TOKEN_PATH = `~/${tokenFileName}`; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "tilde-env-token"; + delete process.env.TDAI_GATEWAY_TOKEN; + + try { + expect(configuredGatewayTokenPath()).toBe(expandedTokenPath); + await expect(ensureGatewayAuthToken()).resolves.toBe("tilde-env-token"); + expect(fs.readFileSync(expandedTokenPath, "utf-8").trim()).toBe("tilde-env-token"); + await expect(readGatewayAuthToken()).resolves.toBe("tilde-env-token"); + } finally { + fs.rmSync(expandedTokenPath, { force: true }); + } + }); + + it("creates generated Gateway tokens atomically across concurrent autostarts", async () => { + const tokenPath = path.join(tmpDir, "concurrent-token"); + process.env.TDAI_TOKEN_PATH = tokenPath; + delete process.env.TDAI_CODEX_GATEWAY_TOKEN; + delete process.env.TDAI_GATEWAY_TOKEN; + + const tokens = await Promise.all(Array.from({ length: 12 }, () => ensureGatewayAuthToken())); + const unique = new Set(tokens); + + expect(unique.size).toBe(1); + expect(fs.readFileSync(tokenPath, "utf-8").trim()).toBe(tokens[0]); + }); + + it("does not send auth or payloads to non-loopback Gateway URLs unless explicitly enabled", async () => { + process.env.TDAI_CODEX_GATEWAY_URL = "https://attacker.example"; + process.env.TDAI_CODEX_GATEWAY_TOKEN = "secret-token"; + delete process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK; + const fetchMock = vi.spyOn(globalThis, "fetch"); + + await expect(healthCheck()).resolves.toBe(false); + await expect(httpPost("/capture", { user_content: "secret" })).resolves.toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); + +function offloadParams(cwd, sessionId, toolUseId) { + const payload = { cwd, session_id: sessionId }; + return { + sessionKey: sessionKeyFromPayload(payload), + sessionId, + cwd, + toolName: "test-tool", + toolUseId, + inputSummary: "input", + redactedOutput: "output".repeat(100), + storedText: "stored output", + policy: { name: "mild", score: 8 }, + }; +} + +function mode(filePath) { + return (fs.statSync(filePath).mode & 0o777).toString(8); +} diff --git a/codex-plugin/scripts/gateway.mjs b/codex-plugin/scripts/gateway.mjs new file mode 100644 index 0000000..423e214 --- /dev/null +++ b/codex-plugin/scripts/gateway.mjs @@ -0,0 +1,37 @@ +#!/usr/bin/env node +import { + ensureGateway, + gatewayUrl, + healthCheck, + resolveTdaiRoot, + startGatewayDetached, + stopGateway, + tdaiDataDir +} from "./lib.mjs"; + +const command = process.argv[2] || "status"; + +if (command === "status") { + const healthy = await healthCheck(); + console.log(JSON.stringify({ + healthy, + gatewayUrl: gatewayUrl(), + tdaiRoot: resolveTdaiRoot(), + dataDir: tdaiDataDir() + }, null, 2)); +} else if (command === "start") { + const ok = await ensureGateway(); + console.log(ok ? "gateway ready" : "gateway unavailable"); + process.exit(ok ? 0 : 1); +} else if (command === "start-detached") { + const ok = await startGatewayDetached(); + console.log(ok ? "gateway start requested" : "gateway start failed"); + process.exit(ok ? 0 : 1); +} else if (command === "stop") { + const ok = await stopGateway(); + console.log(ok ? "gateway stop requested" : "no gateway pid found"); + process.exit(ok ? 0 : 1); +} else { + console.error("Usage: node scripts/gateway.mjs [status|start|start-detached|stop]"); + process.exit(2); +} diff --git a/codex-plugin/scripts/import-codex-history.mjs b/codex-plugin/scripts/import-codex-history.mjs new file mode 100644 index 0000000..220997c --- /dev/null +++ b/codex-plugin/scripts/import-codex-history.mjs @@ -0,0 +1,373 @@ +#!/usr/bin/env node +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + ensureGateway, + expandHome, + httpPost, + sanitizeMemoryText, + sha1 +} from "./lib.mjs"; + +const DEFAULT_SESSIONS_DIR = "~/.codex/sessions"; +const DEFAULT_ARCHIVED_DIR = "~/.codex/archived_sessions"; +const DEFAULT_FULL_PIPELINE_TIMEOUT_MS = 900_000; +const DEFAULT_SEED_TIMEOUT_MS = 960_000; + +export async function importCodexHistoryCli(args = process.argv.slice(2)) { + const opts = parseArgs(args); + if (opts.help) return usage(0); + + const roots = [opts.sessionsDir]; + if (opts.includeArchived) roots.push(opts.archivedDir); + + const files = []; + for (const root of roots) { + files.push(...await collectJsonlFiles(root, root === opts.archivedDir ? "archived" : "active")); + } + files.sort((a, b) => a.file.localeCompare(b.file)); + + const selected = []; + const skipped = { + byDate: 0, + byCwd: 0, + empty: 0, + parseError: 0 + }; + + for (const entry of files) { + const parsed = await parseCodexRollout(entry, opts); + if (!parsed.ok) { + skipped[parsed.reason] = (skipped[parsed.reason] || 0) + 1; + continue; + } + selected.push(parsed.session); + if (opts.limit && selected.length >= opts.limit) break; + } + + const seedData = { sessions: selected }; + const summary = summarize(files, selected, skipped, opts); + + if (opts.out) { + await fs.mkdir(path.dirname(opts.out), { recursive: true }); + await fs.writeFile(opts.out, `${JSON.stringify(seedData, null, 2)}\n`, "utf-8"); + summary.output = opts.out; + } + + if (opts.dryRun || !opts.yes) { + summary.mode = "dry-run"; + summary.next = "Re-run with --yes to import these rounds through Gateway /seed."; + console.log(JSON.stringify(summary, null, 2)); + return; + } + + if (selected.length === 0) { + summary.mode = "import"; + summary.imported = false; + summary.reason = "no_sessions_with_user_assistant_rounds"; + console.log(JSON.stringify(summary, null, 2)); + return; + } + + const ready = await ensureGateway(); + if (!ready) { + console.error("TDAI Gateway unavailable"); + process.exit(1); + } + + const result = await httpPost("/seed", { + data: seedData, + strict_round_role: true, + auto_fill_timestamps: false, + wait_for_full_pipeline: opts.fullPipeline, + full_pipeline_timeout_ms: opts.fullPipelineTimeoutMs + }, Number(process.env.TDAI_CODEX_SEED_TIMEOUT_MS || DEFAULT_SEED_TIMEOUT_MS)); + + console.log(JSON.stringify({ + ...summary, + mode: "import", + imported: !!result, + seedResult: result + }, null, 2)); +} + +async function collectJsonlFiles(root, kind) { + const dir = path.resolve(expandHome(root)); + if (!fsSync.existsSync(dir)) return []; + const found = []; + async function walk(current) { + const entries = await fs.readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) { + await walk(full); + } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + found.push({ file: full, kind }); + } + } + } + await walk(dir); + return found; +} + +async function parseCodexRollout(entry, opts) { + let text; + try { + text = await fs.readFile(entry.file, "utf-8"); + } catch { + return { ok: false, reason: "parseError" }; + } + + let sessionId = path.basename(entry.file, ".jsonl"); + let sessionCwd = ""; + let sessionTimestamp = 0; + let source = ""; + const messages = []; + + for (const line of text.split(/\r?\n/)) { + if (!line.trim()) continue; + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + + const payload = row.payload || {}; + if (row.type === "session_meta") { + sessionId = payload.id || sessionId; + sessionCwd = payload.cwd || sessionCwd; + sessionTimestamp = timestampMs(payload.timestamp || row.timestamp) || sessionTimestamp; + source = sourceLabel(payload.source); + continue; + } + + if (row.type !== "response_item" || payload.type !== "message") continue; + if (payload.role !== "user" && payload.role !== "assistant") continue; + + const content = sanitizeMemoryText(contentToText(payload.content)); + if (shouldSkipMessage(payload.role, content)) continue; + + messages.push({ + role: payload.role, + content, + timestamp: timestampMs(row.timestamp) || sessionTimestamp || Date.now() + }); + } + + if (opts.since && sessionTimestamp && sessionTimestamp < opts.since) { + return { ok: false, reason: "byDate" }; + } + + if (opts.cwd) { + const wanted = path.resolve(expandHome(opts.cwd)); + const actual = sessionCwd ? path.resolve(expandHome(sessionCwd)) : ""; + if (actual !== wanted) return { ok: false, reason: "byCwd" }; + } + + const conversations = pairMessages(messages); + if (conversations.length === 0) return { ok: false, reason: "empty" }; + + const cwdLabel = sessionCwd || "unknown-cwd"; + return { + ok: true, + session: { + sessionKey: `codex-import:${sha1(cwdLabel).slice(0, 10)}:${safeKey(sessionId)}`, + sessionId, + conversations, + metadata: { + source: "codex-jsonl", + codexSource: source || undefined, + codexCwd: sessionCwd || undefined, + codexArchiveKind: entry.kind, + codexPath: entry.file + } + } + }; +} + +function pairMessages(messages) { + const rounds = []; + let pendingUser = null; + + for (const msg of messages) { + if (msg.role === "user") { + if (pendingUser) { + pendingUser.content = `${pendingUser.content}\n\n${msg.content}`; + pendingUser.timestamp = Math.min(pendingUser.timestamp, msg.timestamp); + } else { + pendingUser = { ...msg }; + } + continue; + } + + if (msg.role === "assistant" && pendingUser) { + rounds.push([ + pendingUser, + msg + ]); + pendingUser = null; + } + } + + return rounds; +} + +function contentToText(content) { + if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; + return content.map((part) => { + if (!part || typeof part !== "object") return ""; + if (typeof part.text === "string") return part.text; + if (typeof part.content === "string") return part.content; + return ""; + }).filter(Boolean).join("\n"); +} + +function shouldSkipMessage(role, text) { + const value = String(text || "").trim(); + if (!value) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (value.includes("")) return true; + if (role === "user" && value.startsWith("# AGENTS.md instructions")) return true; + if (role === "user" && value.startsWith("")) return true; + if (role === "user" && value.startsWith("")) return true; + return false; +} + +function summarize(files, sessions, skipped, opts) { + const rounds = sessions.reduce((sum, session) => sum + session.conversations.length, 0); + const messages = sessions.reduce((sum, session) => ( + sum + session.conversations.reduce((inner, round) => inner + round.length, 0) + ), 0); + const byKind = sessions.reduce((acc, session) => { + const kind = session.metadata?.codexArchiveKind || "unknown"; + acc[kind] = (acc[kind] || 0) + 1; + return acc; + }, {}); + return { + source: "codex-jsonl", + sessionsDir: opts.sessionsDir, + archivedDir: opts.includeArchived ? opts.archivedDir : null, + includeArchived: opts.includeArchived, + waitForFullPipeline: opts.fullPipeline, + fullPipelineTimeoutMs: opts.fullPipelineTimeoutMs, + cwd: opts.cwd || null, + since: opts.since ? new Date(opts.since).toISOString() : null, + filesScanned: files.length, + sessionsPrepared: sessions.length, + roundsPrepared: rounds, + messagesPrepared: messages, + sessionsByKind: byKind, + skipped + }; +} + +function parseArgs(args) { + const opts = { + sessionsDir: path.resolve(expandHome(process.env.CODEX_SESSIONS_DIR || DEFAULT_SESSIONS_DIR)), + archivedDir: path.resolve(expandHome(process.env.CODEX_ARCHIVED_SESSIONS_DIR || DEFAULT_ARCHIVED_DIR)), + includeArchived: true, + fullPipeline: true, + fullPipelineTimeoutMs: positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, DEFAULT_FULL_PIPELINE_TIMEOUT_MS), + dryRun: false, + yes: false, + cwd: "", + since: 0, + limit: 0, + out: "" + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--help" || arg === "-h") opts.help = true; + else if (arg === "--dry-run") opts.dryRun = true; + else if (arg === "--yes" || arg === "-y") opts.yes = true; + else if (arg === "--no-archived") opts.includeArchived = false; + else if (arg === "--no-full-pipeline") opts.fullPipeline = false; + else if (arg === "--full-pipeline-timeout-ms") opts.fullPipelineTimeoutMs = positiveNumber(next(args, ++i, arg), 0); + else if (arg === "--sessions-dir") opts.sessionsDir = path.resolve(expandHome(next(args, ++i, arg))); + else if (arg === "--archived-dir") opts.archivedDir = path.resolve(expandHome(next(args, ++i, arg))); + else if (arg === "--cwd") opts.cwd = next(args, ++i, arg); + else if (arg === "--since") opts.since = parseSince(next(args, ++i, arg)); + else if (arg === "--limit") opts.limit = Math.max(0, Number(next(args, ++i, arg)) || 0); + else if (arg === "--out") opts.out = path.resolve(expandHome(next(args, ++i, arg))); + else throw new Error(`Unknown option: ${arg}`); + } + + return opts; +} + +function next(args, index, flag) { + if (index >= args.length) throw new Error(`${flag} requires a value`); + return args[index]; +} + +function parseSince(value) { + const raw = String(value || "").trim(); + const days = raw.match(/^(\d+)d$/i); + if (days) return Date.now() - Number(days[1]) * 24 * 60 * 60 * 1000; + const parsed = Date.parse(raw); + if (!Number.isFinite(parsed)) throw new Error(`Invalid --since value: ${value}`); + return parsed; +} + +function positiveNumber(value, fallback) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} + +function timestampMs(value) { + if (typeof value === "number") return value < 10_000_000_000 ? value * 1000 : value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : 0; + } + return 0; +} + +function sourceLabel(value) { + if (!value) return ""; + if (typeof value === "string") return value; + if (value.subagent) return "subagent"; + return "object"; +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function usage(code = 0) { + const message = `Usage: node scripts/import-codex-history.mjs [options] + +Import local Codex JSONL history into TencentDB Agent Memory through Gateway /seed. +By default this is a dry run unless --yes is provided. + +Options: + --dry-run Show the import plan without writing to the Gateway. + --yes, -y Actually import prepared rounds through /seed. + --sessions-dir Active Codex sessions directory. Default: ${DEFAULT_SESSIONS_DIR} + --archived-dir Archived Codex sessions directory. Default: ${DEFAULT_ARCHIVED_DIR} + --no-archived Do not include archived Codex JSONL files. + --no-full-pipeline Only seed through the Gateway's default L0/L1 path. + --full-pipeline-timeout-ms + Max wait for the final L1/L2/L3 flush. Default: ${DEFAULT_FULL_PIPELINE_TIMEOUT_MS} + --cwd Import only sessions whose session_meta.cwd matches this path. + --since Import only sessions newer than an ISO date or relative day window. + --limit Import at most n prepared sessions. + --out Write the generated Gateway /seed JSON to a file. +`; + (code === 0 ? console.log : console.error)(message); + process.exit(code); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + importCodexHistoryCli().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exit(1); + }); +} diff --git a/codex-plugin/scripts/lib.mjs b/codex-plugin/scripts/lib.mjs new file mode 100644 index 0000000..2ffcb5f --- /dev/null +++ b/codex-plugin/scripts/lib.mjs @@ -0,0 +1,1503 @@ +import crypto from "node:crypto"; +import { execFile, spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import { fileURLToPath } from "node:url"; +import os from "node:os"; +import path from "node:path"; +import { createInterface } from "node:readline"; +import { promisify } from "node:util"; +import { + buildCodexOffloadContext, + maxStoreChars as offloadMaxStoreChars, + previewForPolicy, + recordCodexToolOffload, + selectToolOffloadPolicy +} from "./offload-store.mjs"; + +const DEFAULT_GATEWAY_URL = "http://127.0.0.1:8420"; +const DEFAULT_CONTEXT_MAX_CHARS = 12000; +const DEFAULT_RECALL_TIMEOUT_MS = 5000; +const DEFAULT_CAPTURE_TIMEOUT_MS = 8000; +const DEFAULT_HEALTH_TIMEOUT_MS = 700; +const DEFAULT_START_TIMEOUT_MS = 12000; +const DEFAULT_SESSION_END_TIMEOUT_MS = 8000; +const DEFAULT_BREAKER_FAILURE_THRESHOLD = 5; +const DEFAULT_BREAKER_COOLDOWN_MS = 60_000; +const DEFAULT_FLUSH_EVERY_N_TURNS = 5; +const DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS = 6_000; +const DEFAULT_GATEWAY_PACKAGE = "@tencentdb-agent-memory/memory-tencentdb"; +const PRIVATE_DIR_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const execFileAsync = promisify(execFile); +const INJECTED_MEMORY_TAGS = [ + "tdai-codex-memory-context", + "structured-memory-results", + "tdai-recall-context", + "raw-conversation-results", + "tdai-codex-context-offload", + "tdai-codex-tool-memory-hint", + "tdai-codex-tool-output-offload", + "relevant-memories", + "user-persona", + "relevant-scenes", + "scene-navigation", + "memory-tools-guide", + "current_task_context", + "history_task_context", +]; + +export function pluginRoot() { + const configured = process.env.PLUGIN_ROOT || process.env.CLAUDE_PLUGIN_ROOT; + return configured + ? path.resolve(configured) + : path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +} + +export function expandHome(value) { + if (!value) return value; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +export function tdaiDataDir() { + const configured = + process.env.TDAI_CODEX_DATA_DIR || + process.env.TDAI_DATA_DIR || + path.join(os.homedir(), ".memory-tencentdb", "codex-memory-tdai"); + return path.resolve(expandHome(configured)); +} + +export function hookLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "hook.log"); +} + +export function gatewayStdoutLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "gateway.stdout.log"); +} + +export function gatewayStderrLogPath() { + return path.join(tdaiDataDir(), "codex-adapter", "logs", "gateway.stderr.log"); +} + +export function gatewayUrl() { + return (process.env.TDAI_CODEX_GATEWAY_URL || DEFAULT_GATEWAY_URL).replace(/\/+$/, ""); +} + +export function gatewayHostPort() { + const url = new URL(gatewayUrl()); + return { + host: process.env.TDAI_GATEWAY_HOST || url.hostname || "127.0.0.1", + port: process.env.TDAI_GATEWAY_PORT || url.port || "8420" + }; +} + +export function resolveTdaiRoot() { + const root = pluginRoot(); + const candidates = [ + process.env.TDAI_CODEX_TDAI_ROOT, + process.env.TDAI_INSTALL_DIR, + path.join(root, ".."), + path.join(root, "vendor", "TencentDB-Agent-Memory"), + path.join(process.cwd(), "TencentDB-Agent-Memory") + ].filter(Boolean); + + for (const candidate of candidates) { + const resolved = path.resolve(expandHome(candidate)); + const pkg = path.join(resolved, "package.json"); + const gateway = path.join(resolved, "src", "gateway", "server.ts"); + if (!fsSync.existsSync(pkg) || !fsSync.existsSync(gateway)) continue; + try { + const parsed = JSON.parse(fsSync.readFileSync(pkg, "utf-8")); + if (parsed.name === "@tencentdb-agent-memory/memory-tencentdb") { + return resolved; + } + } catch { + return resolved; + } + } + return null; +} + +export async function readHookInput() { + let input = ""; + for await (const chunk of process.stdin) input += chunk; + if (!input.trim()) return {}; + try { + return JSON.parse(input); + } catch { + return {}; + } +} + +export function cwdFromPayload(payload) { + return path.resolve( + payload.cwd || + payload.project || + process.env.CLAUDE_PROJECT_DIR || + process.cwd() + ); +} + +export function sessionIdFromPayload(payload) { + return String(payload.session_id || payload.sessionId || payload.session || "unknown-session"); +} + +export function promptFromPayload(payload) { + return String( + payload.prompt ?? + payload.user_prompt ?? + payload.userPrompt ?? + payload.input ?? + "" + ); +} + +export function sessionKeyFromPayload(payload) { + const cwd = cwdFromPayload(payload); + const sessionId = sessionIdFromPayload(payload); + return `codex:${sha1(cwd).slice(0, 10)}:${safeKey(sessionId)}`; +} + +export function sessionKeyPrefixesForCwd(cwd) { + const cwdHash = sha1(path.resolve(expandHome(cwd))).slice(0, 10); + return [ + `codex:${cwdHash}:`, + `codex-import:${cwdHash}:` + ]; +} + +export function projectLabel(payload) { + const cwd = cwdFromPayload(payload); + return `${path.basename(cwd)} (${cwd})`; +} + +export function sha1(value) { + return crypto.createHash("sha1").update(String(value)).digest("hex"); +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function statePath(sessionKey) { + return path.join(tdaiDataDir(), "codex-adapter", "sessions", `${sha1(sessionKey)}.json`); +} + +async function ensureStateDir() { + await ensurePrivateDir(path.join(tdaiDataDir(), "codex-adapter", "sessions")); +} + +function gatewayCircuitPath() { + return path.join(tdaiDataDir(), "codex-adapter", "gateway-circuit.json"); +} + +export async function loadSessionState(sessionKey) { + await ensureStateDir(); + const file = statePath(sessionKey); + try { + return JSON.parse(await fs.readFile(file, "utf-8")); + } catch { + return { sessionKey, turns: [] }; + } +} + +export async function saveSessionState(sessionKey, state) { + await ensureStateDir(); + const file = statePath(sessionKey); + const tmp = `${file}.${process.pid}.tmp`; + await writePrivateFile(tmp, `${JSON.stringify(state, null, 2)}\n`); + await fs.rename(tmp, file); + await chmodPrivateFile(file); +} + +export async function beginTurn(payload) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const prompt = sanitizeMemoryText(promptFromPayload(payload)); + const now = Date.now(); + + if (state.currentTurn && !state.currentTurn.captured) { + state.turns = state.turns || []; + state.turns.push({ + ...state.currentTurn, + abandonedAt: now, + abandonedReason: "new_user_prompt_before_stop" + }); + } + + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: prompt, + startedAt: now, + events: [ + { + phase: "user_prompt", + timestamp: now, + content: sanitizeMemoryText(prompt) + } + ], + captured: false + }; + + await saveSessionState(sessionKey, state); + return state.currentTurn; +} + +export async function appendToolEvent(payload, phase, extra = {}) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const now = Date.now(); + if (!state.currentTurn) { + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: "", + startedAt: now, + events: [], + captured: false + }; + } + + state.currentTurn.events.push({ + phase, + timestamp: now, + toolName: payload.tool_name || payload.toolName || "", + toolInput: compact(payload.tool_input ?? payload.toolInput ?? payload.input, 2500), + toolOutput: compact(toolOutputFromPayload(payload), 4000), + ...sanitizeEventDetail(extra) + }); + await saveSessionState(sessionKey, state); +} + +export async function appendLifecycleEvent(payload, phase, detail = {}, options = {}) { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const now = Date.now(); + if (!state.currentTurn) { + if (options.createTurn === false) { + return { appended: false, reason: "no_pending_turn" }; + } + state.currentTurn = { + turnId: `turn_${now}_${crypto.randomBytes(3).toString("hex")}`, + sessionKey, + sessionId: sessionIdFromPayload(payload), + cwd: cwdFromPayload(payload), + project: projectLabel(payload), + userPrompt: "", + startedAt: now, + events: [], + captured: false + }; + } + state.currentTurn.events.push({ + phase, + timestamp: now, + ...sanitizeEventDetail(detail) + }); + await saveSessionState(sessionKey, state); + return { appended: true }; +} + +export async function captureCurrentTurn(payload, reason = "stop") { + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const turn = state.currentTurn; + if (!turn || turn.captured) return { captured: false, reason: "no_pending_turn" }; + + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { captured: false, reason: "gateway_unavailable" }; + + const userContent = buildUserContent(turn, payload, reason); + const assistantContent = + await extractAssistantFromTranscript(payload.transcript_path, turn.startedAt) + || buildAssistantSummary(turn, reason); + + const messages = [ + { role: "user", content: userContent, timestamp: turn.startedAt }, + { role: "assistant", content: assistantContent, timestamp: Date.now() } + ]; + + const response = await httpPost("/capture", { + user_content: userContent, + assistant_content: assistantContent, + session_key: sessionKey, + session_id: turn.sessionId, + started_at: Math.max(0, turn.startedAt - 1), + messages + }, DEFAULT_CAPTURE_TIMEOUT_MS); + + if (!response) { + turn.lastCaptureFailure = { + reason, + failedAt: Date.now() + }; + await saveSessionState(sessionKey, state); + return { + captured: false, + reason: "capture_failed" + }; + } + + turn.captured = true; + turn.capturedAt = Date.now(); + turn.captureReason = reason; + turn.captureResponse = { + l0_recorded: response.l0_recorded, + scheduler_notified: response.scheduler_notified + }; + + state.turns = state.turns || []; + state.turns.push(turn); + delete state.currentTurn; + await saveSessionState(sessionKey, state); + + return { + captured: true, + l0Recorded: response?.l0_recorded ?? 0, + schedulerNotified: response?.scheduler_notified ?? false, + turnCount: state.turns.length + }; +} + +export async function maybeFlushCapturedTurns(payload, captureResult, reason = "periodic_turn_flush") { + if (!captureResult?.captured) return { flushed: false, reason: "no_capture" }; + const interval = numericEnv("TDAI_CODEX_FLUSH_EVERY_N_TURNS", DEFAULT_FLUSH_EVERY_N_TURNS); + if (!Number.isFinite(interval) || interval <= 0) return { flushed: false, reason: "disabled" }; + + const sessionKey = sessionKeyFromPayload(payload); + const state = await loadSessionState(sessionKey); + const turnCount = Array.isArray(state.turns) ? state.turns.length : 0; + if (turnCount === 0 || turnCount % interval !== 0 || state.lastPeriodicFlushTurnCount === turnCount) { + return { flushed: false, reason: "not_due", turnCount, interval }; + } + + const result = await sessionEnd(payload, reason); + state.lastPeriodicFlushTurnCount = turnCount; + state.lastPeriodicFlushAt = Date.now(); + state.lastPeriodicFlushResult = result; + await saveSessionState(sessionKey, state); + return { ...result, turnCount, interval }; +} + +function buildUserContent(turn, payload, reason) { + const prompt = sanitizeMemoryText(turn.userPrompt || promptFromPayload(payload)) || "[Codex turn without captured user prompt]"; + return [ + `Codex project: ${turn.project || projectLabel(payload)}`, + `Codex session: ${turn.sessionId || sessionIdFromPayload(payload)}`, + `Capture reason: ${reason}`, + "", + "User request:", + prompt + ].join("\n"); +} + +function buildAssistantSummary(turn, reason) { + const lines = [ + `Codex turn completed. Capture reason: ${reason}.`, + `Project: ${turn.project || turn.cwd || ""}`, + "" + ]; + + const events = Array.isArray(turn.events) ? turn.events : []; + const toolEvents = events.filter((event) => event.phase === "pre_tool_use" || event.phase === "post_tool_use"); + if (toolEvents.length === 0) { + lines.push("No tool events were available from Codex hooks for this turn."); + } else { + lines.push("Captured tool activity:"); + for (const event of toolEvents.slice(-20)) { + lines.push(`- ${event.phase}: ${event.toolName || "(unknown tool)"}`); + if (event.toolInput) lines.push(indentBlock(`input: ${event.toolInput}`, " ")); + if (event.toolOutput) lines.push(indentBlock(`output: ${event.toolOutput}`, " ")); + } + } + + return truncate(sanitizeMemoryText(lines.join("\n")), 10000); +} + +export async function recallForPrompt(payload, prompt, mode = "prompt") { + const sessionKey = sessionKeyFromPayload(payload); + const cwd = cwdFromPayload(payload); + const cleanPrompt = sanitizeMemoryText(prompt || ""); + const offloadContextPromise = buildCodexOffloadContext({ + sessionKey, + sessionId: sessionIdFromPayload(payload), + maxChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_CHARS", DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS) + }); + + const gatewayReady = await ensureGateway(); + const query = [ + `Codex project cwd: ${cwd}`, + `Recall mode: ${mode}`, + "", + cleanPrompt.trim() + ? `Current user request:\n${cleanPrompt.trim()}` + : "Session startup/resume: recover project state, active decisions, pending tasks, and user preferences." + ].join("\n"); + + const [recall, memories, conversations] = gatewayReady + ? await Promise.all([ + httpPost("/recall", { query, session_key: sessionKey }, DEFAULT_RECALL_TIMEOUT_MS), + httpPost("/search/memories", { + query, + limit: numericEnv("TDAI_CODEX_MEMORY_LIMIT", 8), + session_key_prefixes: sessionKeyPrefixesForCwd(cwd) + }, DEFAULT_RECALL_TIMEOUT_MS), + shouldSearchConversations(cleanPrompt, mode) + ? httpPost("/search/conversations", { + query, + limit: numericEnv("TDAI_CODEX_CONVERSATION_LIMIT", 5), + session_key_prefixes: sessionKeyPrefixesForCwd(cwd) + }, DEFAULT_RECALL_TIMEOUT_MS) + : Promise.resolve(null) + ]) + : [null, null, null]; + const directConversations = !hasUsefulGatewayText(memories?.results) && + !recall?.context?.trim() && + !hasUsefulGatewayText(conversations?.results) && + cleanPrompt.trim() + ? await searchL0JsonlDirect({ + query: cleanPrompt, + cwd, + limit: numericEnv("TDAI_CODEX_DIRECT_L0_LIMIT", 3) + }) + : ""; + + const parts = []; + if (hasUsefulGatewayText(memories?.results)) { + parts.push(`\n${memories.results.trim()}\n`); + } + if (recall?.context?.trim()) { + parts.push(`\n${recall.context.trim()}\n`); + } + if (hasUsefulGatewayText(conversations?.results)) { + parts.push(`\n${conversations.results.trim()}\n`); + } + if (directConversations) { + parts.push(`\n${directConversations}\n`); + } + const offloadContext = await offloadContextPromise; + if (offloadContext) { + parts.push(offloadContext); + } + + if (parts.length === 0) return ""; + + const context = ` +Use retrieved memory as operating context; verify drift-prone facts; persist durable new decisions. +${parts.join("\n\n")} +`; + + return truncate(context, numericEnv("TDAI_CODEX_CONTEXT_MAX_CHARS", DEFAULT_CONTEXT_MAX_CHARS)); +} + +export function hookAdditionalContext(hookEventName, context) { + if (!context?.trim()) return ""; + return `${JSON.stringify({ + hookSpecificOutput: { + hookEventName, + additionalContext: context + } + })}\n`; +} + +export function toolMemoryHint(payload) { + const toolName = payload.tool_name || payload.toolName || ""; + if (!toolName) return ""; + const input = compact(payload.tool_input ?? payload.toolInput ?? payload.input, 1200); + return ` +Use injected memory/MCP search for prior decisions or exact history; use tdai_offload_lookup for offload node_id refs. +Tool: ${escapeText(toolName)}${input ? `\nInput: ${escapeText(input)}` : ""} +`; +} + +export async function maybeOffloadToolOutput(payload) { + if (process.env.TDAI_CODEX_TOOL_OFFLOAD === "false") return null; + + const rawOutput = toolOutputFromPayload(payload); + const rendered = renderToolValue(rawOutput); + if (!rendered.trim()) return null; + + const redacted = redact(rendered); + const policy = selectToolOffloadPolicy(redacted.length); + if (!policy) return null; + + const sessionKey = sessionKeyFromPayload(payload); + const sessionId = sessionIdFromPayload(payload); + const cwd = cwdFromPayload(payload); + const toolName = payload.tool_name || payload.toolName || "unknown-tool"; + const toolUseId = payload.tool_use_id || payload.toolUseId || `${Date.now()}-${crypto.randomBytes(3).toString("hex")}`; + const maxStoreChars = offloadMaxStoreChars(); + const storedText = truncate(redacted, maxStoreChars); + const inputSummary = compact(payload.tool_input ?? payload.toolInput ?? payload.input, 5000); + const recorded = await recordCodexToolOffload({ + sessionKey, + sessionId, + cwd, + toolName, + toolUseId, + inputSummary, + redactedOutput: redacted, + storedText, + policy + }); + + const preview = previewForPolicy(redacted, policy); + const nodeId = recorded.entry.node_id || "pending"; + const outputPath = path.join(recorded.paths.root, recorded.entry.result_ref); + const summary = [ + "TencentDB Agent Memory offloaded this large tool result to keep Codex context compact.", + `Tool: ${toolName}`, + `Tool use id: ${toolUseId}`, + `Node id: ${nodeId}`, + `Offload policy: ${policy.name}`, + `Original output size after redaction: ${redacted.length} characters.`, + `Stored output path: ${outputPath}`, + `Offload JSONL: ${recorded.paths.offloadJsonl}`, + `Mermaid canvas: ${recorded.paths.canvasPath}`, + "", + "Use tdai_offload_lookup with the node id or tool use id for exact audit details.", + "Use tdai_memory_search / tdai_conversation_search for later long-term recall.", + "", + "Preview:", + preview + ].join("\n"); + + return { + outputPath, + nodeId, + toolUseId, + policy: policy.name, + offloadJsonlPath: recorded.paths.offloadJsonl, + canvasPath: recorded.paths.canvasPath, + originalChars: redacted.length, + storedChars: storedText.length, + summary: truncate(summary, numericEnv("TDAI_CODEX_TOOL_OFFLOAD_SUMMARY_MAX_CHARS", 7000)) + }; +} + +export function postToolOffloadHookOutput(offload) { + if (!offload) return ""; + return `${JSON.stringify({ + decision: "block", + reason: offload.summary, + hookSpecificOutput: { + hookEventName: "PostToolUse", + additionalContext: ` +${escapeText(offload.nodeId || "")} +${escapeText(offload.policy || "")} +${escapeText(offload.summary)} +` + } + })}\n`; +} + +function shouldSearchConversations(prompt, mode) { + if (mode === "session-start") return true; + if (!prompt) return false; + return /(继续|上次|之前|刚才|resume|continue|previous|last time|where were we|做到哪)/i.test(prompt); +} + +async function searchL0JsonlDirect(params) { + const { query, cwd, limit } = params; + const keywords = queryKeywords(query); + if (keywords.length === 0) return ""; + + const conversationsDir = path.join(tdaiDataDir(), "conversations"); + let entries; + try { + entries = await fs.readdir(conversationsDir, { withFileTypes: true }); + } catch { + return ""; + } + + const files = (await Promise.all(entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) + .map(async (entry) => { + const file = path.join(conversationsDir, entry.name); + try { + const stat = await fs.stat(file); + return { file, mtimeMs: stat.mtimeMs }; + } catch { + return { file, mtimeMs: 0 }; + } + }))) + .sort((a, b) => b.mtimeMs - a.mtimeMs) + .slice(0, numericEnv("TDAI_CODEX_DIRECT_L0_MAX_FILES", 30)); + + const prefixes = sessionKeyPrefixesForCwd(cwd); + const matches = []; + const seen = new Set(); + + for (const { file } of files) { + let stream; + try { + stream = fsSync.createReadStream(file, { encoding: "utf-8" }); + } catch { + continue; + } + + const rl = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of rl) { + if (!line.trim()) continue; + let row; + try { + row = JSON.parse(line); + } catch { + continue; + } + const sessionKey = typeof row.sessionKey === "string" ? row.sessionKey : ""; + if (!prefixes.some((prefix) => sessionKey.startsWith(prefix))) continue; + const content = sanitizeMemoryText(row.content || row.message_text || ""); + if (!content) continue; + const lower = content.toLowerCase(); + const hits = keywords.filter((keyword) => lower.includes(keyword)).length; + if (hits === 0) continue; + const fingerprint = `${row.role || ""}:${content.slice(0, 180)}`; + if (seen.has(fingerprint)) continue; + seen.add(fingerprint); + matches.push({ + role: row.role || "unknown", + content: truncate(content, 2000), + recordedAt: row.recordedAt || row.recorded_at || "", + hits + }); + } + } finally { + rl.close(); + stream.destroy(); + } + } + + if (matches.length === 0) return ""; + matches.sort((a, b) => + rolePriority(b.role) - rolePriority(a.role) || + b.hits - a.hits || + b.content.length - a.content.length + ); + + const lines = [`Found ${Math.min(matches.length, limit)} matching local L0 conversation message(s):`, ""]; + for (const match of matches.slice(0, limit)) { + lines.push("---"); + lines.push(`**[${match.role}]** ${match.recordedAt}`); + lines.push(""); + lines.push(match.content); + lines.push(""); + } + return lines.join("\n"); +} + +function queryKeywords(value) { + const cjkStop = new Set([ + "之前", "前聊", "聊的", "还记", "记得", "得么", "得吗", + "一下", "怎么", "什么", "关于", "知道", "以前", "上次", + "如何", "为何", "为啥", "哪里", "哪些", "为什", + "请问", "请帮", "帮我", "麻烦" + ]); + const keywords = []; + for (const segment of String(value || "").toLowerCase().replace(/[^\w一-鿿]/g, " ").split(/\s+/)) { + if (!segment) continue; + if (/[一-鿿]/.test(segment)) { + for (let i = 0; i <= segment.length - 2; i++) { + const gram = segment.slice(i, i + 2); + if (!cjkStop.has(gram)) keywords.push(gram); + } + } else if (segment.length >= 2) { + keywords.push(segment); + } + } + return [...new Set(keywords)].slice(0, 40); +} + +function rolePriority(role) { + return role === "assistant" ? 1 : 0; +} + +function hasUsefulGatewayText(value) { + const text = String(value || "").trim(); + if (!text) return false; + return !/^No matching (memories|conversations) found\.?$/i.test(text); +} + +export async function ensureGateway() { + if (await healthCheck()) return true; + if (process.env.TDAI_CODEX_AUTOSTART === "false") return false; + + const started = await startGatewayDetached(); + if (!started) return false; + + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + return false; +} + +export async function healthCheck() { + if (!isAllowedGatewayEndpoint()) return false; + try { + const headers = await gatewayAuthHeaders(); + const res = await fetch(`${gatewayUrl()}/health`, { + headers, + signal: AbortSignal.timeout(DEFAULT_HEALTH_TIMEOUT_MS) + }); + return res.ok; + } catch { + return false; + } +} + +export async function startGatewayDetached() { + const { host, port } = gatewayHostPort(); + if (!isLoopbackHost(host) && process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK !== "true") { + debug(`Refusing to autostart gateway on non-loopback host=${host}. Set TDAI_CODEX_ALLOW_NON_LOOPBACK=true to override.`); + return false; + } + + const logDir = path.dirname(hookLogPath()); + await ensurePrivateDir(logDir); + const lock = await acquireGatewaySpawnLock(logDir); + if (!lock) { + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + debug("Gateway spawn lock contention timed out"); + return false; + } + + try { + await ensureGatewayAuthToken(); + if (await healthCheck()) return true; + const outFd = fsSync.openSync(gatewayStdoutLogPath(), "a", PRIVATE_FILE_MODE); + const errFd = fsSync.openSync(gatewayStderrLogPath(), "a", PRIVATE_FILE_MODE); + const pidFile = path.join(logDir, "gateway.pid"); + const pidMetadataFile = path.join(logDir, "gateway.pid.json"); + const launch = gatewayLaunchSpec(); + + const env = { + ...process.env, + TDAI_DATA_DIR: tdaiDataDir(), + TDAI_GATEWAY_CONFIG: process.env.TDAI_GATEWAY_CONFIG || path.join(tdaiDataDir(), "tdai-gateway.json"), + TDAI_GATEWAY_HOST: host, + TDAI_GATEWAY_PORT: port, + TDAI_TOKEN_PATH: configuredGatewayTokenPath(), + TDAI_CODEX_PARENT_PID: String(process.ppid || process.pid) + }; + delete env.TDAI_GATEWAY_TOKEN; + delete env.TDAI_CODEX_GATEWAY_TOKEN; + + if (launch.mode !== "package-bin" || process.env.TDAI_CODEX_HYDRATE_ENV_FOR_PACKAGE_GATEWAY === "true") { + await hydrateLoginShellEnv(env, [ + "DEEPSEEK_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "TDAI_LLM_API_KEY", + "TDAI_LLM_BASE_URL", + "TDAI_LLM_MODEL" + ]); + } + + const child = spawn(launch.command, launch.args, { + cwd: launch.cwd, + detached: true, + env, + stdio: ["ignore", outFd, errFd] + }); + const spawnError = await detectSpawnError(child); + if (spawnError || !child.pid) { + debug(`Gateway spawn failed (${launch.mode}): ${spawnError?.message || "child has no pid"}`); + return false; + } + child.unref(); + await writePrivateFile(pidFile, `${child.pid}\n`); + await writePrivateFile(pidMetadataFile, `${JSON.stringify({ + pid: child.pid, + root: launch.root, + command: [launch.command, ...launch.args], + launchMode: launch.mode, + startedAt: new Date().toISOString() + }, null, 2)}\n`); + debug(`Started TDAI gateway pid=${child.pid} mode=${launch.mode}`); + + const deadline = Date.now() + numericEnv("TDAI_CODEX_START_TIMEOUT_MS", DEFAULT_START_TIMEOUT_MS); + while (Date.now() < deadline) { + await delay(500); + if (await healthCheck()) return true; + } + debug(`Gateway did not become healthy after spawn mode=${launch.mode}`); + return false; + } finally { + await lock.release(); + } +} + +function detectSpawnError(child) { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(null), 50); + child.once("error", (err) => { + clearTimeout(timer); + resolve(err); + }); + }); +} + +function gatewayLaunchSpec() { + const override = process.env.TDAI_CODEX_GATEWAY_COMMAND || process.env.TDAI_GATEWAY_COMMAND; + if (override) { + return { + command: override, + args: [], + cwd: tdaiDataDir(), + root: "", + mode: "override" + }; + } + + const explicitRoot = process.env.TDAI_CODEX_TDAI_ROOT || process.env.TDAI_INSTALL_DIR; + const root = explicitRoot ? resolveTdaiRoot() : null; + if (root && fsSync.existsSync(path.join(root, "node_modules", "tsx"))) { + return { + command: "npx", + args: ["tsx", "src/gateway/server.ts"], + cwd: root, + root, + mode: "local-checkout" + }; + } + + return { + command: "npx", + args: ["--yes", "--ignore-scripts", "--package", gatewayPackageSpec(), "tdai-memory-gateway"], + cwd: tdaiDataDir(), + root: "", + mode: "package-bin" + }; +} + +function gatewayPackageSpec() { + return process.env.TDAI_CODEX_GATEWAY_PACKAGE || DEFAULT_GATEWAY_PACKAGE; +} + +async function acquireGatewaySpawnLock(logDir) { + const lockPath = path.join(logDir, "gateway.spawn.lock"); + const tryCreate = async () => { + try { + const handle = await fs.open(lockPath, "wx", PRIVATE_FILE_MODE); + await handle.writeFile(`${process.pid}\n`); + await handle.close(); + return { + release: async () => { + await fs.rm(lockPath, { force: true }); + } + }; + } catch (err) { + if (err?.code === "EEXIST") return null; + throw err; + } + }; + + const first = await tryCreate(); + if (first) return first; + + try { + const stat = await fs.stat(lockPath); + if (Date.now() - stat.mtimeMs > 60_000) { + await fs.rm(lockPath, { force: true }); + return tryCreate(); + } + } catch { + return tryCreate(); + } + return null; +} + +export async function stopGateway() { + const logDir = path.join(tdaiDataDir(), "codex-adapter", "logs"); + const pidInfo = await readGatewayPidInfo(logDir); + if (!pidInfo) return false; + + if (!(await pidLooksLikeGateway(pidInfo.pid))) { + debug(`Refusing to stop pid=${pidInfo.pid}: process does not look like TDAI gateway`); + return false; + } + + try { + process.kill(pidInfo.pid, "SIGTERM"); + await removeGatewayPidFiles(logDir); + return true; + } catch { + return false; + } +} + +async function readGatewayPidInfo(logDir) { + const pidMetadataFile = path.join(logDir, "gateway.pid.json"); + try { + const metadata = JSON.parse(await fs.readFile(pidMetadataFile, "utf-8")); + const pid = Number(metadata.pid); + if (Number.isFinite(pid) && pid > 0) return { ...metadata, pid }; + } catch { + // Fall back to the legacy plain PID file below. + } + + try { + const pid = Number((await fs.readFile(path.join(logDir, "gateway.pid"), "utf-8")).trim()); + if (Number.isFinite(pid) && pid > 0) return { pid }; + } catch { + return null; + } + return null; +} + +async function pidLooksLikeGateway(pid) { + try { + process.kill(pid, 0); + } catch { + return false; + } + + try { + const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "command="], { + timeout: 1000, + maxBuffer: 4096 + }); + const command = stdout.trim(); + return command.includes("tdai-memory-gateway") || + (command.includes("tsx") && command.includes("src/gateway/server.ts")); + } catch (err) { + debug(`Could not inspect pid=${pid}: ${err instanceof Error ? err.message : String(err)}`); + return false; + } +} + +async function removeGatewayPidFiles(logDir) { + await Promise.all([ + fs.rm(path.join(logDir, "gateway.pid"), { force: true }), + fs.rm(path.join(logDir, "gateway.pid.json"), { force: true }) + ]); +} + +export async function httpPost(route, body, timeoutMs = DEFAULT_RECALL_TIMEOUT_MS) { + if (!isAllowedGatewayEndpoint()) return null; + if (await isGatewayCircuitOpen()) return null; + try { + const headers = { + "Content-Type": "application/json", + ...await gatewayAuthHeaders() + }; + const res = await fetch(`${gatewayUrl()}${route}`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs) + }); + if (!res.ok) { + debug(`Gateway ${route} returned ${res.status}: ${await res.text().catch(() => "")}`); + await recordGatewayFailure(route); + return null; + } + const json = await res.json(); + await recordGatewaySuccess(); + return json; + } catch (err) { + debug(`Gateway ${route} failed: ${err instanceof Error ? err.message : String(err)}`); + await recordGatewayFailure(route); + return null; + } +} + +function isAllowedGatewayEndpoint() { + let url; + try { + url = new URL(gatewayUrl()); + } catch { + debug(`Refusing invalid Gateway URL: ${gatewayUrl()}`); + return false; + } + if (isLoopbackHost(url.hostname)) return true; + if (process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true") return true; + debug(`Refusing non-loopback Gateway URL=${url.origin}. Set TDAI_CODEX_ALLOW_NON_LOOPBACK=true to override.`); + return false; +} + +async function isGatewayCircuitOpen() { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return false; + const state = await readGatewayCircuit(); + const openedUntil = Number(state.openedUntil || 0); + if (openedUntil > Date.now()) { + debug(`Gateway circuit breaker open for ${Math.ceil((openedUntil - Date.now()) / 1000)}s`); + return true; + } + if (openedUntil) { + await writeGatewayCircuit({ failureCount: 0, openedUntil: 0, lastRoute: state.lastRoute || "" }); + } + return false; +} + +async function recordGatewayFailure(route) { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return; + const state = await readGatewayCircuit(); + const threshold = numericEnv("TDAI_CODEX_BREAKER_FAILURES", DEFAULT_BREAKER_FAILURE_THRESHOLD); + const cooldownMs = numericEnv("TDAI_CODEX_BREAKER_COOLDOWN_MS", DEFAULT_BREAKER_COOLDOWN_MS); + const failureCount = Number(state.failureCount || 0) + 1; + const openedUntil = failureCount >= threshold ? Date.now() + cooldownMs : Number(state.openedUntil || 0); + await writeGatewayCircuit({ + failureCount, + openedUntil, + lastRoute: route, + lastFailureAt: Date.now() + }); + if (openedUntil) { + debug(`Gateway circuit breaker opened after ${failureCount} failures; cooldown=${cooldownMs}ms`); + } +} + +async function recordGatewaySuccess() { + if (process.env.TDAI_CODEX_CIRCUIT_BREAKER === "false") return; + const state = await readGatewayCircuit(); + if (!state.failureCount && !state.openedUntil) return; + await writeGatewayCircuit({ failureCount: 0, openedUntil: 0, lastSuccessAt: Date.now() }); +} + +async function readGatewayCircuit() { + try { + return JSON.parse(await fs.readFile(gatewayCircuitPath(), "utf-8")); + } catch { + return { failureCount: 0, openedUntil: 0 }; + } +} + +async function writeGatewayCircuit(state) { + const file = gatewayCircuitPath(); + await ensurePrivateDir(path.dirname(file)); + const tmp = `${file}.${process.pid}.tmp`; + await writePrivateFile(tmp, `${JSON.stringify(state, null, 2)}\n`); + await fs.rename(tmp, file); + await chmodPrivateFile(file); +} + +async function extractAssistantFromTranscript(transcriptPath, sinceMs) { + if (!transcriptPath || !fsSync.existsSync(transcriptPath)) return ""; + try { + const stat = await fs.stat(transcriptPath); + const maxBytes = numericEnv("TDAI_CODEX_TRANSCRIPT_TAIL_BYTES", 2_000_000); + const fh = await fs.open(transcriptPath, "r"); + const start = Math.max(0, stat.size - maxBytes); + const buffer = Buffer.alloc(stat.size - start); + await fh.read(buffer, 0, buffer.length, start); + await fh.close(); + + const candidates = []; + for (const line of buffer.toString("utf-8").split(/\r?\n/)) { + if (!line.trim()) continue; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const ts = timestampFromAny(parsed); + if (sinceMs && ts && ts < sinceMs - 1000) continue; + const text = extractAssistantText(parsed).trim(); + if (text.length > 20) candidates.push(text); + } + return truncate(sanitizeMemoryText(candidates.at(-1) || ""), 10000); + } catch { + return ""; + } +} + +function timestampFromAny(value) { + if (!value || typeof value !== "object") return 0; + const direct = value.timestamp || value.created_at || value.createdAt; + if (typeof direct === "number") return direct; + if (typeof direct === "string") { + const parsed = Date.parse(direct); + return Number.isFinite(parsed) ? parsed : 0; + } + return timestampFromAny(value.item || value.message || value.payload || value.response); +} + +function extractAssistantText(value) { + if (!value) return ""; + if (typeof value === "string") return ""; + if (Array.isArray(value)) return value.map(extractAssistantText).filter(Boolean).join("\n"); + if (typeof value !== "object") return ""; + + const role = value.role || value.author?.role || value.message?.role || value.item?.role; + const type = value.type; + if ((role === "assistant" || type === "assistant_message") && value.content) { + return contentToText(value.content); + } + if (type === "message" && role === "assistant" && value.content) { + return contentToText(value.content); + } + if (type === "response_item" && value.item) return extractAssistantText(value.item); + if (value.item) return extractAssistantText(value.item); + if (value.message) return extractAssistantText(value.message); + if (value.payload) return extractAssistantText(value.payload); + if (value.response) return extractAssistantText(value.response); + return ""; +} + +function contentToText(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content.map((part) => { + if (!part || typeof part !== "object") return ""; + if (typeof part.text === "string") return part.text; + if (typeof part.content === "string") return part.content; + if (part.type === "output_text" && typeof part.text === "string") return part.text; + return ""; + }).filter(Boolean).join("\n"); + } + if (content && typeof content === "object" && typeof content.text === "string") { + return content.text; + } + return ""; +} + +export async function rememberText(payload, text) { + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { remembered: false, reason: "gateway_unavailable" }; + const now = Date.now(); + const sessionKey = sessionKeyFromPayload(payload); + const content = [ + `Codex project: ${projectLabel(payload)}`, + "Explicit memory note:", + sanitizeMemoryText(text) + ].join("\n"); + const response = await httpPost("/capture", { + user_content: "Explicit memory note saved from Codex.", + assistant_content: content, + session_key: sessionKey, + session_id: sessionIdFromPayload(payload), + started_at: Math.max(0, now - 1), + messages: [ + { role: "user", content: "Remember this.", timestamp: now }, + { role: "assistant", content, timestamp: now + 1 } + ] + }, DEFAULT_CAPTURE_TIMEOUT_MS); + return { remembered: !!response, response }; +} + +export async function sessionEnd(payload, reason = "session_end") { + const gatewayReady = await ensureGateway(); + if (!gatewayReady) return { flushed: false, reason: "gateway_unavailable" }; + const sessionKey = sessionKeyFromPayload(payload); + const response = await httpPost("/session/end", { + session_key: sessionKey, + reason + }, DEFAULT_SESSION_END_TIMEOUT_MS); + return { + flushed: !!response?.flushed, + response + }; +} + +export function compact(value, maxChars) { + if (value === undefined || value === null || value === "") return ""; + let str; + try { + str = typeof value === "string" ? value : JSON.stringify(value); + } catch { + str = String(value); + } + return truncate(sanitizeMemoryText(str), maxChars); +} + +export function redactText(value) { + return redact(value); +} + +export function sanitizeMemoryText(value) { + return redact(stripInjectedMemoryTags(value)); +} + +export function stripInjectedMemoryTags(value) { + let cleaned = String(value ?? ""); + for (const tag of INJECTED_MEMORY_TAGS) { + cleaned = stripTagBlock(cleaned, tag); + cleaned = stripHtmlEscapedTagBlock(cleaned, tag); + } + cleaned = cleaned.replace( + /^\s*\{?\s*"hookSpecificOutput"\s*:\s*\{\s*"hookEventName"\s*:\s*"[^"]+"\s*,\s*"additionalContext"\s*:\s*""\s*\}\s*\}?\s*$/gm, + "", + ); + return cleaned.replace(/\n{3,}/g, "\n\n").trim(); +} + +export function toolOutputFromPayload(payload) { + return ( + payload.tool_response ?? + payload.toolResponse ?? + payload.tool_output ?? + payload.tool_result ?? + payload.toolResult ?? + payload.output ?? + "" + ); +} + +function renderToolValue(value) { + if (value === undefined || value === null || value === "") return ""; + if (typeof value === "string") return value; + try { + return JSON.stringify(value, null, 2); + } catch { + return String(value); + } +} + +function previewHeadTail(value, chars) { + const str = String(value ?? ""); + if (str.length <= chars * 2 + 200) return str; + return [ + str.slice(0, chars), + `\n[...omitted ${str.length - chars * 2} chars; full redacted output is stored on disk...]\n`, + str.slice(-chars) + ].join(""); +} + +function truncate(value, maxChars) { + const str = String(value ?? ""); + if (str.length <= maxChars) return str; + return `${str.slice(0, maxChars)}\n[...truncated ${str.length - maxChars} chars]`; +} + +function redact(value) { + const sensitiveJsonKey = "[A-Za-z0-9_-]*(?:api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|secret|token|password|authorization)[A-Za-z0-9_-]*"; + return String(value ?? "") + .replace(/-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY-----/g, "[REDACTED_PRIVATE_KEY]") + .replace(/(^|[\r\n])(\s*(?:proxy-)?authorization\s*[:=]\s*)[^\r\n]*/gi, "$1$2[REDACTED]") + .replace(/Bearer\s+[A-Za-z0-9._~+/-]+=*/g, "Bearer [REDACTED]") + .replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, "[REDACTED_API_KEY]") + .replace(/\b(github_pat_[A-Za-z0-9_]{20,})\b/g, "[REDACTED_GITHUB_TOKEN]") + .replace(/\b(gh[pousr]_[A-Za-z0-9_]{30,})\b/g, "[REDACTED_GITHUB_TOKEN]") + .replace(/\b(xox[baprs]-[A-Za-z0-9-]{10,})\b/g, "[REDACTED_SLACK_TOKEN]") + .replace(/\b((?:AKIA|ASIA)[0-9A-Z]{16})\b/g, "[REDACTED_AWS_ACCESS_KEY]") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*"[^"]*"`, "gi"), "$1$2$1: \"[REDACTED]\"") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*'[^']*'`, "gi"), "$1$2$1: '[REDACTED]'") + .replace(new RegExp(`(["'])(${sensitiveJsonKey})\\1\\s*:\\s*[^,}\\]\\s]+`, "gi"), "$1$2$1: [REDACTED]") + .replace(new RegExp(`\\b(${sensitiveJsonKey})\\b\\s*[:=]\\s*['"]?[^'\"\\s,}]+`, "gi"), "$1=[REDACTED]"); +} + +function stripTagBlock(value, tag) { + const complete = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*?<\\/${tag}>`, "gi"); + const dangling = new RegExp(`<${tag}\\b[^>]*>[\\s\\S]*$`, "gi"); + return value.replace(complete, "").replace(dangling, ""); +} + +function stripHtmlEscapedTagBlock(value, tag) { + const complete = new RegExp(`<${tag}\\b[\\s\\S]*?>[\\s\\S]*?</${tag}>`, "gi"); + const dangling = new RegExp(`<${tag}\\b[\\s\\S]*?>[\\s\\S]*$`, "gi"); + return value.replace(complete, "").replace(dangling, ""); +} + +function sanitizeEventDetail(value) { + if (typeof value === "string") return sanitizeMemoryText(value); + if (Array.isArray(value)) return value.map(sanitizeEventDetail); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, sanitizeEventDetail(item)]), + ); +} + +function indentBlock(text, prefix) { + return String(text).split("\n").map((line) => `${prefix}${line}`).join("\n"); +} + +function numericEnv(name, fallback) { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) ? parsed : fallback; +} + +async function gatewayAuthHeaders() { + const token = await readGatewayAuthToken(); + return token ? { Authorization: `Bearer ${token}` } : {}; +} + +export async function readGatewayAuthToken() { + const direct = process.env.TDAI_CODEX_GATEWAY_TOKEN || process.env.TDAI_GATEWAY_TOKEN; + if (direct) return direct.trim(); + + const tokenPath = configuredGatewayTokenPath(); + try { + const token = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (token) return token; + } catch { + // Missing token files are expected before the adapter autostarts Gateway. + } + return ""; +} + +export async function ensureGatewayAuthToken() { + const tokenPath = configuredGatewayTokenPath(); + const existing = (process.env.TDAI_CODEX_GATEWAY_TOKEN || process.env.TDAI_GATEWAY_TOKEN || "").trim(); + if (existing) { + await writePrivateFile(tokenPath, `${existing}\n`); + return existing; + } + + await ensurePrivateDir(path.dirname(tokenPath)); + + try { + const existingFileToken = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (existingFileToken) return existingFileToken; + } catch { + // Missing token files are expected before first autostart. + } + + const token = crypto.randomBytes(32).toString("base64url"); + let handle; + try { + handle = await fs.open(tokenPath, "wx", PRIVATE_FILE_MODE); + await handle.writeFile(`${token}\n`); + await handle.close(); + handle = null; + await chmodPrivateFile(tokenPath); + return token; + } catch (err) { + if (handle) { + try { + await handle.close(); + } catch { + // Best effort cleanup after failed atomic create. + } + } + if (err?.code !== "EEXIST") throw err; + const racedToken = (await fs.readFile(tokenPath, "utf-8")).trim(); + if (racedToken) return racedToken; + await writePrivateFile(tokenPath, `${token}\n`); + return token; + } +} + +export function configuredGatewayTokenPath() { + return path.resolve(expandHome(process.env.TDAI_TOKEN_PATH || gatewayTokenPath())); +} + +function gatewayTokenPath() { + return path.join(tdaiDataDir(), "codex-adapter", "gateway-token"); +} + +function isLoopbackHost(host) { + const normalized = String(host || "").trim().toLowerCase(); + return normalized === "localhost" || + normalized === "127.0.0.1" || + normalized === "::1" || + normalized === "[::1]"; +} + +async function ensurePrivateDir(dir) { + await fs.mkdir(dir, { recursive: true, mode: PRIVATE_DIR_MODE }); + try { + await fs.chmod(dir, PRIVATE_DIR_MODE); + } catch { + // Best effort: chmod can fail on some mounted filesystems. + } +} + +async function writePrivateFile(file, content) { + await ensurePrivateDir(path.dirname(file)); + await fs.writeFile(file, content, { encoding: "utf-8", mode: PRIVATE_FILE_MODE }); + await chmodPrivateFile(file); +} + +async function chmodPrivateFile(file) { + try { + await fs.chmod(file, PRIVATE_FILE_MODE); + } catch { + // Best effort: chmod can fail on some mounted filesystems. + } +} + +function escapeText(value) { + return String(value).replace(/[&<>]/g, (ch) => ({ "&": "&", "<": "<", ">": ">" }[ch])); +} + +function escapeAttr(value) { + return escapeText(value).replace(/"/g, """); +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function hydrateLoginShellEnv(env, names) { + const missing = names.filter((name) => !env[name]); + if (missing.length === 0) return; + try { + const script = missing.map((name) => `printf '%s=%s\\n' ${shellQuote(name)} "${"$"}${name}"`).join("; "); + const output = await captureCommand("zsh", ["-lc", script], 1500); + for (const line of output.split(/\r?\n/)) { + const idx = line.indexOf("="); + if (idx <= 0) continue; + const name = line.slice(0, idx); + const value = line.slice(idx + 1); + if (value && !env[name]) env[name] = value; + } + } catch (err) { + debug(`Unable to hydrate login-shell env: ${err instanceof Error ? err.message : String(err)}`); + } +} + +function captureCommand(command, args, timeoutMs) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timeout = setTimeout(() => { + child.kill("SIGTERM"); + reject(new Error(`${command} timed out`)); + }, timeoutMs); + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + child.on("close", (code) => { + clearTimeout(timeout); + if (code === 0) resolve(stdout); + else reject(new Error(`${command} exited ${code}: ${stderr.slice(0, 200)}`)); + }); + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +export function debug(message) { + const line = `[${new Date().toISOString()}] ${truncate(redact(String(message)), 2000)}\n`; + try { + const file = hookLogPath(); + fsSync.mkdirSync(path.dirname(file), { recursive: true, mode: PRIVATE_DIR_MODE }); + fsSync.appendFileSync(file, line, { mode: PRIVATE_FILE_MODE }); + try { + fsSync.chmodSync(file, PRIVATE_FILE_MODE); + } catch { + // Best effort: Windows and some filesystems do not expose POSIX modes. + } + } catch { + // Hook diagnostics must never make memory capture/recall fail. + } + if (process.env.TDAI_CODEX_DEBUG === "true") { + process.stderr.write(`[tdai-codex] ${line}`); + } +} diff --git a/codex-plugin/scripts/mcp-server.mjs b/codex-plugin/scripts/mcp-server.mjs new file mode 100644 index 0000000..b93440d --- /dev/null +++ b/codex-plugin/scripts/mcp-server.mjs @@ -0,0 +1,227 @@ +#!/usr/bin/env node +import { + cwdFromPayload, + ensureGateway, + httpPost, + sessionKeyPrefixesForCwd +} from "./lib.mjs"; +import { + formatLookupText, + lookupCodexOffload +} from "./offload-store.mjs"; + +const PROTOCOL_VERSION = "2024-11-05"; +const allProjectsEnabled = process.env.TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS === "true"; +const offloadContentEnabled = process.env.TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT === "true"; + +const tools = [ + { + name: "tdai_memory_search", + description: "Search TencentDB Agent Memory L1 structured memories for user preferences, prior decisions, durable facts, instructions, or scene summaries. Use before asking the user to repeat context.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "Search query. Include current project/path when relevant." }, + limit: { type: "number", description: "Maximum number of results, 1-20.", default: 5 }, + type: { type: "string", enum: ["persona", "episodic", "instruction"], description: "Optional memory type filter." }, + scene: { type: "string", description: "Optional scene name filter." }, + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + }, + required: ["query"] + } + }, + { + name: "tdai_conversation_search", + description: "Search TencentDB Agent Memory L0 raw conversation history for exact prior wording, timelines, paths, commands, or evidence snippets.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "Search query. Use exact phrases, paths, or markers when available." }, + limit: { type: "number", description: "Maximum number of messages, 1-20.", default: 5 }, + session_key: { type: "string", description: "Optional session key filter." }, + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + }, + required: ["query"] + } + }, + { + name: "tdai_offload_lookup", + description: "Look up Codex short-term context offload entries by node_id, tool_call_id, or query. Use this to retrieve the exact redacted tool result behind an injected Mermaid canvas node.", + inputSchema: { + type: "object", + properties: { + node_id: { type: "string", description: "Mermaid node id from the injected context offload canvas." }, + tool_call_id: { type: "string", description: "Original Codex tool call id." }, + query: { type: "string", description: "Optional text query over tool names, summaries, refs, and cwd." }, + limit: { type: "number", description: "Maximum number of entries, 1-20.", default: 5 }, + ...(offloadContentEnabled ? { + include_content: { type: "boolean", description: "Include stored redacted tool output content.", default: false } + } : {}), + ...(allProjectsEnabled ? { + all_projects: { type: "boolean", description: "Search across all projects instead of the current Codex project.", default: false } + } : {}) + } + } + } +]; + +let buffer = ""; + +process.stdin.setEncoding("utf-8"); +process.stdin.on("data", (chunk) => { + buffer += chunk; + let idx; + while ((idx = buffer.indexOf("\n")) >= 0) { + const line = buffer.slice(0, idx).trim(); + buffer = buffer.slice(idx + 1); + if (!line) continue; + handleLine(line).catch((err) => { + writeError(null, -32603, err instanceof Error ? err.message : String(err)); + }); + } +}); + +async function handleLine(line) { + let message; + try { + message = JSON.parse(line); + } catch { + writeError(null, -32700, "Parse error"); + return; + } + + if (message.id === undefined) return; + + try { + switch (message.method) { + case "initialize": + writeResult(message.id, { + protocolVersion: message.params?.protocolVersion || PROTOCOL_VERSION, + capabilities: { tools: {} }, + serverInfo: { name: "tdai-memory-codex", version: "0.1.0" } + }); + break; + case "ping": + writeResult(message.id, {}); + break; + case "tools/list": + writeResult(message.id, { tools }); + break; + case "tools/call": + writeResult(message.id, await callTool(message.params || {})); + break; + default: + writeError(message.id, -32601, `Method not found: ${message.method}`); + } + } catch (err) { + writeError(message.id, -32603, err instanceof Error ? err.message : String(err)); + } +} + +async function callTool(params) { + const name = params.name; + const args = params.arguments || {}; + + if (name === "tdai_offload_lookup") { + const allProjects = allProjectsEnabled && args.all_projects === true; + const includeContent = offloadContentEnabled && args.include_content === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project memory/offload lookup is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + if (args.include_content === true && !offloadContentEnabled) { + return textResult("Exact offload content lookup is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_OFFLOAD_CONTENT=true outside the model context to enable it.", true); + } + const result = await lookupCodexOffload({ + nodeId: optionalString(args.node_id), + toolCallId: optionalString(args.tool_call_id), + query: optionalString(args.query), + cwd: allProjects ? "" : currentProjectCwd(), + includeContent, + limit: clampLimit(args.limit) + }); + return textResult(formatLookupText(result)); + } + + const ready = await ensureGateway(); + if (!ready) return textResult("TencentDB Agent Memory Gateway is unavailable.", true); + + if (name === "tdai_memory_search") { + const allProjects = allProjectsEnabled && args.all_projects === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project memory search is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + const query = scopedQuery(args.query, allProjects); + const result = await httpPost("/search/memories", { + query, + limit: clampLimit(args.limit), + type: optionalString(args.type), + scene: optionalString(args.scene), + session_key_prefixes: allProjects ? undefined : currentProjectPrefixes() + }); + return textResult(result?.results || "No matching memories found."); + } + + if (name === "tdai_conversation_search") { + const allProjects = allProjectsEnabled && args.all_projects === true; + if (args.all_projects === true && !allProjectsEnabled) { + return textResult("Cross-project conversation search is disabled for MCP. Set TDAI_CODEX_MCP_ALLOW_ALL_PROJECTS=true outside the model context to enable it.", true); + } + const query = scopedQuery(args.query, allProjects); + const body = { + query, + limit: clampLimit(args.limit), + session_key_prefixes: allProjects ? undefined : currentProjectPrefixes() + }; + const sessionKey = optionalString(args.session_key); + if (sessionKey) body.session_key = sessionKey; + const result = await httpPost("/search/conversations", body); + return textResult(result?.results || "No matching conversations found."); + } + + return textResult(`Unknown tool: ${name}`, true); +} + +function optionalString(value) { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function clampLimit(value) { + const parsed = Number(value || 5); + if (!Number.isFinite(parsed)) return 5; + return Math.max(1, Math.min(20, Math.floor(parsed))); +} + +function currentProjectCwd() { + return cwdFromPayload({ + cwd: process.env.CLAUDE_PROJECT_DIR || process.env.CODEX_PROJECT_DIR || process.cwd() + }); +} + +function scopedQuery(query, allProjects) { + const text = String(query || ""); + if (allProjects) return text; + return `Codex project cwd: ${currentProjectCwd()}\n${text}`; +} + +function currentProjectPrefixes() { + return sessionKeyPrefixesForCwd(currentProjectCwd()); +} + +function textResult(text, isError = false) { + return { + content: [{ type: "text", text: String(text || "") }], + isError + }; +} + +function writeResult(id, result) { + process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, result })}\n`); +} + +function writeError(id, code, message) { + process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", id, error: { code, message } })}\n`); +} diff --git a/codex-plugin/scripts/offload-store.mjs b/codex-plugin/scripts/offload-store.mjs new file mode 100644 index 0000000..b26b2e3 --- /dev/null +++ b/codex-plugin/scripts/offload-store.mjs @@ -0,0 +1,675 @@ +import crypto from "node:crypto"; +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const DEFAULT_TOOL_OFFLOAD_MIN_CHARS = 20_000; +const DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS = 80_000; +const DEFAULT_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS = 250_000; +const DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS = 2_000; +const DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS = 800; +const DEFAULT_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS = 240; +const DEFAULT_TOOL_OFFLOAD_MAX_STORE_CHARS = 2_000_000; +const DEFAULT_TOOL_OFFLOAD_L2_NULL_THRESHOLD = 1; +const DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS = 6_000; +const DEFAULT_TOOL_OFFLOAD_LOOKUP_CONTENT_CHARS = 20_000; +const CANVAS_FILE = "001-codex-tool-offload.mmd"; +const NODE_INDEX_FILE = "node-index.json"; +const PRIVATE_DIR_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; + +export function selectToolOffloadPolicy(charCount) { + const mildMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_MILD_MIN_CHARS", + numericEnv("TDAI_CODEX_TOOL_OFFLOAD_MIN_CHARS", DEFAULT_TOOL_OFFLOAD_MIN_CHARS), + ); + const aggressiveMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS", + Math.max(DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_MIN_CHARS, mildMin * 4), + ); + const emergencyMin = numericEnv( + "TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS", + Math.max(DEFAULT_TOOL_OFFLOAD_EMERGENCY_MIN_CHARS, aggressiveMin * 3), + ); + + if (charCount >= emergencyMin) { + return { + name: "emergency", + minChars: emergencyMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_EMERGENCY_PREVIEW_CHARS), + score: 10, + }; + } + if (charCount >= aggressiveMin) { + return { + name: "aggressive", + minChars: aggressiveMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_AGGRESSIVE_PREVIEW_CHARS), + score: 9, + }; + } + if (charCount >= mildMin) { + return { + name: "mild", + minChars: mildMin, + previewChars: numericEnv("TDAI_CODEX_TOOL_OFFLOAD_PREVIEW_CHARS", DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS), + score: 8, + }; + } + return null; +} + +export function maxStoreChars() { + return numericEnv("TDAI_CODEX_TOOL_OFFLOAD_MAX_STORE_CHARS", DEFAULT_TOOL_OFFLOAD_MAX_STORE_CHARS); +} + +export async function recordCodexToolOffload(params) { + const { + sessionKey, + sessionId, + cwd, + toolName, + toolUseId, + inputSummary, + redactedOutput, + storedText, + policy, + } = params; + const paths = pathsForSession(sessionKey, sessionId); + await ensureOffloadDirs(paths); + + const entries = await readEntries(paths.offloadJsonl); + const existing = entries.find((entry) => entry.tool_call_id === toolUseId); + if (existing) { + const canvas = await maybeRebuildCanvas(paths, entries, { force: false }); + return { + entry: existing, + paths, + canvas, + duplicated: true, + }; + } + + const now = new Date(); + const fileStem = `${safeKey(toolName)}-${safeKey(toolUseId)}-${now.getTime()}`; + const resultRef = `refs/${fileStem}.md`; + const refPath = path.join(paths.root, resultRef); + const summary = summarizeToolResult(toolName, inputSummary, redactedOutput.length, policy); + const entry = { + timestamp: now.toISOString(), + node_id: null, + tool_call: `${toolName}: ${singleLine(inputSummary, 220) || "(no input captured)"}`, + summary, + result_ref: resultRef, + tool_call_id: toolUseId, + session_key: sessionKey, + score: policy.score, + codex: { + session_id: sessionId, + cwd, + tool_name: toolName, + policy: policy.name, + original_chars_redacted: redactedOutput.length, + stored_chars: storedText.length, + }, + }; + + await writePrivateFile(refPath, buildRefMarkdown({ + entry, + cwd, + inputSummary, + redactedOutput, + storedText, + policy, + })); + + entries.push(entry); + await writeEntries(paths.offloadJsonl, entries); + const canvas = await maybeRebuildCanvas(paths, entries, { force: false }); + const refreshed = await readEntries(paths.offloadJsonl); + const updatedEntry = refreshed.find((candidate) => candidate.tool_call_id === toolUseId) || entry; + if (updatedEntry.node_id !== entry.node_id) { + await writePrivateFile(refPath, buildRefMarkdown({ + entry: updatedEntry, + cwd, + inputSummary, + redactedOutput, + storedText, + policy, + })); + } + + return { + entry: updatedEntry, + paths, + canvas, + duplicated: false, + }; +} + +export async function buildCodexOffloadContext(params) { + const { sessionKey, sessionId } = params; + const maxChars = params.maxChars ?? numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_CHARS", DEFAULT_TOOL_OFFLOAD_CONTEXT_CHARS); + const paths = pathsForSession(sessionKey, sessionId); + const entries = await readEntries(paths.offloadJsonl); + if (entries.length === 0) return ""; + + let canvas = ""; + try { + canvas = await fs.readFile(paths.canvasPath, "utf-8"); + } catch { + const rebuilt = await maybeRebuildCanvas(paths, entries, { force: true }); + canvas = rebuilt?.content || ""; + } + + const recentLimit = numericEnv("TDAI_CODEX_TOOL_OFFLOAD_CONTEXT_RECENT", 8); + const recent = entries.slice(-recentLimit).map((entry) => { + return [ + `- node_id=${entry.node_id || "pending"}`, + `tool_call_id=${entry.tool_call_id}`, + `tool=${entry.codex?.tool_name || toolNameFromCall(entry.tool_call)}`, + `policy=${entry.codex?.policy || "unknown"}`, + `ref=${path.join(paths.root, entry.result_ref)}`, + `summary=${singleLine(entry.summary, 220)}`, + ].join(" "); + }).join("\n"); + + const block = ` + +Large Codex tool results are stored outside the prompt and represented by node ids. +Use tdai_offload_lookup with a node_id or tool_call_id when exact stored output is needed. + + +\`\`\`mermaid +${stripMermaidFence(canvas).trim()} +\`\`\` + + +${escapeText(recent)} + +`; + + return truncate(block, maxChars); +} + +export async function lookupCodexOffload(params = {}) { + const roots = await listSessionRoots(params.sessionKey ? sessionRootFromKey(params.sessionKey) : null); + const nodeId = optionalLower(params.nodeId); + const toolCallId = optionalLower(params.toolCallId); + const query = optionalLower(params.query); + const cwd = normalizePath(params.cwd); + const includeContent = params.includeContent === true; + const contentMaxChars = params.contentMaxChars ?? DEFAULT_TOOL_OFFLOAD_LOOKUP_CONTENT_CHARS; + const limit = clampLimit(params.limit, 10, 50); + const matches = []; + + for (const root of roots) { + const files = await listOffloadJsonlFiles(root); + for (const file of files) { + const entries = await readEntries(file); + for (const entry of entries) { + if (nodeId && optionalLower(entry.node_id) !== nodeId) continue; + if (toolCallId && optionalLower(entry.tool_call_id) !== toolCallId) continue; + if (query && !entryMatchesQuery(entry, query)) continue; + if (cwd && normalizePath(entry.codex?.cwd) !== cwd) continue; + + const resultPath = entry.result_ref ? path.join(root, entry.result_ref) : ""; + const item = { + node_id: entry.node_id, + tool_call_id: entry.tool_call_id, + tool_call: entry.tool_call, + summary: entry.summary, + score: entry.score, + policy: entry.codex?.policy, + timestamp: entry.timestamp, + session_key: entry.session_key, + result_ref: entry.result_ref, + result_path: resultPath, + offload_jsonl: file, + canvas_path: path.join(root, "mmds", CANVAS_FILE), + }; + if (includeContent && resultPath) { + item.content = await readFileIfExists(resultPath, contentMaxChars); + } + matches.push(item); + if (matches.length >= limit) { + return { matches, total: matches.length, truncated: true }; + } + } + } + } + + return { matches, total: matches.length, truncated: false }; +} + +export async function offloadCli(args, context = {}) { + const command = args[0] || "list"; + if (command === "help" || command === "--help" || command === "-h") return offloadUsage(0); + + if (command === "list") { + const opts = parseLookupArgs(args.slice(1)); + const result = await lookupCodexOffload({ + sessionKey: opts.all ? "" : context.sessionKey, + query: opts.query, + includeContent: false, + limit: opts.limit, + }); + console.log(formatLookupText(result)); + return; + } + + if (command === "node") { + const id = args[1]; + if (!id) return offloadUsage(2); + const opts = parseLookupArgs(args.slice(2)); + let result = await lookupCodexOffload({ + nodeId: id, + query: opts.query, + includeContent: opts.content, + limit: opts.limit || 5, + }); + if (result.matches.length === 0) { + result = await lookupCodexOffload({ + toolCallId: id, + query: opts.query, + includeContent: opts.content, + limit: opts.limit || 5, + }); + } + console.log(opts.json ? JSON.stringify(result, null, 2) : formatLookupText(result)); + return; + } + + if (command === "canvas") { + const paths = pathsForSession(context.sessionKey, context.sessionId); + const content = await readFileIfExists(paths.canvasPath, 200_000); + if (!content) { + console.error(`No Codex offload canvas found for session: ${context.sessionKey}`); + process.exit(1); + } + console.log(content); + return; + } + + return offloadUsage(2); +} + +export function formatLookupText(result) { + if (!result.matches.length) return "No matching Codex offload entries found."; + return result.matches.map((entry) => { + const parts = [ + `node_id: ${entry.node_id || "pending"}`, + `tool_call_id: ${entry.tool_call_id}`, + `tool_call: ${entry.tool_call}`, + `policy: ${entry.policy || "unknown"}`, + `score: ${entry.score ?? ""}`, + `timestamp: ${entry.timestamp}`, + `result_path: ${entry.result_path}`, + `canvas_path: ${entry.canvas_path}`, + "", + entry.summary, + ]; + if (entry.content) { + parts.push("", "content:", entry.content); + } + return parts.join("\n"); + }).join("\n\n---\n\n"); +} + +function pathsForSession(sessionKey, sessionId) { + const root = sessionRootFromKey(sessionKey); + return { + root, + refsDir: path.join(root, "refs"), + mmdsDir: path.join(root, "mmds"), + offloadJsonl: path.join(root, `offload-${safeKey(sessionId || "unknown-session")}.jsonl`), + canvasPath: path.join(root, "mmds", CANVAS_FILE), + nodeIndexPath: path.join(root, NODE_INDEX_FILE), + }; +} + +function sessionRootFromKey(sessionKey) { + return path.join(tdaiDataDir(), "codex-adapter", "context-offload", sha1(sessionKey || "all").slice(0, 16)); +} + +async function ensureOffloadDirs(paths) { + await ensurePrivateDir(paths.root); + await ensurePrivateDir(paths.refsDir); + await ensurePrivateDir(paths.mmdsDir); +} + +async function maybeRebuildCanvas(paths, entries, options = {}) { + if (process.env.TDAI_CODEX_TOOL_OFFLOAD_L2_CANVAS === "false") return null; + const nullCount = entries.filter((entry) => !entry.node_id || entry.node_id === "wait").length; + const threshold = numericEnv("TDAI_CODEX_TOOL_OFFLOAD_L2_NULL_THRESHOLD", DEFAULT_TOOL_OFFLOAD_L2_NULL_THRESHOLD); + if (!options.force && nullCount < threshold && fsSync.existsSync(paths.canvasPath)) { + return { + path: paths.canvasPath, + content: await readFileIfExists(paths.canvasPath, 200_000), + rebuilt: false, + }; + } + + const updated = assignNodeIds(entries, paths.root); + const content = buildMermaidCanvas(updated, paths); + await ensurePrivateDir(paths.mmdsDir); + await writePrivateFile(paths.canvasPath, `${content}\n`); + await writePrivateFile(paths.nodeIndexPath, `${JSON.stringify({ + version: 1, + generated_at: new Date().toISOString(), + offload_jsonl: paths.offloadJsonl, + canvas_path: paths.canvasPath, + nodes: updated.map((entry) => ({ + node_id: entry.node_id, + tool_call_id: entry.tool_call_id, + result_ref: entry.result_ref, + result_path: path.join(paths.root, entry.result_ref), + summary: entry.summary, + })), + }, null, 2)}\n`); + await writeEntries(paths.offloadJsonl, updated); + return { path: paths.canvasPath, content, rebuilt: true }; +} + +function assignNodeIds(entries, root) { + const prefix = `C${sha1(root).slice(0, 6)}`; + const used = new Set(entries.map((entry) => entry.node_id).filter(Boolean)); + let next = 1; + return entries.map((entry) => { + if (entry.node_id && entry.node_id !== "wait") return entry; + let nodeId; + do { + nodeId = `${prefix}_N${String(next).padStart(3, "0")}`; + next += 1; + } while (used.has(nodeId)); + used.add(nodeId); + return { ...entry, node_id: nodeId }; + }); +} + +function buildMermaidCanvas(entries, paths) { + const lines = [ + "%% TencentDB Agent Memory Codex context offload canvas", + `%% generated_at: ${new Date().toISOString()}`, + `%% offload_jsonl: ${paths.offloadJsonl}`, + "flowchart TD", + ]; + + if (entries.length === 0) { + lines.push(" EMPTY[\"No offloaded tool results yet\"]"); + return lines.join("\n"); + } + + for (const entry of entries) { + const label = [ + toolNameFromCall(entry.tool_call), + `status: done`, + `policy: ${entry.codex?.policy || "unknown"}`, + `summary: ${singleLine(entry.summary, 130)}`, + `ref: ${path.basename(entry.result_ref || "")}`, + ].join("
"); + lines.push(` ${entry.node_id}["${escapeMermaidLabel(label)}"]`); + } + + for (let i = 1; i < entries.length; i++) { + lines.push(` ${entries[i - 1].node_id} --> ${entries[i].node_id}`); + } + + lines.push(" classDef offloaded fill:#eef6ff,stroke:#3b82f6,color:#0f172a;"); + lines.push(` class ${entries.map((entry) => entry.node_id).join(",")} offloaded;`); + return lines.join("\n"); +} + +async function readEntries(filePath) { + if (!filePath || !fsSync.existsSync(filePath)) return []; + const content = await fs.readFile(filePath, "utf-8"); + const entries = []; + for (const line of content.split(/\r?\n/)) { + if (!line.trim()) continue; + try { + const parsed = JSON.parse(line); + if (parsed && typeof parsed === "object" && parsed.tool_call_id) entries.push(parsed); + } catch { + // Ignore corrupt lines rather than breaking hook execution. + } + } + return entries; +} + +async function writeEntries(filePath, entries) { + await ensurePrivateDir(path.dirname(filePath)); + const tmp = `${filePath}.${process.pid}.tmp`; + await writePrivateFile(tmp, entries.map((entry) => JSON.stringify(entry)).join("\n") + "\n"); + await fs.rename(tmp, filePath); + await chmodPrivateFile(filePath); +} + +async function listSessionRoots(singleRoot) { + if (singleRoot) return fsSync.existsSync(singleRoot) ? [singleRoot] : []; + const base = path.join(tdaiDataDir(), "codex-adapter", "context-offload"); + if (!fsSync.existsSync(base)) return []; + const entries = await fs.readdir(base, { withFileTypes: true }); + return entries.filter((entry) => entry.isDirectory()).map((entry) => path.join(base, entry.name)).sort(); +} + +async function listOffloadJsonlFiles(root) { + try { + const entries = await fs.readdir(root, { withFileTypes: true }); + return entries + .filter((entry) => entry.isFile() && entry.name.startsWith("offload-") && entry.name.endsWith(".jsonl")) + .map((entry) => path.join(root, entry.name)) + .sort(); + } catch { + return []; + } +} + +function buildRefMarkdown(params) { + const { entry, cwd, inputSummary, storedText, redactedOutput, policy } = params; + return [ + "# Codex Tool Result Offload", + "", + `- node_id: ${entry.node_id || "pending_until_l2_canvas"}`, + `- tool_call_id: ${entry.tool_call_id}`, + `- tool_name: ${entry.codex.tool_name}`, + `- session_key: ${entry.session_key}`, + `- cwd: ${cwd}`, + `- captured_at: ${entry.timestamp}`, + `- policy: ${policy.name}`, + `- original_chars_redacted: ${redactedOutput.length}`, + `- stored_chars: ${storedText.length}`, + "", + "## Summary", + "", + entry.summary, + "", + "## Tool Input", + "", + "```text", + inputSummary || "(no input captured)", + "```", + "", + "## Tool Output", + "", + "```text", + storedText, + "```", + ].join("\n"); +} + +function summarizeToolResult(toolName, inputSummary, charCount, policy) { + const input = singleLine(inputSummary, 120); + return [ + `${toolName} produced a ${policy.name} offloaded result (${charCount} redacted characters).`, + input ? `Input: ${input}.` : "", + "Use result_ref for the exact redacted output.", + ].filter(Boolean).join(" "); +} + +function parseLookupArgs(args) { + const opts = { query: "", limit: 10, content: false, json: false, all: false }; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--query") opts.query = args[++i] || ""; + else if (arg === "--limit") opts.limit = clampLimit(Number(args[++i] || 10), 10, 50); + else if (arg === "--content" || arg === "--full") opts.content = true; + else if (arg === "--json") opts.json = true; + else if (arg === "--all") opts.all = true; + else throw new Error(`Unknown offload option: ${arg}`); + } + return opts; +} + +function offloadUsage(code) { + const message = `Usage: + node scripts/query.mjs offload list [--all] [--query ] [--limit ] + node scripts/query.mjs offload node [--content] [--json] + node scripts/query.mjs offload canvas +`; + (code === 0 ? console.log : console.error)(message); + process.exit(code); +} + +function entryMatchesQuery(entry, query) { + return [ + entry.node_id, + entry.tool_call_id, + entry.tool_call, + entry.summary, + entry.result_ref, + entry.codex?.tool_name, + entry.codex?.cwd, + ].filter(Boolean).join("\n").toLowerCase().includes(query); +} + +async function readFileIfExists(filePath, maxChars) { + try { + const content = await fs.readFile(filePath, "utf-8"); + return truncate(content, maxChars); + } catch { + return ""; + } +} + +function tdaiDataDir() { + const configured = + process.env.TDAI_CODEX_DATA_DIR || + process.env.TDAI_DATA_DIR || + path.join(os.homedir(), ".memory-tencentdb", "codex-memory-tdai"); + return path.resolve(expandHome(configured)); +} + +function expandHome(value) { + if (!value) return value; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +function sha1(value) { + return crypto.createHash("sha1").update(String(value)).digest("hex"); +} + +function safeKey(value) { + return String(value).replace(/[^a-zA-Z0-9_.:-]/g, "_").slice(0, 120); +} + +function optionalLower(value) { + return typeof value === "string" && value.trim() ? value.trim().toLowerCase() : ""; +} + +function normalizePath(value) { + return typeof value === "string" && value.trim() + ? path.resolve(expandHome(value.trim())) + : ""; +} + +function clampLimit(value, fallback, max) { + const parsed = Number(value || fallback); + if (!Number.isFinite(parsed)) return fallback; + return Math.max(1, Math.min(max, Math.floor(parsed))); +} + +function numericEnv(name, fallback) { + const raw = process.env[name]; + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function singleLine(value, maxChars) { + const flattened = String(value || "").replace(/\s+/g, " ").trim(); + if (!maxChars || flattened.length <= maxChars) return flattened; + return `${flattened.slice(0, maxChars)}...`; +} + +function toolNameFromCall(value) { + return String(value || "tool").split(":")[0].trim() || "tool"; +} + +function previewHeadTail(value, chars) { + const str = String(value ?? ""); + if (str.length <= chars * 2 + 200) return str; + return [ + str.slice(0, chars), + `\n[...omitted ${str.length - chars * 2} chars; full redacted output is stored on disk...]\n`, + str.slice(-chars), + ].join(""); +} + +export function previewForPolicy(value, policy) { + return previewHeadTail(value, policy?.previewChars ?? DEFAULT_TOOL_OFFLOAD_PREVIEW_CHARS); +} + +function stripMermaidFence(value) { + return String(value || "") + .replace(/^```mermaid\s*/i, "") + .replace(/```\s*$/i, ""); +} + +function truncate(value, maxChars) { + const str = String(value ?? ""); + if (!maxChars || str.length <= maxChars) return str; + return `${str.slice(0, maxChars)}\n[...truncated ${str.length - maxChars} chars]`; +} + +async function ensurePrivateDir(dir) { + await fs.mkdir(dir, { recursive: true, mode: PRIVATE_DIR_MODE }); + try { + await fs.chmod(dir, PRIVATE_DIR_MODE); + } catch { + // Best effort on filesystems that do not support POSIX modes. + } +} + +async function writePrivateFile(file, content) { + await ensurePrivateDir(path.dirname(file)); + await fs.writeFile(file, content, { encoding: "utf-8", mode: PRIVATE_FILE_MODE }); + await chmodPrivateFile(file); +} + +async function chmodPrivateFile(file) { + try { + await fs.chmod(file, PRIVATE_FILE_MODE); + } catch { + // Best effort on filesystems that do not support POSIX modes. + } +} + +function escapeText(value) { + return String(value).replace(/[&<>]/g, (ch) => ({ "&": "&", "<": "<", ">": ">" }[ch])); +} + +function escapeAttr(value) { + return escapeText(value).replace(/"/g, """); +} + +function escapeMermaidLabel(value) { + return String(value) + .replace(/\\/g, "\\\\") + .replace(/"/g, """) + .replace(/\[/g, "[") + .replace(/\]/g, "]") + .replace(/\n/g, "
"); +} diff --git a/codex-plugin/scripts/permission-request.mjs b/codex-plugin/scripts/permission-request.mjs new file mode 100644 index 0000000..c2f2300 --- /dev/null +++ b/codex-plugin/scripts/permission-request.mjs @@ -0,0 +1,15 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, compact, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendLifecycleEvent(payload, "permission_request", { + toolName: payload.tool_name || payload.toolName || payload.tool?.name || "", + permission: compact( + payload.permission || + payload.permission_request || + payload.permissionRequest || + payload.request || + payload, + 3000 + ) +}); diff --git a/codex-plugin/scripts/post-compact.mjs b/codex-plugin/scripts/post-compact.mjs new file mode 100644 index 0000000..457f46c --- /dev/null +++ b/codex-plugin/scripts/post-compact.mjs @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, readHookInput, sessionEnd } from "./lib.mjs"; + +const payload = await readHookInput(); +const reason = payload.reason || "context_compaction"; +await appendLifecycleEvent(payload, "post_compact", { reason }, { createTurn: false }); +await sessionEnd(payload, "post_compact"); diff --git a/codex-plugin/scripts/post-tool-use.mjs b/codex-plugin/scripts/post-tool-use.mjs new file mode 100644 index 0000000..6b4c0e6 --- /dev/null +++ b/codex-plugin/scripts/post-tool-use.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +import { + appendToolEvent, + maybeOffloadToolOutput, + postToolOffloadHookOutput, + readHookInput +} from "./lib.mjs"; + +const payload = await readHookInput(); +const offload = await maybeOffloadToolOutput(payload); +await appendToolEvent(payload, "post_tool_use", offload ? { + toolOutputOffloaded: true, + toolOutputNodeId: offload.nodeId, + toolOutputPolicy: offload.policy, + toolOutputPath: offload.outputPath, + toolOutputJsonlPath: offload.offloadJsonlPath, + toolOutputCanvasPath: offload.canvasPath, + toolOutputOriginalChars: offload.originalChars, + toolOutputStoredChars: offload.storedChars, + toolOutputSummary: offload.summary +} : {}); +process.stdout.write(postToolOffloadHookOutput(offload)); diff --git a/codex-plugin/scripts/pre-compact.mjs b/codex-plugin/scripts/pre-compact.mjs new file mode 100644 index 0000000..61cde94 --- /dev/null +++ b/codex-plugin/scripts/pre-compact.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { appendLifecycleEvent, captureCurrentTurn, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendLifecycleEvent(payload, "pre_compact", { reason: payload.reason || "context_compaction" }); +await captureCurrentTurn(payload, "pre_compact"); diff --git a/codex-plugin/scripts/pre-tool-use.mjs b/codex-plugin/scripts/pre-tool-use.mjs new file mode 100644 index 0000000..8818e42 --- /dev/null +++ b/codex-plugin/scripts/pre-tool-use.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { appendToolEvent, hookAdditionalContext, readHookInput, toolMemoryHint } from "./lib.mjs"; + +const payload = await readHookInput(); +await appendToolEvent(payload, "pre_tool_use"); +process.stdout.write(hookAdditionalContext("PreToolUse", toolMemoryHint(payload))); diff --git a/codex-plugin/scripts/query.mjs b/codex-plugin/scripts/query.mjs new file mode 100644 index 0000000..03d731b --- /dev/null +++ b/codex-plugin/scripts/query.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { + cwdFromPayload, + ensureGateway, + expandHome, + gatewayStderrLogPath, + gatewayStdoutLogPath, + gatewayUrl, + healthCheck, + hookLogPath, + httpPost, + projectLabel, + rememberText, + sessionEnd, + sessionIdFromPayload, + sessionKeyFromPayload, + sessionKeyPrefixesForCwd +} from "./lib.mjs"; + +const command = process.argv[2] || "status"; +const args = process.argv.slice(3); +const payload = { + cwd: process.env.CLAUDE_PROJECT_DIR || process.cwd(), + session_id: process.env.TDAI_CODEX_MANUAL_SESSION_ID || "manual" +}; + +if (command === "status") { + console.log(JSON.stringify({ + healthy: await healthCheck(), + gatewayUrl: gatewayUrl(), + logs: { + hook: hookLogPath(), + gatewayStdout: gatewayStdoutLogPath(), + gatewayStderr: gatewayStderrLogPath(), + }, + sessionKey: sessionKeyFromPayload(payload), + project: projectLabel(payload) + }, null, 2)); +} else if (command === "memory") { + const query = args.join(" ").trim(); + if (!query) usage(2); + const result = await httpPost("/search/memories", { + query: `Codex project cwd: ${cwdFromPayload(payload)}\n${query}`, + limit: 10, + session_key_prefixes: sessionKeyPrefixesForCwd(cwdFromPayload(payload)) + }); + console.log(result?.results || ""); +} else if (command === "conversation") { + const query = args.join(" ").trim(); + if (!query) usage(2); + const result = await httpPost("/search/conversations", { + query, + limit: 10, + session_key_prefixes: sessionKeyPrefixesForCwd(cwdFromPayload(payload)) + }); + console.log(result?.results || ""); +} else if (command === "remember") { + const text = args.join(" ").trim() || (await readTextFromStdin()); + if (!text) usage(2); + const result = await rememberText(payload, text); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "flush") { + const result = await sessionEnd(payload, "manual_flush"); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "seed") { + const file = args[0]; + if (!file) usage(2); + const ok = await ensureGateway(); + if (!ok) { + console.error("TDAI Gateway unavailable"); + process.exit(1); + } + const fullPipelineTimeoutMs = positiveNumber(process.env.TDAI_CODEX_FULL_PIPELINE_TIMEOUT_MS, 900000); + const seedTimeoutMs = positiveNumber(process.env.TDAI_CODEX_SEED_TIMEOUT_MS, 960000); + const dataPath = path.resolve(expandHome(file)); + const data = JSON.parse(await readFile(dataPath, "utf-8")); + const result = await httpPost("/seed", { + data, + session_key: sessionKeyFromPayload(payload), + wait_for_full_pipeline: process.env.TDAI_CODEX_SEED_FULL_PIPELINE !== "false", + full_pipeline_timeout_ms: fullPipelineTimeoutMs + }, seedTimeoutMs); + console.log(JSON.stringify(result, null, 2)); +} else if (command === "import-codex-history") { + const { importCodexHistoryCli } = await import("./import-codex-history.mjs"); + await importCodexHistoryCli(args); +} else if (command === "offload") { + const { offloadCli } = await import("./offload-store.mjs"); + await offloadCli(args, { + sessionKey: sessionKeyFromPayload(payload), + sessionId: sessionIdFromPayload(payload) + }); +} else { + usage(2); +} + +function usage(code = 0) { + console.error("Usage: node scripts/query.mjs [status|memory |conversation |remember |flush|seed |import-codex-history [options]|offload ]"); + process.exit(code); +} + +async function readTextFromStdin() { + if (process.stdin.isTTY) return ""; + let input = ""; + for await (const chunk of process.stdin) input += chunk; + return input.trim(); +} + +function positiveNumber(value, fallback) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : fallback; +} diff --git a/codex-plugin/scripts/session-start.mjs b/codex-plugin/scripts/session-start.mjs new file mode 100644 index 0000000..9d4acc6 --- /dev/null +++ b/codex-plugin/scripts/session-start.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { hookAdditionalContext, readHookInput, recallForPrompt } from "./lib.mjs"; + +const payload = await readHookInput(); +const context = await recallForPrompt(payload, "", "session-start"); +process.stdout.write(hookAdditionalContext("SessionStart", context)); diff --git a/codex-plugin/scripts/stop.mjs b/codex-plugin/scripts/stop.mjs new file mode 100644 index 0000000..49c3f74 --- /dev/null +++ b/codex-plugin/scripts/stop.mjs @@ -0,0 +1,6 @@ +#!/usr/bin/env node +import { captureCurrentTurn, maybeFlushCapturedTurns, readHookInput } from "./lib.mjs"; + +const payload = await readHookInput(); +const capture = await captureCurrentTurn(payload, "stop"); +await maybeFlushCapturedTurns(payload, capture, "periodic_stop_flush"); diff --git a/codex-plugin/scripts/user-prompt-submit.mjs b/codex-plugin/scripts/user-prompt-submit.mjs new file mode 100644 index 0000000..fbbdd43 --- /dev/null +++ b/codex-plugin/scripts/user-prompt-submit.mjs @@ -0,0 +1,8 @@ +#!/usr/bin/env node +import { beginTurn, hookAdditionalContext, promptFromPayload, readHookInput, recallForPrompt } from "./lib.mjs"; + +const payload = await readHookInput(); +const prompt = promptFromPayload(payload); +await beginTurn(payload); +const context = await recallForPrompt(payload, prompt, "prompt"); +process.stdout.write(hookAdditionalContext("UserPromptSubmit", context)); diff --git a/codex-plugin/skills/tdai-memory/SKILL.md b/codex-plugin/skills/tdai-memory/SKILL.md new file mode 100644 index 0000000..c903511 --- /dev/null +++ b/codex-plugin/skills/tdai-memory/SKILL.md @@ -0,0 +1,35 @@ +--- +name: tdai-memory +description: Use TencentDB Agent Memory from Codex for manual inspection, explicit remembering, and diagnostics. Automatic hooks are the primary path; use this skill only when the user asks about memory state, recall, or saving a specific note. +--- + +# TDAI Memory For Codex + +This plugin is designed to be automatic: + +- `SessionStart` injects project memory and recent raw-conversation hints. +- `UserPromptSubmit` starts a turn, recalls relevant L1/L2/L3 memory, and injects it as context. +- If Gateway recall/search has no useful context, prompt recall falls back to a project-scoped local L0 JSONL scan. +- `PreToolUse` injects a lightweight model-visible memory hint and collects tool activity. +- `PermissionRequest` / `PostToolUse` collect permission and tool result activity. +- Large `PostToolUse` results are offloaded into `context-offload//offload-*.jsonl`, `refs/*.md`, and `mmds/001-codex-tool-offload.mmd`; later prompts inject the compact canvas. +- `PreCompact` captures pending turn state before compaction. +- `PostCompact` flushes session-scoped memory pipeline work through `/session/end`. +- `Stop` captures the completed turn through the TencentDB Agent Memory Gateway and flushes every `TDAI_CODEX_FLUSH_EVERY_N_TURNS` turns. +- Adapter-controlled capture and import paths strip injected TencentDB/Codex memory tags before persistence, matching the original `before_message_write` cleanup goal even though Codex does not expose that exact hook. +- `tdai_memory_search` / `tdai_conversation_search` / `tdai_offload_lookup` are available as Codex MCP tools when `scripts/mcp-server.mjs` is registered. + +Manual commands are only for inspection or explicit notes: + +```bash +node "${PLUGIN_ROOT}/scripts/query.mjs" status +node "${PLUGIN_ROOT}/scripts/query.mjs" memory "query terms" +node "${PLUGIN_ROOT}/scripts/query.mjs" conversation "query terms" +node "${PLUGIN_ROOT}/scripts/query.mjs" remember "durable note to save" +node "${PLUGIN_ROOT}/scripts/query.mjs" flush +node "${PLUGIN_ROOT}/scripts/query.mjs" seed ./historical-conversations.json +node "${PLUGIN_ROOT}/scripts/query.mjs" offload list --all --limit 10 +node "${PLUGIN_ROOT}/scripts/query.mjs" offload node Cxxxxxx_N001 --content +``` + +When memory is retrieved, use it as operating context. Verify path- and data-dependent claims against the current filesystem before acting. If current evidence contradicts memory, trust the current evidence and save the correction. diff --git a/codex-plugin/tdai-gateway.example.json b/codex-plugin/tdai-gateway.example.json new file mode 100644 index 0000000..89e7cf9 --- /dev/null +++ b/codex-plugin/tdai-gateway.example.json @@ -0,0 +1,47 @@ +{ + "server": { + "host": "127.0.0.1", + "port": 8420 + }, + "data": { + "baseDir": "${TDAI_DATA_DIR}" + }, + "llm": { + "baseUrl": "${TDAI_LLM_BASE_URL}", + "apiKey": "${TDAI_LLM_API_KEY}", + "model": "${TDAI_LLM_MODEL}", + "maxTokens": 4096, + "timeoutMs": 120000 + }, + "memory": { + "capture": { + "enabled": true, + "l0l1RetentionDays": 0 + }, + "recall": { + "enabled": true, + "maxResults": 8, + "strategy": "hybrid", + "timeoutMs": 5000 + }, + "pipeline": { + "everyNConversations": 3, + "enableWarmup": true, + "l1IdleTimeoutSeconds": 300 + }, + "extraction": { + "enabled": true, + "enableDedup": true, + "maxMemoriesPerSession": 20 + }, + "persona": { + "triggerEveryN": 30, + "maxScenes": 20 + }, + "embedding": { + "provider": "none", + "enabled": false + }, + "storeBackend": "sqlite" + } +} diff --git a/package.json b/package.json index 2d74158..e514876 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "bin": { "migrate-sqlite-to-tcvdb": "./bin/migrate-sqlite-to-tcvdb.mjs", "export-tencent-vdb": "./bin/export-tencent-vdb.mjs", - "read-local-memory": "./bin/read-local-memory.mjs" + "read-local-memory": "./bin/read-local-memory.mjs", + "tdai-memory-gateway": "./dist/src/gateway/cli.mjs" }, "exports": { ".": { @@ -18,7 +19,7 @@ "scripts": { "build": "npm run build:plugin && npm run build:scripts", "build:plugin": "tsdown", - "build:scripts": "npm run build:migrate-sqlite-to-vdb && npm run build:export-tencent-vdb && npm run build:read-local-memory", + "build:scripts": "node scripts/build-optional-bin-scripts.mjs", "prepack": "npm run build", "build:migrate-sqlite-to-vdb": "tsc -p scripts/migrate-sqlite-to-tcvdb/tsconfig.json --noEmitOnError false", "migrate-sqlite-to-tcvdb": "node ./bin/migrate-sqlite-to-tcvdb.mjs", @@ -38,6 +39,7 @@ "scripts/migrate-sqlite-to-tcvdb/dist/", "scripts/export-tencent-vdb/dist/", "scripts/read-local-memory/dist/", + "scripts/build-optional-bin-scripts.mjs", "scripts/memory-tencentdb-ctl.sh", "scripts/install_hermes_memory_tencentdb.sh", "scripts/README.memory-tencentdb-ctl.md", @@ -45,10 +47,12 @@ "scripts/openclaw-after-tool-call-messages.patch.sh", "scripts/setup-offload.sh", "hermes-plugin/", + "codex-plugin/", "openclaw.plugin.json", "README.md", "CHANGELOG.md", "LICENSE", + "!codex-plugin/**/*.test.mjs", "!src/**/*.test.ts", "!src/**/*.spec.ts", "!src/**/__tests__/" diff --git a/scripts/build-optional-bin-scripts.mjs b/scripts/build-optional-bin-scripts.mjs new file mode 100644 index 0000000..9960b59 --- /dev/null +++ b/scripts/build-optional-bin-scripts.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +const jobs = [ + { + name: "migrate-sqlite-to-tcvdb", + tsconfig: "scripts/migrate-sqlite-to-tcvdb/tsconfig.json", + args: ["-p", "scripts/migrate-sqlite-to-tcvdb/tsconfig.json", "--noEmitOnError", "false"], + }, + { + name: "export-tencent-vdb", + tsconfig: "scripts/export-tencent-vdb/tsconfig.json", + args: ["--project", "scripts/export-tencent-vdb/tsconfig.json"], + }, + { + name: "read-local-memory", + tsconfig: "scripts/read-local-memory/tsconfig.json", + args: ["--project", "scripts/read-local-memory/tsconfig.json"], + }, +]; + +for (const job of jobs) { + if (!existsSync(job.tsconfig)) { + console.warn(`[build:scripts] skipping ${job.name}: missing ${job.tsconfig}`); + continue; + } + + const result = spawnSync("tsc", job.args, { stdio: "inherit" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} diff --git a/src/adapters/standalone/llm-runner.test.ts b/src/adapters/standalone/llm-runner.test.ts new file mode 100644 index 0000000..2bbaea5 --- /dev/null +++ b/src/adapters/standalone/llm-runner.test.ts @@ -0,0 +1,51 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + resolveSandboxedExistingPath, + resolveSandboxedPath, + resolveSandboxedWritablePath, +} from "./llm-runner.js"; + +describe("StandaloneLLMRunner file-tool sandbox", () => { + it("rejects sibling-prefix traversal outside workspaceDir", () => { + const root = path.resolve("/tmp/scene_blocks"); + + expect(resolveSandboxedPath(root, "../scene_blocks_backup/file.md")).toBeNull(); + expect(resolveSandboxedPath(root, "../scene_blocks2/file.md")).toBeNull(); + expect(resolveSandboxedPath(root, "inside/file.md")).toBe(path.join(root, "inside/file.md")); + }); + + it("rejects symlink escapes for existing and writable paths", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-sandbox-")); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-outside-")); + try { + fs.writeFileSync(path.join(outside, "secret.txt"), "secret"); + fs.symlinkSync(outside, path.join(root, "outside-link"), "dir"); + + await expect(resolveSandboxedExistingPath(root, "outside-link/secret.txt")).resolves.toBeNull(); + await expect(resolveSandboxedWritablePath(root, "outside-link/new.txt")).resolves.toBeNull(); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); + + it("rejects existing file symlinks for writable paths", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-sandbox-")); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-llm-outside-")); + try { + const outsideFile = path.join(outside, "secret.txt"); + fs.writeFileSync(outsideFile, "secret"); + fs.symlinkSync(outsideFile, path.join(root, "out.md")); + + await expect(resolveSandboxedExistingPath(root, "out.md")).resolves.toBeNull(); + await expect(resolveSandboxedWritablePath(root, "out.md")).resolves.toBeNull(); + expect(fs.readFileSync(outsideFile, "utf-8")).toBe("secret"); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/src/adapters/standalone/llm-runner.ts b/src/adapters/standalone/llm-runner.ts index 2f7c9e1..372d83f 100644 --- a/src/adapters/standalone/llm-runner.ts +++ b/src/adapters/standalone/llm-runner.ts @@ -16,6 +16,7 @@ * All file paths are resolved relative to `workspaceDir`, enforcing sandbox boundaries. */ +import { constants as fsConstants } from "node:fs"; import fsPromises from "node:fs/promises"; import path from "node:path"; import { generateText, tool, stepCountIs, jsonSchema } from "ai"; @@ -55,14 +56,90 @@ export interface StandaloneLLMConfig { // Sandboxed tool execution helpers // ============================ -function resolveSandboxedPath(workspaceDir: string, relativePath: string): string | null { - const resolved = path.resolve(workspaceDir, relativePath); - if (!resolved.startsWith(path.resolve(workspaceDir))) { +export function resolveSandboxedPath(workspaceDir: string, relativePath: string): string | null { + const root = path.resolve(workspaceDir); + const resolved = path.resolve(root, relativePath); + const relative = path.relative(root, resolved); + if (relative.startsWith("..") || path.isAbsolute(relative)) { return null; } return resolved; } +export async function resolveSandboxedExistingPath(workspaceDir: string, relativePath: string): Promise { + const resolved = resolveSandboxedPath(workspaceDir, relativePath); + if (!resolved) return null; + + try { + const linkStat = await fsPromises.lstat(resolved); + if (linkStat.isSymbolicLink()) return null; + const [realRoot, realResolved] = await Promise.all([ + fsPromises.realpath(workspaceDir), + fsPromises.realpath(resolved), + ]); + return isWithinPath(realRoot, realResolved) ? realResolved : null; + } catch { + return null; + } +} + +export async function resolveSandboxedWritablePath(workspaceDir: string, relativePath: string): Promise { + const resolved = resolveSandboxedPath(workspaceDir, relativePath); + if (!resolved) return null; + + try { + const linkStat = await fsPromises.lstat(resolved); + if (linkStat.isSymbolicLink()) return null; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") return null; + } + + const parent = await nearestExistingParent(path.dirname(resolved)); + if (!parent) return null; + + try { + const [realRoot, realParent] = await Promise.all([ + fsPromises.realpath(workspaceDir), + fsPromises.realpath(parent), + ]); + return isWithinPath(realRoot, realParent) ? resolved : null; + } catch { + return null; + } +} + +async function nearestExistingParent(dir: string): Promise { + let current = path.resolve(dir); + while (true) { + try { + const stat = await fsPromises.stat(current); + return stat.isDirectory() ? current : null; + } catch { + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } + } +} + +async function writeSandboxedUtf8File(filePath: string, content: string): Promise { + const flags = fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_TRUNC | + ((fsConstants as Record).O_NOFOLLOW ?? 0); + const handle = await fsPromises.open(filePath, flags, 0o600); + try { + await handle.writeFile(content, "utf-8"); + } finally { + await handle.close(); + } +} + +function isWithinPath(root: string, target: string): boolean { + const relative = path.relative(path.resolve(root), path.resolve(target)); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + // ============================ // Tool definitions (Vercel AI SDK `tool()` format) // ============================ @@ -79,7 +156,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path"], }), execute: (async (args: { path: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedExistingPath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); try { return await fsPromises.readFile(resolved, "utf-8"); @@ -102,11 +179,11 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path", "content"], }), execute: (async (args: { path: string; content: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedWritablePath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); try { await fsPromises.mkdir(path.dirname(resolved), { recursive: true }); - await fsPromises.writeFile(resolved, args.content, "utf-8"); + await writeSandboxedUtf8File(resolved, args.content); return JSON.stringify({ success: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -128,7 +205,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { required: ["path", "old_str", "new_str"], }), execute: (async (args: { path: string; old_str: string; new_str: string }) => { - const resolved = resolveSandboxedPath(workspaceDir, args.path); + const resolved = await resolveSandboxedExistingPath(workspaceDir, args.path); if (!resolved) return JSON.stringify({ error: `Path "${args.path}" escapes workspace boundary.` }); if (!args.old_str) return JSON.stringify({ error: "old_str cannot be empty." }); try { @@ -137,7 +214,7 @@ function createSandboxedTools(workspaceDir: string, logger?: Logger) { return JSON.stringify({ error: `old_str not found in file "${args.path}".` }); } const updated = existing.replace(args.old_str, args.new_str); - await fsPromises.writeFile(resolved, updated, "utf-8"); + await writeSandboxedUtf8File(resolved, updated); return JSON.stringify({ success: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); diff --git a/src/cli/README.md b/src/cli/README.md index f314e6e..1252692 100644 --- a/src/cli/README.md +++ b/src/cli/README.md @@ -4,7 +4,8 @@ ## seed — 导入历史对话数据 -将历史对话 JSON 文件导入到记忆管线中，完整执行 L0→L1→L2→L3 流程。适用于： +将历史对话 JSON 文件导入到记忆管线中。默认等待 L0→L1；如果需要 seed 返回前立即产出 +L2 场景块和 L3 persona，使用 `--wait-for-full-pipeline` 完整等待 L0→L1→L2→L3。适用于： - 将已有对话数据灌入记忆系统 - 批量测试记忆提取效果 @@ -25,6 +26,8 @@ openclaw memory-tdai seed --input [options] | `--session-key ` | — | 回退 session key（当输入数据缺少时使用） | | `--config ` | — | 配置覆盖文件（JSON，与 openclaw.json 插件配置深度合并） | | `--strict-round-role` | — | 严格校验每轮对话必须包含 user 和 assistant 消息 | +| `--wait-for-full-pipeline` | — | 返回前等待最终 L1→L2→L3 flush 完成 | +| `--full-pipeline-timeout-ms ` | — | 最终 L1→L2→L3 flush 的最长等待时间（默认 900000） | | `--yes` | — | 跳过交互确认（如时间戳自动填充确认） | ### 示例 @@ -42,6 +45,9 @@ openclaw memory-tdai seed --input data.json --config seed-config.json # 跳过所有确认 openclaw memory-tdai seed --input data.json --yes +# 返回前完整等待 L1/L2/L3 +openclaw memory-tdai seed --input data.json --wait-for-full-pipeline --yes + # 严格模式 + 自定义配置 openclaw memory-tdai seed --input data.json --config seed-config.json --strict-round-role --yes ``` @@ -167,6 +173,7 @@ Seed 完成后，`manifest.json` 会记录本次运行信息： "sessions": 3, "rounds": 42, "messages": 128, + "fullPipelineFlushed": true, "startedAt": "2026-04-01T22:00:00.000Z", "completedAt": "2026-04-01T22:05:30.000Z" } diff --git a/src/cli/commands/seed.ts b/src/cli/commands/seed.ts index 611af05..ee68944 100644 --- a/src/cli/commands/seed.ts +++ b/src/cli/commands/seed.ts @@ -25,12 +25,14 @@ const TAG = "[memory-tdai] [seed-cmd]"; export function registerSeedCommand(parent: Command, ctx: SeedCliContext): void { parent .command("seed") - .description("Seed historical conversation data into the memory pipeline (L0 → L1)") + .description("Seed historical conversation data into the memory pipeline (L0 → L1, optionally L2/L3)") .requiredOption("--input ", "Path to input JSON file") .option("--output-dir ", "Output directory for pipeline data (default: auto-generated)") .option("--session-key ", "Fallback session key when input lacks one") .option("--config ", "Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config)") .option("--strict-round-role", "Require each round to have both user and assistant messages", false) + .option("--wait-for-full-pipeline", "Wait for final L1→L2→L3 processing before returning", false) + .option("--full-pipeline-timeout-ms ", "Max wait time for final L1→L2→L3 processing", "900000") .option("--yes", "Skip interactive confirmations (e.g. timestamp auto-fill)", false) .addHelpText("after", ` Examples: @@ -47,6 +49,8 @@ Examples: strictRoundRole: rawOpts.strictRoundRole === true, yes: rawOpts.yes === true, configFile: rawOpts.config as string | undefined, + waitForFullPipeline: rawOpts.waitForFullPipeline === true, + fullPipelineTimeoutMs: Number(rawOpts.fullPipelineTimeoutMs) || undefined, }; await runSeedCommand(opts, ctx); @@ -66,6 +70,7 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr logger.info(`${TAG} sessionKey: ${opts.sessionKey ?? "(from input)"}`); logger.info(`${TAG} config: ${opts.configFile ?? "(default)"}`); logger.info(`${TAG} strict: ${opts.strictRoundRole}`); + logger.info(`${TAG} full: ${opts.waitForFullPipeline === true}`); logger.info(`${TAG} yes: ${opts.yes}`); // 0. Load config override file and deep-merge with base plugin config @@ -149,6 +154,8 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr openclawConfig: ctx.config, pluginConfig: mergedPluginConfig, inputFile: opts.input, + waitForFullPipeline: opts.waitForFullPipeline === true, + fullPipelineFlushTimeoutMs: opts.fullPipelineTimeoutMs, logger, onProgress: (progress) => { const pct = ((progress.currentRound / progress.totalRounds) * 100).toFixed(0); @@ -168,6 +175,7 @@ async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Pr console.log(`║ Rounds: ${String(summary.roundsProcessed).padStart(11)} ║`); console.log(`║ Messages: ${String(summary.messagesProcessed).padStart(11)} ║`); console.log(`║ L0 recorded: ${String(summary.l0RecordedCount).padStart(11)} ║`); + console.log(`║ Full flush: ${String(summary.fullPipelineFlushed ? "yes" : "no").padStart(11)} ║`); console.log(`║ Duration: ${(summary.durationMs / 1000).toFixed(1).padStart(10)}s ║`); console.log("╚══════════════════════════════════════════╝"); console.log(`\n📁 Output: ${summary.outputDir}\n`); diff --git a/src/core/seed/seed-runtime.ts b/src/core/seed/seed-runtime.ts index 46e6647..34ec09a 100644 --- a/src/core/seed/seed-runtime.ts +++ b/src/core/seed/seed-runtime.ts @@ -5,15 +5,14 @@ * L1 runner, L2 runner, L3 runner, and persister wiring — keeping this * module focused on seed-specific concerns: * - Synchronous per-round L0 capture with progress reporting - * - waitForL1Idle polling (L1 only — see FIXME below) + * - waitForL1Idle polling at batch boundaries + * - Optional final full-pipeline flush for callers that need L2/L3 artifacts * - Ctrl+C graceful shutdown * - * FIXME: Currently we only wait for L1 to become idle before destroying the - * pipeline. L2 (scene extraction) and L3 (persona generation) may still be - * in-flight when `pipeline.destroy()` is called. This is intentional for now - * to avoid excessively long seed runs, but means seed output may not include - * the latest L2/L3 artifacts. Re-evaluate adding a full L1+L2+L3 idle wait - * once pipeline-manager exposes reliable L2/L3 idle signals. + * By default, seed preserves the historical CLI behavior and waits for L1 at + * batch boundaries. Callers such as the Codex history importer can opt into a + * final L1→L2→L3 flush before shutdown when they need higher-level artifacts + * to be immediately available for recall injection. */ import path from "node:path"; @@ -47,6 +46,10 @@ export interface SeedRuntimeOptions { pluginConfig?: Record; /** Original input file path (for manifest traceability). */ inputFile?: string; + /** Wait for a final L1→L2→L3 flush before returning. */ + waitForFullPipeline?: boolean; + /** Max time for the final L1→L2→L3 flush. */ + fullPipelineFlushTimeoutMs?: number; /** Logger instance. */ logger: PipelineLogger; /** Progress callback (called after each round). */ @@ -202,9 +205,9 @@ async function waitForL1Idle( /** * Execute the seed pipeline: feed normalized input through L0 → L1. * - * L2/L3 runners are wired but their completion is **not** awaited — see the - * module-level FIXME. The pipeline is destroyed after L1 idle, so L2/L3 may - * be interrupted mid-run. + * L2/L3 runners are wired. Their completion is awaited only when + * `waitForFullPipeline` is true; otherwise seed preserves the faster historical + * behavior and returns after L1 drains. * * This is the core runtime called by `src/cli/commands/seed.ts` after * all input validation and user confirmation are complete. @@ -232,6 +235,7 @@ export async function executeSeed( let pipeline: PipelineInstance | undefined; let totalL0Recorded = 0; let roundsProcessed = 0; + let fullPipelineFlushed = false; try { // Create and start pipeline (returns both the pipeline instance and the @@ -364,6 +368,25 @@ export async function executeSeed( { pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 }, ); } + + if (!interrupted && opts.waitForFullPipeline) { + onProgress?.({ + currentRound: roundsProcessed, + totalRounds: input.totalRounds, + sessionKey: "*", + stage: "l1_l2_l3_flushing", + }); + + logger.info(`${TAG} Final full pipeline flush requested (L1→L2→L3)...`); + await pipeline.scheduler.flushPendingWork({ + reason: "seed", + timeoutMs: opts.fullPipelineFlushTimeoutMs ?? 900_000, + pollIntervalMs: 100, + stableRounds: 3, + armFollowUpL2Timers: false, + }); + fullPipelineFlushed = true; + } } finally { process.removeListener("SIGINT", onSigint); @@ -384,6 +407,7 @@ export async function executeSeed( roundsProcessed, messagesProcessed: input.totalMessages, l0RecordedCount: totalL0Recorded, + fullPipelineFlushed, durationMs, outputDir: opts.outputDir, }; @@ -407,6 +431,7 @@ export async function executeSeed( sessions: summary.sessionsProcessed, rounds: summary.roundsProcessed, messages: summary.messagesProcessed, + fullPipelineFlushed: summary.fullPipelineFlushed, startedAt: new Date(startTime).toISOString(), completedAt: new Date().toISOString(), }; diff --git a/src/core/seed/types.ts b/src/core/seed/types.ts index 2cb4ab8..eb09477 100644 --- a/src/core/seed/types.ts +++ b/src/core/seed/types.ts @@ -111,6 +111,10 @@ export interface SeedCommandOptions { yes: boolean; /** Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config). */ configFile?: string; + /** Wait for final L1→L2→L3 processing before returning. */ + waitForFullPipeline?: boolean; + /** Max wait time for final L1→L2→L3 processing. */ + fullPipelineTimeoutMs?: number; } // ============================ @@ -135,6 +139,8 @@ export interface SeedSummary { roundsProcessed: number; messagesProcessed: number; l0RecordedCount: number; + /** True when the caller requested and completed a final L1→L2→L3 flush. */ + fullPipelineFlushed?: boolean; durationMs: number; outputDir: string; } diff --git a/src/core/tdai-core.ts b/src/core/tdai-core.ts index 977d4a2..1a53732 100644 --- a/src/core/tdai-core.ts +++ b/src/core/tdai-core.ts @@ -293,6 +293,7 @@ export class TdaiCore { limit: params.limit ?? 5, type: params.type, scene: params.scene, + sessionKeyPrefixes: params.sessionKeyPrefixes, vectorStore: this.vectorStore, embeddingService: this.embeddingService, logger: this.logger, @@ -314,6 +315,7 @@ export class TdaiCore { query: params.query, limit: params.limit ?? 5, sessionKey: params.sessionKey, + sessionKeyPrefixes: params.sessionKeyPrefixes, vectorStore: this.vectorStore, embeddingService: this.embeddingService, logger: this.logger, diff --git a/src/core/tools/conversation-search.ts b/src/core/tools/conversation-search.ts index ac4f3b1..423a702 100644 --- a/src/core/tools/conversation-search.ts +++ b/src/core/tools/conversation-search.ts @@ -46,6 +46,8 @@ export interface ConversationSearchResult { } const TAG = "[memory-tdai][tdai_conversation_search]"; +const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; +const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -90,6 +92,7 @@ export async function executeConversationSearch(params: { query: string; limit: number; sessionKey?: string; + sessionKeyPrefixes?: string[]; vectorStore?: IMemoryStore; embeddingService?: EmbeddingService; logger?: Logger; @@ -98,14 +101,17 @@ export async function executeConversationSearch(params: { query, limit, sessionKey: sessionFilter, + sessionKeyPrefixes, vectorStore, embeddingService, logger, } = params; + const normalizedSessionPrefixes = normalizeSessionPrefixes(sessionKeyPrefixes); logger?.debug?.( `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + `sessionFilter=${sessionFilter ?? "(none)"}, ` + + `sessionPrefixFilter=${normalizedSessionPrefixes.join("|") || "(none)"}, ` + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, ); @@ -137,12 +143,96 @@ export async function executeConversationSearch(params: { }; } - // ── Over-retrieve for later filtering and RRF merging ── - const candidateK = sessionFilter ? limit * 4 : limit * 3; + const hasSessionScope = !!sessionFilter || normalizedSessionPrefixes.length > 0; + let candidateK = hasSessionScope + ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) + : limit * 3; + const maxCandidateK = hasSessionScope + ? await scopedSearchMaxCandidates({ + count: () => vectorStore.countL0(), + candidateK, + logger, + }) + : candidateK; + + while (true) { + const search = await collectConversationCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + logger, + }); + + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + const filtered = filterConversationResults(search.results, { + sessionFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + if ( + trimmed.length >= limit || + !hasSessionScope || + !search.mayHaveMore || + candidateK >= maxCandidateK + ) { + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} messages ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; + } + + candidateK = Math.min(candidateK * 2, maxCandidateK); + logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + } +} + +async function scopedSearchMaxCandidates(params: { + count: () => number | Promise; + candidateK: number; + logger?: Logger; +}): Promise { + const { count, candidateK, logger } = params; + try { + const total = await count(); + if (Number.isFinite(total) && total > 0) { + return Math.max(candidateK, Math.floor(total)); + } + } catch (err) { + logger?.warn?.( + `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); +} + +async function collectConversationCandidates(params: { + query: string; + candidateK: number; + hasFts: boolean; + hasEmbedding: boolean; + vectorStore: IMemoryStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise<{ results: ConversationSearchResultItem[]; strategy: string; mayHaveMore: boolean }> { + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; - // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search on L0 (async (): Promise => { if (!hasFts) return []; try { @@ -154,14 +244,7 @@ export async function executeConversationSearch(params: { logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); - return ftsResults.map((r) => ({ - id: r.record_id, - session_key: r.session_key, - role: r.role, - content: r.message_text, - score: r.score, - recorded_at: r.recorded_at, - })); + return ftsResults.map(conversationResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -169,8 +252,6 @@ export async function executeConversationSearch(params: { return []; } })(), - - // Vector embedding search on L0 (async (): Promise => { if (!hasEmbedding) return []; try { @@ -181,14 +262,7 @@ export async function executeConversationSearch(params: { ); const vecResults: L0SearchResult[] = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); - return vecResults.map((r) => ({ - id: r.record_id, - session_key: r.session_key, - role: r.role, - content: r.message_text, - score: r.score, - recorded_at: r.recorded_at, - })); + return vecResults.map(conversationResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -198,56 +272,70 @@ export async function executeConversationSearch(params: { })(), ]); - // ── Determine effective strategy ── const ftsOk = ftsItems.length > 0; const vecOk = vecItems.length > 0; - let strategy: string; - - if (ftsOk && vecOk) { - strategy = "hybrid"; - } else if (vecOk) { - strategy = "embedding"; - } else if (ftsOk) { - strategy = "fts"; - } else { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } + const strategy = ftsOk && vecOk ? "hybrid" : vecOk ? "embedding" : ftsOk ? "fts" : "none"; + const results = strategy === "hybrid" + ? rrfMergeL0(ftsItems, vecItems) + : ftsOk ? ftsItems : vecItems; - // ── Merge results ── - let results: ConversationSearchResultItem[]; if (strategy === "hybrid") { - results = rrfMergeL0(ftsItems, vecItems); logger?.debug?.( `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, ); - } else { - // Single-source: use whichever list has results (already sorted by score) - results = ftsOk ? ftsItems : vecItems; } - // ── Apply session key filter ── - if (sessionFilter) { - const preFilterCount = results.length; - results = results.filter((r) => r.session_key === sessionFilter); - logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`); - } - - // ── Trim to requested limit ── - const trimmed = results.slice(0, limit); - - logger?.debug?.( - `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - return { - results: trimmed, - total: trimmed.length, + results, strategy, + mayHaveMore: (hasFts && ftsItems.length >= candidateK) || (hasEmbedding && vecItems.length >= candidateK), + }; +} + +function conversationResultItemFromStore(r: L0SearchResult): ConversationSearchResultItem { + return { + id: r.record_id, + session_key: r.session_key, + role: r.role, + content: r.message_text, + score: r.score, + recorded_at: r.recorded_at, }; } +function filterConversationResults( + results: ConversationSearchResultItem[], + filters: { + sessionFilter?: string; + sessionPrefixes: string[]; + logger?: Logger; + }, +): ConversationSearchResultItem[] { + const { sessionFilter, sessionPrefixes, logger } = filters; + let filtered = results; + if (sessionFilter) { + const preFilterCount = filtered.length; + filtered = filtered.filter((r) => r.session_key === sessionFilter); + logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${filtered.length}/${preFilterCount}`); + } + if (sessionPrefixes.length > 0) { + const preFilterCount = filtered.length; + filtered = filtered.filter((r) => + sessionPrefixes.some((prefix) => r.session_key.startsWith(prefix)), + ); + logger?.debug?.(`${TAG} After session-prefix filter: ${filtered.length}/${preFilterCount}`); + } + return filtered; +} + +function normalizeSessionPrefixes(prefixes: string[] | undefined): string[] { + if (!Array.isArray(prefixes)) return []; + return prefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20); +} + // ============================ // Tool response formatter // ============================ diff --git a/src/core/tools/memory-search.ts b/src/core/tools/memory-search.ts index dc9d2c2..76145c0 100644 --- a/src/core/tools/memory-search.ts +++ b/src/core/tools/memory-search.ts @@ -31,6 +31,8 @@ export interface MemorySearchResultItem { type: string; priority: number; scene_name: string; + session_key: string; + session_id: string; score: number; created_at: string; updated_at: string; @@ -45,6 +47,8 @@ export interface MemorySearchResult { } const TAG = "[memory-tdai][tdai_memory_search]"; +const FILTERED_SEARCH_INITIAL_CANDIDATES = 50; +const FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES = 500; // ============================ // RRF (Reciprocal Rank Fusion) @@ -90,6 +94,7 @@ export async function executeMemorySearch(params: { limit: number; type?: string; scene?: string; + sessionKeyPrefixes?: string[]; vectorStore?: IMemoryStore; embeddingService?: EmbeddingService; logger?: Logger; @@ -99,14 +104,17 @@ export async function executeMemorySearch(params: { limit, type: typeFilter, scene: sceneFilter, + sessionKeyPrefixes, vectorStore, embeddingService, logger, } = params; + const normalizedSessionPrefixes = normalizeSessionPrefixes(sessionKeyPrefixes); logger?.debug?.( `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + `typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` + + `sessionPrefixFilter=${normalizedSessionPrefixes.join("|") || "(none)"}, ` + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, ); @@ -138,12 +146,96 @@ export async function executeMemorySearch(params: { }; } - // ── Over-retrieve for later filtering and RRF merging ── - const candidateK = limit * 3; + let candidateK = normalizedSessionPrefixes.length > 0 + ? Math.max(limit * 6, FILTERED_SEARCH_INITIAL_CANDIDATES) + : limit * 3; + const maxCandidateK = normalizedSessionPrefixes.length > 0 + ? await scopedSearchMaxCandidates({ + count: () => vectorStore.countL1(), + candidateK, + logger, + }) + : candidateK; + + while (true) { + const search = await collectMemoryCandidates({ + query, + candidateK, + hasFts, + hasEmbedding, + vectorStore, + embeddingService, + logger, + }); + + if (search.results.length === 0) { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + const filtered = filterMemoryResults(search.results, { + typeFilter, + sceneFilter, + sessionPrefixes: normalizedSessionPrefixes, + logger, + }); + const trimmed = filtered.slice(0, limit); + + if ( + trimmed.length >= limit || + normalizedSessionPrefixes.length === 0 || + !search.mayHaveMore || + candidateK >= maxCandidateK + ) { + logger?.debug?.( + `${TAG} RESULT (strategy=${search.strategy}, candidateK=${candidateK}): returning ${trimmed.length} memories ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy: search.strategy, + }; + } + + candidateK = Math.min(candidateK * 2, maxCandidateK); + logger?.debug?.(`${TAG} Expanding scoped search window to candidateK=${candidateK}`); + } +} + +async function scopedSearchMaxCandidates(params: { + count: () => number | Promise; + candidateK: number; + logger?: Logger; +}): Promise { + const { count, candidateK, logger } = params; + try { + const total = await count(); + if (Number.isFinite(total) && total > 0) { + return Math.max(candidateK, Math.floor(total)); + } + } catch (err) { + logger?.warn?.( + `${TAG} Scoped search could not count records; falling back to ${FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES} candidates: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + return Math.max(candidateK, FILTERED_SEARCH_FALLBACK_MAX_CANDIDATES); +} + +async function collectMemoryCandidates(params: { + query: string; + candidateK: number; + hasFts: boolean; + hasEmbedding: boolean; + vectorStore: IMemoryStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise<{ results: MemorySearchResultItem[]; strategy: string; mayHaveMore: boolean }> { + const { query, candidateK, hasFts, hasEmbedding, vectorStore, embeddingService, logger } = params; - // ── Run available search strategies in parallel ── const [ftsItems, vecItems] = await Promise.all([ - // FTS5 keyword search (async (): Promise => { if (!hasFts) return []; try { @@ -155,16 +247,7 @@ export async function executeMemorySearch(params: { logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK); logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); - return ftsResults.map((r) => ({ - id: r.record_id, - content: r.content, - type: r.type, - priority: r.priority, - scene_name: r.scene_name, - score: r.score, - created_at: r.timestamp_start, - updated_at: r.timestamp_end, - })); + return ftsResults.map(memoryResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -172,8 +255,6 @@ export async function executeMemorySearch(params: { return []; } })(), - - // Vector embedding search (async (): Promise => { if (!hasEmbedding) return []; try { @@ -184,16 +265,7 @@ export async function executeMemorySearch(params: { ); const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query); logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); - return vecResults.map((r) => ({ - id: r.record_id, - content: r.content, - type: r.type, - priority: r.priority, - scene_name: r.scene_name, - score: r.score, - created_at: r.timestamp_start, - updated_at: r.timestamp_end, - })); + return vecResults.map(memoryResultItemFromStore); } catch (err) { logger?.warn?.( `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, @@ -203,61 +275,80 @@ export async function executeMemorySearch(params: { })(), ]); - // ── Determine effective strategy ── const ftsOk = ftsItems.length > 0; const vecOk = vecItems.length > 0; - let strategy: string; - - if (ftsOk && vecOk) { - strategy = "hybrid"; - } else if (vecOk) { - strategy = "embedding"; - } else if (ftsOk) { - strategy = "fts"; - } else { - logger?.debug?.(`${TAG} Both search paths returned 0 results`); - return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; - } + const strategy = ftsOk && vecOk ? "hybrid" : vecOk ? "embedding" : ftsOk ? "fts" : "none"; + const results = strategy === "hybrid" + ? rrfMergeL1(ftsItems, vecItems) + : ftsOk ? ftsItems : vecItems; - // ── Merge results ── - let results: MemorySearchResultItem[]; if (strategy === "hybrid") { - results = rrfMergeL1(ftsItems, vecItems); logger?.debug?.( `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, ); - } else { - // Single-source: use whichever list has results (already sorted by score) - results = ftsOk ? ftsItems : vecItems; } - // ── Apply secondary filters (type, scene) ── + return { + results, + strategy, + mayHaveMore: (hasFts && ftsItems.length >= candidateK) || (hasEmbedding && vecItems.length >= candidateK), + }; +} + +function memoryResultItemFromStore(r: L1SearchResult): MemorySearchResultItem { + return { + id: r.record_id, + content: r.content, + type: r.type, + priority: r.priority, + scene_name: r.scene_name, + session_key: r.session_key, + session_id: r.session_id, + score: r.score, + created_at: r.timestamp_start, + updated_at: r.timestamp_end, + }; +} + +function filterMemoryResults( + results: MemorySearchResultItem[], + filters: { + typeFilter?: string; + sceneFilter?: string; + sessionPrefixes: string[]; + logger?: Logger; + }, +): MemorySearchResultItem[] { + const { typeFilter, sceneFilter, sessionPrefixes, logger } = filters; const preFilterCount = results.length; + let filtered = results; if (typeFilter) { - results = results.filter((r) => r.type === typeFilter); - logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`); + filtered = filtered.filter((r) => r.type === typeFilter); + logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${filtered.length}/${preFilterCount}`); } if (sceneFilter) { const normalizedScene = sceneFilter.toLowerCase(); - results = results.filter((r) => + filtered = filtered.filter((r) => r.scene_name.toLowerCase().includes(normalizedScene), ); - logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`); + logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${filtered.length}/${preFilterCount}`); } + if (sessionPrefixes.length > 0) { + const beforeSessionFilter = filtered.length; + filtered = filtered.filter((r) => + sessionPrefixes.some((prefix) => r.session_key.startsWith(prefix)), + ); + logger?.debug?.(`${TAG} After session-prefix filter: ${filtered.length}/${beforeSessionFilter}`); + } + return filtered; +} - // ── Trim to requested limit ── - const trimmed = results.slice(0, limit); - - logger?.debug?.( - `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` + - `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, - ); - - return { - results: trimmed, - total: trimmed.length, - strategy, - }; +function normalizeSessionPrefixes(prefixes: string[] | undefined): string[] { + if (!Array.isArray(prefixes)) return []; + return prefixes + .map((prefix) => typeof prefix === "string" ? prefix.trim() : "") + .filter(Boolean) + .slice(0, 20); } // ============================ diff --git a/src/core/tools/search-prefix-filter.test.ts b/src/core/tools/search-prefix-filter.test.ts new file mode 100644 index 0000000..04fcd4d --- /dev/null +++ b/src/core/tools/search-prefix-filter.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { executeConversationSearch } from "./conversation-search.js"; +import { executeMemorySearch } from "./memory-search.js"; + +const logger = { + info: () => {}, + warn: () => {}, + error: () => {}, +}; + +describe("session-prefix search filters", () => { + it("filters L1 memory search results by session-key prefix", async () => { + const rows = [ + ...Array.from({ length: 700 }, (_, i) => l1Result(`other-${i}`, "codex:def456:session-b")), + l1Result("a", "codex:abc123:session-a"), + l1Result("c", "codex-import:abc123:session-c"), + ]; + const vectorStore = { + isFtsAvailable: () => true, + countL1: () => rows.length, + searchL1Fts: (_query: string, limit: number) => rows.slice(0, limit), + }; + + const result = await executeMemorySearch({ + query: "project note", + limit: 2, + sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + vectorStore: vectorStore as any, + logger, + }); + + expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + }); + + it("filters L0 conversation search results by session-key prefix", async () => { + const rows = [ + ...Array.from({ length: 700 }, (_, i) => l0Result(`other-${i}`, "codex:def456:session-b")), + l0Result("a", "codex:abc123:session-a"), + l0Result("c", "codex-import:abc123:session-c"), + ]; + const vectorStore = { + isFtsAvailable: () => true, + countL0: () => rows.length, + searchL0Fts: (_query: string, limit: number) => rows.slice(0, limit), + }; + + const result = await executeConversationSearch({ + query: "previous command", + limit: 2, + sessionKeyPrefixes: ["codex:abc123:", "codex-import:abc123:"], + vectorStore: vectorStore as any, + logger, + }); + + expect(result.results.map((item) => item.id)).toEqual(["a", "c"]); + }); +}); + +function l1Result(id: string, sessionKey: string) { + return { + record_id: id, + content: `memory ${id}`, + type: "episodic", + priority: 2, + scene_name: "test", + score: 1, + timestamp_str: "", + timestamp_start: "", + timestamp_end: "", + session_key: sessionKey, + session_id: id, + metadata_json: "{}", + }; +} + +function l0Result(id: string, sessionKey: string) { + return { + record_id: id, + session_key: sessionKey, + role: "assistant", + message_text: `conversation ${id}`, + score: 1, + recorded_at: "", + }; +} diff --git a/src/core/types.ts b/src/core/types.ts index 8585b50..5444129 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -231,6 +231,7 @@ export interface MemorySearchParams { limit?: number; type?: string; scene?: string; + sessionKeyPrefixes?: string[]; } /** Search parameters for L0 conversation search. */ @@ -238,4 +239,5 @@ export interface ConversationSearchParams { query: string; limit?: number; sessionKey?: string; + sessionKeyPrefixes?: string[]; } diff --git a/src/gateway/auth.test.ts b/src/gateway/auth.test.ts new file mode 100644 index 0000000..3372671 --- /dev/null +++ b/src/gateway/auth.test.ts @@ -0,0 +1,331 @@ +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { TdaiGateway } from "./server.js"; + +async function request(params: { + port: number; + path: string; + method?: string; + headers?: Record; + body?: unknown; +}): Promise<{ status: number; body: string; wwwAuth?: string }> { + return new Promise((resolve, reject) => { + const body = params.body === undefined ? "" : JSON.stringify(params.body); + const req = http.request( + { + host: "127.0.0.1", + port: params.port, + path: params.path, + method: params.method ?? "GET", + headers: { + ...(body ? { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(body).toString() } : {}), + ...(params.headers ?? {}), + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("end", () => resolve({ + status: res.statusCode ?? 0, + body: Buffer.concat(chunks).toString("utf-8"), + wwwAuth: res.headers["www-authenticate"] as string | undefined, + })); + }, + ); + req.on("error", reject); + if (body) req.write(body); + req.end(); + }); +} + +describe("Gateway bearer auth", () => { + const port = 18451; + const token = "test-token-abc-123"; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", token); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", token); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("requires the bearer token when a token is configured", async () => { + const missing = await request({ port, path: "/health" }); + expect(missing.status).toBe(401); + expect(missing.wwwAuth).toMatch(/^Bearer\s+realm=/); + + const wrong = await request({ port, path: "/health", headers: { Authorization: "Bearer wrong" } }); + expect(wrong.status).toBe(401); + + const ok = await request({ port, path: "/health", headers: { Authorization: `Bearer ${token}` } }); + expect(ok.status).toBe(200); + }); + + it("requires bearer auth across all POST endpoints", async () => { + const endpoints = [ + "/recall", + "/capture", + "/search/memories", + "/search/conversations", + "/session/end", + "/seed", + ]; + + for (const path of endpoints) { + const missing = await request({ port, path, method: "POST", body: {} }); + expect(missing.status, `${path} missing token`).toBe(401); + + const wrong = await request({ + port, + path, + method: "POST", + headers: { Authorization: "Bearer wrong" }, + body: {}, + }); + expect(wrong.status, `${path} wrong token`).toBe(401); + + const authorized = await request({ + port, + path, + method: "POST", + headers: { Authorization: `Bearer ${token}` }, + body: {}, + }); + expect(authorized.status, `${path} valid token`).not.toBe(401); + } + }); + + it("accepts RFC 6750 bearer scheme case variants", async () => { + for (const scheme of ["Bearer", "bearer", "BEARER", "BeArEr"]) { + const result = await request({ + port, + path: "/health", + headers: { Authorization: `${scheme} ${token}` }, + }); + expect(result.status, scheme).toBe(200); + } + }); + + it("rejects malformed authorization headers", async () => { + const headers = [ + "Basic dGVzdA==", + "", + "Bearer", + "Bearer wrong", + `Bearer ${token} trailing`, + `prefix Bearer ${token}`, + `${token}`, + ]; + + for (const authorization of headers) { + const result = await request({ + port, + path: "/health", + headers: authorization ? { Authorization: authorization } : {}, + }); + expect(result.status, authorization || "(empty)").toBe(401); + expect(result.wwwAuth).toMatch(/^Bearer\s+realm=/); + } + }); + + it("rejects non-loopback CORS origins before route handling", async () => { + const result = await request({ + port, + path: "/seed", + method: "OPTIONS", + headers: { Origin: "https://example.invalid" }, + }); + expect(result.status).toBe(403); + }); +}); + +describe("Gateway loopback compatibility without a token", () => { + const port = 18452; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-none-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("preserves tokenless loopback health checks when no token is configured", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(200); + }); + + it("requires a token for tokenless loopback POST routes by default", async () => { + const result = await request({ + port, + path: "/search/memories", + method: "POST", + body: {}, + }); + expect(result.status).toBe(401); + expect(result.wwwAuth).toMatch(/^Bearer\s+realm=/); + }); +}); + +describe("Gateway explicit tokenless loopback development mode", () => { + const port = 18456; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-disabled-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("allows POST routes only when explicitly disabled on loopback", async () => { + const result = await request({ + port, + path: "/search/memories", + method: "POST", + body: {}, + }); + expect(result.status).toBe(400); + expect(result.body).toContain("Missing required field: query"); + }); +}); + +describe("Gateway token file safety", () => { + const port = 18453; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-missing-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", path.join(tmpDir, "missing-token")); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", path.join(tmpDir, "missing-token")); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("does not silently downgrade to tokenless mode when TDAI_TOKEN_PATH is configured but unreadable", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); + +describe("Gateway empty token file safety", () => { + const port = 18455; + let gateway: TdaiGateway; + let tmpDir: string; + let tokenPath: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-empty-")); + tokenPath = path.join(tmpDir, "empty-token"); + fs.writeFileSync(tokenPath, "", { mode: 0o600 }); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", tokenPath); + gateway = new TdaiGateway({ server: { port, host: "127.0.0.1" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", tokenPath); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("does not silently downgrade to tokenless mode when TDAI_TOKEN_PATH is empty", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); + +describe("Gateway non-loopback tokenless safety", () => { + const port = 18454; + let gateway: TdaiGateway; + let tmpDir: string; + + beforeAll(async () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tdai-gateway-auth-remote-")); + vi.stubEnv("TDAI_DATA_DIR", tmpDir); + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + gateway = new TdaiGateway({ server: { port, host: "0.0.0.0" } } as never); + await gateway.start(); + }); + + beforeEach(() => { + vi.stubEnv("TDAI_GATEWAY_TOKEN", ""); + vi.stubEnv("TDAI_TOKEN_PATH", ""); + vi.stubEnv("TDAI_GATEWAY_AUTH_DISABLED", "true"); + }); + + afterAll(async () => { + await gateway.stop(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("rejects tokenless non-loopback access even when auth-disabled is set", async () => { + const result = await request({ port, path: "/health" }); + expect(result.status).toBe(401); + }); +}); diff --git a/src/gateway/cli.ts b/src/gateway/cli.ts new file mode 100644 index 0000000..b8e98e8 --- /dev/null +++ b/src/gateway/cli.ts @@ -0,0 +1,107 @@ +#!/usr/bin/env node +/** + * Standalone Gateway daemon entry for host adapters. + * + * This bin keeps host plugins small: Codex/Claude-style plugins can spawn + * `tdai-memory-gateway` from the installed npm package instead of importing + * package dependencies from the copied plugin directory. + */ + +import { readFileSync, statSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { TdaiGateway } from "./server.js"; + +const LOOPBACK_HOSTS = new Set(["127.0.0.1", "localhost", "::1", "[::1]", "::ffff:127.0.0.1"]); + +function assertSafeHost(): void { + const host = process.env.TDAI_GATEWAY_HOST?.trim(); + if (!host || LOOPBACK_HOSTS.has(host)) return; + if (process.env.TDAI_GATEWAY_ALLOW_REMOTE === "1" || process.env.TDAI_CODEX_ALLOW_NON_LOOPBACK === "true") { + return; + } + process.stderr.write( + `tdai-memory-gateway: refusing non-loopback TDAI_GATEWAY_HOST=${host}. ` + + "Set TDAI_GATEWAY_ALLOW_REMOTE=1 to opt in.\n", + ); + process.exit(2); +} + +function loadTokenFromFile(): void { + const tokenPath = expandHome(process.env.TDAI_TOKEN_PATH); + if (!tokenPath) return; + try { + const stat = statSync(tokenPath); + if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) { + process.stderr.write(`tdai-memory-gateway: token file permissions are too loose: ${tokenPath}\n`); + process.exit(2); + } + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid !== process.getuid()) { + process.stderr.write(`tdai-memory-gateway: token file owner mismatch: ${tokenPath}\n`); + process.exit(2); + } + const token = readFileSync(tokenPath, "utf-8").trim(); + if (!token) { + process.stderr.write(`tdai-memory-gateway: token file is empty: ${tokenPath}\n`); + process.exit(2); + } + // This mutates only Node's in-process env object, not the execve env block. + process.env.TDAI_GATEWAY_TOKEN = token; + } catch (err) { + process.stderr.write(`tdai-memory-gateway: failed to read TDAI_TOKEN_PATH=${tokenPath}: ${String(err)}\n`); + process.exit(2); + } +} + +function expandHome(value: string | undefined): string { + if (!value) return ""; + if (value === "~") return os.homedir(); + if (value.startsWith("~/")) return path.join(os.homedir(), value.slice(2)); + return value; +} + +async function main(): Promise { + assertSafeHost(); + loadTokenFromFile(); + + const gateway = new TdaiGateway(); + await gateway.start(); + + let shuttingDown = false; + const shutdown = async (reason: string): Promise => { + if (shuttingDown) return; + shuttingDown = true; + try { + await Promise.race([ + gateway.stop(), + new Promise((resolve) => setTimeout(resolve, 5_000)), + ]); + } catch { + // Best effort shutdown. + } + process.exit(reason === "error" ? 1 : 0); + }; + + process.on("SIGTERM", () => void shutdown("SIGTERM")); + process.on("SIGINT", () => void shutdown("SIGINT")); + + const parentPid = Number(process.env.TDAI_CODEX_PARENT_PID || process.env.TDAI_CC_PID || 0); + if (Number.isFinite(parentPid) && parentPid > 0) { + const timer = setInterval(() => { + try { + process.kill(parentPid, 0); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ESRCH") { + clearInterval(timer); + void shutdown("parent-exit"); + } + } + }, 15_000); + timer.unref(); + } +} + +main().catch((err) => { + process.stderr.write(`tdai-memory-gateway failed: ${String(err)}\n`); + process.exit(1); +}); diff --git a/src/gateway/server.ts b/src/gateway/server.ts index bd7d0a0..35f5c25 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -2,18 +2,21 @@ * TDAI Gateway — HTTP server for the Hermes sidecar. * * Exposes TDAI Core capabilities as HTTP endpoints: + * GET / — Service metadata for browser/local preview probes * GET /health — Health check * POST /recall — Memory recall (prefetch) * POST /capture — Conversation capture (sync_turn) * POST /search/memories — L1 memory search * POST /search/conversations — L0 conversation search * POST /session/end — Session end + flush - * POST /seed — Batch seed historical conversations (L0 → L1) + * POST /seed — Batch seed historical conversations (L0 → L1, optionally L2/L3) * * Built with Node.js native `http` module — no Express/Fastify dependency. * Designed to run as a managed sidecar alongside Hermes. */ +import crypto from "node:crypto"; +import fs from "node:fs"; import http from "node:http"; import { URL } from "node:url"; import { TdaiCore } from "../core/tdai-core.js"; @@ -23,6 +26,7 @@ import type { GatewayConfig } from "./config.js"; import { initDataDirectories } from "../utils/pipeline-factory.js"; import { SessionFilter } from "../utils/session-filter.js"; import type { + RootResponse, HealthResponse, RecallRequest, RecallResponse, @@ -173,10 +177,7 @@ export class TdaiGateway { const method = req.method?.toUpperCase() ?? "GET"; const pathname = url.pathname; - // CORS headers (for development) - res.setHeader("Access-Control-Allow-Origin", "*"); - res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); - res.setHeader("Access-Control-Allow-Headers", "Content-Type"); + if (!this.applyCors(req, res)) return; if (method === "OPTIONS") { res.writeHead(204); @@ -184,8 +185,12 @@ export class TdaiGateway { return; } + if (!this.authorizeRequest(req, res, method)) return; + try { switch (`${method} ${pathname}`) { + case "GET /": + return this.handleRoot(res); case "GET /health": return this.handleHealth(res); case "POST /recall": @@ -214,6 +219,73 @@ export class TdaiGateway { // Route handlers // ============================ + private applyCors(req: http.IncomingMessage, res: http.ServerResponse): boolean { + res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization"); + + const origin = String(req.headers.origin ?? ""); + if (!origin) return true; + if (!isAllowedCorsOrigin(origin)) { + sendError(res, 403, "CORS origin not allowed"); + return false; + } + + res.setHeader("Access-Control-Allow-Origin", origin); + res.setHeader("Vary", "Origin"); + return true; + } + + private authorizeRequest(req: http.IncomingMessage, res: http.ServerResponse, method: string): boolean { + const token = expectedGatewayToken(); + + if (!token) { + if (!isLoopbackHost(this.config.server.host)) { + sendError(res, 401, "Unauthorized: Gateway token is required for non-loopback routes"); + return false; + } + + if (method === "GET") { + return true; + } + + if (process.env.TDAI_GATEWAY_AUTH_DISABLED === "true") { + return true; + } + + res.setHeader("WWW-Authenticate", 'Bearer realm="tdai-gateway"'); + sendError(res, 401, "Unauthorized: Gateway token is required for POST routes; set TDAI_GATEWAY_AUTH_DISABLED=true only for trusted loopback development"); + return false; + } + + const authorization = String(req.headers.authorization ?? ""); + const match = authorization.match(/^Bearer\s+(\S+)\s*$/i); + if (!match || !safeTokenEqual(match[1], token)) { + res.setHeader("WWW-Authenticate", 'Bearer realm="tdai-gateway"'); + sendError(res, 401, "Unauthorized"); + return false; + } + return true; + } + + private handleRoot(res: http.ServerResponse): void { + const response: RootResponse = { + service: "TencentDB Agent Memory Gateway", + kind: "api", + version: VERSION, + message: "This local service is an API sidecar, not a web UI. Use GET /health for readiness.", + endpoints: [ + { method: "GET", path: "/health", description: "Gateway readiness and store status" }, + { method: "POST", path: "/recall", description: "Memory recall for a session query" }, + { method: "POST", path: "/capture", description: "Conversation turn capture" }, + { method: "POST", path: "/search/memories", description: "Structured memory search" }, + { method: "POST", path: "/search/conversations", description: "Raw conversation search" }, + { method: "POST", path: "/session/end", description: "Session flush" }, + { method: "POST", path: "/seed", description: "Batch seed historical conversations" }, + ], + }; + sendJson(res, 200, response); + } + private handleHealth(res: http.ServerResponse): void { const response: HealthResponse = { status: this.core.getVectorStore() ? "ok" : "degraded", @@ -267,6 +339,7 @@ export class TdaiGateway { ], sessionKey: body.session_key, sessionId: body.session_id, + startedAt: typeof body.started_at === "number" ? body.started_at : undefined, }); const elapsed = Date.now() - startMs; @@ -292,6 +365,7 @@ export class TdaiGateway { limit: body.limit, type: body.type, scene: body.scene, + sessionKeyPrefixes: body.session_key_prefixes, }); const response: MemorySearchResponse = { @@ -314,6 +388,7 @@ export class TdaiGateway { query: body.query, limit: body.limit, sessionKey: body.session_key, + sessionKeyPrefixes: body.session_key_prefixes, }); const response: ConversationSearchResponse = { @@ -366,7 +441,8 @@ export class TdaiGateway { this.logger.info( `Seed request: ${input.sessions.length} session(s), ` + - `${input.totalRounds} round(s), ${input.totalMessages} message(s)`, + `${input.totalRounds} round(s), ${input.totalMessages} message(s), ` + + `waitFullPipeline=${body.wait_for_full_pipeline === true}`, ); // Resolve output directory: use gateway's data dir with a timestamped subfolder @@ -392,6 +468,15 @@ export class TdaiGateway { }, }; if (body.config_override) { + const blockedOverridePaths = findBlockedConfigOverridePaths(body.config_override); + if (blockedOverridePaths.length > 0) { + sendJson(res, 400, { + error: "config_override contains blocked credential-bearing or network-routing keys", + blocked_paths: blockedOverridePaths, + }); + return; + } + for (const key of Object.keys(body.config_override)) { const baseVal = pluginConfig[key]; const overVal = body.config_override[key]; @@ -409,6 +494,10 @@ export class TdaiGateway { outputDir, openclawConfig: {}, pluginConfig, + waitForFullPipeline: body.wait_for_full_pipeline === true, + fullPipelineFlushTimeoutMs: typeof body.full_pipeline_timeout_ms === "number" + ? body.full_pipeline_timeout_ms + : undefined, logger: this.logger as import("../utils/pipeline-factory.js").PipelineLogger, onProgress: (progress: SeedProgress) => { this.logger.debug?.( @@ -428,6 +517,7 @@ export class TdaiGateway { rounds_processed: summary.roundsProcessed, messages_processed: summary.messagesProcessed, l0_recorded: summary.l0RecordedCount, + full_pipeline_flushed: summary.fullPipelineFlushed, duration_ms: summary.durationMs, output_dir: summary.outputDir, }; @@ -435,6 +525,79 @@ export class TdaiGateway { } } +function expectedGatewayToken(): string { + const direct = process.env.TDAI_GATEWAY_TOKEN?.trim(); + if (direct) return direct; + + const tokenPath = process.env.TDAI_TOKEN_PATH?.trim(); + if (!tokenPath) return ""; + try { + const stat = fs.statSync(tokenPath); + if (process.platform !== "win32" && (stat.mode & 0o077) !== 0) return "\0"; + if (process.platform !== "win32" && typeof process.getuid === "function" && stat.uid !== process.getuid()) { + return "\0"; + } + return fs.readFileSync(tokenPath, "utf-8").trim() || "\0"; + } catch { + return "\0"; + } +} + +function isAllowedCorsOrigin(origin: string): boolean { + const explicit = process.env.TDAI_GATEWAY_CORS_ORIGINS?.trim(); + if (explicit) { + const allowed = explicit.split(",").map((item) => item.trim()).filter(Boolean); + if (allowed.includes("*")) return true; + return allowed.includes(origin); + } + + try { + const url = new URL(origin); + return isLoopbackHost(url.hostname); + } catch { + return false; + } +} + +function isLoopbackHost(host: string): boolean { + const normalized = String(host || "").trim().toLowerCase(); + return normalized === "localhost" || + normalized === "127.0.0.1" || + normalized === "::1" || + normalized === "[::1]"; +} + +function findBlockedConfigOverridePaths(value: unknown, prefix = ""): string[] { + if (!value || typeof value !== "object" || Array.isArray(value)) return []; + const blocked: string[] = []; + for (const [key, child] of Object.entries(value as Record)) { + const current = prefix ? `${prefix}.${key}` : key; + if (isBlockedConfigOverridePath(current)) { + blocked.push(current); + continue; + } + blocked.push(...findBlockedConfigOverridePaths(child, current)); + } + return blocked; +} + +function isBlockedConfigOverridePath(path: string): boolean { + const parts = path.toLowerCase().split("."); + const leaf = parts.at(-1) || ""; + if (parts[0] === "tcvdb") return true; + if (parts[0] === "embedding") return true; + if (parts[0] === "llm" && ["apikey", "baseurl", "enabled"].includes(leaf)) return true; + if (parts[0] === "offload" && ["backendurl", "backendapikey", "mode"].includes(leaf)) return true; + return /(?:apikey|secret|token|password|authorization|credential|baseurl|backendurl|proxyurl)$/.test(leaf); +} + +function safeTokenEqual(actual: string, expected: string): boolean { + const actualBuffer = Buffer.from(actual); + const expectedBuffer = Buffer.from(expected); + if (actualBuffer.length !== expectedBuffer.length) return false; + return crypto.timingSafeEqual(actualBuffer, expectedBuffer); +} + // ============================ // CLI entry point // ============================ diff --git a/src/gateway/types.ts b/src/gateway/types.ts index 50b2ff4..13fd50f 100644 --- a/src/gateway/types.ts +++ b/src/gateway/types.ts @@ -11,6 +11,22 @@ export interface GatewayErrorResponse { code?: string; } +// ============================ +// / +// ============================ + +export interface RootResponse { + service: "TencentDB Agent Memory Gateway"; + kind: "api"; + version: string; + message: string; + endpoints: { + method: "GET" | "POST"; + path: string; + description: string; + }[]; +} + // ============================ // /health // ============================ @@ -50,6 +66,8 @@ export interface CaptureRequest { assistant_content: string; session_key: string; session_id?: string; + /** Epoch ms when the captured turn began. Hosts that reconstruct turns out-of-process should set this. */ + started_at?: number; user_id?: string; messages?: unknown[]; } @@ -68,6 +86,8 @@ export interface MemorySearchRequest { limit?: number; type?: string; scene?: string; + /** Optional session-key prefixes for host/project scoped search. */ + session_key_prefixes?: string[]; } export interface MemorySearchResponse { @@ -84,6 +104,8 @@ export interface ConversationSearchRequest { query: string; limit?: number; session_key?: string; + /** Optional session-key prefixes for host/project scoped search. */ + session_key_prefixes?: string[]; } export interface ConversationSearchResponse { @@ -129,6 +151,10 @@ export interface SeedRequest { strict_round_role?: boolean; /** Auto-fill missing timestamps (default: true). */ auto_fill_timestamps?: boolean; + /** Wait for final L1→L2→L3 processing before returning (default: false). */ + wait_for_full_pipeline?: boolean; + /** Max wait time for final L1→L2→L3 processing. */ + full_pipeline_timeout_ms?: number; /** Plugin config overrides (deep-merged on top of gateway memory config). */ config_override?: Record; } @@ -138,6 +164,7 @@ export interface SeedResponse { rounds_processed: number; messages_processed: number; l0_recorded: number; + full_pipeline_flushed?: boolean; duration_ms: number; output_dir: string; } diff --git a/src/offload/backend-client.ts b/src/offload/backend-client.ts index 30dc622..9daf096 100644 --- a/src/offload/backend-client.ts +++ b/src/offload/backend-client.ts @@ -296,6 +296,10 @@ export class BackendClient { const parsed = new URL(url); const isHttps = parsed.protocol === "https:"; const transport = isHttps ? https : http; + const allowInsecureTls = process.env.TDAI_OFFLOAD_INSECURE_TLS === "true"; + if (isHttps && allowInsecureTls) { + this.logger.warn("[context-offload] TDAI_OFFLOAD_INSECURE_TLS=true disables backend TLS certificate verification"); + } return new Promise((resolve, reject) => { const timer = setTimeout(() => { @@ -309,7 +313,7 @@ export class BackendClient { path: parsed.pathname + parsed.search, method: "POST", headers: reqHeaders, - ...(isHttps ? { rejectUnauthorized: false } : {}), + ...(isHttps && allowInsecureTls ? { rejectUnauthorized: false } : {}), }, (res) => { let data = ""; diff --git a/src/utils/pipeline-manager.ts b/src/utils/pipeline-manager.ts index b56234c..d8d2d75 100644 --- a/src/utils/pipeline-manager.ts +++ b/src/utils/pipeline-manager.ts @@ -146,6 +146,40 @@ export interface PipelineConfig { }; } +export interface PipelineQueueSizes { + l1: number; + l2: number; + l3: number; + l1Pending: boolean; + l2Pending: boolean; + l3Pending: boolean; + l1Idle: boolean; + l2Idle: boolean; + l3Idle: boolean; +} + +export interface PipelineFlushOptions { + /** Human-readable reason for diagnostics. */ + reason?: string; + /** Maximum time to wait before rejecting. Omit or set to 0 for no timeout. */ + timeoutMs?: number; + /** Poll interval for the final stability check. */ + pollIntervalMs?: number; + /** Number of consecutive idle polls required before returning. */ + stableRounds?: number; + /** + * Whether L2 should arm the follow-up max-interval timer after this flush. + * Seed/import callers should set this false because they are about to tear + * down the pipeline and do not need a periodic L2 timer. + */ + armFollowUpL2Timers?: boolean; +} + +export interface PipelineFlushResult { + durationMs: number; + queueSizes: PipelineQueueSizes; +} + /** Result returned by the L1 runner. */ export interface L1RunnerResult { /** Number of messages successfully processed */ @@ -216,6 +250,7 @@ export class MemoryPipelineManager { // L3 dedup flag private l3Pending = false; private l3Running = false; + private suppressL2MaxInterval = false; // Per-session state private readonly sessionStates = new Map(); @@ -502,6 +537,48 @@ export class MemoryPipelineManager { this.logger?.debug?.(`${TAG} [${sessionKey}] flushSession: complete`); } + /** + * Flush immediate pipeline work without destroying the scheduler. + * + * This is stronger than {@link flushSession}: it drains pending L1 work, + * flushes currently scheduled L2 timers, waits for L2, then waits for the + * L3 persona runner triggered by L2. It intentionally ignores future + * max-interval L2 timers, which are periodic maintenance rather than work + * created by the current seed/import batch. + */ + async flushPendingWork(options: PipelineFlushOptions = {}): Promise { + const start = Date.now(); + const reason = options.reason ?? "manual"; + + this.logger?.info(`${TAG} Flushing pending work (${reason})...`); + + let timeoutId: ReturnType | undefined; + const flushPromise = this._doFlush(options); + + if (options.timeoutMs && options.timeoutMs > 0) { + await Promise.race([ + flushPromise, + new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`flush timeout after ${options.timeoutMs}ms`)), + options.timeoutMs, + ); + }), + ]).finally(() => { + if (timeoutId !== undefined) clearTimeout(timeoutId); + }); + } else { + await flushPromise; + } + + const result: PipelineFlushResult = { + durationMs: Date.now() - start, + queueSizes: this.getQueueSizes(), + }; + this.logger?.info(`${TAG} Pending work flushed (${reason}) in ${result.durationMs}ms`); + return result; + } + /** * Maximum time (ms) to wait for pipeline flush during destroy. * Must be shorter than the gateway_stop hook timeout (3 s) to leave @@ -560,37 +637,77 @@ export class MemoryPipelineManager { * Internal: attempt to flush all pending pipeline work (L1 → L2 → L3). * Extracted from destroy() so it can be wrapped with a timeout. */ - private async _doFlush(): Promise { - // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages - for (const [sessionKey, timers] of this.sessionTimers) { - if (timers.l1Idle.pending) { - timers.l1Idle.cancel(); // don't fire the idle callback directly - const buffer = this.messageBuffers.get(sessionKey); - if (buffer && buffer.length > 0) { - this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); - this.enqueueL1(sessionKey, "flush"); + private async _doFlush(options: PipelineFlushOptions = {}): Promise { + const previousSuppressL2MaxInterval = this.suppressL2MaxInterval; + if (options.armFollowUpL2Timers === false) { + this.suppressL2MaxInterval = true; + } + + try { + // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l1Idle.pending) { + timers.l1Idle.cancel(); // don't fire the idle callback directly + const buffer = this.messageBuffers.get(sessionKey); + if (buffer && buffer.length > 0) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); + this.enqueueL1(sessionKey, "flush"); + } } } - } - // Step 2: Wait for L1 queue to drain - this.logger?.debug?.(`${TAG} Waiting for L1 queue to drain (size=${this.l1Queue.size})`); - await this.l1Queue.onIdle(); + // Step 2: Wait for L1 queue to drain + this.logger?.debug?.(`${TAG} Waiting for L1 queue to drain (size=${this.l1Queue.size})`); + await this.l1Queue.onIdle(); - // Step 3: Flush all L2 schedule timers - for (const [sessionKey, timers] of this.sessionTimers) { - if (timers.l2Schedule.pending) { - this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: triggering L2 schedule timer`); - timers.l2Schedule.flush(); + // Step 3: Flush all L2 schedule timers + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l2Schedule.pending) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: triggering L2 schedule timer`); + timers.l2Schedule.flush(); + } } + + // Step 4: Wait for all remaining queues to drain + this.logger?.debug?.(`${TAG} Waiting for queues to drain (l2=${this.l2Queue.size}, l3=${this.l3Queue.size})`); + await this.l2Queue.onIdle(); + await this.l3Queue.onIdle(); + + // L3 is enqueued by L2 completion. Depending on microtask ordering, an + // early l3Queue.onIdle() observer can miss a just-enqueued follow-up run. + // Require a short stable idle window that also checks the L3 dedupe flags. + await this.waitForImmediateQueuesIdle({ + pollIntervalMs: options.pollIntervalMs, + stableRounds: options.stableRounds, + }); + } finally { + this.suppressL2MaxInterval = previousSuppressL2MaxInterval; } + } + + private async waitForImmediateQueuesIdle(options: Pick = {}): Promise { + const pollIntervalMs = options.pollIntervalMs ?? 50; + const stableRounds = options.stableRounds ?? 2; + let consecutiveIdle = 0; + + while (true) { + const queues = this.getQueueSizes(); + const idle = + queues.l1Idle && + queues.l2Idle && + queues.l3Idle && + !this.l3Running && + !this.l3Pending; + + if (idle) { + consecutiveIdle += 1; + if (consecutiveIdle >= stableRounds) return; + } else { + consecutiveIdle = 0; + } - // Step 4: Wait for all remaining queues to drain - this.logger?.debug?.(`${TAG} Waiting for queues to drain (l2=${this.l2Queue.size}, l3=${this.l3Queue.size})`); - await Promise.all([ - this.l2Queue.onIdle(), - this.l3Queue.onIdle(), - ]); + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); + } } // ============================ @@ -915,8 +1032,11 @@ export class MemoryPipelineManager { this.logger?.debug?.(`${TAG} [${sessionKey}] L2 complete`); - // Arm the maxInterval timer for the next cycle - this.armL2MaxInterval(sessionKey); + // Arm the maxInterval timer for the next cycle unless this L2 was forced + // by a one-shot seed/import flush that is about to tear the pipeline down. + if (!this.suppressL2MaxInterval) { + this.armL2MaxInterval(sessionKey); + } // Trigger L3 this.triggerL3(); @@ -1139,11 +1259,7 @@ export class MemoryPipelineManager { } /** Queue sizes and running state for monitoring. */ - getQueueSizes(): { - l1: number; l2: number; l3: number; - l1Pending: boolean; l2Pending: boolean; l3Pending: boolean; - l1Idle: boolean; l2Idle: boolean; l3Idle: boolean; - } { + getQueueSizes(): PipelineQueueSizes { return { l1: this.l1Queue.size, l2: this.l2Queue.size, diff --git a/src/utils/sanitize.ts b/src/utils/sanitize.ts index 80ee636..3e7b55f 100644 --- a/src/utils/sanitize.ts +++ b/src/utils/sanitize.ts @@ -17,6 +17,18 @@ export function sanitizeText(text: string): string { cleaned = cleaned.replace(/[\s\S]*?<\/user-persona>/g, ""); cleaned = cleaned.replace(/[\s\S]*?<\/relevant-scenes>/g, ""); cleaned = cleaned.replace(/[\s\S]*?<\/scene-navigation>/g, ""); + cleaned = cleaned.replace(//g, ""); + + // Remove Codex adapter injected context before L0/L1 persistence. Codex does + // not expose an OpenClaw-style before_message_write rewrite hook, so the + // Gateway-side sanitizer is the final guard against recall feedback loops. + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); + cleaned = cleaned.replace(//g, ""); // Remove offload-injected task context blocks (MMD mermaid diagrams) cleaned = cleaned.replace(/[\s\S]*?<\/current_task_context>/g, ""); diff --git a/tsdown.config.ts b/tsdown.config.ts index 16b0073..89b0e06 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -11,7 +11,7 @@ function collectExternalDependencies(): string[] { } export default defineConfig({ - entry: ["./index.ts"], + entry: ["./index.ts", "./src/gateway/cli.ts"], outDir: "./dist", format: "esm", platform: "node", diff --git a/vitest.config.ts b/vitest.config.ts index 1f9ce29..3f5446c 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,7 +4,7 @@ export default defineConfig({ test: { environment: "node", pool: "forks", - include: ["src/**/*.test.ts", "__tests__/**/*.test.ts"], + include: ["src/**/*.test.ts", "__tests__/**/*.test.ts", "codex-plugin/**/*.test.mjs"], exclude: ["dist/**", "node_modules/**", "**/*.e2e.test.ts"], testTimeout: 120_000, hookTimeout: 120_000,