From 7e06a202dd1e107c311a2834a57a0a57269b46e8 Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 06:52:45 +0000 Subject: [PATCH 01/14] docs(plan): add app-level telemetry plan --- .../docs/plans/app-level-telemetry-plan.md | 1716 +++++++++++++++++ 1 file changed, 1716 insertions(+) create mode 100644 packages/core/docs/plans/app-level-telemetry-plan.md diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md new file mode 100644 index 000000000..9f21726fb --- /dev/null +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -0,0 +1,1716 @@ +# App-Level Telemetry Plan for boring-ui-v2 + +**Status:** draft plan — ready for review, then bead conversion +**Branch/worktree:** `plan/telemetry` at `/home/ubuntu/projects/worktrees/boring-ui-v2-telemetry` +**Primary decision:** the **child app declares telemetry providers and routing**; package code emits typed vendor-neutral events only. +**Packages touched:** `@hachej/boring-core`, `@hachej/boring-agent`, `@hachej/boring-workspace`, app examples under `apps/*` +**Last updated:** 2026-05-21 + +--- + +## 0. Executive summary + +boring-ui-v2 needs telemetry for product usage and reliability: + +- usage: who opens workspaces, starts chat sessions, uses panels, runs tools, triggers commands +- reliability: chat stream failures, tool failures, sandbox/runtime failures, server 5xx, frontend error boundaries +- operations: durations, status codes, model/runtime/tool metadata, request IDs + +But telemetry cannot be owned by `@hachej/boring-core`, `@hachej/boring-agent`, or `@hachej/boring-workspace` as vendor integrations. A final child app may need to send different events to different PostHog accounts, Sentry projects, OpenTelemetry collectors, or customer-specific analytics destinations. + +Therefore: + +> Packages expose and use a small `TelemetrySink` interface. The child app constructs the actual telemetry adapter and passes it into core/agent/workspace server and front entrypoints. + +Packages never import PostHog or Sentry directly. Packages do not read `POSTHOG_KEY` or `SENTRY_DSN`. The final app does. + +--- + +## 1. Goals + +### 1.1 Product analytics + +Capture enough events to answer: + +- How many users open the app daily? +- How many workspaces are active? +- How often do users start chat sessions? +- How many messages per session? +- Which agent tools are used most? +- Which workspace panels and commands are used? +- Which plugins produce actual engagement? + +### 1.2 Reliability analytics + +Capture enough events to answer: + +- Are chat streams failing? +- Which runtime mode fails most: `direct`, `local`, or `vercel-sandbox`? +- Which tools fail most? +- Are specific workspaces/users seeing repeated failures? +- Are frontend panel/plugin crashes happening? +- Are server 5xx errors correlated with agent sessions? + +### 1.3 App-level provider control + +The child app must be able to: + +- disable telemetry entirely +- send all events to one destination +- route frontend and backend events differently +- route events per deployment or environment +- route events per workspace/customer/tenant +- use PostHog for analytics and Sentry for exceptions +- use only Sentry, only PostHog, only OTLP, or a custom internal endpoint + +### 1.4 Privacy-safe by default + +Telemetry must default to operational metadata only. + +Do capture: + +- IDs: user/workspace/session/request +- route/method/status/duration +- runtime mode, model provider, tool name +- counts and lengths +- error codes and sanitized messages + +Do not capture by default: + +- user prompt text +- assistant output text/code +- file contents +- command stdout/stderr +- environment variables +- raw headers/cookies/tokens +- secret values + +--- + +## 2. Non-goals + +- No built-in billing/metering system in this feature. +- No core-owned analytics database. +- No mandatory SaaS vendor. +- No automatic capture of prompt/content bodies. +- No full distributed tracing rollout in phase 1. +- No hard dependency from agent/workspace to core. +- No environment variables in packages like `POSTHOG_KEY`; those belong to child apps. +- No “telemetry SDK singleton” hidden in package internals. + +--- + +## 3. Core architectural decision + +### 3.1 Decision + +Telemetry is declared at the child-app level. + +```ts +const telemetry = createAppTelemetry({ + appId: 'full-app', + environment: process.env.NODE_ENV, + posthog: { + defaultProjectKey: process.env.POSTHOG_PROJECT_KEY, + }, + sentry: { + dsn: process.env.SENTRY_DSN, + }, + routeEvent: async (event) => { + // Optional: choose destination by workspace/customer/deployment. + return chooseTelemetryDestination(event.context?.workspaceId) + }, +}) + +const app = await createCoreWorkspaceAgentServer({ + telemetry, + plugins, + mode, + workspaceRoot, +}) +``` + +Package code only emits: + +```ts +telemetry.capture({ + name: 'agent.chat.stream.failed', + context: { workspaceId, sessionId, requestId, runtimeMode }, + properties: { durationMs, errorCode }, +}) +``` + +The app decides whether that becomes: + +- `posthog.capture(...)` +- `Sentry.captureException(...)` +- an OTLP span/event +- a database insert +- a no-op + +### 3.2 Why this is better + +#### Package composability + +`@hachej/boring-agent` must remain standalone. It cannot depend on core or a specific analytics vendor. + +#### Tenant-specific telemetry + +Future apps may be white-labeled or multi-tenant. Workspace A may need one PostHog account and workspace B another. + +#### Deployment flexibility + +Self-hosters may want no telemetry. Enterprise users may require a private collector. Local CLI users should not be forced to configure anything. + +#### CSP flexibility + +Browser analytics vendors require CSP `connect-src` changes. The app shell owns CSP policy and should decide which endpoints to allow. + +#### Privacy policy ownership + +The final app owns privacy policy, user consent, and data retention. Packages should not surprise-capture content. + +--- + +## 4. Package boundaries and invariants + +### 4.1 Existing package graph + +Current project shape: + +```txt +apps/* + ├─→ @hachej/boring-core + ├─→ @hachej/boring-workspace + └─→ @hachej/boring-agent + +@hachej/boring-workspace → @hachej/boring-core only where allowed by app composition rules +@hachej/boring-agent → standalone leaf +``` + +Telemetry must not invert or tangle this graph. + +### 4.2 Invariants + +Telemetry work must preserve these project rules: + +1. No `node:*` imports in `src/shared/**`. +2. No `Buffer` in `src/shared/**`. +3. Workspace base front/shared code has no value imports from `@hachej/boring-agent`. +4. Agent remains usable as standalone `createAgentApp()` and CLI. +5. Core remains the package that knows auth/workspace identity, but it does not own vendor telemetry credentials. +6. Error codes remain stable and canonical. +7. Telemetry failures never break user flows. + +--- + +## 5. Telemetry contract + +Each package should expose a compatible structural type from its own shared layer. + +Why per-package instead of a new shared package right now: + +- avoids adding a new publishable package before needed +- avoids forcing agent to depend on core/workspace +- keeps shared code browser-safe +- TypeScript structural typing lets one child-app object satisfy all package contracts + +Potential future extraction: + +- `@hachej/boring-telemetry` can be added later if the contract grows. +- Do not start there unless the contract becomes large or duplicated logic hurts. + +### 5.1 Shared type + +```ts +export type TelemetrySeverity = 'debug' | 'info' | 'warn' | 'error' + +export interface TelemetryContext { + appId?: string + environment?: string + deploymentId?: string + userId?: string + workspaceId?: string + sessionId?: string + requestId?: string + route?: string + method?: string + runtimeMode?: string + pluginId?: string +} + +export interface TelemetryEvent { + name: string + timestamp?: string + severity?: TelemetrySeverity + context?: TelemetryContext + properties?: Record +} + +export interface TelemetryErrorInfo { + code?: string + statusCode?: number + className?: string + // Optional, already-sanitized, short message. No raw upstream error text by default. + message?: string +} + +export interface TelemetryErrorEvent extends TelemetryEvent { + error?: TelemetryErrorInfo +} + +export interface TelemetrySink { + capture(event: TelemetryEvent): void | Promise + captureError(event: TelemetryErrorEvent): void | Promise + flush?(): Promise +} +``` + +**Simplicity rule:** package code passes sanitized error metadata, not raw `unknown` exceptions. App adapters may capture raw exceptions only inside app-owned code paths where the app has its own scrubbers and consent policy. + +### 5.2 Noop sink + +```ts +export const noopTelemetry: TelemetrySink = { + capture() {}, + captureError() {}, + async flush() {}, +} +``` + +### 5.3 Safe capture wrapper + +Package code should not `await telemetry.capture(...)` directly in hot paths unless needed. Use a helper that swallows sink errors and optionally logs them. + +```ts +export function safeCaptureTelemetry( + telemetry: TelemetrySink, + event: TelemetryEvent, + logger?: { warn?: (data: unknown, message?: string) => void }, +): void { + try { + void Promise.resolve(telemetry.capture(event)).catch((error) => { + logger?.warn?.({ err: error, eventName: event.name }, 'telemetry capture failed') + }) + } catch (error) { + logger?.warn?.({ err: error, eventName: event.name }, 'telemetry capture failed') + } +} +``` + +A similar `safeCaptureTelemetryError` should exist. It accepts `TelemetryErrorInfo`, not raw `unknown`. + +### 5.4 Why failures are swallowed + +Telemetry is observability, not product logic. If PostHog/Sentry/collector is down, chat should still work. + +--- + +## 6. Server integration design + +### 6.1 Core server + +Add optional telemetry to `CreateCoreAppOptions`: + +```ts +export interface CreateCoreAppOptions { + authProvider?: AuthProvider + userStore?: UserStore + workspaceStore?: WorkspaceStore + provisioner?: WorkspaceProvisioner + manageShutdown?: boolean + telemetry?: TelemetrySink +} +``` + +Decorate Fastify: + +```ts +app.decorate('telemetry', options?.telemetry ?? noopTelemetry) +``` + +Fastify module augmentation: + +```ts +declare module 'fastify' { + interface FastifyInstance { + telemetry: TelemetrySink + } +} +``` + +Capture: + +- request failures +- unhandled errors +- shutdown errors if useful +- workspace CRUD lifecycle events +- auth lifecycle events only after privacy review + +### 6.2 Core workspace-agent app + +`createCoreWorkspaceAgentServer()` is the key child-app composition path. It should accept either a ready sink or a small factory. + +Keep the common case simple: + +```ts +createCoreWorkspaceAgentServer({ telemetry }) +``` + +Allow the advanced case without making every package know about stores: + +```ts +export interface CreateCoreWorkspaceAgentServerOptions + extends Omit { + telemetry?: TelemetrySink + createTelemetry?: (ctx: { + config: CoreConfig + userStore: UserStore + workspaceStore: WorkspaceStore + }) => TelemetrySink | Promise + // existing options... +} +``` + +Implementation shape: + +```ts +const runtime = await createCoreRuntime(config, { + telemetry: options.telemetry, + createTelemetry: options.createTelemetry, +}) + +await app.register(registerAgentRoutes, { + telemetry: runtime.telemetry, + // existing options... +}) +``` + +Use `telemetry` for one-account apps. Use `createTelemetry` only when routing needs `workspaceStore`/customer settings. The same resolved sink flows through the composed server unless the child app intentionally passes separate sinks. + +### 6.3 Agent standalone server + +Add telemetry to `CreateAgentAppOptions`: + +```ts +export interface CreateAgentAppOptions { + telemetry?: TelemetrySink + // existing options... +} +``` + +Standalone mode defaults to noop. + +```ts +const telemetry = opts.telemetry ?? noopTelemetry +``` + +### 6.4 Agent embedded routes + +Add telemetry to `RegisterAgentRoutesOptions`: + +```ts +export interface RegisterAgentRoutesOptions { + telemetry?: TelemetrySink + // existing options... +} +``` + +When embedded in core, agent should use: + +```ts +const telemetry = opts.telemetry ?? maybeAppTelemetry(app) ?? noopTelemetry +``` + +This must be structural; agent cannot import core types. + +```ts +function maybeAppTelemetry(app: FastifyInstance): TelemetrySink | undefined { + const value = (app as FastifyInstance & { telemetry?: unknown }).telemetry + return isTelemetrySink(value) ? value : undefined +} +``` + +--- + +## 7. Frontend integration design + +### 7.1 Core front + +`CoreFront` currently owns providers and `AppErrorBoundary`. Add: + +```ts +export interface CoreFrontProps { + children?: ReactNode + authPages?: CoreFrontAuthPagesOverride + cspNonce?: string + telemetry?: TelemetrySink +} +``` + +Wire error boundary: + +```tsx + { + safeCaptureTelemetryError(telemetry, { + name: 'core.frontend.error', + severity: 'error', + error: sanitizeError(error), + context: { route: window.location.pathname }, + properties: { + componentStackHash: hashString(errorInfo.componentStack ?? ''), + }, + }) + }} +> +``` + +Do not capture raw component stack by default. It can include component names but may still be noisy. Hash is enough for grouping in product telemetry. If a child app wants Sentry raw exceptions, it can do that inside its own `AppErrorBoundary` or adapter after applying its own scrubbers. + +### 7.2 Workspace front + +Add telemetry to `WorkspaceProvider` and context. + +```tsx + + + +``` + +Capture: + +- provider mounted +- panel opened/closed +- left tab selected +- command executed +- catalog row opened +- UI command posted/failed +- plugin/panel error boundary + +### 7.3 Agent front + +Add telemetry to `ChatPanel` and/or `useAgentChat` options. + +```tsx + +``` + +Frontend can capture user-intent events immediately: + +- message submit clicked +- attachment rejected +- chat UI error shown +- stream disconnected in browser + +Backend remains source of truth for stream/tool success/failure. + +### 7.4 Composed frontend helpers + +The app-level composition helpers must forward telemetry too. Otherwise child apps using the default boring app shell would have to manually re-compose everything. + +Required pass-throughs: + +- `CoreWorkspaceAgentFront telemetry` → `CoreFront telemetry` +- `CoreWorkspaceAgentFront telemetry` → `WorkspaceAgentFront telemetry` +- `WorkspaceAgentFront telemetry` → `WorkspaceProvider telemetry` +- `WorkspaceAgentFront telemetry` → default `ChatPanel`/chat params when it renders the default agent panel + +Keep this as simple prop forwarding. Do not make workspace context the only way for agent front code to receive telemetry, because base workspace code must not value-import agent. + +--- + +## 8. Child app telemetry adapters + +### 8.1 Server adapter example + +Location for example: + +```txt +apps/full-app/src/telemetry/server.ts +``` + +Shape: + +```ts +export interface ServerTelemetryConfig { + appId: string + environment: string + posthog?: { + defaultProjectKey?: string + host?: string + } + sentry?: { + dsn?: string + } + routeEvent?: (event: TelemetryEvent | TelemetryErrorEvent) => Promise | TelemetryDestination +} +``` + +The example should be dependency-light. If adding actual vendor packages is too much for first pass, use documented extension points and maybe a minimal `fetch`-based PostHog capture example. + +### 8.2 Browser adapter example + +Location: + +```txt +apps/full-app/src/telemetry/browser.ts +``` + +Browser env vars are public: + +```bash +VITE_POSTHOG_KEY= +VITE_POSTHOG_HOST=https://us.i.posthog.com +VITE_SENTRY_DSN= +``` + +CSP notes must be documented because browser telemetry needs `connect-src` additions. + +### 8.3 Multi-account routing + +Example routing: + +```ts +const app = await createCoreWorkspaceAgentServer({ + createTelemetry: ({ config, workspaceStore }) => createServerTelemetry({ + appId: config.appId, + environment: process.env.NODE_ENV ?? 'development', + routeEvent: async (event) => { + const workspaceId = event.context?.workspaceId + if (!workspaceId) return defaultDestination + + const settings = await workspaceStore.getWorkspaceSettings(workspaceId) + const posthogProjectKey = settings.find((s) => s.key === 'POSTHOG_PROJECT_KEY') + if (posthogProjectKey?.configured) { + return { posthog: { projectKey: await decryptSetting(workspaceId, 'POSTHOG_PROJECT_KEY') } } + } + + return defaultDestination + }, + }), +}) +``` + +This is why package-level provider initialization is wrong. Most apps should use plain `telemetry`; apps that need store-backed routing use `createTelemetry`. + +--- + +## 9. Event taxonomy + +Event names use stable dotted names. + +Naming rules: + +- prefix by package/domain: `core.*`, `agent.*`, `workspace.*` +- use past-tense lifecycle names: `created`, `started`, `completed`, `failed` +- use stable low-cardinality event names +- put high-cardinality values in properties only if safe +- no raw user text in event names or properties + +### 9.1 Core events + +| Event | When | Context | Properties | +|---|---|---|---| +| `core.app.started` | server boot completes | appId, environment, deploymentId | packageVersion | +| `core.http.request.failed` | 5xx/unhandled route error | requestId, route, method, userId, workspaceId | statusCode, errorCode, durationMs | +| `core.auth.sign_in.completed` | sign-in succeeds | userId | provider | +| `core.auth.sign_in.failed` | sign-in fails | requestId | statusCode, reason | +| `core.auth.sign_up.completed` | sign-up succeeds | userId | provider | +| `core.workspace.created` | workspace created | userId, workspaceId | isDefault | +| `core.workspace.updated` | workspace changed | userId, workspaceId | changedFieldsCount | +| `core.workspace.deleted` | workspace removed | userId, workspaceId | hadRuntime | +| `core.workspace.opened` | front loads active workspace | userId, workspaceId | source | +| `core.frontend.error` | React boundary catches error | route, userId, workspaceId | componentStackHash | + +Auth events may be phased later if better-auth hooks are awkward or privacy review wants fewer identity events first. + +### 9.2 Agent events + +| Event | When | Context | Properties | +|---|---|---|---| +| `agent.chat.session.created` | session created | workspaceId, sessionId, userId | source | +| `agent.chat.message.sent` | user message accepted | workspaceId, sessionId, userId | messageLength, attachmentCount | +| `agent.chat.stream.started` | backend stream opens | workspaceId, sessionId, requestId, runtimeMode | modelProvider, modelName | +| `agent.chat.stream.completed` | stream finishes normally | workspaceId, sessionId, requestId, runtimeMode | durationMs, chunkCount, toolCallCount | +| `agent.chat.stream.failed` | stream errors | workspaceId, sessionId, requestId, runtimeMode | durationMs, errorCode, statusCode | +| `agent.tool.started` | tool call begins | workspaceId, sessionId, runtimeMode | toolName | +| `agent.tool.completed` | tool call succeeds | workspaceId, sessionId, runtimeMode | toolName, durationMs | +| `agent.tool.failed` | tool call fails | workspaceId, sessionId, runtimeMode | toolName, durationMs, errorCode | +| `agent.runtime.binding.created` | runtime binding created | workspaceId, runtimeMode | fsCapability | +| `agent.runtime.binding.recreated` | expired sandbox recreated | workspaceId, runtimeMode | reason | +| `agent.sandbox.failed` | sandbox/runtime operation failed | workspaceId, runtimeMode | statusCode, errorCode | +| `agent.plugin.load.failed` | pi plugin failed to load | workspaceId | pluginSourceHash, errorCode | + +### 9.3 Workspace events + +| Event | When | Context | Properties | +|---|---|---|---| +| `workspace.provider.mounted` | provider initializes | userId, workspaceId | pluginCount | +| `workspace.panel.opened` | dockview opens panel | workspaceId | panelId, placement, source | +| `workspace.panel.closed` | dockview closes panel | workspaceId | panelId, durationMs | +| `workspace.left_tab.selected` | left tab selected | workspaceId | tabId | +| `workspace.command.executed` | command palette command runs | workspaceId | commandId, source | +| `workspace.catalog.searched` | catalog search submitted | workspaceId | catalogId, queryLength, resultCount | +| `workspace.catalog.row.opened` | catalog row opens surface | workspaceId | catalogId, kind | +| `workspace.ui_command.posted` | server posts UI command | workspaceId | commandType, source | +| `workspace.ui_command.dispatched` | front dispatches command | workspaceId | commandType | +| `workspace.ui_command.failed` | command cannot dispatch | workspaceId | commandType, reason | +| `workspace.plugin.error` | plugin boundary catches error | workspaceId, pluginId | panelId, componentStackHash | + +--- + +## 10. Privacy and content policy + +### 10.1 Default capture matrix + +| Data | Default | Rationale | +|---|---:|---| +| userId | yes | app already owns auth; useful for support | +| workspaceId | yes | needed for workspace-level reliability | +| sessionId | yes | needed for chat debugging | +| requestId | yes | joins logs/errors | +| route template | yes | operational metadata | +| raw URL query string | no | may include secrets/search text | +| status code | yes | operational metadata | +| durationMs | yes | performance | +| runtimeMode | yes | compare direct/local/vercel-sandbox | +| toolName | yes | low-cardinality product metadata | +| model provider/name | yes | cost/reliability analysis | +| prompt text | no | sensitive content | +| assistant text/code | no | sensitive content/IP | +| file contents | no | sensitive/IP | +| file path | no by default | may reveal project structure; app can opt into hash/basename | +| command stdout/stderr | no | may include secrets | +| command string | no by default | may include secrets; maybe hash only later | +| env vars | never | secrets | +| cookies/auth headers | never | secrets | + +### 10.2 Sanitization helpers + +Add helpers where needed: + +```ts +function sanitizeError(error: unknown): TelemetryErrorInfo +function sanitizeErrorCode(error: unknown): string | undefined +function sanitizeStatusCode(error: unknown): number | undefined +function lengthOnly(value: string | undefined): number +function safeRoute(request: FastifyRequest): string +function truncateTelemetryString(value: string, maxLength?: number): string +``` + +Do not add generic deep object telemetry serialization. That invites accidental content capture. + +### 10.3 Event allowlists + +Privacy tests should validate allowlisted fields per event name, not only `not.toContain('secret')` snapshots. + +Simple rule: each package keeps a tiny test-only map of expected event names and allowed `context`/`properties` keys for the events it emits. Tests fail if code starts spreading raw request/body/error objects into telemetry. + +Do not build a big runtime schema system in v1. Test allowlists are enough. + +### 10.3 User consent + +The package-level sink contract should not enforce consent. The child app adapter should decide: + +- disabled until user opts in +- enabled for operational error events only +- enabled for all product events +- disabled for enterprise deployments + +This can be a `beforeSend` hook in the child-app adapter. + +```ts +beforeSend(event) { + if (!userConsentAllows(event)) return null + return sanitize(event) +} +``` + +--- + +## 11. Core implementation plan + +### 11.1 Shared contract + +Files: + +```txt +packages/core/src/shared/telemetry.ts +packages/core/src/shared/index.ts +``` + +Exports: + +- `TelemetrySeverity` +- `TelemetryContext` +- `TelemetryEvent` +- `TelemetryErrorEvent` +- `TelemetrySink` +- `noopTelemetry` +- `safeCaptureTelemetry` +- `safeCaptureTelemetryError` + +Tests: + +```txt +packages/core/src/shared/__tests__/telemetry.test.ts +``` + +Acceptance: + +- noop accepts events +- safe helpers swallow sync throw +- safe helpers swallow async rejection +- helpers call logger warn when provided + +### 11.2 Fastify decoration + +Files: + +```txt +packages/core/src/server/app/types.ts +packages/core/src/server/app/createCoreApp.ts +``` + +Add `telemetry` option and decoration. + +Acceptance: + +- `createCoreApp(config)` exposes noop telemetry +- `createCoreApp(config, { telemetry })` exposes supplied sink +- no behavior change when omitted + +### 11.3 Error handler capture + +File: + +```txt +packages/core/src/server/app/errorHandler.ts +``` + +Capture only server failures and validation/rate-limit failures if useful. + +First pass recommendation: + +- capture 5xx in `captureError` +- optionally capture 4xx auth/rate-limit later + +Payload: + +```ts +{ + name: 'core.http.request.failed', + severity: 'error', + error: sanitizeError(error), + context: { + requestId: request.id, + route: request.routeOptions?.url, + method: request.method, + userId: request.user?.id, + }, + properties: { + statusCode: 500, + errorCode: 'internal_error', + }, +} +``` + +Acceptance: + +- HTTP response unchanged +- telemetry sink receives event on 500 +- telemetry sink failure does not change response + +### 11.4 Front error boundary capture + +Files: + +```txt +packages/core/src/front/CoreFront.tsx +packages/core/src/front/AppErrorBoundary.tsx +``` + +`AppErrorBoundary` already supports `onError`; wire from `CoreFront` prop. + +Acceptance: + +- rendering crash calls telemetry +- no telemetry prop means no crash in tests +- no raw component stack in product properties unless explicitly accepted + +--- + +## 12. Agent implementation plan + +### 12.1 Shared contract + +Files: + +```txt +packages/agent/src/shared/telemetry.ts +packages/agent/src/shared/index.ts +``` + +Same structural contract as core. + +### 12.2 Options + +Files: + +```txt +packages/agent/src/server/createAgentApp.ts +packages/agent/src/server/registerAgentRoutes.ts +``` + +Add: + +```ts +telemetry?: TelemetrySink +``` + +Embedded routes fallback to `app.telemetry` if structurally present. + +### 12.3 Chat route events + +Likely files: + +```txt +packages/agent/src/server/http/routes/chat.ts +packages/agent/src/server/http/routes/sessions.ts +``` + +Capture: + +- message accepted +- stream started +- stream completed +- stream failed +- stream aborted +- session created + +Important: do not capture message content. Only length/count. + +Current chat streaming can handle generator errors inside the stream executor, write an error chunk, and still finish HTTP plumbing. Therefore telemetry must live inside the stream execution path, not only an outer route `try/catch`. + +Simple terminal rule: + +- emit `agent.chat.stream.started` once when backend execution starts +- emit exactly one terminal event per backend execution: + - `agent.chat.stream.completed` for normal completion + - `agent.chat.stream.failed` for harness/generator/tool error + - `agent.chat.stream.aborted` for client abort/cancel when detectable +- resume/replay endpoints must not emit a second terminal event for an already-finished stream + +Acceptance: + +- test sink receives stream lifecycle events +- failed stream emits failure event with sanitized error code/status +- successful stream emits completion duration +- aborted stream emits aborted, not completed +- no duplicate terminal events on resume/replay +- no event snapshot includes prompt text + +### 12.4 Tool lifecycle events + +Preferred design: wrap tools once when building the tool list, not inside every tool implementation. + +```ts +function withToolTelemetry( + tools: AgentTool[], + telemetry: TelemetrySink, + getContext: () => TelemetryContext, +): AgentTool[] +``` + +Wrapper captures: + +- `agent.tool.started` +- `agent.tool.completed` +- `agent.tool.failed` + +The exact `AgentTool` shape must be read before implementation. Do not assume handler property name. + +Acceptance: + +- every standard/extra/plugin tool is wrapped once +- wrapper preserves tool schema/name/description/onUpdate behavior +- wrapper does not alter tool result/errors +- failures still propagate to harness +- `agent.tool.failed` is emitted for thrown errors and for tool results with `isError === true` +- wrapper combines binding context (`workspaceId`, `runtimeMode`) with tool execution context (`sessionId`) + +### 12.5 Runtime/sandbox events + +Locations: + +```txt +packages/agent/src/server/registerAgentRoutes.ts +packages/agent/src/server/runtime/* +packages/agent/src/server/sandbox/vercel-sandbox/* +``` + +First pass can capture at route registration layer: + +- runtime binding created +- runtime binding recreated because sandbox expired +- runtime binding creation failed + +Avoid deep invasive instrumentation in sandbox adapters until needed. + +--- + +## 13. Workspace implementation plan + +### 13.1 Shared contract + +Files: + +```txt +packages/workspace/src/shared/telemetry.ts +packages/workspace/src/shared/index.ts +``` + +Browser-safe only. No `node:*`, no `Buffer`. + +### 13.2 Provider wiring + +Likely files: + +```txt +packages/workspace/src/front/WorkspaceProvider.tsx +packages/workspace/src/front/context/* +``` + +Add telemetry to workspace runtime context. + +Acceptance: + +- omitted telemetry defaults noop +- plugins/panels can access telemetry through workspace context if needed +- no agent import added to base workspace code + +### 13.3 UI events + +Capture at central dispatch points, not scattered everywhere. + +Preferred central points: + +- panel open API / surface resolver dispatch +- command execution registry +- catalog row open helper +- UI bridge command dispatcher +- plugin/panel error boundaries + +Acceptance: + +- opening a panel emits one event +- command execution emits one event +- failed UI command emits failure event +- plugin error boundary emits error event + +### 13.4 Workspace server events + +A few workspace events are server-owned, not front-owned. Keep this small and focused. + +Wire optional telemetry into: + +- `uiRoutes` for UI command HTTP route failures +- `createWorkspaceUiTools` / `exec_ui` for server-posted UI commands +- `createWorkspaceAgentServer` composition helper +- core composed server when it registers workspace UI routes/tools + +Acceptance: + +- server-posted UI command emits `workspace.ui_command.posted` +- `exec_ui` failure emits `workspace.ui_command.failed` +- no new dependency from workspace server to agent/core internals + +--- + +## 14. App example plan + +### 14.1 `apps/full-app` server example + +Files: + +```txt +apps/full-app/src/telemetry/server.ts +apps/full-app/src/server/main.ts +apps/full-app/src/server/vercel-entry.ts +apps/full-app/.env.example +apps/full-app/README.md +``` + +Implement a minimal app-level adapter. Depending on dependency appetite: + +Option A — dependency-free example: + +- use `fetch` to PostHog capture API if configured +- use console or placeholder for Sentry +- document where to plug actual SDK + +Option B — real adapters: + +- add `posthog-node` +- add `@sentry/node` +- add `@sentry/react` for browser + +Recommendation for first implementation: start with dependency-free or optional adapters. Do not force new vendor deps into packages. + +### 14.2 Browser example + +Files: + +```txt +apps/full-app/src/telemetry/browser.ts +apps/full-app/src/front/main.tsx +``` + +Initialize browser telemetry only if public env vars exist. + +### 14.3 CSP updates + +Core CSP currently has `connectSrc: ['self']`. If browser telemetry is enabled, child app needs a way to extend CSP connect sources. + +Possible approaches: + +1. Add core config for CSP extra connect sources. +2. Document that app must override config/security. +3. Server injects CSP based on app config. + +This is a blocker for browser PostHog/Sentry in production. Server-side telemetry is not blocked. + +Recommended first bead for CSP: + +- add config field `security.csp.connectSrcExtra?: string[]` +- wire it through core config schema/loadConfig/types +- use it in `createCoreApp` helmet config +- app example sets PostHog/Sentry hosts when env vars exist + +Keep this small: only extend `connect-src`; do not redesign the whole CSP system. + +--- + +## 15. Data flow examples + +### 15.1 Chat stream failure + +```txt +User sends message + ↓ +ChatPanel/useAgentChat posts to /api/v1/agent/chat/:sessionId + ↓ +agent chat route emits agent.chat.message.sent + ↓ +stream starts: agent.chat.stream.started + ↓ +harness/tool/sandbox throws + ↓ +chat stream executor marks stream failed and emits sanitized terminal event + ↓ +telemetry.captureError(agent.chat.stream.failed) + ↓ +child app adapter sends sanitized event to PostHog/Sentry +``` + +Captured data: + +```json +{ + "name": "agent.chat.stream.failed", + "context": { + "workspaceId": "ws_123", + "sessionId": "sess_456", + "requestId": "req_789", + "runtimeMode": "vercel-sandbox" + }, + "properties": { + "durationMs": 1832, + "errorCode": "sandbox_expired", + "statusCode": 410 + } +} +``` + +Not captured: + +- prompt body +- generated code +- command output + +### 15.2 Workspace panel open + +```txt +User runs command palette command + ↓ +workspace command registry executes open panel + ↓ +panel open central function emits workspace.panel.opened + ↓ +child app adapter sends to PostHog +``` + +### 15.3 Multi-account event routing + +```txt +agent.chat.stream.failed workspaceId=customer_a + ↓ +app routeEvent reads workspace/customer mapping + ↓ +send to Customer A PostHog project and shared Sentry project + +agent.chat.stream.failed workspaceId=customer_b + ↓ +send to Customer B PostHog project and shared Sentry project +``` + +--- + +## 16. Performance considerations + +Telemetry must be low overhead. + +Rules: + +- Do not block chat streaming on telemetry network calls. +- Use fire-and-forget safe wrappers for most events. +- Only `flush()` on graceful shutdown if available. +- Avoid serializing large objects. +- Keep properties low-cardinality and small. +- Avoid hashing huge strings/content. +- Batch in child-app adapters if vendor supports it. + +Shutdown: + +`createCoreApp` already has graceful shutdown handling. If telemetry has `flush`, call it during shutdown/onClose, but do not hang beyond grace window. + +--- + +## 17. Error handling rules + +Telemetry must never cause: + +- failed HTTP request +- broken chat stream +- broken React render +- failed tool call +- failed app boot, unless the child app adapter intentionally throws during construction + +Package helpers swallow runtime telemetry sink errors. + +The only errors that should fail boot are explicit child-app configuration errors, e.g. malformed DSN in `createAppTelemetry`, and those happen in app code. + +--- + +## 18. Testing strategy + +### 18.1 Unit tests + +Core: + +- noop/safe helper behavior +- Fastify decoration with noop/custom sink +- error handler emits event on 500 +- frontend boundary emits event + +Agent: + +- noop/safe helper behavior +- `createAgentApp({ telemetry })` wires sink +- `registerAgentRoutes({ telemetry })` wires sink +- chat success/failure events +- tool wrapper success/failure events +- no prompt text in snapshots +- exactly-one terminal stream event for success/failure/abort +- resume/replay does not duplicate terminal events + +Workspace: + +- provider defaults noop +- panel open event +- command event +- UI command failure event +- plugin error boundary event + +### 18.2 Integration tests + +- composed `createCoreWorkspaceAgentServer({ telemetry })` receives both core and agent events through one sink +- request workspace ID appears in agent events +- user ID appears when auth hook has populated `request.user` +- telemetry sink rejection does not alter response status/body + +### 18.3 Privacy/allowlist tests + +Build representative captured events and assert they do not contain forbidden strings: + +```ts +expect(JSON.stringify(events)).not.toContain('my secret prompt') +expect(JSON.stringify(events)).not.toContain('DATABASE_URL') +expect(JSON.stringify(events)).not.toContain('file contents') +expect(JSON.stringify(events)).not.toContain('stdout with token') +``` + +Also assert each emitted event uses only allowed context/property keys for that event name. This is the main guard against accidental object spreading. + +Examples: + +```ts +expectEventKeys(events, { + 'agent.chat.stream.failed': { + context: ['workspaceId', 'sessionId', 'requestId', 'runtimeMode'], + properties: ['durationMs', 'errorCode', 'statusCode'], + error: ['code', 'statusCode', 'className'], + }, +}) +``` + +### 18.4 Quality gates + +Run relevant scoped tests while implementing: + +```bash +pnpm --filter @hachej/boring-core test +pnpm --filter @hachej/boring-agent test +pnpm --filter @hachej/boring-workspace test +``` + +Before close: + +```bash +pnpm typecheck +pnpm lint +pnpm test +pnpm lint:invariants +``` + +--- + +## 19. Rollout strategy + +### Phase 1 — contracts/noop + +Lowest risk. No behavior changes. + +Deliver: + +- shared telemetry types in all packages +- sanitized error metadata type +- noop and safe helpers +- public exports +- tests + +### Phase 2 — core server/front + +Deliver: + +- `createCoreApp({ telemetry })` +- Fastify decoration +- sanitized error handler capture +- `CoreFront telemetry` prop +- frontend boundary capture + +### Phase 3 — agent chat events + +Deliver: + +- `createAgentApp({ telemetry })` +- `registerAgentRoutes({ telemetry })` +- chat/session stream lifecycle events +- exactly-one terminal stream event semantics +- privacy/allowlist tests + +### Phase 4 — composed passthrough + +Deliver: + +- `createCoreWorkspaceAgentServer({ telemetry })` +- optional `createTelemetry({ config, userStore, workspaceStore })` +- composed frontend telemetry prop forwarding + +### Phase 5 — agent tool/runtime events + +Deliver: + +- tool wrapper +- thrown-error and `isError` failure detection +- runtime binding created/recreated/failed +- sandbox expired/recreated event + +### Phase 6 — workspace UI/server UI-command events + +Deliver: + +- provider context +- panel/command/catalog/UI bridge/plugin error events +- `uiRoutes` and `exec_ui` server-side events + +### Phase 7 — CSP support + +Deliver: + +- `security.csp.connectSrcExtra?: string[]` +- Helmet `connectSrc` append support +- tests for default and extra hosts + +### Phase 8 — app examples and docs + +Deliver: + +- `apps/full-app` app-level adapter example +- README docs +- event taxonomy docs + +--- + +## 20. Risks and mitigations + +### Risk: accidental content capture + +Mitigation: + +- explicit allowlisted properties only +- tests asserting forbidden content absent +- no generic object spreading into telemetry payloads + +### Risk: telemetry breaks streaming performance + +Mitigation: + +- fire-and-forget safe wrappers +- no await in stream hot path unless using in-memory sink in tests +- child-app adapter handles batching + +### Risk: event cardinality explosion + +Mitigation: + +- stable event names +- avoid raw paths/URLs/queries +- avoid arbitrary error messages as dimensions + +### Risk: duplicate events frontend/backend + +Mitigation: + +- define source-of-truth per event +- backend owns chat/tool success/failure +- frontend owns UI interaction events +- if both emit, use different event names (`message.submitted` vs `message.sent`) + +### Risk: package dependency tangle + +Mitigation: + +- structural per-package contracts +- no imports between packages for telemetry types +- no vendor SDKs in packages + +### Risk: CSP blocks browser analytics + +Mitigation: + +- app-level CSP extension config +- docs list required `connect-src` hosts + +--- + +## 21. Open questions + +1. Should auth success/failure events be in the first implementation, or deferred until core auth hooks are reviewed? +2. Should file paths be omitted entirely by default or captured as stable salted hashes? +3. Should event names be exported constants, or keep string literals plus docs? +4. Should server request telemetry capture all 4xx or only 5xx? +5. Should we add an explicit user consent API in core front, or leave consent entirely to child apps? +6. Should agent use existing `@opentelemetry/api` dependency for optional spans later? +7. Should child-app examples use actual vendor SDKs or dependency-free fetch adapters? +8. Should telemetry sink support `identify(user)` and `group(workspace)` helpers, or keep only event capture? + +Recommendation for v1: + +- keep only `capture`, `captureError`, `flush` +- defer `identify/group` to app adapter or future expansion +- start with 5xx/chat/tool/UI events before auth analytics + +--- + +## 22. Bead breakdown + +The beads should stay small. Do not build a tracing platform. Ship a simple sink contract, a few high-value events, and tests that prevent content leaks. + +### Bead 1 — `telemetry-contracts-noop` + +**Goal:** add vendor-neutral telemetry contracts and noop/safe helpers to core, agent, workspace. + +Files: + +```txt +packages/core/src/shared/telemetry.ts +packages/core/src/shared/index.ts +packages/core/src/shared/__tests__/telemetry.test.ts +packages/agent/src/shared/telemetry.ts +packages/agent/src/shared/index.ts +packages/agent/src/shared/__tests__/telemetry.test.ts +packages/workspace/src/shared/telemetry.ts +packages/workspace/src/shared/index.ts +packages/workspace/src/shared/__tests__/telemetry.test.ts +``` + +Acceptance: + +- types exported from public shared barrels +- noop works +- safe helpers swallow sync/async failures +- `TelemetryErrorEvent.error` is sanitized metadata, not raw `unknown` +- no `node:*` or `Buffer` in shared files + +Dependencies: none. + +### Bead 2 — `core-telemetry-hooks` + +**Goal:** wire telemetry into core server and frontend error boundary. + +Files: + +```txt +packages/core/src/server/app/types.ts +packages/core/src/server/app/createCoreApp.ts +packages/core/src/server/app/errorHandler.ts +packages/core/src/front/CoreFront.tsx +packages/core/src/front/__tests__/*telemetry*.test.tsx +packages/core/src/server/app/__tests__/*telemetry*.test.ts +``` + +Acceptance: + +- `createCoreApp(config, { telemetry })` decorates app +- 500 handler emits `core.http.request.failed` with sanitized error metadata +- `CoreFront telemetry` captures boundary crash without raw component stack in properties +- telemetry failure does not change responses/rendering + +Dependencies: Bead 1. + +### Bead 3 — `composed-app-telemetry-passthrough` + +**Goal:** pass app-level telemetry through default composed server and front helpers. + +Files: + +```txt +packages/core/src/app/server/createCoreWorkspaceAgentServer.ts +packages/core/src/app/server/__tests__/*telemetry*.test.ts +packages/core/src/app/front/CoreWorkspaceAgentFront.tsx +packages/core/src/app/front/__tests__/*telemetry*.test.tsx +packages/workspace/src/app/front/WorkspaceAgentFront.tsx +packages/workspace/src/app/front/__tests__/*telemetry*.test.tsx +``` + +Acceptance: + +- simple `telemetry` sink works for composed server +- optional `createTelemetry({ config, userStore, workspaceStore })` works for store-backed routing +- resolved sink is passed to core and agent route registration +- frontend telemetry prop forwards to `CoreFront`, `WorkspaceAgentFront`, `WorkspaceProvider`, and default chat panel params +- omitted sink remains noop + +Dependencies: Beads 1-2 and agent options from Bead 4 if done together. + +### Bead 4 — `agent-chat-telemetry` + +**Goal:** wire telemetry into agent server routes and capture chat/session lifecycle. + +Files: + +```txt +packages/agent/src/server/createAgentApp.ts +packages/agent/src/server/registerAgentRoutes.ts +packages/agent/src/server/http/routes/chat.ts +packages/agent/src/server/http/routes/sessions.ts +packages/agent/src/server/http/routes/__tests__/*telemetry*.test.ts +``` + +Acceptance: + +- `createAgentApp({ telemetry })` accepted +- `registerAgentRoutes({ telemetry })` accepted and can fall back to structural `app.telemetry` +- chat started/completed/failed/aborted events emitted from the stream execution path +- exactly one terminal stream event per backend execution +- resume/replay does not duplicate terminal events +- message/session events use counts/lengths only +- allowlist/privacy tests pass + +Dependencies: Bead 1. + +### Bead 5 — `agent-tool-runtime-telemetry` + +**Goal:** capture tool and runtime/sandbox failures. + +Files: + +```txt +packages/agent/src/server/tools/* or new telemetry wrapper location +packages/agent/src/server/registerAgentRoutes.ts +packages/agent/src/server/__tests__/*tool-telemetry*.test.ts +``` + +Acceptance: + +- all tools wrapped once +- started/completed/failed emitted +- failed means thrown error OR `ToolResult.isError === true` +- wrapper preserves schema/name/description/onUpdate/result/error behavior +- runtime binding recreate/failure emits event +- allowlist/privacy tests pass + +Dependencies: Bead 4. + +### Bead 6 — `workspace-ui-telemetry` + +**Goal:** capture workspace UI usage and the small set of workspace-owned server UI command events. + +Files: + +```txt +packages/workspace/src/front/** +packages/workspace/src/front/components/PanelErrorBoundary.tsx +packages/workspace/src/front/plugin/PluginErrorBoundary.tsx +packages/workspace/src/server/ui-control/http/uiRoutes.ts +packages/workspace/src/server/ui-control/tools/uiTools.ts +packages/workspace/src/app/server/createWorkspaceAgentServer.ts +packages/workspace/src/front/**/__tests__/*telemetry*.test.tsx +packages/workspace/src/server/**/__tests__/*telemetry*.test.ts +``` + +Acceptance: + +- `WorkspaceProvider telemetry` accepted +- panel open/close emits events +- commands emit events +- UI command posted/failure emits events server-side where appropriate +- plugin/panel boundary emits sanitized error event +- no workspace base front/shared value import from agent + +Dependencies: Bead 1. + +### Bead 7 — `core-csp-connect-src-extra` + +**Goal:** let child apps opt into browser telemetry endpoints without weakening CSP globally. + +Files: + +```txt +packages/core/src/shared/types.ts +packages/core/src/server/config/* +packages/core/src/server/app/createCoreApp.ts +packages/core/src/server/app/__tests__/*csp*.test.ts +``` + +Acceptance: + +- config supports `security.csp.connectSrcExtra?: string[]` +- values append to Helmet `connectSrc` +- default remains `connectSrc: ["'self'"]` +- tests cover default and extra PostHog/Sentry-style hosts + +Dependencies: none, but needed before browser telemetry example is useful in production. + +### Bead 8 — `full-app-telemetry-example` + +**Goal:** show child-app-owned PostHog/Sentry-style routing with minimal dependencies. + +Files: + +```txt +apps/full-app/src/telemetry/server.ts +apps/full-app/src/telemetry/browser.ts +apps/full-app/src/server/main.ts +apps/full-app/src/server/vercel-entry.ts +apps/full-app/src/front/main.tsx +apps/full-app/README.md +apps/full-app/.env.example +``` + +Acceptance: + +- app creates telemetry adapter +- app passes it into composed server/front +- docs explain plain `telemetry` vs advanced `createTelemetry` +- adapter has `beforeSend`, bounded queue or clear drop behavior, and flush timeout +- docs explain privacy defaults + +Dependencies: Beads 2-7. + +### Bead 9 — `telemetry-docs-event-taxonomy` + +**Goal:** publish stable event taxonomy and privacy policy docs. + +Files: + +```txt +packages/core/docs/CORE.md or docs/telemetry.md +packages/agent/docs/plans/agent-package-spec.md or package docs +packages/workspace/docs/INTERFACES.md or package docs +README.md +``` + +Acceptance: + +- event names documented +- sanitized payload rules documented +- child app integration documented +- no vendor is presented as mandatory + +Dependencies: Beads 1-8. + +--- + +## 23. Recommended implementation order + +1. Bead 1 — contracts/noop. +2. Bead 4 — agent chat telemetry, because original user need includes chat session errors. +3. Bead 2 — core error/front hooks. +4. Bead 3 — composed app passthrough. +5. Bead 5 — tool/runtime events. +6. Bead 6 — workspace UI/server UI-command events. +7. Bead 7 — CSP connect-src extra. +8. Bead 8 — app examples. +9. Bead 9 — docs finalization. + +Reasoning: + +- Chat errors are the urgent user value. +- Contracts are required first. +- Core passthrough is what makes app-level declaration real. +- CSP can wait until browser vendor examples, but must land before production browser telemetry docs. +- Workspace UI events can follow once backend reliability is covered. + +--- + +## 24. Definition of done + +Telemetry work is done when: + +- child app can pass one telemetry sink into composed boring app +- core emits server/frontend error events +- agent emits chat stream failure/success events +- agent emits tool failure/success events +- workspace emits central UI usage/error events +- no package imports PostHog/Sentry directly +- no prompt/file/command output content is captured by default +- examples show PostHog/Sentry-style routing at app level +- tests prove telemetry failures do not break product behavior +- docs explain event taxonomy and privacy guarantees + +--- + +## 25. Review prompt for next model pass + +Use this exact review prompt with a stronger reasoning model: + +```txt +Carefully review this entire plan for me and come up with your best revisions in terms of better architecture, new features, changed features, etc. to make it better, more robust/reliable, more performant, more compelling/useful, etc. For each proposed change, give me your detailed analysis and rationale/justification for why it would make the project better along with the git-diff style change versus the original plan shown below: + + +``` + +After receiving review, integrate revisions in-place and then convert to beads. From 07ded72702c600fa997ba6129ed895628d9c2458 Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 14:22:35 +0000 Subject: [PATCH 02/14] docs(plan): simplify PostHog telemetry plan --- .../docs/plans/app-level-telemetry-plan.md | 1802 ++--------------- 1 file changed, 223 insertions(+), 1579 deletions(-) diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md index 9f21726fb..2523b1f7e 100644 --- a/packages/core/docs/plans/app-level-telemetry-plan.md +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -1,1716 +1,360 @@ -# App-Level Telemetry Plan for boring-ui-v2 +# Simple PostHog Telemetry Plan for boring-ui-v2 -**Status:** draft plan — ready for review, then bead conversion +**Status:** simplified draft for PR 65 **Branch/worktree:** `plan/telemetry` at `/home/ubuntu/projects/worktrees/boring-ui-v2-telemetry` -**Primary decision:** the **child app declares telemetry providers and routing**; package code emits typed vendor-neutral events only. -**Packages touched:** `@hachej/boring-core`, `@hachej/boring-agent`, `@hachej/boring-workspace`, app examples under `apps/*` -**Last updated:** 2026-05-21 +**Primary decision:** use **PostHog**, configured by the child app through env vars. +**Last updated:** 2026-05-22 --- -## 0. Executive summary +## 0. What changed from the original plan -boring-ui-v2 needs telemetry for product usage and reliability: +The old plan was too big for the current need. It designed a vendor-neutral telemetry layer with Sentry, OTLP, routing, multi-account selection, CSP planning, and nine beads. -- usage: who opens workspaces, starts chat sessions, uses panels, runs tools, triggers commands -- reliability: chat stream failures, tool failures, sandbox/runtime failures, server 5xx, frontend error boundaries -- operations: durations, status codes, model/runtime/tool metadata, request IDs +We do not need that now. -But telemetry cannot be owned by `@hachej/boring-core`, `@hachej/boring-agent`, or `@hachej/boring-workspace` as vendor integrations. A final child app may need to send different events to different PostHog accounts, Sentry projects, OpenTelemetry collectors, or customer-specific analytics destinations. +New shape: -Therefore: - -> Packages expose and use a small `TelemetrySink` interface. The child app constructs the actual telemetry adapter and passes it into core/agent/workspace server and front entrypoints. - -Packages never import PostHog or Sentry directly. Packages do not read `POSTHOG_KEY` or `SENTRY_DSN`. The final app does. - ---- - -## 1. Goals - -### 1.1 Product analytics - -Capture enough events to answer: - -- How many users open the app daily? -- How many workspaces are active? -- How often do users start chat sessions? -- How many messages per session? -- Which agent tools are used most? -- Which workspace panels and commands are used? -- Which plugins produce actual engagement? - -### 1.2 Reliability analytics - -Capture enough events to answer: - -- Are chat streams failing? -- Which runtime mode fails most: `direct`, `local`, or `vercel-sandbox`? -- Which tools fail most? -- Are specific workspaces/users seeing repeated failures? -- Are frontend panel/plugin crashes happening? -- Are server 5xx errors correlated with agent sessions? - -### 1.3 App-level provider control - -The child app must be able to: - -- disable telemetry entirely -- send all events to one destination -- route frontend and backend events differently -- route events per deployment or environment -- route events per workspace/customer/tenant -- use PostHog for analytics and Sentry for exceptions -- use only Sentry, only PostHog, only OTLP, or a custom internal endpoint - -### 1.4 Privacy-safe by default - -Telemetry must default to operational metadata only. - -Do capture: - -- IDs: user/workspace/session/request -- route/method/status/duration -- runtime mode, model provider, tool name -- counts and lengths -- error codes and sanitized messages - -Do not capture by default: - -- user prompt text -- assistant output text/code -- file contents -- command stdout/stderr -- environment variables -- raw headers/cookies/tokens -- secret values - ---- - -## 2. Non-goals - -- No built-in billing/metering system in this feature. -- No core-owned analytics database. -- No mandatory SaaS vendor. -- No automatic capture of prompt/content bodies. -- No full distributed tracing rollout in phase 1. -- No hard dependency from agent/workspace to core. -- No environment variables in packages like `POSTHOG_KEY`; those belong to child apps. -- No “telemetry SDK singleton” hidden in package internals. - ---- - -## 3. Core architectural decision - -### 3.1 Decision - -Telemetry is declared at the child-app level. - -```ts -const telemetry = createAppTelemetry({ - appId: 'full-app', - environment: process.env.NODE_ENV, - posthog: { - defaultProjectKey: process.env.POSTHOG_PROJECT_KEY, - }, - sentry: { - dsn: process.env.SENTRY_DSN, - }, - routeEvent: async (event) => { - // Optional: choose destination by workspace/customer/deployment. - return chooseTelemetryDestination(event.context?.workspaceId) - }, -}) - -const app = await createCoreWorkspaceAgentServer({ - telemetry, - plugins, - mode, - workspaceRoot, -}) -``` - -Package code only emits: - -```ts -telemetry.capture({ - name: 'agent.chat.stream.failed', - context: { workspaceId, sessionId, requestId, runtimeMode }, - properties: { durationMs, errorCode }, -}) -``` - -The app decides whether that becomes: - -- `posthog.capture(...)` -- `Sentry.captureException(...)` -- an OTLP span/event -- a database insert -- a no-op - -### 3.2 Why this is better - -#### Package composability - -`@hachej/boring-agent` must remain standalone. It cannot depend on core or a specific analytics vendor. - -#### Tenant-specific telemetry - -Future apps may be white-labeled or multi-tenant. Workspace A may need one PostHog account and workspace B another. - -#### Deployment flexibility - -Self-hosters may want no telemetry. Enterprise users may require a private collector. Local CLI users should not be forced to configure anything. - -#### CSP flexibility - -Browser analytics vendors require CSP `connect-src` changes. The app shell owns CSP policy and should decide which endpoints to allow. - -#### Privacy policy ownership - -The final app owns privacy policy, user consent, and data retention. Packages should not surprise-capture content. +- PostHog is the only planned provider. +- The child app owns PostHog env vars and client creation. +- Packages receive a tiny optional telemetry sink and default to no-op. +- Shared PostHog accounts are handled with a stable project prefix/property, not routing logic. +- No Sentry, OTLP, tenant routing, event bus, or analytics database in this pass. --- -## 4. Package boundaries and invariants - -### 4.1 Existing package graph - -Current project shape: - -```txt -apps/* - ├─→ @hachej/boring-core - ├─→ @hachej/boring-workspace - └─→ @hachej/boring-agent - -@hachej/boring-workspace → @hachej/boring-core only where allowed by app composition rules -@hachej/boring-agent → standalone leaf -``` - -Telemetry must not invert or tangle this graph. +## 1. Goal -### 4.2 Invariants +Add enough telemetry to answer simple product and reliability questions: -Telemetry work must preserve these project rules: +- app opened +- workspace opened +- chat session started +- user message submitted, without content +- agent/tool run completed or failed +- workspace panel/command used +- server/frontend error happened, with stable error code only -1. No `node:*` imports in `src/shared/**`. -2. No `Buffer` in `src/shared/**`. -3. Workspace base front/shared code has no value imports from `@hachej/boring-agent`. -4. Agent remains usable as standalone `createAgentApp()` and CLI. -5. Core remains the package that knows auth/workspace identity, but it does not own vendor telemetry credentials. -6. Error codes remain stable and canonical. -7. Telemetry failures never break user flows. +Telemetry must never capture prompts, assistant output, file contents, command output, secrets, headers, cookies, or raw env vars. --- -## 5. Telemetry contract +## 2. Decision -Each package should expose a compatible structural type from its own shared layer. +Telemetry is app-level and PostHog-focused. -Why per-package instead of a new shared package right now: - -- avoids adding a new publishable package before needed -- avoids forcing agent to depend on core/workspace -- keeps shared code browser-safe -- TypeScript structural typing lets one child-app object satisfy all package contracts - -Potential future extraction: - -- `@hachej/boring-telemetry` can be added later if the contract grows. -- Do not start there unless the contract becomes large or duplicated logic hurts. - -### 5.1 Shared type +Packages expose only this tiny structural contract: ```ts -export type TelemetrySeverity = 'debug' | 'info' | 'warn' | 'error' - -export interface TelemetryContext { - appId?: string - environment?: string - deploymentId?: string - userId?: string - workspaceId?: string - sessionId?: string - requestId?: string - route?: string - method?: string - runtimeMode?: string - pluginId?: string -} - -export interface TelemetryEvent { - name: string - timestamp?: string - severity?: TelemetrySeverity - context?: TelemetryContext - properties?: Record -} - -export interface TelemetryErrorInfo { - code?: string - statusCode?: number - className?: string - // Optional, already-sanitized, short message. No raw upstream error text by default. - message?: string -} - -export interface TelemetryErrorEvent extends TelemetryEvent { - error?: TelemetryErrorInfo -} - export interface TelemetrySink { capture(event: TelemetryEvent): void | Promise - captureError(event: TelemetryErrorEvent): void | Promise - flush?(): Promise } -``` - -**Simplicity rule:** package code passes sanitized error metadata, not raw `unknown` exceptions. App adapters may capture raw exceptions only inside app-owned code paths where the app has its own scrubbers and consent policy. -### 5.2 Noop sink - -```ts -export const noopTelemetry: TelemetrySink = { - capture() {}, - captureError() {}, - async flush() {}, +export interface TelemetryEvent { + name: string + distinctId?: string + properties?: Record } ``` -### 5.3 Safe capture wrapper +Every package option that needs telemetry accepts `telemetry?: TelemetrySink`. +If absent, it uses a no-op sink. -Package code should not `await telemetry.capture(...)` directly in hot paths unless needed. Use a helper that swallows sink errors and optionally logs them. +Package code emits events like: ```ts -export function safeCaptureTelemetry( - telemetry: TelemetrySink, - event: TelemetryEvent, - logger?: { warn?: (data: unknown, message?: string) => void }, -): void { - try { - void Promise.resolve(telemetry.capture(event)).catch((error) => { - logger?.warn?.({ err: error, eventName: event.name }, 'telemetry capture failed') - }) - } catch (error) { - logger?.warn?.({ err: error, eventName: event.name }, 'telemetry capture failed') - } -} +telemetry.capture({ + name: 'agent.chat.started', + distinctId: userId, + properties: { + workspaceId, + sessionId, + runtimeMode, + }, +}) ``` -A similar `safeCaptureTelemetryError` should exist. It accepts `TelemetryErrorInfo`, not raw `unknown`. - -### 5.4 Why failures are swallowed - -Telemetry is observability, not product logic. If PostHog/Sentry/collector is down, chat should still work. +The final app decides how to build the PostHog sink from env vars. --- -## 6. Server integration design +## 3. Env vars -### 6.1 Core server - -Add optional telemetry to `CreateCoreAppOptions`: - -```ts -export interface CreateCoreAppOptions { - authProvider?: AuthProvider - userStore?: UserStore - workspaceStore?: WorkspaceStore - provisioner?: WorkspaceProvisioner - manageShutdown?: boolean - telemetry?: TelemetrySink -} -``` +Use boring-ui env names for boring-ui behavior, and PostHog env names for PostHog credentials. -Decorate Fastify: +```bash +# Required to send telemetry. +POSTHOG_KEY=phc_... -```ts -app.decorate('telemetry', options?.telemetry ?? noopTelemetry) -``` +# Optional. Defaults in the app helper if omitted. +POSTHOG_HOST=https://us.i.posthog.com -Fastify module augmentation: +# Optional. If unset, telemetry is enabled when POSTHOG_KEY is set. +BORING_TELEMETRY_ENABLED=true -```ts -declare module 'fastify' { - interface FastifyInstance { - telemetry: TelemetrySink - } -} +# Optional, but recommended when several apps share one PostHog account/project. +# Used both as an event-name prefix and as a property. +BORING_TELEMETRY_PROJECT=full-app ``` -Capture: - -- request failures -- unhandled errors -- shutdown errors if useful -- workspace CRUD lifecycle events -- auth lifecycle events only after privacy review +### Project prefix rule -### 6.2 Core workspace-agent app +If `BORING_TELEMETRY_PROJECT=full-app`, event names sent to PostHog become: -`createCoreWorkspaceAgentServer()` is the key child-app composition path. It should accept either a ready sink or a small factory. - -Keep the common case simple: - -```ts -createCoreWorkspaceAgentServer({ telemetry }) +```txt +full-app.app.opened +full-app.workspace.opened +full-app.agent.chat.started +full-app.agent.tool.completed ``` -Allow the advanced case without making every package know about stores: +The same value is also sent as a property: ```ts -export interface CreateCoreWorkspaceAgentServerOptions - extends Omit { - telemetry?: TelemetrySink - createTelemetry?: (ctx: { - config: CoreConfig - userStore: UserStore - workspaceStore: WorkspaceStore - }) => TelemetrySink | Promise - // existing options... +{ + boringProject: 'full-app', + eventName: 'agent.chat.started' } ``` -Implementation shape: - -```ts -const runtime = await createCoreRuntime(config, { - telemetry: options.telemetry, - createTelemetry: options.createTelemetry, -}) - -await app.register(registerAgentRoutes, { - telemetry: runtime.telemetry, - // existing options... -}) -``` - -Use `telemetry` for one-account apps. Use `createTelemetry` only when routing needs `workspaceStore`/customer settings. The same resolved sink flows through the composed server unless the child app intentionally passes separate sinks. +Why both: -### 6.3 Agent standalone server - -Add telemetry to `CreateAgentAppOptions`: - -```ts -export interface CreateAgentAppOptions { - telemetry?: TelemetrySink - // existing options... -} -``` +- prefixed event names make PostHog dashboards easy when projects share an account/project +- the `boringProject` property makes filtering and grouping easy +- no need for multi-account routing in v1 -Standalone mode defaults to noop. +If the prefix is unset, send the raw event name. -```ts -const telemetry = opts.telemetry ?? noopTelemetry -``` +--- -### 6.4 Agent embedded routes +## 4. PostHog sink helper -Add telemetry to `RegisterAgentRoutesOptions`: +Implement this helper in the app/example layer, not deep inside agent/workspace internals: ```ts -export interface RegisterAgentRoutesOptions { - telemetry?: TelemetrySink - // existing options... +export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink { + const enabled = env.BORING_TELEMETRY_ENABLED !== 'false' && Boolean(env.POSTHOG_KEY) + + if (!enabled) return noopTelemetry + + const posthog = new PostHog(env.POSTHOG_KEY!, { + host: env.POSTHOG_HOST, + }) + + const prefix = env.BORING_TELEMETRY_PROJECT?.trim() + + return { + capture(event) { + const name = prefix ? `${prefix}.${event.name}` : event.name + + posthog.capture({ + distinctId: event.distinctId ?? 'anonymous', + event: name, + properties: { + ...event.properties, + boringProject: prefix, + eventName: event.name, + }, + }) + }, + } } ``` -When embedded in core, agent should use: - -```ts -const telemetry = opts.telemetry ?? maybeAppTelemetry(app) ?? noopTelemetry -``` +Exact file location can be chosen during implementation. Preferred: -This must be structural; agent cannot import core types. +- reusable helper: `packages/core/src/server/telemetry/posthog.ts`, if core already owns app env/config helpers +- app-only helper: `apps/full-app/src/server/telemetry.ts`, if we want zero PostHog dependency in packages -```ts -function maybeAppTelemetry(app: FastifyInstance): TelemetrySink | undefined { - const value = (app as FastifyInstance & { telemetry?: unknown }).telemetry - return isTelemetrySink(value) ? value : undefined -} -``` +For the first pass, prefer app-only helper unless there is a clear need to publish it. --- -## 7. Frontend integration design +## 5. Package boundaries -### 7.1 Core front +### Core -`CoreFront` currently owns providers and `AppErrorBoundary`. Add: - -```ts -export interface CoreFrontProps { - children?: ReactNode - authPages?: CoreFrontAuthPagesOverride - cspNonce?: string - telemetry?: TelemetrySink -} -``` +Core may pass telemetry through app creation options and capture generic app/server events: -Wire error boundary: - -```tsx - { - safeCaptureTelemetryError(telemetry, { - name: 'core.frontend.error', - severity: 'error', - error: sanitizeError(error), - context: { route: window.location.pathname }, - properties: { - componentStackHash: hashString(errorInfo.componentStack ?? ''), - }, - }) - }} -> -``` +- `app.opened` +- `server.request.failed` +- `auth.user.signed_in` if auth hooks already exist naturally -Do not capture raw component stack by default. It can include component names but may still be noisy. Hash is enough for grouping in product telemetry. If a child app wants Sentry raw exceptions, it can do that inside its own `AppErrorBoundary` or adapter after applying its own scrubbers. +Core must not require PostHog env vars to boot. -### 7.2 Workspace front +### Agent -Add telemetry to `WorkspaceProvider` and context. +Agent accepts `telemetry?: TelemetrySink` in server/app options and emits agent events: -```tsx - - - -``` +- `agent.chat.started` +- `agent.chat.message.submitted` +- `agent.chat.completed` +- `agent.chat.failed` +- `agent.tool.started` +- `agent.tool.completed` +- `agent.tool.failed` -Capture: +No prompt text, assistant text, stdout, stderr, file contents, or command args by default. -- provider mounted -- panel opened/closed -- left tab selected -- command executed -- catalog row opened -- UI command posted/failed -- plugin/panel error boundary +### Workspace -### 7.3 Agent front +Workspace accepts `telemetry?: TelemetrySink` in provider/server options and emits UI/workspace events: -Add telemetry to `ChatPanel` and/or `useAgentChat` options. +- `workspace.opened` +- `workspace.panel.opened` +- `workspace.command.executed` +- `workspace.ui_command.posted` +- `workspace.plugin.error` -```tsx - -``` +No panel params unless explicitly allowlisted and known safe. -Frontend can capture user-intent events immediately: +--- -- message submit clicked -- attachment rejected -- chat UI error shown -- stream disconnected in browser +## 6. Minimal event properties -Backend remains source of truth for stream/tool success/failure. +Use low-cardinality metadata only. -### 7.4 Composed frontend helpers +Allowed by default: -The app-level composition helpers must forward telemetry too. Otherwise child apps using the default boring app shell would have to manually re-compose everything. +- `workspaceId` +- `sessionId` +- `requestId` +- `userId` only as `distinctId` or a safe hashed/id value already used by auth +- `runtimeMode` +- `modelProvider` +- `toolName` +- `panelId` +- `commandId` +- `status` +- `durationMs` +- `errorCode` +- `packageName` +- `packageVersion` -Required pass-throughs: +Not allowed by default: -- `CoreWorkspaceAgentFront telemetry` → `CoreFront telemetry` -- `CoreWorkspaceAgentFront telemetry` → `WorkspaceAgentFront telemetry` -- `WorkspaceAgentFront telemetry` → `WorkspaceProvider telemetry` -- `WorkspaceAgentFront telemetry` → default `ChatPanel`/chat params when it renders the default agent panel +- prompts/messages +- file paths unless explicitly normalized/approved later +- command strings +- command output +- stack traces +- raw errors +- headers/cookies/tokens +- env vars -Keep this as simple prop forwarding. Do not make workspace context the only way for agent front code to receive telemetry, because base workspace code must not value-import agent. +If a future event needs richer data, add it intentionally with a small allowlist. --- -## 8. Child app telemetry adapters +## 7. Error handling -### 8.1 Server adapter example +Telemetry must never break user flows. -Location for example: - -```txt -apps/full-app/src/telemetry/server.ts -``` - -Shape: +Rules: -```ts -export interface ServerTelemetryConfig { - appId: string - environment: string - posthog?: { - defaultProjectKey?: string - host?: string - } - sentry?: { - dsn?: string - } - routeEvent?: (event: TelemetryEvent | TelemetryErrorEvent) => Promise | TelemetryDestination -} -``` +- `capture()` calls are best-effort. +- Package call sites must not `await` telemetry on hot streaming paths unless already async and safe. +- Sink failures are swallowed or logged at debug level. +- Error events send `errorCode`, not raw error messages/stacks by default. -The example should be dependency-light. If adding actual vendor packages is too much for first pass, use documented extension points and maybe a minimal `fetch`-based PostHog capture example. +--- -### 8.2 Browser adapter example +## 8. Browser telemetry -Location: +Keep browser telemetry simple: -```txt -apps/full-app/src/telemetry/browser.ts -``` +- app shell initializes PostHog in the browser only if a public key/config is provided +- workspace/agent front providers accept the same `TelemetrySink` shape +- no package reads `VITE_POSTHOG_KEY` directly -Browser env vars are public: +The app can expose config however it already exposes runtime config. +For Vite demo apps, use: ```bash -VITE_POSTHOG_KEY= +VITE_POSTHOG_KEY=phc_... VITE_POSTHOG_HOST=https://us.i.posthog.com -VITE_SENTRY_DSN= -``` - -CSP notes must be documented because browser telemetry needs `connect-src` additions. - -### 8.3 Multi-account routing - -Example routing: - -```ts -const app = await createCoreWorkspaceAgentServer({ - createTelemetry: ({ config, workspaceStore }) => createServerTelemetry({ - appId: config.appId, - environment: process.env.NODE_ENV ?? 'development', - routeEvent: async (event) => { - const workspaceId = event.context?.workspaceId - if (!workspaceId) return defaultDestination - - const settings = await workspaceStore.getWorkspaceSettings(workspaceId) - const posthogProjectKey = settings.find((s) => s.key === 'POSTHOG_PROJECT_KEY') - if (posthogProjectKey?.configured) { - return { posthog: { projectKey: await decryptSetting(workspaceId, 'POSTHOG_PROJECT_KEY') } } - } - - return defaultDestination - }, - }), -}) -``` - -This is why package-level provider initialization is wrong. Most apps should use plain `telemetry`; apps that need store-backed routing use `createTelemetry`. - ---- - -## 9. Event taxonomy - -Event names use stable dotted names. - -Naming rules: - -- prefix by package/domain: `core.*`, `agent.*`, `workspace.*` -- use past-tense lifecycle names: `created`, `started`, `completed`, `failed` -- use stable low-cardinality event names -- put high-cardinality values in properties only if safe -- no raw user text in event names or properties - -### 9.1 Core events - -| Event | When | Context | Properties | -|---|---|---|---| -| `core.app.started` | server boot completes | appId, environment, deploymentId | packageVersion | -| `core.http.request.failed` | 5xx/unhandled route error | requestId, route, method, userId, workspaceId | statusCode, errorCode, durationMs | -| `core.auth.sign_in.completed` | sign-in succeeds | userId | provider | -| `core.auth.sign_in.failed` | sign-in fails | requestId | statusCode, reason | -| `core.auth.sign_up.completed` | sign-up succeeds | userId | provider | -| `core.workspace.created` | workspace created | userId, workspaceId | isDefault | -| `core.workspace.updated` | workspace changed | userId, workspaceId | changedFieldsCount | -| `core.workspace.deleted` | workspace removed | userId, workspaceId | hadRuntime | -| `core.workspace.opened` | front loads active workspace | userId, workspaceId | source | -| `core.frontend.error` | React boundary catches error | route, userId, workspaceId | componentStackHash | - -Auth events may be phased later if better-auth hooks are awkward or privacy review wants fewer identity events first. - -### 9.2 Agent events - -| Event | When | Context | Properties | -|---|---|---|---| -| `agent.chat.session.created` | session created | workspaceId, sessionId, userId | source | -| `agent.chat.message.sent` | user message accepted | workspaceId, sessionId, userId | messageLength, attachmentCount | -| `agent.chat.stream.started` | backend stream opens | workspaceId, sessionId, requestId, runtimeMode | modelProvider, modelName | -| `agent.chat.stream.completed` | stream finishes normally | workspaceId, sessionId, requestId, runtimeMode | durationMs, chunkCount, toolCallCount | -| `agent.chat.stream.failed` | stream errors | workspaceId, sessionId, requestId, runtimeMode | durationMs, errorCode, statusCode | -| `agent.tool.started` | tool call begins | workspaceId, sessionId, runtimeMode | toolName | -| `agent.tool.completed` | tool call succeeds | workspaceId, sessionId, runtimeMode | toolName, durationMs | -| `agent.tool.failed` | tool call fails | workspaceId, sessionId, runtimeMode | toolName, durationMs, errorCode | -| `agent.runtime.binding.created` | runtime binding created | workspaceId, runtimeMode | fsCapability | -| `agent.runtime.binding.recreated` | expired sandbox recreated | workspaceId, runtimeMode | reason | -| `agent.sandbox.failed` | sandbox/runtime operation failed | workspaceId, runtimeMode | statusCode, errorCode | -| `agent.plugin.load.failed` | pi plugin failed to load | workspaceId | pluginSourceHash, errorCode | - -### 9.3 Workspace events - -| Event | When | Context | Properties | -|---|---|---|---| -| `workspace.provider.mounted` | provider initializes | userId, workspaceId | pluginCount | -| `workspace.panel.opened` | dockview opens panel | workspaceId | panelId, placement, source | -| `workspace.panel.closed` | dockview closes panel | workspaceId | panelId, durationMs | -| `workspace.left_tab.selected` | left tab selected | workspaceId | tabId | -| `workspace.command.executed` | command palette command runs | workspaceId | commandId, source | -| `workspace.catalog.searched` | catalog search submitted | workspaceId | catalogId, queryLength, resultCount | -| `workspace.catalog.row.opened` | catalog row opens surface | workspaceId | catalogId, kind | -| `workspace.ui_command.posted` | server posts UI command | workspaceId | commandType, source | -| `workspace.ui_command.dispatched` | front dispatches command | workspaceId | commandType | -| `workspace.ui_command.failed` | command cannot dispatch | workspaceId | commandType, reason | -| `workspace.plugin.error` | plugin boundary catches error | workspaceId, pluginId | panelId, componentStackHash | - ---- - -## 10. Privacy and content policy - -### 10.1 Default capture matrix - -| Data | Default | Rationale | -|---|---:|---| -| userId | yes | app already owns auth; useful for support | -| workspaceId | yes | needed for workspace-level reliability | -| sessionId | yes | needed for chat debugging | -| requestId | yes | joins logs/errors | -| route template | yes | operational metadata | -| raw URL query string | no | may include secrets/search text | -| status code | yes | operational metadata | -| durationMs | yes | performance | -| runtimeMode | yes | compare direct/local/vercel-sandbox | -| toolName | yes | low-cardinality product metadata | -| model provider/name | yes | cost/reliability analysis | -| prompt text | no | sensitive content | -| assistant text/code | no | sensitive content/IP | -| file contents | no | sensitive/IP | -| file path | no by default | may reveal project structure; app can opt into hash/basename | -| command stdout/stderr | no | may include secrets | -| command string | no by default | may include secrets; maybe hash only later | -| env vars | never | secrets | -| cookies/auth headers | never | secrets | - -### 10.2 Sanitization helpers - -Add helpers where needed: - -```ts -function sanitizeError(error: unknown): TelemetryErrorInfo -function sanitizeErrorCode(error: unknown): string | undefined -function sanitizeStatusCode(error: unknown): number | undefined -function lengthOnly(value: string | undefined): number -function safeRoute(request: FastifyRequest): string -function truncateTelemetryString(value: string, maxLength?: number): string +VITE_BORING_TELEMETRY_PROJECT=workspace-playground ``` -Do not add generic deep object telemetry serialization. That invites accidental content capture. - -### 10.3 Event allowlists - -Privacy tests should validate allowlisted fields per event name, not only `not.toContain('secret')` snapshots. - -Simple rule: each package keeps a tiny test-only map of expected event names and allowed `context`/`properties` keys for the events it emits. Tests fail if code starts spreading raw request/body/error objects into telemetry. - -Do not build a big runtime schema system in v1. Test allowlists are enough. - -### 10.3 User consent - -The package-level sink contract should not enforce consent. The child app adapter should decide: - -- disabled until user opts in -- enabled for operational error events only -- enabled for all product events -- disabled for enterprise deployments - -This can be a `beforeSend` hook in the child-app adapter. - -```ts -beforeSend(event) { - if (!userConsentAllows(event)) return null - return sanitize(event) -} -``` +The browser helper should follow the same prefix rule as the server helper. --- -## 11. Core implementation plan - -### 11.1 Shared contract - -Files: - -```txt -packages/core/src/shared/telemetry.ts -packages/core/src/shared/index.ts -``` - -Exports: - -- `TelemetrySeverity` -- `TelemetryContext` -- `TelemetryEvent` -- `TelemetryErrorEvent` -- `TelemetrySink` -- `noopTelemetry` -- `safeCaptureTelemetry` -- `safeCaptureTelemetryError` +## 9. Implementation plan -Tests: +### Bead 1 — telemetry contract and no-op -```txt -packages/core/src/shared/__tests__/telemetry.test.ts -``` +- Add `TelemetrySink`, `TelemetryEvent`, and `noopTelemetry` where each package can use them without adding forbidden dependencies. +- Add a safe wrapper/helper if useful. +- Add unit tests for no-op and prefix formatting if helper exists. Acceptance: -- noop accepts events -- safe helpers swallow sync throw -- safe helpers swallow async rejection -- helpers call logger warn when provided - -### 11.2 Fastify decoration - -Files: +- packages compile without PostHog installed unless app/helper needs it +- no `node:*` or `Buffer` in shared files +- no telemetry call can throw into product code -```txt -packages/core/src/server/app/types.ts -packages/core/src/server/app/createCoreApp.ts -``` +### Bead 2 — PostHog env helper in app/example -Add `telemetry` option and decoration. +- Add `createPostHogTelemetryFromEnv()` for server usage. +- Add browser equivalent for Vite/demo usage if needed. +- Support `POSTHOG_KEY`, `POSTHOG_HOST`, `BORING_TELEMETRY_ENABLED`, and `BORING_TELEMETRY_PROJECT`. +- Wire it in `apps/full-app` or the canonical demo shell. Acceptance: -- `createCoreApp(config)` exposes noop telemetry -- `createCoreApp(config, { telemetry })` exposes supplied sink -- no behavior change when omitted - -### 11.3 Error handler capture - -File: - -```txt -packages/core/src/server/app/errorHandler.ts -``` - -Capture only server failures and validation/rate-limit failures if useful. - -First pass recommendation: +- unset env = no-op +- `BORING_TELEMETRY_ENABLED=false` = no-op even with key +- project prefix changes event names and adds `boringProject` -- capture 5xx in `captureError` -- optionally capture 4xx auth/rate-limit later +### Bead 3 — core/agent/workspace event calls -Payload: - -```ts -{ - name: 'core.http.request.failed', - severity: 'error', - error: sanitizeError(error), - context: { - requestId: request.id, - route: request.routeOptions?.url, - method: request.method, - userId: request.user?.id, - }, - properties: { - statusCode: 500, - errorCode: 'internal_error', - }, -} -``` +- Thread `telemetry?: TelemetrySink` through existing app/provider/server options. +- Add only the minimal event list from this plan. +- Keep privacy allowlist strict. Acceptance: -- HTTP response unchanged -- telemetry sink receives event on 500 -- telemetry sink failure does not change response +- tests prove expected events are emitted with safe metadata +- tests prove prompts/output/file contents are not included +- quality gates pass -### 11.4 Front error boundary capture +### Bead 4 — docs -Files: - -```txt -packages/core/src/front/CoreFront.tsx -packages/core/src/front/AppErrorBoundary.tsx -``` - -`AppErrorBoundary` already supports `onError`; wire from `CoreFront` prop. +- Document env vars. +- Document event names. +- Document privacy rules. +- Add a short “shared PostHog account” example using `BORING_TELEMETRY_PROJECT`. Acceptance: -- rendering crash calls telemetry -- no telemetry prop means no crash in tests -- no raw component stack in product properties unless explicitly accepted +- docs match implementation +- no extra vendor/routing/Sentry/OTLP scope sneaks back in --- -## 12. Agent implementation plan - -### 12.1 Shared contract - -Files: - -```txt -packages/agent/src/shared/telemetry.ts -packages/agent/src/shared/index.ts -``` - -Same structural contract as core. - -### 12.2 Options - -Files: - -```txt -packages/agent/src/server/createAgentApp.ts -packages/agent/src/server/registerAgentRoutes.ts -``` - -Add: - -```ts -telemetry?: TelemetrySink -``` - -Embedded routes fallback to `app.telemetry` if structurally present. - -### 12.3 Chat route events - -Likely files: - -```txt -packages/agent/src/server/http/routes/chat.ts -packages/agent/src/server/http/routes/sessions.ts -``` - -Capture: - -- message accepted -- stream started -- stream completed -- stream failed -- stream aborted -- session created - -Important: do not capture message content. Only length/count. - -Current chat streaming can handle generator errors inside the stream executor, write an error chunk, and still finish HTTP plumbing. Therefore telemetry must live inside the stream execution path, not only an outer route `try/catch`. - -Simple terminal rule: - -- emit `agent.chat.stream.started` once when backend execution starts -- emit exactly one terminal event per backend execution: - - `agent.chat.stream.completed` for normal completion - - `agent.chat.stream.failed` for harness/generator/tool error - - `agent.chat.stream.aborted` for client abort/cancel when detectable -- resume/replay endpoints must not emit a second terminal event for an already-finished stream - -Acceptance: - -- test sink receives stream lifecycle events -- failed stream emits failure event with sanitized error code/status -- successful stream emits completion duration -- aborted stream emits aborted, not completed -- no duplicate terminal events on resume/replay -- no event snapshot includes prompt text +## 10. Explicit non-goals for this PR -### 12.4 Tool lifecycle events +Do not implement now: -Preferred design: wrap tools once when building the tool list, not inside every tool implementation. +- Sentry integration +- OpenTelemetry/OTLP +- per-tenant provider routing +- database-backed telemetry storage +- billing/metering +- content capture +- complex consent system +- automatic CSP management +- multi-account PostHog routing -```ts -function withToolTelemetry( - tools: AgentTool[], - telemetry: TelemetrySink, - getContext: () => TelemetryContext, -): AgentTool[] -``` +If any of those become necessary, make separate beads later. -Wrapper captures: +--- -- `agent.tool.started` -- `agent.tool.completed` -- `agent.tool.failed` +## 11. Definition of done -The exact `AgentTool` shape must be read before implementation. Do not assume handler property name. - -Acceptance: - -- every standard/extra/plugin tool is wrapped once -- wrapper preserves tool schema/name/description/onUpdate behavior -- wrapper does not alter tool result/errors -- failures still propagate to harness -- `agent.tool.failed` is emitted for thrown errors and for tool results with `isError === true` -- wrapper combines binding context (`workspaceId`, `runtimeMode`) with tool execution context (`sessionId`) - -### 12.5 Runtime/sandbox events - -Locations: - -```txt -packages/agent/src/server/registerAgentRoutes.ts -packages/agent/src/server/runtime/* -packages/agent/src/server/sandbox/vercel-sandbox/* -``` - -First pass can capture at route registration layer: - -- runtime binding created -- runtime binding recreated because sandbox expired -- runtime binding creation failed - -Avoid deep invasive instrumentation in sandbox adapters until needed. - ---- - -## 13. Workspace implementation plan - -### 13.1 Shared contract - -Files: - -```txt -packages/workspace/src/shared/telemetry.ts -packages/workspace/src/shared/index.ts -``` - -Browser-safe only. No `node:*`, no `Buffer`. - -### 13.2 Provider wiring - -Likely files: - -```txt -packages/workspace/src/front/WorkspaceProvider.tsx -packages/workspace/src/front/context/* -``` - -Add telemetry to workspace runtime context. - -Acceptance: - -- omitted telemetry defaults noop -- plugins/panels can access telemetry through workspace context if needed -- no agent import added to base workspace code - -### 13.3 UI events - -Capture at central dispatch points, not scattered everywhere. - -Preferred central points: - -- panel open API / surface resolver dispatch -- command execution registry -- catalog row open helper -- UI bridge command dispatcher -- plugin/panel error boundaries - -Acceptance: - -- opening a panel emits one event -- command execution emits one event -- failed UI command emits failure event -- plugin error boundary emits error event - -### 13.4 Workspace server events - -A few workspace events are server-owned, not front-owned. Keep this small and focused. - -Wire optional telemetry into: - -- `uiRoutes` for UI command HTTP route failures -- `createWorkspaceUiTools` / `exec_ui` for server-posted UI commands -- `createWorkspaceAgentServer` composition helper -- core composed server when it registers workspace UI routes/tools - -Acceptance: - -- server-posted UI command emits `workspace.ui_command.posted` -- `exec_ui` failure emits `workspace.ui_command.failed` -- no new dependency from workspace server to agent/core internals - ---- - -## 14. App example plan - -### 14.1 `apps/full-app` server example - -Files: - -```txt -apps/full-app/src/telemetry/server.ts -apps/full-app/src/server/main.ts -apps/full-app/src/server/vercel-entry.ts -apps/full-app/.env.example -apps/full-app/README.md -``` - -Implement a minimal app-level adapter. Depending on dependency appetite: - -Option A — dependency-free example: - -- use `fetch` to PostHog capture API if configured -- use console or placeholder for Sentry -- document where to plug actual SDK - -Option B — real adapters: - -- add `posthog-node` -- add `@sentry/node` -- add `@sentry/react` for browser - -Recommendation for first implementation: start with dependency-free or optional adapters. Do not force new vendor deps into packages. - -### 14.2 Browser example - -Files: - -```txt -apps/full-app/src/telemetry/browser.ts -apps/full-app/src/front/main.tsx -``` - -Initialize browser telemetry only if public env vars exist. - -### 14.3 CSP updates - -Core CSP currently has `connectSrc: ['self']`. If browser telemetry is enabled, child app needs a way to extend CSP connect sources. - -Possible approaches: - -1. Add core config for CSP extra connect sources. -2. Document that app must override config/security. -3. Server injects CSP based on app config. - -This is a blocker for browser PostHog/Sentry in production. Server-side telemetry is not blocked. - -Recommended first bead for CSP: - -- add config field `security.csp.connectSrcExtra?: string[]` -- wire it through core config schema/loadConfig/types -- use it in `createCoreApp` helmet config -- app example sets PostHog/Sentry hosts when env vars exist - -Keep this small: only extend `connect-src`; do not redesign the whole CSP system. - ---- - -## 15. Data flow examples - -### 15.1 Chat stream failure - -```txt -User sends message - ↓ -ChatPanel/useAgentChat posts to /api/v1/agent/chat/:sessionId - ↓ -agent chat route emits agent.chat.message.sent - ↓ -stream starts: agent.chat.stream.started - ↓ -harness/tool/sandbox throws - ↓ -chat stream executor marks stream failed and emits sanitized terminal event - ↓ -telemetry.captureError(agent.chat.stream.failed) - ↓ -child app adapter sends sanitized event to PostHog/Sentry -``` - -Captured data: - -```json -{ - "name": "agent.chat.stream.failed", - "context": { - "workspaceId": "ws_123", - "sessionId": "sess_456", - "requestId": "req_789", - "runtimeMode": "vercel-sandbox" - }, - "properties": { - "durationMs": 1832, - "errorCode": "sandbox_expired", - "statusCode": 410 - } -} -``` - -Not captured: - -- prompt body -- generated code -- command output - -### 15.2 Workspace panel open - -```txt -User runs command palette command - ↓ -workspace command registry executes open panel - ↓ -panel open central function emits workspace.panel.opened - ↓ -child app adapter sends to PostHog -``` - -### 15.3 Multi-account event routing - -```txt -agent.chat.stream.failed workspaceId=customer_a - ↓ -app routeEvent reads workspace/customer mapping - ↓ -send to Customer A PostHog project and shared Sentry project - -agent.chat.stream.failed workspaceId=customer_b - ↓ -send to Customer B PostHog project and shared Sentry project -``` - ---- - -## 16. Performance considerations - -Telemetry must be low overhead. - -Rules: - -- Do not block chat streaming on telemetry network calls. -- Use fire-and-forget safe wrappers for most events. -- Only `flush()` on graceful shutdown if available. -- Avoid serializing large objects. -- Keep properties low-cardinality and small. -- Avoid hashing huge strings/content. -- Batch in child-app adapters if vendor supports it. - -Shutdown: - -`createCoreApp` already has graceful shutdown handling. If telemetry has `flush`, call it during shutdown/onClose, but do not hang beyond grace window. - ---- - -## 17. Error handling rules - -Telemetry must never cause: - -- failed HTTP request -- broken chat stream -- broken React render -- failed tool call -- failed app boot, unless the child app adapter intentionally throws during construction - -Package helpers swallow runtime telemetry sink errors. - -The only errors that should fail boot are explicit child-app configuration errors, e.g. malformed DSN in `createAppTelemetry`, and those happen in app code. - ---- - -## 18. Testing strategy - -### 18.1 Unit tests - -Core: - -- noop/safe helper behavior -- Fastify decoration with noop/custom sink -- error handler emits event on 500 -- frontend boundary emits event - -Agent: - -- noop/safe helper behavior -- `createAgentApp({ telemetry })` wires sink -- `registerAgentRoutes({ telemetry })` wires sink -- chat success/failure events -- tool wrapper success/failure events -- no prompt text in snapshots -- exactly-one terminal stream event for success/failure/abort -- resume/replay does not duplicate terminal events - -Workspace: - -- provider defaults noop -- panel open event -- command event -- UI command failure event -- plugin error boundary event - -### 18.2 Integration tests - -- composed `createCoreWorkspaceAgentServer({ telemetry })` receives both core and agent events through one sink -- request workspace ID appears in agent events -- user ID appears when auth hook has populated `request.user` -- telemetry sink rejection does not alter response status/body - -### 18.3 Privacy/allowlist tests - -Build representative captured events and assert they do not contain forbidden strings: - -```ts -expect(JSON.stringify(events)).not.toContain('my secret prompt') -expect(JSON.stringify(events)).not.toContain('DATABASE_URL') -expect(JSON.stringify(events)).not.toContain('file contents') -expect(JSON.stringify(events)).not.toContain('stdout with token') -``` - -Also assert each emitted event uses only allowed context/property keys for that event name. This is the main guard against accidental object spreading. - -Examples: - -```ts -expectEventKeys(events, { - 'agent.chat.stream.failed': { - context: ['workspaceId', 'sessionId', 'requestId', 'runtimeMode'], - properties: ['durationMs', 'errorCode', 'statusCode'], - error: ['code', 'statusCode', 'className'], - }, -}) -``` - -### 18.4 Quality gates - -Run relevant scoped tests while implementing: - -```bash -pnpm --filter @hachej/boring-core test -pnpm --filter @hachej/boring-agent test -pnpm --filter @hachej/boring-workspace test -``` - -Before close: - -```bash -pnpm typecheck -pnpm lint -pnpm test -pnpm lint:invariants -``` - ---- - -## 19. Rollout strategy - -### Phase 1 — contracts/noop - -Lowest risk. No behavior changes. - -Deliver: - -- shared telemetry types in all packages -- sanitized error metadata type -- noop and safe helpers -- public exports -- tests - -### Phase 2 — core server/front - -Deliver: - -- `createCoreApp({ telemetry })` -- Fastify decoration -- sanitized error handler capture -- `CoreFront telemetry` prop -- frontend boundary capture - -### Phase 3 — agent chat events - -Deliver: - -- `createAgentApp({ telemetry })` -- `registerAgentRoutes({ telemetry })` -- chat/session stream lifecycle events -- exactly-one terminal stream event semantics -- privacy/allowlist tests - -### Phase 4 — composed passthrough - -Deliver: - -- `createCoreWorkspaceAgentServer({ telemetry })` -- optional `createTelemetry({ config, userStore, workspaceStore })` -- composed frontend telemetry prop forwarding - -### Phase 5 — agent tool/runtime events - -Deliver: - -- tool wrapper -- thrown-error and `isError` failure detection -- runtime binding created/recreated/failed -- sandbox expired/recreated event - -### Phase 6 — workspace UI/server UI-command events - -Deliver: - -- provider context -- panel/command/catalog/UI bridge/plugin error events -- `uiRoutes` and `exec_ui` server-side events - -### Phase 7 — CSP support - -Deliver: - -- `security.csp.connectSrcExtra?: string[]` -- Helmet `connectSrc` append support -- tests for default and extra hosts - -### Phase 8 — app examples and docs - -Deliver: - -- `apps/full-app` app-level adapter example -- README docs -- event taxonomy docs - ---- - -## 20. Risks and mitigations - -### Risk: accidental content capture - -Mitigation: - -- explicit allowlisted properties only -- tests asserting forbidden content absent -- no generic object spreading into telemetry payloads - -### Risk: telemetry breaks streaming performance - -Mitigation: - -- fire-and-forget safe wrappers -- no await in stream hot path unless using in-memory sink in tests -- child-app adapter handles batching - -### Risk: event cardinality explosion - -Mitigation: - -- stable event names -- avoid raw paths/URLs/queries -- avoid arbitrary error messages as dimensions - -### Risk: duplicate events frontend/backend - -Mitigation: - -- define source-of-truth per event -- backend owns chat/tool success/failure -- frontend owns UI interaction events -- if both emit, use different event names (`message.submitted` vs `message.sent`) - -### Risk: package dependency tangle - -Mitigation: - -- structural per-package contracts -- no imports between packages for telemetry types -- no vendor SDKs in packages - -### Risk: CSP blocks browser analytics - -Mitigation: - -- app-level CSP extension config -- docs list required `connect-src` hosts - ---- - -## 21. Open questions - -1. Should auth success/failure events be in the first implementation, or deferred until core auth hooks are reviewed? -2. Should file paths be omitted entirely by default or captured as stable salted hashes? -3. Should event names be exported constants, or keep string literals plus docs? -4. Should server request telemetry capture all 4xx or only 5xx? -5. Should we add an explicit user consent API in core front, or leave consent entirely to child apps? -6. Should agent use existing `@opentelemetry/api` dependency for optional spans later? -7. Should child-app examples use actual vendor SDKs or dependency-free fetch adapters? -8. Should telemetry sink support `identify(user)` and `group(workspace)` helpers, or keep only event capture? - -Recommendation for v1: - -- keep only `capture`, `captureError`, `flush` -- defer `identify/group` to app adapter or future expansion -- start with 5xx/chat/tool/UI events before auth analytics - ---- - -## 22. Bead breakdown - -The beads should stay small. Do not build a tracing platform. Ship a simple sink contract, a few high-value events, and tests that prevent content leaks. - -### Bead 1 — `telemetry-contracts-noop` - -**Goal:** add vendor-neutral telemetry contracts and noop/safe helpers to core, agent, workspace. - -Files: - -```txt -packages/core/src/shared/telemetry.ts -packages/core/src/shared/index.ts -packages/core/src/shared/__tests__/telemetry.test.ts -packages/agent/src/shared/telemetry.ts -packages/agent/src/shared/index.ts -packages/agent/src/shared/__tests__/telemetry.test.ts -packages/workspace/src/shared/telemetry.ts -packages/workspace/src/shared/index.ts -packages/workspace/src/shared/__tests__/telemetry.test.ts -``` - -Acceptance: - -- types exported from public shared barrels -- noop works -- safe helpers swallow sync/async failures -- `TelemetryErrorEvent.error` is sanitized metadata, not raw `unknown` -- no `node:*` or `Buffer` in shared files - -Dependencies: none. - -### Bead 2 — `core-telemetry-hooks` - -**Goal:** wire telemetry into core server and frontend error boundary. - -Files: - -```txt -packages/core/src/server/app/types.ts -packages/core/src/server/app/createCoreApp.ts -packages/core/src/server/app/errorHandler.ts -packages/core/src/front/CoreFront.tsx -packages/core/src/front/__tests__/*telemetry*.test.tsx -packages/core/src/server/app/__tests__/*telemetry*.test.ts -``` - -Acceptance: - -- `createCoreApp(config, { telemetry })` decorates app -- 500 handler emits `core.http.request.failed` with sanitized error metadata -- `CoreFront telemetry` captures boundary crash without raw component stack in properties -- telemetry failure does not change responses/rendering - -Dependencies: Bead 1. - -### Bead 3 — `composed-app-telemetry-passthrough` - -**Goal:** pass app-level telemetry through default composed server and front helpers. - -Files: - -```txt -packages/core/src/app/server/createCoreWorkspaceAgentServer.ts -packages/core/src/app/server/__tests__/*telemetry*.test.ts -packages/core/src/app/front/CoreWorkspaceAgentFront.tsx -packages/core/src/app/front/__tests__/*telemetry*.test.tsx -packages/workspace/src/app/front/WorkspaceAgentFront.tsx -packages/workspace/src/app/front/__tests__/*telemetry*.test.tsx -``` - -Acceptance: - -- simple `telemetry` sink works for composed server -- optional `createTelemetry({ config, userStore, workspaceStore })` works for store-backed routing -- resolved sink is passed to core and agent route registration -- frontend telemetry prop forwards to `CoreFront`, `WorkspaceAgentFront`, `WorkspaceProvider`, and default chat panel params -- omitted sink remains noop - -Dependencies: Beads 1-2 and agent options from Bead 4 if done together. - -### Bead 4 — `agent-chat-telemetry` - -**Goal:** wire telemetry into agent server routes and capture chat/session lifecycle. - -Files: - -```txt -packages/agent/src/server/createAgentApp.ts -packages/agent/src/server/registerAgentRoutes.ts -packages/agent/src/server/http/routes/chat.ts -packages/agent/src/server/http/routes/sessions.ts -packages/agent/src/server/http/routes/__tests__/*telemetry*.test.ts -``` - -Acceptance: - -- `createAgentApp({ telemetry })` accepted -- `registerAgentRoutes({ telemetry })` accepted and can fall back to structural `app.telemetry` -- chat started/completed/failed/aborted events emitted from the stream execution path -- exactly one terminal stream event per backend execution -- resume/replay does not duplicate terminal events -- message/session events use counts/lengths only -- allowlist/privacy tests pass - -Dependencies: Bead 1. - -### Bead 5 — `agent-tool-runtime-telemetry` - -**Goal:** capture tool and runtime/sandbox failures. - -Files: - -```txt -packages/agent/src/server/tools/* or new telemetry wrapper location -packages/agent/src/server/registerAgentRoutes.ts -packages/agent/src/server/__tests__/*tool-telemetry*.test.ts -``` - -Acceptance: - -- all tools wrapped once -- started/completed/failed emitted -- failed means thrown error OR `ToolResult.isError === true` -- wrapper preserves schema/name/description/onUpdate/result/error behavior -- runtime binding recreate/failure emits event -- allowlist/privacy tests pass - -Dependencies: Bead 4. - -### Bead 6 — `workspace-ui-telemetry` - -**Goal:** capture workspace UI usage and the small set of workspace-owned server UI command events. - -Files: - -```txt -packages/workspace/src/front/** -packages/workspace/src/front/components/PanelErrorBoundary.tsx -packages/workspace/src/front/plugin/PluginErrorBoundary.tsx -packages/workspace/src/server/ui-control/http/uiRoutes.ts -packages/workspace/src/server/ui-control/tools/uiTools.ts -packages/workspace/src/app/server/createWorkspaceAgentServer.ts -packages/workspace/src/front/**/__tests__/*telemetry*.test.tsx -packages/workspace/src/server/**/__tests__/*telemetry*.test.ts -``` - -Acceptance: - -- `WorkspaceProvider telemetry` accepted -- panel open/close emits events -- commands emit events -- UI command posted/failure emits events server-side where appropriate -- plugin/panel boundary emits sanitized error event -- no workspace base front/shared value import from agent - -Dependencies: Bead 1. - -### Bead 7 — `core-csp-connect-src-extra` - -**Goal:** let child apps opt into browser telemetry endpoints without weakening CSP globally. - -Files: - -```txt -packages/core/src/shared/types.ts -packages/core/src/server/config/* -packages/core/src/server/app/createCoreApp.ts -packages/core/src/server/app/__tests__/*csp*.test.ts -``` - -Acceptance: - -- config supports `security.csp.connectSrcExtra?: string[]` -- values append to Helmet `connectSrc` -- default remains `connectSrc: ["'self'"]` -- tests cover default and extra PostHog/Sentry-style hosts - -Dependencies: none, but needed before browser telemetry example is useful in production. - -### Bead 8 — `full-app-telemetry-example` - -**Goal:** show child-app-owned PostHog/Sentry-style routing with minimal dependencies. - -Files: - -```txt -apps/full-app/src/telemetry/server.ts -apps/full-app/src/telemetry/browser.ts -apps/full-app/src/server/main.ts -apps/full-app/src/server/vercel-entry.ts -apps/full-app/src/front/main.tsx -apps/full-app/README.md -apps/full-app/.env.example -``` - -Acceptance: - -- app creates telemetry adapter -- app passes it into composed server/front -- docs explain plain `telemetry` vs advanced `createTelemetry` -- adapter has `beforeSend`, bounded queue or clear drop behavior, and flush timeout -- docs explain privacy defaults - -Dependencies: Beads 2-7. - -### Bead 9 — `telemetry-docs-event-taxonomy` - -**Goal:** publish stable event taxonomy and privacy policy docs. - -Files: - -```txt -packages/core/docs/CORE.md or docs/telemetry.md -packages/agent/docs/plans/agent-package-spec.md or package docs -packages/workspace/docs/INTERFACES.md or package docs -README.md -``` - -Acceptance: - -- event names documented -- sanitized payload rules documented -- child app integration documented -- no vendor is presented as mandatory - -Dependencies: Beads 1-8. - ---- - -## 23. Recommended implementation order - -1. Bead 1 — contracts/noop. -2. Bead 4 — agent chat telemetry, because original user need includes chat session errors. -3. Bead 2 — core error/front hooks. -4. Bead 3 — composed app passthrough. -5. Bead 5 — tool/runtime events. -6. Bead 6 — workspace UI/server UI-command events. -7. Bead 7 — CSP connect-src extra. -8. Bead 8 — app examples. -9. Bead 9 — docs finalization. - -Reasoning: - -- Chat errors are the urgent user value. -- Contracts are required first. -- Core passthrough is what makes app-level declaration real. -- CSP can wait until browser vendor examples, but must land before production browser telemetry docs. -- Workspace UI events can follow once backend reliability is covered. - ---- - -## 24. Definition of done - -Telemetry work is done when: - -- child app can pass one telemetry sink into composed boring app -- core emits server/frontend error events -- agent emits chat stream failure/success events -- agent emits tool failure/success events -- workspace emits central UI usage/error events -- no package imports PostHog/Sentry directly -- no prompt/file/command output content is captured by default -- examples show PostHog/Sentry-style routing at app level -- tests prove telemetry failures do not break product behavior -- docs explain event taxonomy and privacy guarantees - ---- - -## 25. Review prompt for next model pass - -Use this exact review prompt with a stronger reasoning model: - -```txt -Carefully review this entire plan for me and come up with your best revisions in terms of better architecture, new features, changed features, etc. to make it better, more robust/reliable, more performant, more compelling/useful, etc. For each proposed change, give me your detailed analysis and rationale/justification for why it would make the project better along with the git-diff style change versus the original plan shown below: - - -``` +This plan is done when: -After receiving review, integrate revisions in-place and then convert to beads. +- PR 65 describes the simplified PostHog-only direction. +- The event prefix/env-var contract is clear. +- The implementation work is small enough to start without another architecture pass. From eed6de1147c1f6d5a4c9584ee6dcb6f4e318a162 Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 20:28:36 +0000 Subject: [PATCH 03/14] docs(plan): make telemetry env opt-in via core --- .../docs/plans/app-level-telemetry-plan.md | 62 ++++++++++++------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md index 2523b1f7e..ea48c9591 100644 --- a/packages/core/docs/plans/app-level-telemetry-plan.md +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -16,8 +16,9 @@ We do not need that now. New shape: - PostHog is the only planned provider. -- The child app owns PostHog env vars and client creation. -- Packages receive a tiny optional telemetry sink and default to no-op. +- Core owns the PostHog env helper for core-composed apps. +- Child apps enable telemetry and pass credentials with env vars; no app code is required for the common path. +- Agent/workspace internals receive a tiny optional telemetry sink and default to no-op. - Shared PostHog accounts are handled with a stable project prefix/property, not routing logic. - No Sentry, OTLP, tenant routing, event bus, or analytics database in this pass. @@ -60,6 +61,16 @@ export interface TelemetryEvent { Every package option that needs telemetry accepts `telemetry?: TelemetrySink`. If absent, it uses a no-op sink. +For core-composed apps, core creates the sink from env automatically unless a custom `telemetry` sink is passed. This gives the common child-app usage: + +```bash +BORING_TELEMETRY_ENABLED=true +POSTHOG_KEY=phc_... +BORING_TELEMETRY_PROJECT=full-app +``` + +No extra child-app code is required. Advanced apps can still pass `telemetry` explicitly to override the env helper. + Package code emits events like: ```ts @@ -74,7 +85,7 @@ telemetry.capture({ }) ``` -The final app decides how to build the PostHog sink from env vars. +Core builds the default PostHog sink from env vars for core-composed apps. Advanced apps can override this by passing a custom `TelemetrySink`. --- @@ -83,20 +94,22 @@ The final app decides how to build the PostHog sink from env vars. Use boring-ui env names for boring-ui behavior, and PostHog env names for PostHog credentials. ```bash -# Required to send telemetry. +# Required to send telemetry. If omitted, telemetry is no-op. +BORING_TELEMETRY_ENABLED=true + +# Required when telemetry is enabled. POSTHOG_KEY=phc_... -# Optional. Defaults in the app helper if omitted. +# Optional. Defaults in the core helper if omitted. POSTHOG_HOST=https://us.i.posthog.com -# Optional. If unset, telemetry is enabled when POSTHOG_KEY is set. -BORING_TELEMETRY_ENABLED=true - # Optional, but recommended when several apps share one PostHog account/project. # Used both as an event-name prefix and as a property. BORING_TELEMETRY_PROJECT=full-app ``` +Telemetry is **off by default**. If `BORING_TELEMETRY_ENABLED` is unset or set to `false`, core returns `noopTelemetry` even if other app code imports the helper. + ### Project prefix rule If `BORING_TELEMETRY_PROJECT=full-app`, event names sent to PostHog become: @@ -129,11 +142,12 @@ If the prefix is unset, send the raw event name. ## 4. PostHog sink helper -Implement this helper in the app/example layer, not deep inside agent/workspace internals: +Implement this helper in core's app/server layer, not inside agent/workspace internals: ```ts +// packages/core/src/server/telemetry/posthog.ts export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink { - const enabled = env.BORING_TELEMETRY_ENABLED !== 'false' && Boolean(env.POSTHOG_KEY) + const enabled = env.BORING_TELEMETRY_ENABLED === 'true' && Boolean(env.POSTHOG_KEY) if (!enabled) return noopTelemetry @@ -161,12 +175,15 @@ export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink } ``` -Exact file location can be chosen during implementation. Preferred: +Core-composed app entrypoints use it by default: + +```ts +const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) +``` -- reusable helper: `packages/core/src/server/telemetry/posthog.ts`, if core already owns app env/config helpers -- app-only helper: `apps/full-app/src/server/telemetry.ts`, if we want zero PostHog dependency in packages +Child apps that want telemetry set env vars. Child apps that do not want telemetry leave `BORING_TELEMETRY_ENABLED` unset or set it to `false`. -For the first pass, prefer app-only helper unless there is a clear need to publish it. +Advanced apps may still pass their own `telemetry` sink to bypass the PostHog env helper. --- @@ -174,13 +191,13 @@ For the first pass, prefer app-only helper unless there is a clear need to publi ### Core -Core may pass telemetry through app creation options and capture generic app/server events: +Core owns the PostHog env helper and may pass telemetry through app creation options. Core captures generic app/server events: - `app.opened` - `server.request.failed` - `auth.user.signed_in` if auth hooks already exist naturally -Core must not require PostHog env vars to boot. +Core must not require PostHog env vars to boot. Missing env or disabled env means no-op telemetry. ### Agent @@ -263,9 +280,9 @@ Rules: Keep browser telemetry simple: -- app shell initializes PostHog in the browser only if a public key/config is provided +- core-composed browser entrypoints initialize PostHog only if public telemetry env/config is explicitly enabled - workspace/agent front providers accept the same `TelemetrySink` shape -- no package reads `VITE_POSTHOG_KEY` directly +- workspace/agent packages do not read `VITE_POSTHOG_KEY` directly The app can expose config however it already exposes runtime config. For Vite demo apps, use: @@ -294,17 +311,18 @@ Acceptance: - no `node:*` or `Buffer` in shared files - no telemetry call can throw into product code -### Bead 2 — PostHog env helper in app/example +### Bead 2 — core PostHog env helper -- Add `createPostHogTelemetryFromEnv()` for server usage. -- Add browser equivalent for Vite/demo usage if needed. +- Add `createPostHogTelemetryFromEnv()` in `packages/core/src/server/telemetry/posthog.ts`. +- Wire core-composed server entrypoints to use `options.telemetry ?? createPostHogTelemetryFromEnv(process.env)`. +- Add browser equivalent only if a core-composed browser entrypoint needs it. - Support `POSTHOG_KEY`, `POSTHOG_HOST`, `BORING_TELEMETRY_ENABLED`, and `BORING_TELEMETRY_PROJECT`. -- Wire it in `apps/full-app` or the canonical demo shell. Acceptance: - unset env = no-op - `BORING_TELEMETRY_ENABLED=false` = no-op even with key +- `BORING_TELEMETRY_ENABLED=true` plus `POSTHOG_KEY` sends events - project prefix changes event names and adds `boringProject` ### Bead 3 — core/agent/workspace event calls From f07f0302455757ae26f24f7db03b70798460f2ea Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 20:36:32 +0000 Subject: [PATCH 04/14] docs(plan): rewrite PostHog telemetry plan --- .../docs/plans/app-level-telemetry-plan.md | 464 +++++++++++------- 1 file changed, 284 insertions(+), 180 deletions(-) diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md index ea48c9591..da7bb2213 100644 --- a/packages/core/docs/plans/app-level-telemetry-plan.md +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -1,118 +1,142 @@ -# Simple PostHog Telemetry Plan for boring-ui-v2 +# PostHog Telemetry Plan for boring-ui-v2 -**Status:** simplified draft for PR 65 -**Branch/worktree:** `plan/telemetry` at `/home/ubuntu/projects/worktrees/boring-ui-v2-telemetry` -**Primary decision:** use **PostHog**, configured by the child app through env vars. +**Status:** PR 65 rewrite — simplified and coherent +**Primary decision:** core owns the PostHog env integration. Child apps enable telemetry with env vars only. **Last updated:** 2026-05-22 --- -## 0. What changed from the original plan +## 0. Summary -The old plan was too big for the current need. It designed a vendor-neutral telemetry layer with Sentry, OTLP, routing, multi-account selection, CSP planning, and nine beads. +Telemetry should be boring to use: -We do not need that now. +```bash +BORING_TELEMETRY_ENABLED=true +POSTHOG_KEY=phc_... +BORING_TELEMETRY_PROJECT=full-app +``` -New shape: +That is the common path. A child app should not need to write PostHog setup code. -- PostHog is the only planned provider. -- Core owns the PostHog env helper for core-composed apps. -- Child apps enable telemetry and pass credentials with env vars; no app code is required for the common path. -- Agent/workspace internals receive a tiny optional telemetry sink and default to no-op. -- Shared PostHog accounts are handled with a stable project prefix/property, not routing logic. -- No Sentry, OTLP, tenant routing, event bus, or analytics database in this pass. +If telemetry is not wanted, do nothing. Telemetry is off by default. + +```bash +# Either omit BORING_TELEMETRY_ENABLED, or set: +BORING_TELEMETRY_ENABLED=false +``` + +Core provides the PostHog env helper and wires it into core-composed apps. Agent and workspace stay PostHog-free and receive only a small optional telemetry sink. --- -## 1. Goal +## 1. Goals -Add enough telemetry to answer simple product and reliability questions: +Capture simple product and reliability signals: - app opened - workspace opened - chat session started - user message submitted, without content - agent/tool run completed or failed -- workspace panel/command used +- workspace panel or command used - server/frontend error happened, with stable error code only -Telemetry must never capture prompts, assistant output, file contents, command output, secrets, headers, cookies, or raw env vars. +Support several boring-ui apps using the same PostHog account/project by adding a project prefix/property. + +Keep telemetry safe: + +- no prompts +- no assistant output +- no file contents +- no command strings +- no stdout/stderr +- no raw paths unless explicitly allowlisted later +- no headers/cookies/tokens/env dumps +- no stack traces by default --- -## 2. Decision +## 2. Non-goals -Telemetry is app-level and PostHog-focused. +Not in this pass: -Packages expose only this tiny structural contract: +- Sentry +- OpenTelemetry/OTLP +- billing/metering +- per-tenant PostHog routing +- multiple PostHog accounts selected at runtime +- database-backed analytics storage +- content capture +- complex consent management +- package-level PostHog singletons inside agent/workspace -```ts -export interface TelemetrySink { - capture(event: TelemetryEvent): void | Promise -} +--- -export interface TelemetryEvent { - name: string - distinctId?: string - properties?: Record -} -``` +## 3. Architecture decision -Every package option that needs telemetry accepts `telemetry?: TelemetrySink`. -If absent, it uses a no-op sink. +### 3.1 Common path: env only -For core-composed apps, core creates the sink from env automatically unless a custom `telemetry` sink is passed. This gives the common child-app usage: +Core-composed apps should automatically create telemetry from env: -```bash -BORING_TELEMETRY_ENABLED=true -POSTHOG_KEY=phc_... -BORING_TELEMETRY_PROJECT=full-app +```ts +const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) ``` -No extra child-app code is required. Advanced apps can still pass `telemetry` explicitly to override the env helper. +Child apps enable telemetry by setting env vars. They do not need to call `createPostHogTelemetryFromEnv()` manually unless they are building a custom composition. -Package code emits events like: +### 3.2 Escape hatch: custom sink + +Advanced apps can still pass a custom sink: ```ts -telemetry.capture({ - name: 'agent.chat.started', - distinctId: userId, - properties: { - workspaceId, - sessionId, - runtimeMode, - }, +createCoreWorkspaceAgentServer({ + telemetry: myTelemetrySink, }) ``` -Core builds the default PostHog sink from env vars for core-composed apps. Advanced apps can override this by passing a custom `TelemetrySink`. +If `telemetry` is provided, core uses it and does not create the PostHog env sink. ---- +### 3.3 Package boundary -## 3. Env vars +- `@hachej/boring-core` owns the PostHog helper and core-composed wiring. +- `@hachej/boring-agent` does not import PostHog or core. +- `@hachej/boring-workspace` base code does not import PostHog or agent. +- Agent/workspace accept a structural `TelemetrySink` and default to no-op. -Use boring-ui env names for boring-ui behavior, and PostHog env names for PostHog credentials. +--- + +## 4. Env vars ```bash -# Required to send telemetry. If omitted, telemetry is no-op. +# Required to enable telemetry. If unset, telemetry is off. BORING_TELEMETRY_ENABLED=true # Required when telemetry is enabled. POSTHOG_KEY=phc_... -# Optional. Defaults in the core helper if omitted. +# Optional. Defaults to PostHog Cloud US unless overridden. POSTHOG_HOST=https://us.i.posthog.com -# Optional, but recommended when several apps share one PostHog account/project. -# Used both as an event-name prefix and as a property. +# Optional. Recommended when several apps share one PostHog account/project. BORING_TELEMETRY_PROJECT=full-app ``` -Telemetry is **off by default**. If `BORING_TELEMETRY_ENABLED` is unset or set to `false`, core returns `noopTelemetry` even if other app code imports the helper. +Behavior: + +| Env state | Result | +|---|---| +| `BORING_TELEMETRY_ENABLED` unset | no-op telemetry | +| `BORING_TELEMETRY_ENABLED=false` | no-op telemetry | +| `BORING_TELEMETRY_ENABLED=true`, `POSTHOG_KEY` missing | no-op telemetry, with a safe warning | +| `BORING_TELEMETRY_ENABLED=true`, `POSTHOG_KEY` set | send to PostHog | -### Project prefix rule +Telemetry must be explicit opt-in. `POSTHOG_KEY` alone is not enough. -If `BORING_TELEMETRY_PROJECT=full-app`, event names sent to PostHog become: +--- + +## 5. Project prefix rule + +If `BORING_TELEMETRY_PROJECT=full-app`, core sends event names like: ```txt full-app.app.opened @@ -121,7 +145,7 @@ full-app.agent.chat.started full-app.agent.tool.completed ``` -The same value is also sent as a property: +Core also sends the raw event name and project as properties: ```ts { @@ -132,30 +156,80 @@ The same value is also sent as a property: Why both: -- prefixed event names make PostHog dashboards easy when projects share an account/project -- the `boringProject` property makes filtering and grouping easy -- no need for multi-account routing in v1 +- prefixed event names keep shared PostHog projects readable +- `boringProject` makes filtering/grouping easy +- no runtime multi-account routing needed in v1 -If the prefix is unset, send the raw event name. +If `BORING_TELEMETRY_PROJECT` is unset, core sends the raw event name. --- -## 4. PostHog sink helper +## 6. Shared telemetry contract -Implement this helper in core's app/server layer, not inside agent/workspace internals: +Each package may define or import a compatible structural type: + +```ts +export interface TelemetrySink { + capture(event: TelemetryEvent): void | Promise +} + +export interface TelemetryEvent { + name: string + distinctId?: string + properties?: Record +} +``` + +No package call site should depend on PostHog types. + +Provide: + +```ts +export const noopTelemetry: TelemetrySink = { + capture() {}, +} +``` + +And optionally: + +```ts +export function safeCapture(telemetry: TelemetrySink, event: TelemetryEvent): void { + try { + void telemetry.capture(event) + } catch { + // telemetry must never break product behavior + } +} +``` + +--- + +## 7. Core PostHog helper + +Location: + +```txt +packages/core/src/server/telemetry/posthog.ts +``` + +Shape: ```ts -// packages/core/src/server/telemetry/posthog.ts export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink { - const enabled = env.BORING_TELEMETRY_ENABLED === 'true' && Boolean(env.POSTHOG_KEY) + const enabled = env.BORING_TELEMETRY_ENABLED === 'true' if (!enabled) return noopTelemetry - const posthog = new PostHog(env.POSTHOG_KEY!, { - host: env.POSTHOG_HOST, + if (!env.POSTHOG_KEY) { + // warn once, then no-op + return noopTelemetry + } + + const posthog = new PostHog(env.POSTHOG_KEY, { + host: env.POSTHOG_HOST ?? 'https://us.i.posthog.com', }) - const prefix = env.BORING_TELEMETRY_PROJECT?.trim() + const prefix = env.BORING_TELEMETRY_PROJECT?.trim() || undefined return { capture(event) { @@ -175,68 +249,48 @@ export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink } ``` -Core-composed app entrypoints use it by default: - -```ts -const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) -``` - -Child apps that want telemetry set env vars. Child apps that do not want telemetry leave `BORING_TELEMETRY_ENABLED` unset or set it to `false`. +Rules: -Advanced apps may still pass their own `telemetry` sink to bypass the PostHog env helper. +- helper lives in core server code only +- agent/workspace do not import it +- disabled/misconfigured env returns no-op +- capture failures are swallowed or logged at debug level +- shutdown should flush PostHog if the SDK requires it --- -## 5. Package boundaries +## 8. Frontend telemetry -### Core +Frontend telemetry should also work from env-only setup. -Core owns the PostHog env helper and may pass telemetry through app creation options. Core captures generic app/server events: +Preferred first pass: -- `app.opened` -- `server.request.failed` -- `auth.user.signed_in` if auth hooks already exist naturally - -Core must not require PostHog env vars to boot. Missing env or disabled env means no-op telemetry. - -### Agent - -Agent accepts `telemetry?: TelemetrySink` in server/app options and emits agent events: - -- `agent.chat.started` -- `agent.chat.message.submitted` -- `agent.chat.completed` -- `agent.chat.failed` -- `agent.tool.started` -- `agent.tool.completed` -- `agent.tool.failed` +1. Core server creates the PostHog sink from env. +2. Core exposes a small internal telemetry endpoint for browser events, only when telemetry is enabled. +3. Core-composed frontend uses an HTTP telemetry sink that posts safe events to that endpoint. +4. The server forwards those events to PostHog with the same prefix/property rule. -No prompt text, assistant text, stdout, stderr, file contents, or command args by default. - -### Workspace +This avoids requiring child apps to expose `VITE_POSTHOG_KEY` or initialize PostHog in browser code. -Workspace accepts `telemetry?: TelemetrySink` in provider/server options and emits UI/workspace events: +Endpoint rules: -- `workspace.opened` -- `workspace.panel.opened` -- `workspace.command.executed` -- `workspace.ui_command.posted` -- `workspace.plugin.error` +- bounded request body +- only known event names or known prefixes +- property allowlist/drop unsafe fields +- authenticated when the app is authenticated +- telemetry endpoint failures never break UI behavior -No panel params unless explicitly allowlisted and known safe. +If direct browser PostHog becomes necessary later, add it as a separate bead. --- -## 6. Minimal event properties - -Use low-cardinality metadata only. +## 9. Event property policy Allowed by default: - `workspaceId` - `sessionId` - `requestId` -- `userId` only as `distinctId` or a safe hashed/id value already used by auth - `runtimeMode` - `modelProvider` - `toolName` @@ -248,131 +302,181 @@ Allowed by default: - `packageName` - `packageVersion` +Identity: + +- use a safe auth/user id as `distinctId` when available +- otherwise use `anonymous` +- do not send emails by default + Not allowed by default: - prompts/messages -- file paths unless explicitly normalized/approved later +- assistant output +- file contents - command strings - command output +- raw file paths - stack traces - raw errors - headers/cookies/tokens - env vars -If a future event needs richer data, add it intentionally with a small allowlist. +If a future event needs richer data, add a small explicit allowlist in the same PR. --- -## 7. Error handling +## 10. Initial event list -Telemetry must never break user flows. +### Core -Rules: +- `app.opened` +- `server.request.failed` +- `auth.user.signed_in` if there is already a natural auth hook + +### Workspace + +- `workspace.opened` +- `workspace.panel.opened` +- `workspace.command.executed` +- `workspace.ui_command.posted` +- `workspace.plugin.error` + +### Agent + +- `agent.chat.started` +- `agent.chat.message.submitted` +- `agent.chat.completed` +- `agent.chat.failed` +- `agent.tool.started` +- `agent.tool.completed` +- `agent.tool.failed` -- `capture()` calls are best-effort. -- Package call sites must not `await` telemetry on hot streaming paths unless already async and safe. -- Sink failures are swallowed or logged at debug level. -- Error events send `errorCode`, not raw error messages/stacks by default. +Keep names stable once shipped. --- -## 8. Browser telemetry +## 11. Package wiring -Keep browser telemetry simple: +### Core-composed server -- core-composed browser entrypoints initialize PostHog only if public telemetry env/config is explicitly enabled -- workspace/agent front providers accept the same `TelemetrySink` shape -- workspace/agent packages do not read `VITE_POSTHOG_KEY` directly +Core server entrypoints accept: -The app can expose config however it already exposes runtime config. -For Vite demo apps, use: +```ts +telemetry?: TelemetrySink +``` -```bash -VITE_POSTHOG_KEY=phc_... -VITE_POSTHOG_HOST=https://us.i.posthog.com -VITE_BORING_TELEMETRY_PROJECT=workspace-playground +Then resolve: + +```ts +const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) ``` -The browser helper should follow the same prefix rule as the server helper. +Core passes the resolved sink into workspace/agent composition. + +### Agent standalone + +Agent standalone remains no-op by default. It accepts `telemetry?: TelemetrySink` for embedders. + +No PostHog env helper in agent for this pass. + +### Workspace standalone + +Workspace remains no-op by default. It accepts `telemetry?: TelemetrySink` in provider/server composition where needed. + +No PostHog env helper in workspace for this pass. + +--- + +## 12. Error handling + +Telemetry must never break user flows. + +Rules: + +- never throw from no-op or safe capture helpers +- do not await telemetry in hot streaming paths unless already async and safe +- swallow or debug-log sink failures +- send stable `errorCode`, not raw stack/message by default +- if PostHog is down, product behavior is unchanged --- -## 9. Implementation plan +## 13. Implementation beads ### Bead 1 — telemetry contract and no-op -- Add `TelemetrySink`, `TelemetryEvent`, and `noopTelemetry` where each package can use them without adding forbidden dependencies. -- Add a safe wrapper/helper if useful. -- Add unit tests for no-op and prefix formatting if helper exists. +- Add `TelemetrySink`, `TelemetryEvent`, `noopTelemetry`, and safe capture helper where needed. +- Keep shared files platform-neutral. +- Add unit tests for no-op and safe capture. Acceptance: -- packages compile without PostHog installed unless app/helper needs it - no `node:*` or `Buffer` in shared files -- no telemetry call can throw into product code +- telemetry capture cannot throw into product code +- agent/workspace do not import PostHog or core ### Bead 2 — core PostHog env helper -- Add `createPostHogTelemetryFromEnv()` in `packages/core/src/server/telemetry/posthog.ts`. -- Wire core-composed server entrypoints to use `options.telemetry ?? createPostHogTelemetryFromEnv(process.env)`. -- Add browser equivalent only if a core-composed browser entrypoint needs it. -- Support `POSTHOG_KEY`, `POSTHOG_HOST`, `BORING_TELEMETRY_ENABLED`, and `BORING_TELEMETRY_PROJECT`. +- Add `packages/core/src/server/telemetry/posthog.ts`. +- Add `posthog-node` dependency to core if needed. +- Implement explicit opt-in via `BORING_TELEMETRY_ENABLED=true`. +- Implement `POSTHOG_KEY`, `POSTHOG_HOST`, and `BORING_TELEMETRY_PROJECT`. +- Add tests for disabled, missing key, enabled, host override, and project prefix. Acceptance: - unset env = no-op -- `BORING_TELEMETRY_ENABLED=false` = no-op even with key -- `BORING_TELEMETRY_ENABLED=true` plus `POSTHOG_KEY` sends events -- project prefix changes event names and adds `boringProject` +- `BORING_TELEMETRY_ENABLED=false` = no-op +- `POSTHOG_KEY` without `BORING_TELEMETRY_ENABLED=true` = no-op +- enabled env sends prefixed event and properties -### Bead 3 — core/agent/workspace event calls +### Bead 3 — core wiring and frontend bridge -- Thread `telemetry?: TelemetrySink` through existing app/provider/server options. -- Add only the minimal event list from this plan. -- Keep privacy allowlist strict. +- Wire core-composed server entrypoints to resolve telemetry from `options.telemetry ?? env helper`. +- Pass the resolved sink to workspace/agent composition. +- Add internal frontend telemetry endpoint if frontend events are included in this pass. +- Add HTTP frontend sink in core-composed frontend if the endpoint is added. Acceptance: -- tests prove expected events are emitted with safe metadata -- tests prove prompts/output/file contents are not included -- quality gates pass +- child app can enable telemetry with env only +- child app can disable telemetry by omitting env +- custom `telemetry` option overrides env helper +- frontend endpoint, if added, validates/drops unsafe properties -### Bead 4 — docs +### Bead 4 — event emitters -- Document env vars. -- Document event names. -- Document privacy rules. -- Add a short “shared PostHog account” example using `BORING_TELEMETRY_PROJECT`. +- Add the initial core/workspace/agent event calls. +- Use only the allowed property list. +- Avoid hot-path awaits. Acceptance: -- docs match implementation -- no extra vendor/routing/Sentry/OTLP scope sneaks back in - ---- +- tests prove expected safe events are emitted +- tests prove prompts/output/file contents/command output are not emitted +- package boundaries remain intact -## 10. Explicit non-goals for this PR +### Bead 5 — docs -Do not implement now: +- Document env vars. +- Document event names. +- Document privacy rules. +- Add shared PostHog account example with `BORING_TELEMETRY_PROJECT`. -- Sentry integration -- OpenTelemetry/OTLP -- per-tenant provider routing -- database-backed telemetry storage -- billing/metering -- content capture -- complex consent system -- automatic CSP management -- multi-account PostHog routing +Acceptance: -If any of those become necessary, make separate beads later. +- docs match implementation +- no Sentry/OTLP/routing scope reappears --- -## 11. Definition of done +## 14. Definition of done -This plan is done when: +Telemetry v1 is done when: -- PR 65 describes the simplified PostHog-only direction. -- The event prefix/env-var contract is clear. -- The implementation work is small enough to start without another architecture pass. +- a core-composed app can enable PostHog with env vars only +- telemetry is off by default +- shared PostHog accounts can use `BORING_TELEMETRY_PROJECT` +- agent/workspace remain PostHog-free +- no content or secret-bearing data is captured by default +- tests cover opt-in, opt-out, prefixing, and privacy behavior From fab67e307f2675da68d1b2776eb88f44799eec27 Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 20:57:51 +0000 Subject: [PATCH 05/14] docs(plan): apply telemetry planning review --- .../docs/plans/app-level-telemetry-plan.md | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md index da7bb2213..38eb8dc8e 100644 --- a/packages/core/docs/plans/app-level-telemetry-plan.md +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -171,6 +171,7 @@ Each package may define or import a compatible structural type: ```ts export interface TelemetrySink { capture(event: TelemetryEvent): void | Promise + flush?(): void | Promise } export interface TelemetryEvent { @@ -195,7 +196,7 @@ And optionally: ```ts export function safeCapture(telemetry: TelemetrySink, event: TelemetryEvent): void { try { - void telemetry.capture(event) + void Promise.resolve(telemetry.capture(event)).catch(() => {}) } catch { // telemetry must never break product behavior } @@ -229,7 +230,7 @@ export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink host: env.POSTHOG_HOST ?? 'https://us.i.posthog.com', }) - const prefix = env.BORING_TELEMETRY_PROJECT?.trim() || undefined + const prefix = parseTelemetryProject(env.BORING_TELEMETRY_PROJECT) return { capture(event) { @@ -239,23 +240,30 @@ export function createPostHogTelemetryFromEnv(env = process.env): TelemetrySink distinctId: event.distinctId ?? 'anonymous', event: name, properties: { - ...event.properties, + ...sanitizeTelemetryProperties(event.properties), boringProject: prefix, eventName: event.name, }, }) }, + async flush() { + await posthog.shutdown() + }, } } ``` +`parseTelemetryProject()` accepts only a slug safe for event-name prefixing, such as `full-app` or `customer-portal`. Invalid values warn once and disable the prefix instead of sending surprising event names. + +`sanitizeTelemetryProperties()` is the central allowlist from this plan. It drops unknown keys before events reach PostHog, so one mistaken emitter cannot leak prompts, command output, raw paths, headers, or stack traces. + Rules: - helper lives in core server code only - agent/workspace do not import it - disabled/misconfigured env returns no-op -- capture failures are swallowed or logged at debug level -- shutdown should flush PostHog if the SDK requires it +- capture failures and rejected promises are swallowed or logged at debug level +- PostHog `shutdown()` is exposed through optional `telemetry.flush()` and wired into server shutdown --- @@ -266,17 +274,18 @@ Frontend telemetry should also work from env-only setup. Preferred first pass: 1. Core server creates the PostHog sink from env. -2. Core exposes a small internal telemetry endpoint for browser events, only when telemetry is enabled. -3. Core-composed frontend uses an HTTP telemetry sink that posts safe events to that endpoint. -4. The server forwards those events to PostHog with the same prefix/property rule. +2. Core exposes non-secret runtime config such as `{ telemetry: { enabled, endpoint } }`. +3. Core exposes a small internal telemetry endpoint for browser events, only when telemetry is enabled. +4. Core-composed frontend installs an HTTP telemetry sink only when `telemetry.enabled === true`. +5. The server forwards those events to PostHog with the same prefix/property rule. This avoids requiring child apps to expose `VITE_POSTHOG_KEY` or initialize PostHog in browser code. Endpoint rules: - bounded request body -- only known event names or known prefixes -- property allowlist/drop unsafe fields +- only known event names from this plan +- reuse the same central property allowlist as the server PostHog sink - authenticated when the app is authenticated - telemetry endpoint failures never break UI behavior @@ -321,6 +330,8 @@ Not allowed by default: - headers/cookies/tokens - env vars +The allowlist should be enforced centrally by `sanitizeTelemetryProperties()` before any event reaches PostHog or the frontend telemetry endpoint. + If a future event needs richer data, add a small explicit allowlist in the same PR. --- @@ -405,9 +416,9 @@ Rules: ### Bead 1 — telemetry contract and no-op -- Add `TelemetrySink`, `TelemetryEvent`, `noopTelemetry`, and safe capture helper where needed. +- Add `TelemetrySink`, `TelemetryEvent`, optional `flush()`, `noopTelemetry`, and safe capture helper where needed. - Keep shared files platform-neutral. -- Add unit tests for no-op and safe capture. +- Add unit tests for no-op, sync throw, async rejection, and safe capture. Acceptance: @@ -421,7 +432,9 @@ Acceptance: - Add `posthog-node` dependency to core if needed. - Implement explicit opt-in via `BORING_TELEMETRY_ENABLED=true`. - Implement `POSTHOG_KEY`, `POSTHOG_HOST`, and `BORING_TELEMETRY_PROJECT`. -- Add tests for disabled, missing key, enabled, host override, and project prefix. +- Add central property sanitization and project-prefix slug validation. +- Add optional `flush()` support and wire it into server shutdown. +- Add tests for disabled, missing key, enabled, host override, project prefix, invalid prefix, property sanitization, and flush. Acceptance: @@ -434,6 +447,7 @@ Acceptance: - Wire core-composed server entrypoints to resolve telemetry from `options.telemetry ?? env helper`. - Pass the resolved sink to workspace/agent composition. +- Add non-secret runtime config for `telemetry.enabled` and `telemetry.endpoint`. - Add internal frontend telemetry endpoint if frontend events are included in this pass. - Add HTTP frontend sink in core-composed frontend if the endpoint is added. From 93da80d653d93534de4508b79603a1fb8aac5bb5 Mon Sep 17 00:00:00 2001 From: hachej Date: Fri, 22 May 2026 21:01:59 +0000 Subject: [PATCH 06/14] docs(plan): simplify telemetry v1 scope --- .../docs/plans/app-level-telemetry-plan.md | 70 ++++++++----------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/packages/core/docs/plans/app-level-telemetry-plan.md b/packages/core/docs/plans/app-level-telemetry-plan.md index 38eb8dc8e..664f5a5b5 100644 --- a/packages/core/docs/plans/app-level-telemetry-plan.md +++ b/packages/core/docs/plans/app-level-telemetry-plan.md @@ -84,17 +84,11 @@ const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env Child apps enable telemetry by setting env vars. They do not need to call `createPostHogTelemetryFromEnv()` manually unless they are building a custom composition. -### 3.2 Escape hatch: custom sink +### 3.2 Package seam: structural sink -Advanced apps can still pass a custom sink: +Core passes a tiny structural `TelemetrySink` into composed packages so agent/workspace remain PostHog-free. This is a package boundary seam, not a v1 provider framework. -```ts -createCoreWorkspaceAgentServer({ - telemetry: myTelemetrySink, -}) -``` - -If `telemetry` is provided, core uses it and does not create the PostHog env sink. +If a caller already has a custom sink for tests or advanced composition, it may pass `telemetry`; otherwise core uses the PostHog env helper. ### 3.3 Package boundary @@ -263,33 +257,23 @@ Rules: - agent/workspace do not import it - disabled/misconfigured env returns no-op - capture failures and rejected promises are swallowed or logged at debug level -- PostHog `shutdown()` is exposed through optional `telemetry.flush()` and wired into server shutdown +- PostHog `shutdown()` is exposed through optional `telemetry.flush()` +- wire `flush()` into server shutdown only if an existing close hook makes it a small change; otherwise defer lifecycle wiring --- ## 8. Frontend telemetry -Frontend telemetry should also work from env-only setup. - -Preferred first pass: - -1. Core server creates the PostHog sink from env. -2. Core exposes non-secret runtime config such as `{ telemetry: { enabled, endpoint } }`. -3. Core exposes a small internal telemetry endpoint for browser events, only when telemetry is enabled. -4. Core-composed frontend installs an HTTP telemetry sink only when `telemetry.enabled === true`. -5. The server forwards those events to PostHog with the same prefix/property rule. +Deferred. -This avoids requiring child apps to expose `VITE_POSTHOG_KEY` or initialize PostHog in browser code. +V1 only emits events from server/core-composed code paths that can use the core PostHog env helper directly. Do not add runtime config, a browser telemetry endpoint, or a frontend HTTP sink in this pass. -Endpoint rules: +If browser-originated events are needed later, add a separate bead for: -- bounded request body -- only known event names from this plan -- reuse the same central property allowlist as the server PostHog sink -- authenticated when the app is authenticated -- telemetry endpoint failures never break UI behavior - -If direct browser PostHog becomes necessary later, add it as a separate bead. +- non-secret runtime config such as `{ telemetry: { enabled, endpoint } }` +- an authenticated telemetry endpoint +- a small HTTP frontend sink +- endpoint body limits, event allowlist, and property sanitization --- @@ -330,9 +314,9 @@ Not allowed by default: - headers/cookies/tokens - env vars -The allowlist should be enforced centrally by `sanitizeTelemetryProperties()` before any event reaches PostHog or the frontend telemetry endpoint. +The allowlist should be enforced centrally by `sanitizeTelemetryProperties()` before any event reaches PostHog. -If a future event needs richer data, add a small explicit allowlist in the same PR. +If a future event needs richer data, add a small explicit allowlist in that future PR/bead. --- @@ -342,15 +326,12 @@ If a future event needs richer data, add a small explicit allowlist in the same - `app.opened` - `server.request.failed` -- `auth.user.signed_in` if there is already a natural auth hook ### Workspace - `workspace.opened` - `workspace.panel.opened` - `workspace.command.executed` -- `workspace.ui_command.posted` -- `workspace.plugin.error` ### Agent @@ -358,10 +339,11 @@ If a future event needs richer data, add a small explicit allowlist in the same - `agent.chat.message.submitted` - `agent.chat.completed` - `agent.chat.failed` -- `agent.tool.started` - `agent.tool.completed` - `agent.tool.failed` +Defer auth hooks, browser/frontend events, UI-command events, plugin-error events, and tool-started events unless an existing hook makes them zero-touch. + Keep names stable once shipped. --- @@ -433,8 +415,9 @@ Acceptance: - Implement explicit opt-in via `BORING_TELEMETRY_ENABLED=true`. - Implement `POSTHOG_KEY`, `POSTHOG_HOST`, and `BORING_TELEMETRY_PROJECT`. - Add central property sanitization and project-prefix slug validation. -- Add optional `flush()` support and wire it into server shutdown. -- Add tests for disabled, missing key, enabled, host override, project prefix, invalid prefix, property sanitization, and flush. +- Add optional `flush()` support on the PostHog sink. +- Wire `flush()` into server shutdown only if an existing core close hook makes this a small change; otherwise defer lifecycle wiring. +- Add tests for disabled, missing key, enabled, host override, project prefix, invalid prefix, and property sanitization. Acceptance: @@ -443,20 +426,17 @@ Acceptance: - `POSTHOG_KEY` without `BORING_TELEMETRY_ENABLED=true` = no-op - enabled env sends prefixed event and properties -### Bead 3 — core wiring and frontend bridge +### Bead 3 — core wiring - Wire core-composed server entrypoints to resolve telemetry from `options.telemetry ?? env helper`. - Pass the resolved sink to workspace/agent composition. -- Add non-secret runtime config for `telemetry.enabled` and `telemetry.endpoint`. -- Add internal frontend telemetry endpoint if frontend events are included in this pass. -- Add HTTP frontend sink in core-composed frontend if the endpoint is added. +- Do not add runtime config, frontend endpoint, or frontend HTTP sink in v1. Acceptance: - child app can enable telemetry with env only - child app can disable telemetry by omitting env - custom `telemetry` option overrides env helper -- frontend endpoint, if added, validates/drops unsafe properties ### Bead 4 — event emitters @@ -494,3 +474,11 @@ Telemetry v1 is done when: - agent/workspace remain PostHog-free - no content or secret-bearing data is captured by default - tests cover opt-in, opt-out, prefixing, and privacy behavior + +Deferred after v1: + +- browser-originated telemetry +- auth hook telemetry +- plugin-error telemetry +- UI-command telemetry +- mandatory PostHog flush lifecycle wiring From b111c4580c6e77e5c4d7fb109ce981bf897d6f42 Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 06:57:24 +0000 Subject: [PATCH 07/14] feat(core): add telemetry sink contract (boring-ui-v2-reorg-x0ek.1) --- .../src/shared/__tests__/telemetry.test.ts | 67 +++++++++++++++++++ packages/agent/src/shared/index.ts | 2 + packages/agent/src/shared/telemetry.ts | 20 ++++++ .../src/shared/__tests__/telemetry.test.ts | 67 +++++++++++++++++++ packages/core/src/shared/index.ts | 3 + packages/core/src/shared/telemetry.ts | 20 ++++++ .../src/shared/__tests__/telemetry.test.ts | 67 +++++++++++++++++++ packages/workspace/src/shared/index.ts | 2 + packages/workspace/src/shared/telemetry.ts | 20 ++++++ 9 files changed, 268 insertions(+) create mode 100644 packages/agent/src/shared/__tests__/telemetry.test.ts create mode 100644 packages/agent/src/shared/telemetry.ts create mode 100644 packages/core/src/shared/__tests__/telemetry.test.ts create mode 100644 packages/core/src/shared/telemetry.ts create mode 100644 packages/workspace/src/shared/__tests__/telemetry.test.ts create mode 100644 packages/workspace/src/shared/telemetry.ts diff --git a/packages/agent/src/shared/__tests__/telemetry.test.ts b/packages/agent/src/shared/__tests__/telemetry.test.ts new file mode 100644 index 000000000..fdb56eb5a --- /dev/null +++ b/packages/agent/src/shared/__tests__/telemetry.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, expectTypeOf, it, vi } from 'vitest' + +import { + noopTelemetry, + safeCapture, + type TelemetryEvent, + type TelemetrySink, +} from '../telemetry' + +describe('telemetry contract', () => { + const event: TelemetryEvent = { + name: 'telemetry.test', + distinctId: 'user-1', + properties: { status: 'ok' }, + } + + it('noop capture accepts events without side effects', () => { + expect(() => noopTelemetry.capture(event)).not.toThrow() + }) + + it('safeCapture forwards normal captures', () => { + const capture = vi.fn() + const telemetry: TelemetrySink = { capture } + + safeCapture(telemetry, event) + + expect(capture).toHaveBeenCalledWith(event) + }) + + it('safeCapture swallows synchronous capture failures', () => { + const telemetry: TelemetrySink = { + capture() { + throw new Error('sync telemetry failure') + }, + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + }) + + it('safeCapture swallows asynchronous capture rejections', async () => { + const telemetry: TelemetrySink = { + capture: vi.fn().mockRejectedValue(new Error('async telemetry failure')), + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + await Promise.resolve() + }) + + it('allows sinks to expose an optional flush hook', async () => { + const flush = vi.fn().mockResolvedValue(undefined) + const telemetry: TelemetrySink = { + capture() {}, + flush, + } + + await telemetry.flush?.() + + expect(flush).toHaveBeenCalledOnce() + }) + + it('keeps the TelemetrySink shape structural', () => { + expectTypeOf().toEqualTypeOf<{ + capture: (event: TelemetryEvent) => void | Promise + flush?: () => void | Promise + }>() + }) +}) diff --git a/packages/agent/src/shared/index.ts b/packages/agent/src/shared/index.ts index e92034bbf..e627a8287 100644 --- a/packages/agent/src/shared/index.ts +++ b/packages/agent/src/shared/index.ts @@ -22,6 +22,8 @@ export type { } from './session' export type { UIMessage, UIMessageChunk } from './message' export type { FileSearch } from './file-search' +export type { TelemetryEvent, TelemetrySink } from './telemetry' +export { noopTelemetry, safeCapture } from './telemetry' export type { SandboxHandleRecord, SandboxHandleStore, diff --git a/packages/agent/src/shared/telemetry.ts b/packages/agent/src/shared/telemetry.ts new file mode 100644 index 000000000..d7579ec52 --- /dev/null +++ b/packages/agent/src/shared/telemetry.ts @@ -0,0 +1,20 @@ +export interface TelemetrySink { + capture(event: TelemetryEvent): void | Promise + flush?(): void | Promise +} + +export interface TelemetryEvent { + name: string + distinctId?: string + properties?: Record +} + +export const noopTelemetry: TelemetrySink = { + capture() {}, +} + +export function safeCapture(telemetry: TelemetrySink, event: TelemetryEvent): void { + try { + void Promise.resolve(telemetry.capture(event)).catch(() => {}) + } catch {} +} diff --git a/packages/core/src/shared/__tests__/telemetry.test.ts b/packages/core/src/shared/__tests__/telemetry.test.ts new file mode 100644 index 000000000..fdb56eb5a --- /dev/null +++ b/packages/core/src/shared/__tests__/telemetry.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, expectTypeOf, it, vi } from 'vitest' + +import { + noopTelemetry, + safeCapture, + type TelemetryEvent, + type TelemetrySink, +} from '../telemetry' + +describe('telemetry contract', () => { + const event: TelemetryEvent = { + name: 'telemetry.test', + distinctId: 'user-1', + properties: { status: 'ok' }, + } + + it('noop capture accepts events without side effects', () => { + expect(() => noopTelemetry.capture(event)).not.toThrow() + }) + + it('safeCapture forwards normal captures', () => { + const capture = vi.fn() + const telemetry: TelemetrySink = { capture } + + safeCapture(telemetry, event) + + expect(capture).toHaveBeenCalledWith(event) + }) + + it('safeCapture swallows synchronous capture failures', () => { + const telemetry: TelemetrySink = { + capture() { + throw new Error('sync telemetry failure') + }, + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + }) + + it('safeCapture swallows asynchronous capture rejections', async () => { + const telemetry: TelemetrySink = { + capture: vi.fn().mockRejectedValue(new Error('async telemetry failure')), + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + await Promise.resolve() + }) + + it('allows sinks to expose an optional flush hook', async () => { + const flush = vi.fn().mockResolvedValue(undefined) + const telemetry: TelemetrySink = { + capture() {}, + flush, + } + + await telemetry.flush?.() + + expect(flush).toHaveBeenCalledOnce() + }) + + it('keeps the TelemetrySink shape structural', () => { + expectTypeOf().toEqualTypeOf<{ + capture: (event: TelemetryEvent) => void | Promise + flush?: () => void | Promise + }>() + }) +}) diff --git a/packages/core/src/shared/index.ts b/packages/core/src/shared/index.ts index ed7e9a500..3def1b461 100644 --- a/packages/core/src/shared/index.ts +++ b/packages/core/src/shared/index.ts @@ -22,3 +22,6 @@ export { ConfigValidationError, } from './errors.js' export type { ErrorCode } from './errors.js' + +export { noopTelemetry, safeCapture } from './telemetry.js' +export type { TelemetryEvent, TelemetrySink } from './telemetry.js' diff --git a/packages/core/src/shared/telemetry.ts b/packages/core/src/shared/telemetry.ts new file mode 100644 index 000000000..d7579ec52 --- /dev/null +++ b/packages/core/src/shared/telemetry.ts @@ -0,0 +1,20 @@ +export interface TelemetrySink { + capture(event: TelemetryEvent): void | Promise + flush?(): void | Promise +} + +export interface TelemetryEvent { + name: string + distinctId?: string + properties?: Record +} + +export const noopTelemetry: TelemetrySink = { + capture() {}, +} + +export function safeCapture(telemetry: TelemetrySink, event: TelemetryEvent): void { + try { + void Promise.resolve(telemetry.capture(event)).catch(() => {}) + } catch {} +} diff --git a/packages/workspace/src/shared/__tests__/telemetry.test.ts b/packages/workspace/src/shared/__tests__/telemetry.test.ts new file mode 100644 index 000000000..27eb47a8d --- /dev/null +++ b/packages/workspace/src/shared/__tests__/telemetry.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, expectTypeOf, it, vi } from "vitest" + +import { + noopTelemetry, + safeCapture, + type TelemetryEvent, + type TelemetrySink, +} from "../telemetry" + +describe("telemetry contract", () => { + const event: TelemetryEvent = { + name: "telemetry.test", + distinctId: "user-1", + properties: { status: "ok" }, + } + + it("noop capture accepts events without side effects", () => { + expect(() => noopTelemetry.capture(event)).not.toThrow() + }) + + it("safeCapture forwards normal captures", () => { + const capture = vi.fn() + const telemetry: TelemetrySink = { capture } + + safeCapture(telemetry, event) + + expect(capture).toHaveBeenCalledWith(event) + }) + + it("safeCapture swallows synchronous capture failures", () => { + const telemetry: TelemetrySink = { + capture() { + throw new Error("sync telemetry failure") + }, + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + }) + + it("safeCapture swallows asynchronous capture rejections", async () => { + const telemetry: TelemetrySink = { + capture: vi.fn().mockRejectedValue(new Error("async telemetry failure")), + } + + expect(() => safeCapture(telemetry, event)).not.toThrow() + await Promise.resolve() + }) + + it("allows sinks to expose an optional flush hook", async () => { + const flush = vi.fn().mockResolvedValue(undefined) + const telemetry: TelemetrySink = { + capture() {}, + flush, + } + + await telemetry.flush?.() + + expect(flush).toHaveBeenCalledOnce() + }) + + it("keeps the TelemetrySink shape structural", () => { + expectTypeOf().toEqualTypeOf<{ + capture: (event: TelemetryEvent) => void | Promise + flush?: () => void | Promise + }>() + }) +}) diff --git a/packages/workspace/src/shared/index.ts b/packages/workspace/src/shared/index.ts index 32a9cabf9..e30efd703 100644 --- a/packages/workspace/src/shared/index.ts +++ b/packages/workspace/src/shared/index.ts @@ -22,3 +22,5 @@ export type { export { WORKSPACE_OPEN_PATH_SURFACE_KIND } from "./types/surface" export { definePanel } from "./types/panel" export type { AgentTool, JSONSchema, ToolExecContext, ToolResult } from "./types/agent-tool" +export type { TelemetryEvent, TelemetrySink } from "./telemetry" +export { noopTelemetry, safeCapture } from "./telemetry" diff --git a/packages/workspace/src/shared/telemetry.ts b/packages/workspace/src/shared/telemetry.ts new file mode 100644 index 000000000..d7579ec52 --- /dev/null +++ b/packages/workspace/src/shared/telemetry.ts @@ -0,0 +1,20 @@ +export interface TelemetrySink { + capture(event: TelemetryEvent): void | Promise + flush?(): void | Promise +} + +export interface TelemetryEvent { + name: string + distinctId?: string + properties?: Record +} + +export const noopTelemetry: TelemetrySink = { + capture() {}, +} + +export function safeCapture(telemetry: TelemetrySink, event: TelemetryEvent): void { + try { + void Promise.resolve(telemetry.capture(event)).catch(() => {}) + } catch {} +} From b484d790a6d7a4b7b9660ec02a7843dd6d9c4182 Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 07:06:25 +0000 Subject: [PATCH 08/14] feat(core): add PostHog telemetry env helper (boring-ui-v2-reorg-x0ek.2) --- packages/core/package.json | 1 + .../telemetry/__tests__/posthog.test.ts | 256 ++++++++++++++++++ packages/core/src/server/telemetry/posthog.ts | 93 +++++++ pnpm-lock.yaml | 28 ++ 4 files changed, 378 insertions(+) create mode 100644 packages/core/src/server/telemetry/__tests__/posthog.test.ts create mode 100644 packages/core/src/server/telemetry/posthog.ts diff --git a/packages/core/package.json b/packages/core/package.json index 8a4594d2b..cc3cf1c91 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -89,6 +89,7 @@ "nodemailer": "^8.0.6", "pino": "^10.3.1", "postgres": "^3.4.9", + "posthog-node": "^5.35.1", "react-helmet-async": "^2.0.5", "react-hook-form": "^7.74.0", "react-router-dom": "^7.14.2", diff --git a/packages/core/src/server/telemetry/__tests__/posthog.test.ts b/packages/core/src/server/telemetry/__tests__/posthog.test.ts new file mode 100644 index 000000000..e15fd48b0 --- /dev/null +++ b/packages/core/src/server/telemetry/__tests__/posthog.test.ts @@ -0,0 +1,256 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const posthogMock = vi.hoisted(() => { + type MockPostHogClient = { + apiKey: string + options: Record | undefined + capture: ReturnType + shutdown: ReturnType + } + + const clients: MockPostHogClient[] = [] + const PostHog = vi.fn((apiKey: string, options?: Record) => { + const client: MockPostHogClient = { + apiKey, + options, + capture: vi.fn(), + shutdown: vi.fn(), + } + clients.push(client) + return client + }) + + return { clients, PostHog } +}) + +vi.mock('posthog-node', () => ({ + PostHog: posthogMock.PostHog, +})) + +import { + createPostHogTelemetryFromEnv, + parseTelemetryProject, + sanitizeTelemetryProperties, +} from '../posthog' + +function env(overrides: Record = {}): NodeJS.ProcessEnv { + return { ...overrides } +} + +describe('createPostHogTelemetryFromEnv', () => { + let warn: ReturnType + + beforeEach(() => { + posthogMock.clients.length = 0 + posthogMock.PostHog.mockClear() + warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('uses noop telemetry when BORING_TELEMETRY_ENABLED is unset', () => { + const telemetry = createPostHogTelemetryFromEnv(env({ POSTHOG_KEY: 'phc_secret' })) + + telemetry.capture({ name: 'app.opened' }) + + expect(posthogMock.PostHog, 'POSTHOG_KEY alone must not construct PostHog').not.toHaveBeenCalled() + expect(warn).not.toHaveBeenCalled() + }) + + it('uses noop telemetry when BORING_TELEMETRY_ENABLED is false', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'false', POSTHOG_KEY: 'phc_secret' }), + ) + + telemetry.capture({ name: 'app.opened' }) + + expect(posthogMock.PostHog).not.toHaveBeenCalled() + }) + + it('uses noop telemetry and warns without secrets when enabled but POSTHOG_KEY is missing', () => { + const telemetry = createPostHogTelemetryFromEnv(env({ BORING_TELEMETRY_ENABLED: 'true' })) + + telemetry.capture({ name: 'app.opened' }) + + expect(posthogMock.PostHog).not.toHaveBeenCalled() + expect(warn).toHaveBeenCalledOnce() + expect(String(warn.mock.calls[0]?.[0])).toContain('POSTHOG_KEY is missing') + expect(String(warn.mock.calls[0]?.[0])).not.toContain('phc_') + }) + + it('constructs PostHog with the default host when enabled', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), + ) + + telemetry.capture({ name: 'app.opened' }) + + expect(posthogMock.PostHog).toHaveBeenCalledWith('phc_secret', { + host: 'https://us.i.posthog.com', + }) + expect(posthogMock.clients[0]?.capture).toHaveBeenCalledWith({ + distinctId: 'anonymous', + event: 'app.opened', + properties: { eventName: 'app.opened' }, + }) + }) + + it('passes POSTHOG_HOST overrides to PostHog', () => { + createPostHogTelemetryFromEnv( + env({ + BORING_TELEMETRY_ENABLED: 'true', + POSTHOG_KEY: 'phc_secret', + POSTHOG_HOST: 'https://eu.i.posthog.com', + }), + ) + + expect(posthogMock.clients[0]?.options).toEqual({ host: 'https://eu.i.posthog.com' }) + }) + + it('prefixes event names and adds project properties for safe BORING_TELEMETRY_PROJECT values', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ + BORING_TELEMETRY_ENABLED: 'true', + POSTHOG_KEY: 'phc_secret', + BORING_TELEMETRY_PROJECT: 'full-app', + }), + ) + + telemetry.capture({ + name: 'agent.chat.started', + distinctId: 'user_123', + properties: { workspaceId: 'workspace_1', durationMs: 12, prompt: 'do not send' }, + }) + + expect(posthogMock.clients[0]?.capture).toHaveBeenCalledWith({ + distinctId: 'user_123', + event: 'full-app.agent.chat.started', + properties: { + workspaceId: 'workspace_1', + durationMs: 12, + boringProject: 'full-app', + eventName: 'agent.chat.started', + }, + }) + }) + + it('disables invalid project prefixes and warns without secrets', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ + BORING_TELEMETRY_ENABLED: 'true', + POSTHOG_KEY: 'phc_secret', + BORING_TELEMETRY_PROJECT: '../bad project', + }), + ) + + telemetry.capture({ name: 'workspace.opened' }) + + expect(posthogMock.clients[0]?.capture).toHaveBeenCalledWith({ + distinctId: 'anonymous', + event: 'workspace.opened', + properties: { eventName: 'workspace.opened' }, + }) + expect(warn).toHaveBeenCalledOnce() + expect(String(warn.mock.calls[0]?.[0])).toContain('prefix disabled') + expect(String(warn.mock.calls[0]?.[0])).not.toContain('phc_secret') + expect(String(warn.mock.calls[0]?.[0])).not.toContain('../bad project') + }) + + it('swallows PostHog capture failures', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), + ) + posthogMock.clients[0]!.capture.mockImplementation(() => { + throw new Error('network down') + }) + + expect(() => telemetry.capture({ name: 'server.request.failed' })).not.toThrow() + }) + + it('flushes via PostHog shutdown when requested', async () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), + ) + + await telemetry.flush?.() + + expect(posthogMock.clients[0]?.shutdown).toHaveBeenCalledOnce() + }) +}) + +describe('parseTelemetryProject', () => { + it('accepts lowercase slugs and trims whitespace', () => { + expect(parseTelemetryProject(' full-app ')).toBe('full-app') + expect(parseTelemetryProject('customer-portal')).toBe('customer-portal') + }) + + it('rejects empty or unsafe values', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + + expect(parseTelemetryProject(undefined)).toBeUndefined() + expect(parseTelemetryProject('')).toBeUndefined() + expect(parseTelemetryProject('Full App')).toBeUndefined() + expect(parseTelemetryProject('../escape')).toBeUndefined() + + expect(warn).toHaveBeenCalledTimes(2) + }) +}) + +describe('sanitizeTelemetryProperties', () => { + it('keeps only allowlisted primitive telemetry properties', () => { + expect( + sanitizeTelemetryProperties({ + workspaceId: 'workspace_1', + sessionId: 'session_1', + requestId: 'request_1', + runtimeMode: 'local', + modelProvider: 'anthropic', + toolName: 'bash', + panelId: 'files', + commandId: 'open', + status: 'ok', + durationMs: 42, + errorCode: 'WORKSPACE_NOT_READY', + packageName: '@hachej/boring-core', + packageVersion: '0.1.0', + invalidDuration: Number.NaN, + prompt: 'secret prompt', + assistantOutput: 'secret answer', + command: 'cat .env', + stdout: 'secret output', + path: '/tmp/private', + stack: 'stack trace', + headers: { authorization: 'Bearer secret' }, + nestedAllowedKey: { workspaceId: 'nested' }, + env: 'SECRET=value', + objectValueOnAllowedKey: { unsafe: true }, + durationMsUnsafe: Number.POSITIVE_INFINITY, + }), + ).toEqual({ + workspaceId: 'workspace_1', + sessionId: 'session_1', + requestId: 'request_1', + runtimeMode: 'local', + modelProvider: 'anthropic', + toolName: 'bash', + panelId: 'files', + commandId: 'open', + status: 'ok', + durationMs: 42, + errorCode: 'WORKSPACE_NOT_READY', + packageName: '@hachej/boring-core', + packageVersion: '0.1.0', + }) + }) + + it('drops non-finite numbers even on allowlisted keys', () => { + expect( + sanitizeTelemetryProperties({ + durationMs: Number.POSITIVE_INFINITY, + requestId: 'request_1', + }), + ).toEqual({ requestId: 'request_1' }) + }) +}) diff --git a/packages/core/src/server/telemetry/posthog.ts b/packages/core/src/server/telemetry/posthog.ts new file mode 100644 index 000000000..9832aa239 --- /dev/null +++ b/packages/core/src/server/telemetry/posthog.ts @@ -0,0 +1,93 @@ +import { PostHog } from 'posthog-node' + +import { noopTelemetry, type TelemetryEvent, type TelemetrySink } from '../../shared/telemetry.js' + +const DEFAULT_POSTHOG_HOST = 'https://us.i.posthog.com' +const PROJECT_SLUG_PATTERN = /^[a-z0-9][a-z0-9-]{0,62}$/ + +const ALLOWED_PROPERTY_KEYS = new Set([ + 'workspaceId', + 'sessionId', + 'requestId', + 'runtimeMode', + 'modelProvider', + 'toolName', + 'panelId', + 'commandId', + 'status', + 'durationMs', + 'errorCode', + 'packageName', + 'packageVersion', +]) + +type SafeTelemetryProperty = string | number | boolean | null + +export function createPostHogTelemetryFromEnv( + env: NodeJS.ProcessEnv = process.env, +): TelemetrySink { + if (env.BORING_TELEMETRY_ENABLED !== 'true') return noopTelemetry + + const posthogKey = env.POSTHOG_KEY + if (!posthogKey) { + console.warn('PostHog telemetry is enabled but POSTHOG_KEY is missing; using noop telemetry.') + return noopTelemetry + } + + const posthog = new PostHog(posthogKey, { + host: env.POSTHOG_HOST ?? DEFAULT_POSTHOG_HOST, + }) + const project = parseTelemetryProject(env.BORING_TELEMETRY_PROJECT) + + return { + capture(event: TelemetryEvent) { + const properties = sanitizeTelemetryProperties(event.properties) + if (project) properties.boringProject = project + properties.eventName = event.name + + try { + posthog.capture({ + distinctId: event.distinctId ?? 'anonymous', + event: project ? `${project}.${event.name}` : event.name, + properties, + }) + } catch {} + }, + async flush() { + await Promise.resolve(posthog.shutdown()) + }, + } +} + +export function parseTelemetryProject(value: string | undefined): string | undefined { + const project = value?.trim() + if (!project) return undefined + if (PROJECT_SLUG_PATTERN.test(project)) return project + + console.warn('BORING_TELEMETRY_PROJECT must be a lowercase slug; telemetry project prefix disabled.') + return undefined +} + +export function sanitizeTelemetryProperties( + properties: Record | undefined, +): Record { + const sanitized: Record = {} + if (!properties) return sanitized + + for (const [key, value] of Object.entries(properties)) { + if (!ALLOWED_PROPERTY_KEYS.has(key)) continue + if (!isSafeTelemetryProperty(value)) continue + sanitized[key] = value + } + + return sanitized +} + +function isSafeTelemetryProperty(value: unknown): value is SafeTelemetryProperty { + return ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ) +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ae61357ab..5ec8b4526 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -496,6 +496,9 @@ importers: postgres: specifier: ^3.4.9 version: 3.4.9 + posthog-node: + specifier: ^5.35.1 + version: 5.35.1 react-helmet-async: specifier: ^2.0.5 version: 2.0.5(react@19.2.5) @@ -2415,6 +2418,12 @@ packages: engines: {node: '>=18'} hasBin: true + '@posthog/core@1.29.9': + resolution: {integrity: sha512-DjvuIyBZ2Z/gBhtZlITlM2D8PlnMsHSQ1D78dbUYoVsgGguvanpJTobZObjLlFkybyvfZFYkpoJkFNI/2Pw4IQ==} + + '@posthog/types@1.376.0': + resolution: {integrity: sha512-gbFfxCuZDs/D4QZMwdE+smD1jsuqgGpS6yKGHZZ19foxMy8RYHsU1E47iG1b88n/uN02fAabLibVwuxLtq8juw==} + '@protobufjs/aspromise@1.1.2': resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} @@ -6684,6 +6693,15 @@ packages: resolution: {integrity: sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw==} engines: {node: '>=12'} + posthog-node@5.35.1: + resolution: {integrity: sha512-F9S3pEIYfGEVjLYIFHKaqfTIhn5IpS02Dkp7C/f1rqr4Z67Iqbt4jbKO8raWsT0veEI3rUp+DKuXLW1hN07FQA==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + prettier@3.8.3: resolution: {integrity: sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==} engines: {node: '>=14'} @@ -9505,6 +9523,12 @@ snapshots: dependencies: playwright: 1.59.1 + '@posthog/core@1.29.9': + dependencies: + '@posthog/types': 1.376.0 + + '@posthog/types@1.376.0': {} + '@protobufjs/aspromise@1.1.2': {} '@protobufjs/base64@1.1.2': {} @@ -14201,6 +14225,10 @@ snapshots: postgres@3.4.9: {} + posthog-node@5.35.1: + dependencies: + '@posthog/core': 1.29.9 + prettier@3.8.3: {} pretty-format@27.5.1: From 2300929eb1c9221ab06cdd413c79d64453d59abc Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 07:23:35 +0000 Subject: [PATCH 09/14] feat(core): wire telemetry through core app server (boring-ui-v2-reorg-x0ek.3) --- packages/agent/src/server/createAgentApp.ts | 4 + .../agent/src/server/registerAgentRoutes.ts | 4 + packages/agent/src/shared/harness.ts | 3 + ...CoreWorkspaceAgentServer.telemetry.test.ts | 225 ++++++++++++++++++ .../server/createCoreWorkspaceAgentServer.ts | 12 + 5 files changed, 248 insertions(+) create mode 100644 packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts diff --git a/packages/agent/src/server/createAgentApp.ts b/packages/agent/src/server/createAgentApp.ts index 243ed312b..a91f665a9 100644 --- a/packages/agent/src/server/createAgentApp.ts +++ b/packages/agent/src/server/createAgentApp.ts @@ -2,6 +2,7 @@ import Fastify, { type FastifyInstance } from 'fastify' import type { AgentTool } from '../shared/tool' import type { AgentHarnessFactory } from '../shared/harness' import type { SessionStore } from '../shared/session' +import type { TelemetrySink } from '../shared/telemetry' import { getEnv } from './config/env' import type { RuntimeModeAdapter, RuntimeModeId } from './runtime/mode' import { resolveMode, autoDetectMode } from './runtime/resolveMode' @@ -57,6 +58,8 @@ export interface CreateAgentAppOptions { pi?: PiHarnessOptions /** Optional stable namespace for file-backed session storage. */ sessionNamespace?: string + /** Optional best-effort telemetry sink supplied by an embedding host. */ + telemetry?: TelemetrySink /** Optional explicit file-backed session directory. Mostly for tests/hosts. */ sessionDir?: string /** @@ -134,6 +137,7 @@ export async function createAgentApp( sessionDir: opts.sessionDir, systemPromptAppend: opts.systemPromptAppend, systemPromptDynamic: opts.systemPromptDynamic, + telemetry: opts.telemetry, }) const sessionChangesTracker = new InMemorySessionChangesTracker() diff --git a/packages/agent/src/server/registerAgentRoutes.ts b/packages/agent/src/server/registerAgentRoutes.ts index 563cb9732..d640ef869 100644 --- a/packages/agent/src/server/registerAgentRoutes.ts +++ b/packages/agent/src/server/registerAgentRoutes.ts @@ -4,6 +4,7 @@ import type { AgentTool } from '../shared/tool' import type { AgentHarnessFactory } from '../shared/harness' import type { SessionStore } from '../shared/session' import type { SandboxHandleStore } from '../shared/sandbox-handle-store' +import type { TelemetrySink } from '../shared/telemetry' import { AuthStorage, ModelRegistry } from '@mariozechner/pi-coding-agent' import { getEnv } from './config/env' import type { RuntimeBundle, RuntimeModeAdapter, RuntimeModeId } from './runtime/mode' @@ -175,6 +176,8 @@ export interface RegisterAgentRoutesOptions { request?: FastifyRequest }) => PiHarnessOptions | undefined | Promise sessionNamespace?: string + /** Optional best-effort telemetry sink supplied by an embedding host. */ + telemetry?: TelemetrySink getSessionNamespace?: (ctx: { workspaceId: string workspaceRoot: string @@ -333,6 +336,7 @@ export const registerAgentRoutes: FastifyPluginAsync cwd: root, sessionNamespace: scope.sessionNamespace, systemPromptAppend: opts.systemPromptAppend, + telemetry: opts.telemetry, }) const readyTracker = new ReadyStatusTracker({ sandboxReady: resolvedMode !== 'vercel-sandbox', diff --git a/packages/agent/src/shared/harness.ts b/packages/agent/src/shared/harness.ts index 991fd1f36..b92018e0d 100644 --- a/packages/agent/src/shared/harness.ts +++ b/packages/agent/src/shared/harness.ts @@ -1,5 +1,6 @@ import type { UIMessageChunk } from './message' import type { SessionStore } from './session' +import type { TelemetrySink } from './telemetry' import type { AgentTool } from './tool' export interface AgentHarnessFactoryInput { @@ -15,6 +16,8 @@ export interface AgentHarnessFactoryInput { * prompt context without a workspace-injected harness extension. */ systemPromptDynamic?: () => string | undefined | Promise + /** Host-provided telemetry sink. Optional and best-effort; harnesses may ignore it. */ + telemetry?: TelemetrySink } export type AgentHarnessFactory = (input: AgentHarnessFactoryInput) => AgentHarness | Promise diff --git a/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts new file mode 100644 index 000000000..565195e32 --- /dev/null +++ b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts @@ -0,0 +1,225 @@ +import Fastify from 'fastify' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { TelemetrySink } from '../../../shared/telemetry.js' +import type { CoreConfig } from '../../../shared/types.js' + +const posthogMock = vi.hoisted(() => { + type MockPostHogClient = { + capture: ReturnType + shutdown: ReturnType + } + + const clients: MockPostHogClient[] = [] + const PostHog = vi.fn(() => { + const client = { + capture: vi.fn(), + shutdown: vi.fn(), + } + clients.push(client) + return client + }) + + return { clients, PostHog } +}) + +const agentMock = vi.hoisted(() => ({ + registerOptions: [] as Array>, +})) + +const coreAppMock = vi.hoisted(() => ({ + debugLogs: [] as unknown[][], +})) + +vi.mock('posthog-node', () => ({ + PostHog: posthogMock.PostHog, +})) + +vi.mock('@hachej/boring-agent/server', () => ({ + compactPiPackages: (packages: unknown[]) => packages, + registerAgentRoutes: async (_app: unknown, opts: Record) => { + agentMock.registerOptions.push(opts) + }, +})) + +vi.mock('@hachej/boring-workspace/app/server', () => ({ + collectWorkspaceAgentServerPlugins: () => ({ + agentOptions: { + extraTools: [], + pi: undefined, + systemPromptAppend: undefined, + }, + preservedUiStateKeys: [], + provisioningContributions: [], + routeContributions: [], + }), + hasDirServerPlugin: () => false, + provisionWorkspaceAgentServer: vi.fn(), + readWorkspacePluginPackagePiSnapshot: () => ({ + additionalSkillPaths: [], + extensionFactories: [], + extensionPaths: [], + packages: [], + systemPromptAppend: undefined, + }), + resolveDefaultWorkspacePluginPackagePaths: () => [], + resolveOnePluginEntry: async (entry: unknown) => entry, +})) + +vi.mock('@hachej/boring-workspace/server', () => ({ + createInMemoryBridge: () => ({ + drainCommands: vi.fn(), + getState: vi.fn(), + postCommand: vi.fn(), + setState: vi.fn(), + subscribeCommands: vi.fn(), + }), + createWorkspaceUiTools: () => [], + uiRoutes: async () => {}, +})) + +vi.mock('../../../server/auth/index.js', () => ({ + authHook: async () => {}, + createAuth: () => ({ + handler: vi.fn(), + }), +})) + +vi.mock('../../../server/app/index.js', () => ({ + createCoreApp: async (config: CoreConfig) => { + const app = Fastify({ logger: false }) + app.decorate('config', config) + app.log.debug = (...args: unknown[]) => { + coreAppMock.debugLogs.push(args) + } + return app + }, + registerRoutes: async () => {}, +})) + +vi.mock('../../../server/routes/index.js', () => ({ + registerInviteRoutes: async () => {}, + registerMemberRoutes: async () => {}, + registerSettingsRoutes: async () => {}, + registerWorkspaceRoutes: async () => {}, +})) + +vi.mock('../../../server/db/index.js', () => ({ + createDatabase: () => ({ + db: {}, + sql: { end: vi.fn() }, + }), + PostgresUserStore: class PostgresUserStore {}, + PostgresWorkspaceStore: class PostgresWorkspaceStore {}, +})) + +vi.mock('../../../server/config/index.js', () => ({ + loadConfig: async () => ({ + auth: { url: 'http://localhost:3000' }, + encryption: { workspaceSettingsKey: 'test-key' }, + stores: 'postgres', + }), +})) + +vi.mock('../../../server/runtime/index.js', () => ({ + WorkspaceRuntimeSandboxHandleStore: class WorkspaceRuntimeSandboxHandleStore {}, +})) + +const { createCoreWorkspaceAgentServer } = await import('../createCoreWorkspaceAgentServer.js') + +function resetTelemetryEnv(): void { + delete process.env.BORING_TELEMETRY_ENABLED + delete process.env.POSTHOG_KEY + delete process.env.POSTHOG_HOST + delete process.env.BORING_TELEMETRY_PROJECT +} + +describe('createCoreWorkspaceAgentServer telemetry wiring', () => { + beforeEach(() => { + resetTelemetryEnv() + agentMock.registerOptions.length = 0 + coreAppMock.debugLogs.length = 0 + posthogMock.clients.length = 0 + posthogMock.PostHog.mockClear() + }) + + afterEach(() => { + resetTelemetryEnv() + vi.clearAllMocks() + }) + + it('uses the core PostHog env helper by default and passes the sink to agent routes', async () => { + process.env.BORING_TELEMETRY_ENABLED = 'true' + process.env.POSTHOG_KEY = 'phc_redacted_test_key' + process.env.BORING_TELEMETRY_PROJECT = 'full-app' + + const app = await createCoreWorkspaceAgentServer({ serveFrontend: false }) + try { + const telemetry = agentMock.registerOptions[0]?.telemetry as TelemetrySink | undefined + telemetry?.capture({ + name: 'agent.chat.started', + properties: { workspaceId: 'workspace_1', prompt: 'do not send' }, + }) + + expect(agentMock.registerOptions, 'agent routes should be registered once').toHaveLength(1) + expect(telemetry, 'resolved telemetry sink should be passed to agent routes').toBeDefined() + expect(posthogMock.PostHog, 'enabled env should construct PostHog from the core helper').toHaveBeenCalledOnce() + expect(posthogMock.clients[0]?.capture).toHaveBeenCalledWith({ + distinctId: 'anonymous', + event: 'full-app.agent.chat.started', + properties: { + workspaceId: 'workspace_1', + boringProject: 'full-app', + eventName: 'agent.chat.started', + }, + }) + expect(coreAppMock.debugLogs).toContainEqual([ + { telemetry: { source: 'posthog-env' } }, + 'resolved telemetry sink', + ]) + } finally { + await app.close() + } + }) + + it('lets a custom telemetry sink override env helper creation', async () => { + process.env.BORING_TELEMETRY_ENABLED = 'true' + process.env.POSTHOG_KEY = 'phc_redacted_test_key' + const customTelemetry: TelemetrySink = { capture: vi.fn() } + + const app = await createCoreWorkspaceAgentServer({ + serveFrontend: false, + telemetry: customTelemetry, + }) + try { + expect(agentMock.registerOptions[0]?.telemetry).toBe(customTelemetry) + expect(posthogMock.PostHog, 'custom sink should bypass PostHog env helper').not.toHaveBeenCalled() + expect(coreAppMock.debugLogs).toContainEqual([ + { telemetry: { source: 'custom' } }, + 'resolved telemetry sink', + ]) + } finally { + await app.close() + } + }) + + it('passes noop telemetry when env telemetry is disabled', async () => { + process.env.POSTHOG_KEY = 'phc_redacted_test_key' + + const app = await createCoreWorkspaceAgentServer({ serveFrontend: false }) + try { + const telemetry = agentMock.registerOptions[0]?.telemetry as TelemetrySink | undefined + telemetry?.capture({ name: 'app.opened' }) + + expect(telemetry, 'disabled env still passes a safe noop sink').toBeDefined() + expect(posthogMock.PostHog, 'POSTHOG_KEY alone must not construct PostHog').not.toHaveBeenCalled() + expect(posthogMock.clients[0]?.capture).toBeUndefined() + expect(coreAppMock.debugLogs).toContainEqual([ + { telemetry: { source: 'noop-env' } }, + 'resolved telemetry sink', + ]) + } finally { + await app.close() + } + }) +}) diff --git a/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts b/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts index c11b07fa7..eb5483a0c 100644 --- a/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts +++ b/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts @@ -29,6 +29,7 @@ import { import type { FastifyInstance } from 'fastify' import type postgres from 'postgres' import type { CoreConfig } from '../../shared/types.js' +import type { TelemetrySink } from '../../shared/telemetry.js' import { authHook, createAuth, @@ -54,6 +55,7 @@ import { } from '../../server/db/index.js' import { loadConfig, type LoadConfigOptions } from '../../server/config/index.js' import { WorkspaceRuntimeSandboxHandleStore } from '../../server/runtime/index.js' +import { createPostHogTelemetryFromEnv } from '../../server/telemetry/posthog.js' const MIME_TYPES: Record = { '.css': 'text/css; charset=utf-8', @@ -127,6 +129,8 @@ export interface CreateCoreWorkspaceAgentServerOptions extraTools?: RegisterAgentRoutesOptions['extraTools'] systemPromptAppend?: string serveFrontend?: boolean + /** Optional best-effort telemetry sink. Defaults to core's PostHog env helper. */ + telemetry?: TelemetrySink } type AgentPiOptions = RegisterAgentRoutesOptions['pi'] @@ -549,6 +553,13 @@ export async function createCoreWorkspaceAgentServer( const serveFrontend = options.serveFrontend ?? (process.env.NODE_ENV !== 'development' && Boolean(appRoot)) const workspaceRoot = options.workspaceRoot ?? process.cwd() + const telemetrySource = options.telemetry + ? 'custom' + : process.env.BORING_TELEMETRY_ENABLED === 'true' && process.env.POSTHOG_KEY + ? 'posthog-env' + : 'noop-env' + const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) + app.log.debug({ telemetry: { source: telemetrySource } }, 'resolved telemetry sink') await registerCoreRoutes({ app, sql, db, userStore, workspaceStore }) @@ -693,6 +704,7 @@ export async function createCoreWorkspaceAgentServer( getWorkspaceId: resolveWorkspaceId, getWorkspaceRoot: resolveRoot, registerHealthRoute: options.registerHealthRoute ?? false, + telemetry, }) await app.register(uiRoutes, { From 3fbe2aebc5596c648091a2349c63e20fe31edece Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 08:03:43 +0000 Subject: [PATCH 10/14] feat(agent): emit minimal telemetry events (boring-ui-v2-reorg-x0ek.4) --- .../server/__tests__/createAgentApp.test.ts | 21 +++ packages/agent/src/server/createAgentApp.ts | 1 + .../__tests__/tool-adapter.telemetry.test.ts | 131 ++++++++++++++++++ .../harness/pi-coding-agent/createHarness.ts | 5 +- .../harness/pi-coding-agent/tool-adapter.ts | 87 +++++++++--- .../server/http/routes/__tests__/chat.test.ts | 123 +++++++++++++++- packages/agent/src/server/http/routes/chat.ts | 74 +++++++++- .../agent/src/server/registerAgentRoutes.ts | 1 + ...CoreWorkspaceAgentServer.telemetry.test.ts | 79 +++++++++++ .../server/createCoreWorkspaceAgentServer.ts | 35 ++++- 10 files changed, 526 insertions(+), 31 deletions(-) create mode 100644 packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts diff --git a/packages/agent/src/server/__tests__/createAgentApp.test.ts b/packages/agent/src/server/__tests__/createAgentApp.test.ts index fd7ab478f..008f741b2 100644 --- a/packages/agent/src/server/__tests__/createAgentApp.test.ts +++ b/packages/agent/src/server/__tests__/createAgentApp.test.ts @@ -85,6 +85,12 @@ test('createAgentApp falls back to BORING_AGENT_TEMPLATE_PATH', async () => { test('createAgentApp can use a custom harness factory for non-pi runtimes', async () => { const workspaceRoot = await makeTempDir('boring-ui-custom-harness-') const reloadSession = vi.fn(async () => true) + const telemetryEvents: Array<{ name: string; properties?: Record }> = [] + const telemetry = { + capture(event: { name: string; properties?: Record }) { + telemetryEvents.push(event) + }, + } const harnessFactory = vi.fn(async (input) => ({ id: 'custom-test-harness', placement: 'server' as const, @@ -109,6 +115,7 @@ test('createAgentApp can use a custom harness factory for non-pi runtimes', asyn mode: 'direct', logger: false, harnessFactory, + telemetry, extraTools: [{ name: 'custom_runtime_tool', description: 'Provided to harness factory.', @@ -119,8 +126,22 @@ test('createAgentApp can use a custom harness factory for non-pi runtimes', asyn try { expect(harnessFactory).toHaveBeenCalledTimes(1) expect(harnessFactory.mock.calls[0]?.[0].cwd).toBe(workspaceRoot) + expect(harnessFactory.mock.calls[0]?.[0].telemetry).toBe(telemetry) expect(harnessFactory.mock.calls[0]?.[0].tools.map((tool: { name: string }) => tool.name)).toContain('custom_runtime_tool') + const chatRes = await app.inject({ + method: 'POST', + url: '/api/v1/agent/chat', + payload: { sessionId: 'custom', message: 'secret prompt must not be captured' }, + }) + expect(chatRes.statusCode).toBe(200) + expect(telemetryEvents.map((event) => event.name)).toEqual([ + 'agent.chat.started', + 'agent.chat.message.submitted', + 'agent.chat.completed', + ]) + expect(JSON.stringify(telemetryEvents)).not.toContain('secret prompt') + const res = await app.inject({ method: 'POST', url: '/api/v1/agent/reload', payload: { sessionId: 'custom' } }) expect(res.statusCode).toBe(200) expect(res.json()).toEqual({ ok: true, sessionId: 'custom', reloaded: true }) diff --git a/packages/agent/src/server/createAgentApp.ts b/packages/agent/src/server/createAgentApp.ts index a91f665a9..4183a8ec2 100644 --- a/packages/agent/src/server/createAgentApp.ts +++ b/packages/agent/src/server/createAgentApp.ts @@ -175,6 +175,7 @@ export async function createAgentApp( harness, workdir: runtimeBundle.workspace.root, sessionChangesTracker, + telemetry: opts.telemetry, }) await app.register(sessionRoutes, { sessionStore: harness.sessions as unknown as SessionStore, diff --git a/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts b/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts new file mode 100644 index 000000000..fee6e88c4 --- /dev/null +++ b/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it, vi } from 'vitest' + +import { ErrorCode } from '../../../../shared/error-codes' +import type { AgentTool } from '../../../../shared/tool' +import type { TelemetryEvent, TelemetrySink } from '../../../../shared/telemetry' +import { adaptToolForPi } from '../tool-adapter' + +function createTelemetryRecorder(): { telemetry: TelemetrySink; events: TelemetryEvent[] } { + const events: TelemetryEvent[] = [] + return { + events, + telemetry: { + capture(event) { + events.push(event) + }, + }, + } +} + +function createTool(overrides: Partial = {}): AgentTool { + return { + name: 'bash', + description: 'test tool', + parameters: {}, + async execute() { + return { content: [{ type: 'text', text: 'ok output' }] } + }, + ...overrides, + } +} + +async function executeAdapted(tool: AgentTool, telemetry: TelemetrySink) { + const adapted = adaptToolForPi(tool, 'sess-tool', telemetry) + return await adapted.execute( + 'tool-call-1', + { command: 'cat .env', path: '/tmp/private-path' }, + new AbortController().signal, + undefined, + {} as never, + ) +} + +describe('tool adapter telemetry', () => { + it('emits safe agent.tool.completed telemetry without args or output', async () => { + const recorder = createTelemetryRecorder() + + await executeAdapted(createTool(), recorder.telemetry) + + expect(recorder.events).toHaveLength(1) + expect(recorder.events[0]).toEqual({ + name: 'agent.tool.completed', + properties: { + toolName: 'bash', + sessionId: 'sess-tool', + status: 'ok', + durationMs: expect.any(Number), + }, + }) + const serialized = JSON.stringify(recorder.events) + expect(serialized).not.toContain('cat .env') + expect(serialized).not.toContain('private-path') + expect(serialized).not.toContain('ok output') + }) + + it('emits safe agent.tool.failed telemetry for tool error results', async () => { + const recorder = createTelemetryRecorder() + const tool = createTool({ + async execute() { + return { + isError: true, + content: [{ type: 'text', text: 'secret stderr output' }], + details: { code: ErrorCode.enum.WORKSPACE_NOT_READY, command: 'cat .env' }, + } + }, + }) + + await expect(executeAdapted(tool, recorder.telemetry)).rejects.toThrow('secret stderr output') + + expect(recorder.events).toHaveLength(1) + expect(recorder.events[0]).toEqual({ + name: 'agent.tool.failed', + properties: { + toolName: 'bash', + sessionId: 'sess-tool', + status: 'error', + durationMs: expect.any(Number), + errorCode: ErrorCode.enum.WORKSPACE_NOT_READY, + }, + }) + const serialized = JSON.stringify(recorder.events) + expect(serialized).not.toContain('secret stderr output') + expect(serialized).not.toContain('cat .env') + }) + + it('emits safe agent.tool.failed telemetry for thrown tool errors', async () => { + const recorder = createTelemetryRecorder() + const tool = createTool({ + async execute() { + throw new Error('raw stack /tmp/private-path secret') + }, + }) + + await expect(executeAdapted(tool, recorder.telemetry)).rejects.toThrow('raw stack') + + expect(recorder.events).toHaveLength(1) + expect(recorder.events[0]).toEqual({ + name: 'agent.tool.failed', + properties: { + toolName: 'bash', + sessionId: 'sess-tool', + status: 'error', + durationMs: expect.any(Number), + errorCode: ErrorCode.enum.TOOL_EXECUTION_ERROR, + }, + }) + expect(JSON.stringify(recorder.events)).not.toContain('private-path') + }) + + it('telemetry sink failures do not change tool behavior', async () => { + const result = await executeAdapted(createTool(), { + capture() { + throw new Error('telemetry down') + }, + }) + + expect(result).toEqual({ + content: [{ type: 'text', text: 'ok output' }], + details: undefined, + }) + }) +}) diff --git a/packages/agent/src/server/harness/pi-coding-agent/createHarness.ts b/packages/agent/src/server/harness/pi-coding-agent/createHarness.ts index 06f2e01f9..64a0951b5 100644 --- a/packages/agent/src/server/harness/pi-coding-agent/createHarness.ts +++ b/packages/agent/src/server/harness/pi-coding-agent/createHarness.ts @@ -18,6 +18,7 @@ import { import type { AgentHarness, SendMessageInput, RunContext, MessageAttachment, FollowUpOptions } from "../../../shared/harness.js"; import { createLogger } from "../../logging.js"; import type { AgentTool } from "../../../shared/tool.js"; +import type { TelemetrySink } from "../../../shared/telemetry.js"; import type { UIMessageChunk } from "../../../shared/message.js"; import { adaptToolsForPi } from "./tool-adapter.js"; import { piEventToChunks } from "./stream-adapter.js"; @@ -327,6 +328,8 @@ export function createPiCodingAgentHarness(opts: { sessionNamespace?: string; /** Optional explicit file-backed session directory. Mostly for tests/hosts. */ sessionDir?: string; + /** Optional best-effort telemetry sink supplied by an embedding host. */ + telemetry?: TelemetrySink; }): AgentHarness { const sessionStore = new PiSessionStore(opts.cwd, { sessionNamespace: opts.sessionNamespace, @@ -473,7 +476,7 @@ export function createPiCodingAgentHarness(opts: { const { session: piSession } = await createAgentSession({ cwd: ctx.workdir, tools: [], - customTools: adaptToolsForPi(opts.tools, input.sessionId), + customTools: adaptToolsForPi(opts.tools, input.sessionId, opts.telemetry), model, thinkingLevel: input.thinkingLevel ?? "off", sessionManager, diff --git a/packages/agent/src/server/harness/pi-coding-agent/tool-adapter.ts b/packages/agent/src/server/harness/pi-coding-agent/tool-adapter.ts index 67838ac26..722641e9e 100644 --- a/packages/agent/src/server/harness/pi-coding-agent/tool-adapter.ts +++ b/packages/agent/src/server/harness/pi-coding-agent/tool-adapter.ts @@ -1,7 +1,31 @@ import type { ToolDefinition } from "@mariozechner/pi-coding-agent"; -import type { AgentTool } from "../../../shared/tool.js"; +import type { AgentTool, ToolResult } from "../../../shared/tool.js"; +import { noopTelemetry, safeCapture, type TelemetrySink } from "../../../shared/telemetry.js"; +import { ErrorCode } from "../../../shared/error-codes.js"; -export function adaptToolForPi(tool: AgentTool, sessionId?: string): ToolDefinition { +function toolTelemetryProperties( + toolName: string, + sessionId: string | undefined, + status: 'ok' | 'error', + startedAt: number, + result?: ToolResult, +): Record { + const properties: Record = { + toolName, + status, + durationMs: Date.now() - startedAt, + } + if (sessionId) properties.sessionId = sessionId + const errorCode = (result?.details as { code?: unknown } | undefined)?.code + if (status === 'error') { + properties.errorCode = ErrorCode.safeParse(errorCode).success + ? (errorCode as string) + : ErrorCode.enum.TOOL_EXECUTION_ERROR + } + return properties +} + +export function adaptToolForPi(tool: AgentTool, sessionId?: string, telemetry: TelemetrySink = noopTelemetry): ToolDefinition { return { name: tool.name, label: tool.name, @@ -9,25 +33,52 @@ export function adaptToolForPi(tool: AgentTool, sessionId?: string): ToolDefinit parameters: tool.parameters as any, promptSnippet: tool.promptSnippet ?? tool.description, async execute(toolCallId, params, signal, onUpdate, _ctx) { - const result = await tool.execute(params as Record, { - toolCallId, - abortSignal: signal ?? new AbortController().signal, - onUpdate: onUpdate - ? (partial) => onUpdate({ content: [{ type: "text", text: partial }], details: undefined }) - : undefined, - sessionId, - }); - if (result.isError) { - throw new Error(result.content.map((c) => c.text).join("\n")); + const startedAt = Date.now(); + let emittedFailure = false; + try { + const result = await tool.execute(params as Record, { + toolCallId, + abortSignal: signal ?? new AbortController().signal, + onUpdate: onUpdate + ? (partial) => onUpdate({ content: [{ type: "text", text: partial }], details: undefined }) + : undefined, + sessionId, + }); + safeCapture(telemetry, { + name: result.isError ? 'agent.tool.failed' : 'agent.tool.completed', + properties: toolTelemetryProperties( + tool.name, + sessionId, + result.isError ? 'error' : 'ok', + startedAt, + result, + ), + }); + if (result.isError) { + emittedFailure = true; + throw new Error(result.content.map((c) => c.text).join("\n")); + } + return { + content: result.content, + details: result.details, + }; + } catch (error) { + if (!emittedFailure) { + safeCapture(telemetry, { + name: 'agent.tool.failed', + properties: toolTelemetryProperties(tool.name, sessionId, 'error', startedAt), + }); + } + throw error; } - return { - content: result.content, - details: result.details, - }; }, } as ToolDefinition; } -export function adaptToolsForPi(tools: AgentTool[], sessionId?: string): ToolDefinition[] { - return tools.map((tool) => adaptToolForPi(tool, sessionId)); +export function adaptToolsForPi( + tools: AgentTool[], + sessionId?: string, + telemetry?: TelemetrySink, +): ToolDefinition[] { + return tools.map((tool) => adaptToolForPi(tool, sessionId, telemetry)); } diff --git a/packages/agent/src/server/http/routes/__tests__/chat.test.ts b/packages/agent/src/server/http/routes/__tests__/chat.test.ts index 42d3f733e..7d22590a5 100644 --- a/packages/agent/src/server/http/routes/__tests__/chat.test.ts +++ b/packages/agent/src/server/http/routes/__tests__/chat.test.ts @@ -4,6 +4,8 @@ import { chatRoutes, type ChatRouteOptions } from '../chat' import type { AgentHarness, SendMessageInput, RunContext } from '../../../../shared/harness' import type { UIMessageChunk } from '../../../../shared/message' import type { SessionStore } from '../../../../shared/session' +import { ErrorCode } from '../../../../shared/error-codes' +import type { TelemetrySink, TelemetryEvent } from '../../../../shared/telemetry' import { ERROR_CODE_VALIDATION_ERROR, ERROR_CODE_INTERNAL, @@ -45,10 +47,23 @@ function buildApp(overrides: Partial = {}) { app.register(chatRoutes, { harness: overrides.harness ?? createMockHarness(), workdir: overrides.workdir ?? '/tmp/test', + telemetry: overrides.telemetry, }) return app.ready().then(() => app) } +function createTelemetryRecorder(): { telemetry: TelemetrySink; events: TelemetryEvent[] } { + const events: TelemetryEvent[] = [] + return { + events, + telemetry: { + capture(event) { + events.push(event) + }, + }, + } +} + const validBody = { sessionId: 'sess-1', message: 'hello', @@ -88,6 +103,95 @@ describe('POST /api/v1/agent/chat', () => { await app.close() }) + test('emits safe chat telemetry for submitted and completed turns', async () => { + const recorder = createTelemetryRecorder() + const app = await buildApp({ telemetry: recorder.telemetry }) + + const res = await app.inject({ + method: 'POST', + url: '/api/v1/agent/chat', + payload: { + sessionId: 'sess-telemetry', + message: 'secret user prompt must not be captured', + model: { provider: 'anthropic', id: 'claude-secret' }, + }, + }) + + expect(res.statusCode).toBe(200) + expect(recorder.events.map((event) => event.name)).toEqual([ + 'agent.chat.started', + 'agent.chat.message.submitted', + 'agent.chat.completed', + ]) + expect(recorder.events.at(-1)?.properties).toEqual(expect.objectContaining({ + sessionId: 'sess-telemetry', + requestId: expect.any(String), + modelProvider: 'anthropic', + status: 'ok', + durationMs: expect.any(Number), + })) + const serialized = JSON.stringify(recorder.events) + expect(serialized).not.toContain('secret user prompt') + expect(serialized).not.toContain('claude-secret') + + await app.close() + }) + + test('emits safe chat failure telemetry without changing the response', async () => { + const recorder = createTelemetryRecorder() + const rawError = new Error('raw failure with /tmp/private-path and secret') as Error & { code?: string } + rawError.code = 'SECRET_TOKEN' + const harness = createMockHarness([], { + throwOnSend: rawError, + }) + const app = await buildApp({ harness, telemetry: recorder.telemetry }) + + const res = await app.inject({ + method: 'POST', + url: '/api/v1/agent/chat', + payload: { sessionId: 'sess-failed', message: 'secret prompt' }, + }) + + expect(res.statusCode).toBe(500) + expect(recorder.events.map((event) => event.name)).toEqual([ + 'agent.chat.started', + 'agent.chat.message.submitted', + 'agent.chat.failed', + ]) + expect(recorder.events.at(-1)?.properties).toEqual(expect.objectContaining({ + sessionId: 'sess-failed', + status: 'error', + durationMs: expect.any(Number), + errorCode: ErrorCode.enum.INTERNAL_ERROR, + })) + const serialized = JSON.stringify(recorder.events) + expect(serialized).not.toContain('secret prompt') + expect(serialized).not.toContain('private-path') + + await app.close() + }) + + test('telemetry sink failures do not break chat streaming', async () => { + const app = await buildApp({ + telemetry: { + capture() { + throw new Error('telemetry down') + }, + }, + }) + + const res = await app.inject({ + method: 'POST', + url: '/api/v1/agent/chat', + payload: validBody, + }) + + expect(res.statusCode).toBe(200) + expect(res.body).toContain('[DONE]') + + await app.close() + }) + test('accepts optional model and thinkingLevel', async () => { const sendMessage = vi.fn(function* () { yield { type: 'start' } @@ -254,18 +358,19 @@ describe('POST /api/v1/agent/chat', () => { await app.close() }) - test('emits SSE error chunk when async iterator throws mid-stream', async () => { + test('emits SSE error chunk and failed telemetry when async iterator throws mid-stream', async () => { + const recorder = createTelemetryRecorder() const harness = createMockHarness() harness.sendMessage = function () { return (async function* () { yield { type: 'start' } yield { type: 'text-start', id: '0' } yield { type: 'text-delta', id: '0', delta: 'before' } - throw new Error('mid-stream kaboom') + throw new Error('mid-stream kaboom with /tmp/private-path secret') })() } - const app = await buildApp({ harness }) + const app = await buildApp({ harness, telemetry: recorder.telemetry }) const res = await app.inject({ method: 'POST', @@ -275,6 +380,18 @@ describe('POST /api/v1/agent/chat', () => { expect(res.statusCode).toBe(200) expect(res.body).toContain('error') + expect(recorder.events.map((event) => event.name)).toEqual([ + 'agent.chat.started', + 'agent.chat.message.submitted', + 'agent.chat.failed', + ]) + expect(recorder.events.at(-1)?.properties).toEqual(expect.objectContaining({ + sessionId: 'sess-1', + status: 'error', + errorCode: ErrorCode.enum.INTERNAL_ERROR, + durationMs: expect.any(Number), + })) + expect(JSON.stringify(recorder.events)).not.toContain('private-path') await app.close() }) diff --git a/packages/agent/src/server/http/routes/chat.ts b/packages/agent/src/server/http/routes/chat.ts index 8b9838e2d..6d00b74ef 100644 --- a/packages/agent/src/server/http/routes/chat.ts +++ b/packages/agent/src/server/http/routes/chat.ts @@ -6,6 +6,8 @@ import type { UIMessageChunk } from '../sse' import type { AgentHarness, RunContext } from '../../../shared/harness' import type { SessionCtx } from '../../../shared/session' import type { UIMessage } from '../../../shared/message' +import { ErrorCode } from '../../../shared/error-codes' +import { noopTelemetry, safeCapture, type TelemetrySink } from '../../../shared/telemetry' import { createBodyValidator, ERROR_CODE_INTERNAL, @@ -60,6 +62,21 @@ export interface ChatRouteOptions { workdir: string }> sessionChangesTracker?: SessionChangesTracker + telemetry?: TelemetrySink +} + +function addTelemetryProperty( + properties: Record, + key: string, + value: unknown, +): void { + if (typeof value === 'string' && value) properties[key] = value + if (typeof value === 'number' && Number.isFinite(value)) properties[key] = value +} + +function safeTelemetryErrorCode(value: unknown): string { + const parsed = ErrorCode.safeParse(value) + return parsed.success ? parsed.data : ErrorCode.enum.INTERNAL_ERROR } export function chatRoutes( @@ -68,6 +85,7 @@ export function chatRoutes( done: (err?: Error) => void, ): void { const { sessionChangesTracker } = opts + const telemetry = opts.telemetry ?? noopTelemetry const validateBody = createBodyValidator(chatBodySchema) const buffers = new StreamBufferStore() // Track last follow-up seq/nonce per session for dedupe detection. @@ -121,9 +139,19 @@ export function chatRoutes( const { sessionId, message, model, thinkingLevel, attachments } = request.body as ChatBody const turnId = randomUUID() + const startedAt = Date.now() + const telemetryProperties: Record = { + sessionId, + requestId: request.id, + } + addTelemetryProperty(telemetryProperties, 'workspaceId', request.workspaceContext?.workspaceId) + addTelemetryProperty(telemetryProperties, 'modelProvider', model?.provider) request.log.info({ sessionId, turnId, model, thinkingLevel }, '[chat] start') - const runtime = await resolveRuntime(request) + safeCapture(telemetry, { + name: 'agent.chat.started', + properties: telemetryProperties, + }) const abortController = new AbortController() let streamStarted = false @@ -137,14 +165,18 @@ export function chatRoutes( } }) - const ctx: RunContext = { - abortSignal: abortController.signal, - workdir: runtime.workdir, - } - const buf = buffers.create(sessionId, turnId) try { + const runtime = await resolveRuntime(request) + const ctx: RunContext = { + abortSignal: abortController.signal, + workdir: runtime.workdir, + } + safeCapture(telemetry, { + name: 'agent.chat.message.submitted', + properties: telemetryProperties, + }) const chunks = runtime.harness.sendMessage( { sessionId, message, model, thinkingLevel, attachments }, ctx, @@ -152,6 +184,7 @@ export function chatRoutes( const stream = createUIMessageStream({ async execute({ writer }: { writer: { write(chunk: UIMessageChunk): void } }) { + let streamFailed = false try { for await (const chunk of chunks) { const c = chunk as UIMessageChunk @@ -163,7 +196,17 @@ export function chatRoutes( writer.write(c) } } catch (err) { + streamFailed = true request.log.error({ err, sessionId }, '[chat] stream error') + safeCapture(telemetry, { + name: 'agent.chat.failed', + properties: { + ...telemetryProperties, + status: 'error', + durationMs: Date.now() - startedAt, + errorCode: ErrorCode.enum.INTERNAL_ERROR, + }, + }) const errChunk = { type: 'error', errorText: 'internal error', @@ -176,6 +219,16 @@ export function chatRoutes( // and may arrive before markComplete's callback) sees the flag // and does NOT abort an already-finished stream. streamCompleted = true + if (!streamFailed) { + safeCapture(telemetry, { + name: 'agent.chat.completed', + properties: { + ...telemetryProperties, + status: 'ok', + durationMs: Date.now() - startedAt, + }, + }) + } buf.markComplete(() => buffers.evict(sessionId, turnId)) } }, @@ -199,6 +252,15 @@ export function chatRoutes( return } catch (err) { request.log.error({ err, sessionId }, '[chat] error') + safeCapture(telemetry, { + name: 'agent.chat.failed', + properties: { + ...telemetryProperties, + status: 'error', + durationMs: Date.now() - startedAt, + errorCode: safeTelemetryErrorCode((err as { code?: unknown })?.code), + }, + }) buf.markComplete(() => buffers.evict(sessionId, turnId)) if (streamStarted) return return reply.code(500).send({ diff --git a/packages/agent/src/server/registerAgentRoutes.ts b/packages/agent/src/server/registerAgentRoutes.ts index d640ef869..f0e9addca 100644 --- a/packages/agent/src/server/registerAgentRoutes.ts +++ b/packages/agent/src/server/registerAgentRoutes.ts @@ -547,6 +547,7 @@ export const registerAgentRoutes: FastifyPluginAsync } }, sessionChangesTracker, + telemetry: opts.telemetry, }) await app.register(sessionRoutes, { getSessionStore: async (request) => { diff --git a/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts index 565195e32..90aa9a3b1 100644 --- a/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts +++ b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts @@ -1,6 +1,10 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import Fastify from 'fastify' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ERROR_CODES } from '../../../shared/errors.js' import type { TelemetrySink } from '../../../shared/telemetry.js' import type { CoreConfig } from '../../../shared/types.js' @@ -134,6 +138,14 @@ function resetTelemetryEnv(): void { delete process.env.BORING_TELEMETRY_PROJECT } +async function createBuiltFrontendRoot(): Promise { + const appRoot = await mkdtemp(join(tmpdir(), 'boring-core-telemetry-')) + const frontDir = join(appRoot, 'dist', 'front') + await mkdir(frontDir, { recursive: true }) + await writeFile(join(frontDir, 'index.html'), 'app') + return appRoot +} + describe('createCoreWorkspaceAgentServer telemetry wiring', () => { beforeEach(() => { resetTelemetryEnv() @@ -222,4 +234,71 @@ describe('createCoreWorkspaceAgentServer telemetry wiring', () => { await app.close() } }) + + it('emits app.opened when the server shell is served without raw URL data', async () => { + const capture = vi.fn() + const app = await createCoreWorkspaceAgentServer({ + appRoot: await createBuiltFrontendRoot(), + serveFrontend: true, + telemetry: { capture }, + }) + try { + const res = await app.inject({ method: 'GET', url: '/workspace/private-path?token=secret' }) + + expect(res.statusCode).toBe(200) + expect(capture).toHaveBeenCalledWith({ + name: 'app.opened', + properties: { requestId: expect.any(String) }, + }) + expect(JSON.stringify(capture.mock.calls)).not.toContain('private-path') + expect(JSON.stringify(capture.mock.calls)).not.toContain('secret') + } finally { + await app.close() + } + }) + + it('emits server.request.failed with stable metadata only', async () => { + const capture = vi.fn() + const app = await createCoreWorkspaceAgentServer({ serveFrontend: false, telemetry: { capture } }) + app.get('/boom/private-path', async () => { + throw new Error('raw secret failure with /tmp/private-path') + }) + try { + const res = await app.inject({ method: 'GET', url: '/boom/private-path?cookie=secret' }) + + expect(res.statusCode).toBe(500) + expect(capture).toHaveBeenCalledWith({ + name: 'server.request.failed', + properties: { + requestId: expect.any(String), + status: 500, + errorCode: ERROR_CODES.INTERNAL_ERROR, + }, + }) + expect(JSON.stringify(capture.mock.calls)).not.toContain('private-path') + expect(JSON.stringify(capture.mock.calls)).not.toContain('secret') + } finally { + await app.close() + } + }) + + it('keeps serving the shell when telemetry capture fails', async () => { + const app = await createCoreWorkspaceAgentServer({ + appRoot: await createBuiltFrontendRoot(), + serveFrontend: true, + telemetry: { + capture() { + throw new Error('telemetry sink down') + }, + }, + }) + try { + const res = await app.inject({ method: 'GET', url: '/' }) + + expect(res.statusCode).toBe(200) + expect(res.body).toContain('') + } finally { + await app.close() + } + }) }) diff --git a/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts b/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts index eb5483a0c..66785bc92 100644 --- a/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts +++ b/packages/core/src/app/server/createCoreWorkspaceAgentServer.ts @@ -29,7 +29,8 @@ import { import type { FastifyInstance } from 'fastify' import type postgres from 'postgres' import type { CoreConfig } from '../../shared/types.js' -import type { TelemetrySink } from '../../shared/telemetry.js' +import { ERROR_CODES } from '../../shared/errors.js' +import { safeCapture, type TelemetrySink } from '../../shared/telemetry.js' import { authHook, createAuth, @@ -394,9 +395,31 @@ async function registerAuthProxy(app: CoreWorkspaceAgentServer) { }) } +function captureAppOpened(telemetry: TelemetrySink, requestId: string): void { + safeCapture(telemetry, { + name: 'app.opened', + properties: { requestId }, + }) +} + +function registerTelemetryHooks(app: CoreWorkspaceAgentServer, telemetry: TelemetrySink): void { + app.addHook('onResponse', async (request, reply) => { + if (reply.statusCode < 500) return + safeCapture(telemetry, { + name: 'server.request.failed', + properties: { + requestId: request.id, + status: reply.statusCode, + errorCode: ERROR_CODES.INTERNAL_ERROR, + }, + }) + }) +} + async function registerFrontendAuthPages( app: CoreWorkspaceAgentServer, appRoot: string, + telemetry: TelemetrySink, ) { const frontDistDir = path.resolve(appRoot, 'dist/front') const indexPath = path.resolve(frontDistDir, 'index.html') @@ -411,6 +434,7 @@ async function registerFrontendAuthPages( } } const html = await readFile(indexPath, 'utf-8') + captureAppOpened(telemetry, request.id) reply.type('text/html; charset=utf-8') return reply.send(injectCspNonceIntoHtml(html, request.cspNonce)) }) @@ -420,6 +444,7 @@ async function registerFrontendAuthPages( async function registerFrontendFallback( app: CoreWorkspaceAgentServer, appRoot: string, + telemetry: TelemetrySink, ) { const frontDistDir = path.resolve(appRoot, 'dist/front') const indexPath = path.resolve(frontDistDir, 'index.html') @@ -434,6 +459,7 @@ async function registerFrontendFallback( } const html = await readFile(indexPath, 'utf-8') + captureAppOpened(telemetry, request.id) reply.type('text/html; charset=utf-8') return reply.send(injectCspNonceIntoHtml(html, request.cspNonce)) }) @@ -465,6 +491,7 @@ async function registerFrontendFallback( } const html = await readFile(indexPath, 'utf-8') + captureAppOpened(telemetry, request.id) reply.type('text/html; charset=utf-8') return reply.send(injectCspNonceIntoHtml(html, request.cspNonce)) }) @@ -561,10 +588,12 @@ export async function createCoreWorkspaceAgentServer( const telemetry = options.telemetry ?? createPostHogTelemetryFromEnv(process.env) app.log.debug({ telemetry: { source: telemetrySource } }, 'resolved telemetry sink') + registerTelemetryHooks(app, telemetry) + await registerCoreRoutes({ app, sql, db, userStore, workspaceStore }) if (serveFrontend && appRoot) { - await registerFrontendAuthPages(app, appRoot) + await registerFrontendAuthPages(app, appRoot, telemetry) } await registerAuthProxy(app) @@ -717,7 +746,7 @@ export async function createCoreWorkspaceAgentServer( } if (serveFrontend && appRoot) { - await registerFrontendFallback(app, appRoot) + await registerFrontendFallback(app, appRoot, telemetry) } return app From 1bf71112e866ad8dab64361c489adb5e81068286 Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 08:09:33 +0000 Subject: [PATCH 11/14] test(core): add telemetry env smoke coverage (boring-ui-v2-reorg-x0ek.6) --- ...rkspaceAgentServer.telemetry-smoke.test.ts | 398 ++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry-smoke.test.ts diff --git a/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry-smoke.test.ts b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry-smoke.test.ts new file mode 100644 index 000000000..07824bf10 --- /dev/null +++ b/packages/core/src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry-smoke.test.ts @@ -0,0 +1,398 @@ +import { mkdir, mkdtemp, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import Fastify from 'fastify' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { CoreConfig } from '../../../shared/types.js' + +type CapturePayload = { + distinctId: string + event: string + properties?: Record +} + +const posthogMock = vi.hoisted(() => { + type MockPostHogClient = { + captures: CapturePayload[] + capture: ReturnType + shutdown: ReturnType + } + + const clients: MockPostHogClient[] = [] + const PostHog = vi.fn((_apiKey: string, _options?: Record) => { + const client: MockPostHogClient = { + captures: [], + capture: vi.fn((payload: CapturePayload) => { + client.captures.push(payload) + }), + shutdown: vi.fn(), + } + clients.push(client) + return client + }) + + return { clients, PostHog } +}) + +const smokeLogs = vi.hoisted(() => ({ + entries: [] as Array>, +})) + +vi.mock('posthog-node', () => ({ + PostHog: posthogMock.PostHog, +})) + +vi.mock('@hachej/boring-agent/server', () => ({ + compactPiPackages: (packages: unknown[]) => packages, + registerAgentRoutes: async (app: { post: (path: string, handler: () => Promise) => void }, opts: { telemetry?: { capture: (event: { name: string; distinctId?: string; properties?: Record }) => void | Promise } }) => { + app.post('/__telemetry-smoke/agent-turn', async () => { + opts.telemetry?.capture({ + name: 'agent.chat.started', + distinctId: 'user_123', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_1', + requestId: 'request_1', + modelProvider: 'anthropic', + prompt: 'secret prompt must not be captured', + rawPath: '/tmp/private-path', + headers: { authorization: 'Bearer secret-token' }, + }, + }) + opts.telemetry?.capture({ + name: 'agent.chat.message.submitted', + distinctId: 'user_123', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_1', + requestId: 'request_1', + command: 'cat .env', + stdout: 'secret command output', + }, + }) + opts.telemetry?.capture({ + name: 'agent.tool.completed', + distinctId: 'user_123', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_1', + toolName: 'bash', + status: 'ok', + durationMs: 12, + commandOutput: 'assistant/file content must not be captured', + }, + }) + opts.telemetry?.capture({ + name: 'agent.chat.completed', + distinctId: 'user_123', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_1', + status: 'ok', + durationMs: 34, + assistantOutput: 'secret assistant output must not be captured', + }, + }) + return { ok: true } + }) + + app.post('/__telemetry-smoke/agent-failed-turn', async () => { + opts.telemetry?.capture({ + name: 'agent.chat.failed', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_failed', + status: 'error', + durationMs: 5, + errorCode: 'INTERNAL_ERROR', + stack: 'Error: stack with /tmp/private-path and secret-token', + }, + }) + opts.telemetry?.capture({ + name: 'agent.tool.failed', + properties: { + workspaceId: 'workspace_1', + sessionId: 'session_failed', + toolName: 'bash', + status: 'error', + durationMs: 6, + errorCode: 'TOOL_EXECUTION_ERROR', + stderr: 'secret stderr', + }, + }) + return { ok: true } + }) + }, +})) + +vi.mock('@hachej/boring-workspace/app/server', () => ({ + collectWorkspaceAgentServerPlugins: () => ({ + agentOptions: { + extraTools: [], + pi: undefined, + systemPromptAppend: undefined, + }, + preservedUiStateKeys: [], + provisioningContributions: [], + routeContributions: [], + }), + hasDirServerPlugin: () => false, + provisionWorkspaceAgentServer: vi.fn(), + readWorkspacePluginPackagePiSnapshot: () => ({ + additionalSkillPaths: [], + extensionFactories: [], + extensionPaths: [], + packages: [], + systemPromptAppend: undefined, + }), + resolveDefaultWorkspacePluginPackagePaths: () => [], + resolveOnePluginEntry: async (entry: unknown) => entry, +})) + +vi.mock('@hachej/boring-workspace/server', () => ({ + createInMemoryBridge: () => ({ + drainCommands: vi.fn(), + getState: vi.fn(), + postCommand: vi.fn(), + setState: vi.fn(), + subscribeCommands: vi.fn(), + }), + createWorkspaceUiTools: () => [], + uiRoutes: async () => {}, +})) + +vi.mock('../../../server/auth/index.js', () => ({ + authHook: async () => {}, + createAuth: () => ({ + handler: vi.fn(), + }), +})) + +vi.mock('../../../server/app/index.js', () => ({ + createCoreApp: async (config: CoreConfig) => { + const app = Fastify({ logger: false }) + app.decorate('config', config) + return app + }, + registerRoutes: async () => {}, +})) + +vi.mock('../../../server/routes/index.js', () => ({ + registerInviteRoutes: async () => {}, + registerMemberRoutes: async () => {}, + registerSettingsRoutes: async () => {}, + registerWorkspaceRoutes: async () => {}, +})) + +vi.mock('../../../server/db/index.js', () => ({ + createDatabase: () => ({ + db: {}, + sql: { end: vi.fn() }, + }), + PostgresUserStore: class PostgresUserStore {}, + PostgresWorkspaceStore: class PostgresWorkspaceStore {}, +})) + +vi.mock('../../../server/config/index.js', () => ({ + loadConfig: async () => ({ + auth: { url: 'http://localhost:3000' }, + encryption: { workspaceSettingsKey: 'test-key' }, + stores: 'postgres', + }), +})) + +vi.mock('../../../server/runtime/index.js', () => ({ + WorkspaceRuntimeSandboxHandleStore: class WorkspaceRuntimeSandboxHandleStore {}, +})) + +const { createCoreWorkspaceAgentServer } = await import('../createCoreWorkspaceAgentServer.js') + +function resetTelemetryEnv(): void { + delete process.env.BORING_TELEMETRY_ENABLED + delete process.env.POSTHOG_KEY + delete process.env.POSTHOG_HOST + delete process.env.BORING_TELEMETRY_PROJECT +} + +async function createBuiltFrontendRoot(): Promise { + const appRoot = await mkdtemp(join(tmpdir(), 'boring-core-telemetry-smoke-')) + const frontDir = join(appRoot, 'dist', 'front') + await mkdir(frontDir, { recursive: true }) + await writeFile(join(frontDir, 'index.html'), 'telemetry smoke') + return appRoot +} + +function currentCaptures(): CapturePayload[] { + return posthogMock.clients.flatMap((client) => client.captures) +} + +function logSmoke(label: string, captures: CapturePayload[]): Record { + const summary = { + label, + env: { + enabled: process.env.BORING_TELEMETRY_ENABLED ?? '', + hasPostHogKey: Boolean(process.env.POSTHOG_KEY), + host: process.env.POSTHOG_HOST ?? '', + project: process.env.BORING_TELEMETRY_PROJECT ?? '', + }, + capturedCount: captures.length, + eventNames: captures.map((capture) => capture.event), + distinctIdKinds: captures.map((capture) => (capture.distinctId === 'anonymous' ? 'anonymous' : typeof capture.distinctId)), + sanitizedPropertyKeys: captures.map((capture) => Object.keys(capture.properties ?? {}).sort()), + } + smokeLogs.entries.push(summary) + console.info('[telemetry-smoke]', JSON.stringify(summary)) + return summary +} + +function expectNoSensitiveTelemetryText(value: unknown): void { + const serialized = JSON.stringify(value) + expect(serialized).not.toContain('phc_never_log_secret') + expect(serialized).not.toContain('secret prompt') + expect(serialized).not.toContain('secret assistant') + expect(serialized).not.toContain('assistant/file content') + expect(serialized).not.toContain('cat .env') + expect(serialized).not.toContain('secret command output') + expect(serialized).not.toContain('/tmp/private-path') + expect(serialized).not.toContain('secret-token') + expect(serialized).not.toContain('secret stderr') + expect(serialized).not.toContain('POSTHOG_KEY') +} + +describe('telemetry v1 env-only e2e smoke', () => { + beforeEach(() => { + resetTelemetryEnv() + posthogMock.clients.length = 0 + posthogMock.PostHog.mockClear() + smokeLogs.entries.length = 0 + }) + + afterEach(() => { + resetTelemetryEnv() + vi.clearAllMocks() + }) + + it('captures prefixed representative events from core-composed env setup with redacted logs', async () => { + process.env.BORING_TELEMETRY_ENABLED = 'true' + process.env.POSTHOG_KEY = 'phc_never_log_secret' + process.env.POSTHOG_HOST = 'https://eu.i.posthog.com' + process.env.BORING_TELEMETRY_PROJECT = 'full-app' + + const app = await createCoreWorkspaceAgentServer({ + appRoot: await createBuiltFrontendRoot(), + serveFrontend: true, + }) + app.get('/__telemetry-smoke/server-failure/private-path', async () => { + throw new Error('raw server failure with /tmp/private-path and secret-token') + }) + + try { + const shell = await app.inject({ method: 'GET', url: '/workspace/private-path?token=secret-token' }) + const chat = await app.inject({ method: 'POST', url: '/__telemetry-smoke/agent-turn' }) + const failedTurn = await app.inject({ method: 'POST', url: '/__telemetry-smoke/agent-failed-turn' }) + const serverFailure = await app.inject({ method: 'GET', url: '/__telemetry-smoke/server-failure/private-path?token=secret-token' }) + + expect(shell.statusCode).toBe(200) + expect(chat.statusCode).toBe(200) + expect(failedTurn.statusCode).toBe(200) + expect(serverFailure.statusCode).toBe(500) + + const captures = currentCaptures() + const smokeLog = logSmoke('enabled-prefixed-core-composed', captures) + + expect(posthogMock.PostHog).toHaveBeenCalledWith('phc_never_log_secret', { + host: 'https://eu.i.posthog.com', + }) + expect(captures.map((capture) => capture.event)).toEqual(expect.arrayContaining([ + 'full-app.app.opened', + 'full-app.agent.chat.started', + 'full-app.agent.chat.message.submitted', + 'full-app.agent.tool.completed', + 'full-app.agent.chat.completed', + 'full-app.agent.chat.failed', + 'full-app.agent.tool.failed', + 'full-app.server.request.failed', + ])) + expect(captures.every((capture) => capture.event.startsWith('full-app.'))).toBe(true) + expect(captures.every((capture) => capture.properties?.boringProject === 'full-app')).toBe(true) + expect(captures.map((capture) => capture.properties?.eventName)).toEqual(expect.arrayContaining([ + 'app.opened', + 'agent.chat.started', + 'agent.chat.message.submitted', + 'agent.tool.completed', + 'agent.chat.completed', + 'agent.chat.failed', + 'agent.tool.failed', + 'server.request.failed', + ])) + expect(captures.find((capture) => capture.event === 'full-app.agent.chat.started')?.properties).toEqual({ + workspaceId: 'workspace_1', + sessionId: 'session_1', + requestId: 'request_1', + modelProvider: 'anthropic', + boringProject: 'full-app', + eventName: 'agent.chat.started', + }) + expect(captures.find((capture) => capture.event === 'full-app.agent.tool.completed')?.properties).toEqual({ + workspaceId: 'workspace_1', + sessionId: 'session_1', + toolName: 'bash', + status: 'ok', + durationMs: 12, + boringProject: 'full-app', + eventName: 'agent.tool.completed', + }) + expect(smokeLog).toMatchObject({ + env: { + enabled: 'true', + hasPostHogKey: true, + host: 'https://eu.i.posthog.com', + project: 'full-app', + }, + capturedCount: captures.length, + }) + expectNoSensitiveTelemetryText(captures) + expectNoSensitiveTelemetryText(smokeLogs.entries) + } finally { + await app.close() + } + }) + + it.each([ + ['unset', undefined], + ['false', 'false'], + ])('captures zero events when telemetry is %s even if POSTHOG_KEY is present', async (_label, enabled) => { + if (enabled) process.env.BORING_TELEMETRY_ENABLED = enabled + process.env.POSTHOG_KEY = 'phc_never_log_secret' + process.env.BORING_TELEMETRY_PROJECT = 'full-app' + + const app = await createCoreWorkspaceAgentServer({ + appRoot: await createBuiltFrontendRoot(), + serveFrontend: true, + }) + + try { + await app.inject({ method: 'GET', url: '/' }) + await app.inject({ method: 'POST', url: '/__telemetry-smoke/agent-turn' }) + + const captures = currentCaptures() + const smokeLog = logSmoke(`disabled-${enabled ?? 'unset'}`, captures) + + expect(posthogMock.PostHog).not.toHaveBeenCalled() + expect(captures).toHaveLength(0) + expect(smokeLog).toMatchObject({ + env: { + enabled: enabled ?? '', + hasPostHogKey: true, + project: 'full-app', + }, + capturedCount: 0, + eventNames: [], + }) + expectNoSensitiveTelemetryText(smokeLogs.entries) + } finally { + await app.close() + } + }) +}) From 4611b17d9a3ab082d75fd98490893ff56a2463e4 Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 08:13:41 +0000 Subject: [PATCH 12/14] docs(core): document PostHog telemetry setup (boring-ui-v2-reorg-x0ek.5) --- packages/core/docs/CORE.md | 82 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/packages/core/docs/CORE.md b/packages/core/docs/CORE.md index 959a73aea..20dcc1a87 100644 --- a/packages/core/docs/CORE.md +++ b/packages/core/docs/CORE.md @@ -326,6 +326,88 @@ invite_ttl_days = 7 | `BODY_LIMIT_BYTES` | no | Override Fastify body limit (default `16777216` / 16MB). | | `SESSION_TTL_SECONDS` | no | Session cookie max-age (default `60*60*24*30` / 30 days, matches v1). | | `SESSION_COOKIE_SECURE` | no | Force `Secure` cookie flag (default `true` when `BETTER_AUTH_URL` is https). | +| `BORING_TELEMETRY_ENABLED` | no | Explicit opt-in for PostHog telemetry. Must be `true`; unset/`false` means no-op telemetry. | +| `POSTHOG_KEY` | when telemetry enabled | PostHog project key. `POSTHOG_KEY` alone does **not** enable telemetry. | +| `POSTHOG_HOST` | no | PostHog host override. Defaults to `https://us.i.posthog.com`. | +| `BORING_TELEMETRY_PROJECT` | no | Lowercase slug used as an event-name prefix and `boringProject` property when multiple boring-ui apps share one PostHog account/project. | + +### PostHog telemetry (v1) + +Core-composed apps get PostHog telemetry from env vars only. Core owns the PostHog helper and resolves telemetry inside `createCoreWorkspaceAgentServer`; child apps do not need to import or initialize PostHog for the common path. + +Enable telemetry: + +```bash +BORING_TELEMETRY_ENABLED=true +POSTHOG_KEY=phc_... +# Optional. Defaults to https://us.i.posthog.com +POSTHOG_HOST=https://us.i.posthog.com +# Optional. Useful when several apps share one PostHog account/project. +BORING_TELEMETRY_PROJECT=full-app +``` + +Disable telemetry: + +```bash +# Either omit BORING_TELEMETRY_ENABLED, or set it explicitly: +BORING_TELEMETRY_ENABLED=false +``` + +`POSTHOG_KEY` by itself is ignored. This keeps telemetry off by default and prevents accidental capture when a secret exists in the environment. + +When `BORING_TELEMETRY_PROJECT=full-app`, event names are prefixed and the raw event name is preserved in properties: + +```txt +full-app.app.opened +full-app.agent.chat.started +full-app.agent.tool.completed +``` + +```ts +{ + boringProject: 'full-app', + eventName: 'agent.chat.started' +} +``` + +Implemented v1 events: + +| Area | Events | +|---|---| +| Core | `app.opened`, `server.request.failed` | +| Agent chat | `agent.chat.started`, `agent.chat.message.submitted`, `agent.chat.completed`, `agent.chat.failed` | +| Agent tools | `agent.tool.completed`, `agent.tool.failed` | + +Allowed event properties are intentionally low-cardinality operational metadata only: + +- `workspaceId` +- `sessionId` +- `requestId` +- `runtimeMode` +- `modelProvider` +- `toolName` +- `panelId` +- `commandId` +- `status` +- `durationMs` +- `errorCode` +- `packageName` +- `packageVersion` + +Privacy exclusions: v1 does not capture prompts, assistant output, file contents, command strings, stdout/stderr, raw file paths, raw errors, stack traces, headers, cookies, tokens, full env dumps, or secret values. Core sanitizes properties before sending to PostHog, and telemetry failures are best-effort only: they do not break app, chat, or tool flows. + +Package boundary: agent/workspace remain PostHog-free. Core passes a tiny structural telemetry sink into composed agent code; `@hachej/boring-agent` standalone remains no-op unless an embedder explicitly provides a sink. + +Deferred scope: browser-originated telemetry, direct browser PostHog setup, workspace frontend events (`workspace.opened`, `workspace.panel.opened`, `workspace.command.executed`), auth hook telemetry, plugin-error telemetry, UI-command telemetry, tool-started events, Sentry, OpenTelemetry/OTLP, routing/multi-account selection, and mandatory lifecycle flush wiring are not shipped in v1. + +Focused validation commands: + +```bash +pnpm --filter @hachej/boring-core test src/shared/__tests__/telemetry.test.ts src/server/telemetry/__tests__/posthog.test.ts src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry.test.ts src/app/server/__tests__/createCoreWorkspaceAgentServer.telemetry-smoke.test.ts +pnpm --filter @hachej/boring-agent test src/shared/__tests__/telemetry.test.ts src/server/http/routes/__tests__/chat.test.ts src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts src/server/__tests__/createAgentApp.test.ts +pnpm --filter @hachej/boring-workspace test src/shared/__tests__/telemetry.test.ts +pnpm lint:invariants +``` ### `CoreConfig` type From ffab9a0f17235192e637504016a2df8ed447993e Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 09:15:09 +0000 Subject: [PATCH 13/14] fix(core): harden telemetry value sanitization --- .../telemetry/__tests__/posthog.test.ts | 64 ++++++++++++++- packages/core/src/server/telemetry/posthog.ts | 78 +++++++++++++++---- 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/packages/core/src/server/telemetry/__tests__/posthog.test.ts b/packages/core/src/server/telemetry/__tests__/posthog.test.ts index e15fd48b0..2761e1dd6 100644 --- a/packages/core/src/server/telemetry/__tests__/posthog.test.ts +++ b/packages/core/src/server/telemetry/__tests__/posthog.test.ts @@ -30,6 +30,7 @@ vi.mock('posthog-node', () => ({ import { createPostHogTelemetryFromEnv, parseTelemetryProject, + sanitizeTelemetryDistinctId, sanitizeTelemetryProperties, } from '../posthog' @@ -158,7 +159,31 @@ describe('createPostHogTelemetryFromEnv', () => { expect(String(warn.mock.calls[0]?.[0])).not.toContain('../bad project') }) - it('swallows PostHog capture failures', () => { + it('falls back to anonymous for unsafe distinct ids', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), + ) + + telemetry.capture({ name: 'app.opened', distinctId: 'user@example.com' }) + + expect(posthogMock.clients[0]?.capture).toHaveBeenCalledWith({ + distinctId: 'anonymous', + event: 'app.opened', + properties: { eventName: 'app.opened' }, + }) + }) + + it('drops unsafe event names without sending to PostHog', () => { + const telemetry = createPostHogTelemetryFromEnv( + env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), + ) + + telemetry.capture({ name: 'secret.token./tmp/private' }) + + expect(posthogMock.clients[0]?.capture).not.toHaveBeenCalled() + }) + + it('swallows PostHog capture failures', async () => { const telemetry = createPostHogTelemetryFromEnv( env({ BORING_TELEMETRY_ENABLED: 'true', POSTHOG_KEY: 'phc_secret' }), ) @@ -167,6 +192,10 @@ describe('createPostHogTelemetryFromEnv', () => { }) expect(() => telemetry.capture({ name: 'server.request.failed' })).not.toThrow() + + posthogMock.clients[0]!.capture.mockRejectedValueOnce(new Error('network still down')) + expect(() => telemetry.capture({ name: 'server.request.failed' })).not.toThrow() + await Promise.resolve() }) it('flushes via PostHog shutdown when requested', async () => { @@ -198,6 +227,16 @@ describe('parseTelemetryProject', () => { }) }) +describe('sanitizeTelemetryDistinctId', () => { + it('keeps safe ids and falls back for emails, tokens, and paths', () => { + expect(sanitizeTelemetryDistinctId('user_123')).toBe('user_123') + expect(sanitizeTelemetryDistinctId('user@example.com')).toBe('anonymous') + expect(sanitizeTelemetryDistinctId('Bearer secret-token')).toBe('anonymous') + expect(sanitizeTelemetryDistinctId('/tmp/private-path')).toBe('anonymous') + expect(sanitizeTelemetryDistinctId(undefined)).toBe('anonymous') + }) +}) + describe('sanitizeTelemetryProperties', () => { it('keeps only allowlisted primitive telemetry properties', () => { expect( @@ -245,6 +284,29 @@ describe('sanitizeTelemetryProperties', () => { }) }) + it('drops suspicious strings even on allowlisted keys', () => { + expect( + sanitizeTelemetryProperties({ + requestId: '/tmp/private-path', + sessionId: 'Bearer secret-token', + workspaceId: 'sk_live_abc123', + toolName: 'ghp_abc123', + modelProvider: 'anthropic', + errorCode: 'SECRET_TOKEN', + packageName: '@hachej/boring-core', + }), + ).toEqual({ + modelProvider: 'anthropic', + packageName: '@hachej/boring-core', + }) + }) + + it('keeps lower-case stable core error codes', () => { + expect(sanitizeTelemetryProperties({ errorCode: 'internal_error' })).toEqual({ + errorCode: 'internal_error', + }) + }) + it('drops non-finite numbers even on allowlisted keys', () => { expect( sanitizeTelemetryProperties({ diff --git a/packages/core/src/server/telemetry/posthog.ts b/packages/core/src/server/telemetry/posthog.ts index 9832aa239..8f7c6cb87 100644 --- a/packages/core/src/server/telemetry/posthog.ts +++ b/packages/core/src/server/telemetry/posthog.ts @@ -4,6 +4,14 @@ import { noopTelemetry, type TelemetryEvent, type TelemetrySink } from '../../sh const DEFAULT_POSTHOG_HOST = 'https://us.i.posthog.com' const PROJECT_SLUG_PATTERN = /^[a-z0-9][a-z0-9-]{0,62}$/ +const EVENT_NAME_PATTERN = /^[a-z][a-z0-9_-]*(?:\.[a-z][a-z0-9_-]*){0,8}$/ +const SAFE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/ +const SAFE_SLUG_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$/ +const SAFE_STATUS_PATTERN = /^[a-z][a-z0-9_-]{0,31}$/ +const SAFE_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_:-]{0,63}$/ +const SAFE_PACKAGE_NAME_PATTERN = /^(?:@[A-Za-z0-9_.-]+\/)?[A-Za-z0-9_.-]{1,96}$/ +const SAFE_PACKAGE_VERSION_PATTERN = /^v?\d+\.\d+\.\d+(?:[-+][A-Za-z0-9.-]+)?$/ +const SUSPICIOUS_STRING_PATTERN = /(?:secret|token|bearer|password|api[_-]?key|private|\.env|sk[_-](?:live|test)|ghp_|github_pat_|glpat-|xox[baprs]-|AKIA|ASIA|ya29\.|eyJ|phc_|npm_)/i const ALLOWED_PROPERTY_KEYS = new Set([ 'workspaceId', @@ -41,16 +49,19 @@ export function createPostHogTelemetryFromEnv( return { capture(event: TelemetryEvent) { + const eventName = parseTelemetryEventName(event.name) + if (!eventName) return + const properties = sanitizeTelemetryProperties(event.properties) if (project) properties.boringProject = project - properties.eventName = event.name + properties.eventName = eventName try { - posthog.capture({ - distinctId: event.distinctId ?? 'anonymous', - event: project ? `${project}.${event.name}` : event.name, + void Promise.resolve(posthog.capture({ + distinctId: sanitizeTelemetryDistinctId(event.distinctId), + event: project ? `${project}.${eventName}` : eventName, properties, - }) + })).catch(() => {}) } catch {} }, async flush() { @@ -68,6 +79,11 @@ export function parseTelemetryProject(value: string | undefined): string | undef return undefined } +export function sanitizeTelemetryDistinctId(value: string | undefined): string { + if (!value) return 'anonymous' + return sanitizeTelemetryString('distinctId', value) ?? 'anonymous' +} + export function sanitizeTelemetryProperties( properties: Record | undefined, ): Record { @@ -76,18 +92,52 @@ export function sanitizeTelemetryProperties( for (const [key, value] of Object.entries(properties)) { if (!ALLOWED_PROPERTY_KEYS.has(key)) continue - if (!isSafeTelemetryProperty(value)) continue - sanitized[key] = value + const sanitizedValue = sanitizeTelemetryProperty(key, value) + if (sanitizedValue === undefined) continue + sanitized[key] = sanitizedValue } return sanitized } -function isSafeTelemetryProperty(value: unknown): value is SafeTelemetryProperty { - return ( - value === null || - typeof value === 'string' || - typeof value === 'boolean' || - (typeof value === 'number' && Number.isFinite(value)) - ) +function parseTelemetryEventName(value: string): string | undefined { + if (value.length > 128) return undefined + if (SUSPICIOUS_STRING_PATTERN.test(value)) return undefined + return EVENT_NAME_PATTERN.test(value) ? value : undefined +} + +function sanitizeTelemetryProperty(key: string, value: unknown): SafeTelemetryProperty | undefined { + if (value === null || typeof value === 'boolean') return value + if (typeof value === 'number') return Number.isFinite(value) ? value : undefined + if (typeof value !== 'string') return undefined + return sanitizeTelemetryString(key, value) +} + +function sanitizeTelemetryString(key: string, value: string): string | undefined { + if (value.length === 0) return undefined + if (SUSPICIOUS_STRING_PATTERN.test(value)) return undefined + + switch (key) { + case 'workspaceId': + case 'sessionId': + case 'requestId': + case 'distinctId': + return SAFE_ID_PATTERN.test(value) ? value : undefined + case 'runtimeMode': + case 'modelProvider': + case 'toolName': + case 'panelId': + case 'commandId': + return SAFE_SLUG_PATTERN.test(value) ? value : undefined + case 'status': + return SAFE_STATUS_PATTERN.test(value) ? value : undefined + case 'errorCode': + return SAFE_ERROR_CODE_PATTERN.test(value) ? value : undefined + case 'packageName': + return SAFE_PACKAGE_NAME_PATTERN.test(value) ? value : undefined + case 'packageVersion': + return SAFE_PACKAGE_VERSION_PATTERN.test(value) ? value : undefined + default: + return undefined + } } From 6207dfc1ae0032e2285d55d8a5e2086f52f6d022 Mon Sep 17 00:00:00 2001 From: hachej Date: Sat, 23 May 2026 09:35:13 +0000 Subject: [PATCH 14/14] test(agent): align telemetry tool error code after main merge --- .../pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts b/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts index fee6e88c4..953805071 100644 --- a/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts +++ b/packages/agent/src/server/harness/pi-coding-agent/__tests__/tool-adapter.telemetry.test.ts @@ -69,7 +69,7 @@ describe('tool adapter telemetry', () => { return { isError: true, content: [{ type: 'text', text: 'secret stderr output' }], - details: { code: ErrorCode.enum.WORKSPACE_NOT_READY, command: 'cat .env' }, + details: { code: ErrorCode.enum.TOOL_EXECUTION_ERROR, command: 'cat .env' }, } }, }) @@ -84,7 +84,7 @@ describe('tool adapter telemetry', () => { sessionId: 'sess-tool', status: 'error', durationMs: expect.any(Number), - errorCode: ErrorCode.enum.WORKSPACE_NOT_READY, + errorCode: ErrorCode.enum.TOOL_EXECUTION_ERROR, }, }) const serialized = JSON.stringify(recorder.events)