From 2a4fa122ae170eaf0625c8cc9d7aef1f61d8bfe3 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 14:41:14 -0400 Subject: [PATCH 01/14] =?UTF-8?q?=E2=9C=A8=20Session=20search=20workbench?= =?UTF-8?q?=20with=20runnable=20command=20primitive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SCO-059 session-knowledge exploration as an interactive studio surface. Six-stage pipeline (discover → drilldown) walking a real Codex or Claude session through preparation. Discover scans ~/.codex/sessions and ~/.claude/projects against the actual filesystem; Normalize opens the selected JSONL and parses the head into uniform records. Introduces a studio-internal primitive: Command + runCommand with per-input TTL caching, plus a CommandSurface chrome component owning copyable shell line, ran/cached badge, and output frame. Renderer registry is parked for the next consumer. Also lands the parallel KnowledgeSearchScreen sketch on the web product side for future cross-reference. --- .../app/studies/session-search/page.tsx | 1546 +++++++++++++++++ .../components/studio/CommandSurface.tsx | 60 + design/studio/lib/inventory.ts | 227 +++ design/studio/lib/studio-pages.ts | 13 + design/studio/lib/studio/command.ts | 87 + .../studio/lib/studio/commands/inventory.ts | 32 + .../lib/studio/commands/parse-session.ts | 328 ++++ design/studio/lib/studio/renderers.ts | 42 + ...59-session-knowledge-search-exploration.md | 135 ++ .../client/screens/KnowledgeSearchScreen.tsx | 659 +++++++ .../web/client/screens/knowledge-search.css | 912 ++++++++++ 11 files changed, 4041 insertions(+) create mode 100644 design/studio/app/studies/session-search/page.tsx create mode 100644 design/studio/components/studio/CommandSurface.tsx create mode 100644 design/studio/lib/inventory.ts create mode 100644 design/studio/lib/studio/command.ts create mode 100644 design/studio/lib/studio/commands/inventory.ts create mode 100644 design/studio/lib/studio/commands/parse-session.ts create mode 100644 design/studio/lib/studio/renderers.ts create mode 100644 docs/eng/sco-059-session-knowledge-search-exploration.md create mode 100644 packages/web/client/screens/KnowledgeSearchScreen.tsx create mode 100644 packages/web/client/screens/knowledge-search.css diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx new file mode 100644 index 00000000..9c6cfd1b --- /dev/null +++ b/design/studio/app/studies/session-search/page.tsx @@ -0,0 +1,1546 @@ +import type { InventoryResult } from "@/lib/inventory"; +import { runCommand, type CommandRun } from "@/lib/studio/command"; +import { inventoryCommand } from "@/lib/studio/commands/inventory"; +import { + parseSessionCommand, + type NormalizedKind, + type NormalizedRecord, + type ParseSessionResult, +} from "@/lib/studio/commands/parse-session"; +import { CommandSurface } from "@/components/studio/CommandSurface"; + +type Harness = "Codex" | "Claude"; +type Tier = "large" | "normal" | "small"; +type StageId = + | "discover" + | "normalize" + | "extract" + | "index" + | "query" + | "drilldown"; + +interface SessionSample { + id: string; + harness: Harness; + tier: Tier; + sizeBytes: number; + events: number; + modified: string; + displayPath: string; + fullPath: string; + focus: string; + codeArea: string; +} + +interface Stage { + id: StageId; + label: string; + verb: string; + input: string; + output: string; + summary: string; +} + +interface ProducedFile { + name: string; + kind: string; + bytes: number; + refs: string; +} + +interface SearchHit { + title: string; + source: string; + score: string; + snippet: string; +} + +interface PageProps { + searchParams: Promise<{ + artifact?: string; + q?: string; + session?: string; + step?: string; + }>; +} + +interface StudySelection { + artifactName: string; + query: string; + sessionId: string; + stageId: StageId; +} + +const SESSIONS: SessionSample[] = [ + { + id: "codex-large", + harness: "Codex", + tier: "large", + sizeBytes: 13_623_669, + events: 4_220, + modified: "2026-05-29 23:29", + displayPath: "~/.codex/sessions/2026/05/29/...019e75fd-a431...jsonl", + fullPath: + "/Users/arach/.codex/sessions/2026/05/29/rollout-2026-05-29T19-06-57-019e75fd-a431-76c3-82f1-8b61e94f613a.jsonl", + focus: "large Codex context with many tool calls and iterative UI work", + codeArea: "packages/web/client and design/studio", + }, + { + id: "codex-normal", + harness: "Codex", + tier: "normal", + sizeBytes: 1_154_910, + events: 494, + modified: "2026-05-25 15:45", + displayPath: "~/.codex/sessions/2026/05/25/...019e609c-4389...jsonl", + fullPath: + "/Users/arach/.codex/sessions/2026/05/25/rollout-2026-05-25T15-28-34-019e609c-4389-70e3-9fa1-9dedeea99410.jsonl", + focus: "normal Codex edit pass with a bounded implementation surface", + codeArea: "packages/runtime and protocol integration", + }, + { + id: "codex-small", + harness: "Codex", + tier: "small", + sizeBytes: 34_921, + events: 12, + modified: "2026-05-29 00:16", + displayPath: "~/.codex/sessions/2026/05/29/...019e71f2-958c...jsonl", + fullPath: + "/Users/arach/.codex/sessions/2026/05/29/rollout-2026-05-29T00-16-24-019e71f2-958c-7943-bc24-a1c2214f8b7a.jsonl", + focus: "tiny Codex clarification thread", + codeArea: "planning notes and command checks", + }, + { + id: "claude-large", + harness: "Claude", + tier: "large", + sizeBytes: 55_431_537, + events: 12_009, + modified: "2026-05-26 02:49", + displayPath: "~/.claude/projects/-Users-arach-dev-openscout/a00198bf...jsonl", + fullPath: + "/Users/arach/.claude/projects/-Users-arach-dev-openscout/a00198bf-0a6f-4011-a35b-8cc35f391868.jsonl", + focus: "very large Claude project session with broad OpenScout context", + codeArea: "apps/macos, packages/web, packages/runtime", + }, + { + id: "claude-normal", + harness: "Claude", + tier: "normal", + sizeBytes: 762_408, + events: 252, + modified: "2026-05-23 13:11", + displayPath: "~/.claude/projects/-Users-arach-dev-openscout/c680a795...jsonl", + fullPath: + "/Users/arach/.claude/projects/-Users-arach-dev-openscout/c680a795-dfc5-4d76-9a49-51b782c6a7dc.jsonl", + focus: "normal Claude session around a focused implementation slice", + codeArea: "docs and package-level implementation notes", + }, + { + id: "claude-small", + harness: "Claude", + tier: "small", + sizeBytes: 2_124, + events: 5, + modified: "2026-05-24 21:57", + displayPath: "~/.claude/projects/-Users-arach-dev-contextual/ada6d81e...jsonl", + fullPath: + "/Users/arach/.claude/projects/-Users-arach-dev-contextual/ada6d81e-bc86-44d9-9421-564768edc650.jsonl", + focus: "micro Claude session with almost no processing pressure", + codeArea: "contextual scratch material", + }, +]; + +const STAGES: Stage[] = [ + { + id: "discover", + label: "Discover", + verb: "inventory", + input: "raw harness JSONL", + output: "session manifest", + summary: + "Scan local harness directories. Capture path, size, event count, mtime, and a stable session id without parsing payloads.", + }, + { + id: "normalize", + label: "Normalize", + verb: "shape", + input: "events and tool payloads", + output: "stable event records", + summary: + "Parse JSONL into a uniform event model so downstream code sees the same record shape regardless of harness.", + }, + { + id: "extract", + label: "Extract", + verb: "derive", + input: "normalized records", + output: "QMD-style markdown files", + summary: + "Write a small sidecar corpus per session: overview, decisions, files, tool calls, and event windows with source refs.", + }, + { + id: "index", + label: "Index", + verb: "compile", + input: "derived markdown", + output: "fuzzy and FTS rows", + summary: + "Compile the derived corpus into local SQLite tables. The index is rebuildable and smaller than a transcript copy.", + }, + { + id: "query", + label: "Query", + verb: "rank", + input: "operator question", + output: "answerable hits", + summary: + "Ask the derived corpus first. The LLM sees compact retrieved evidence instead of the full raw session.", + }, + { + id: "drilldown", + label: "Drilldown", + verb: "anchor", + input: "source refs", + output: "raw log coordinates", + summary: + "Follow a qmd source ref back to the raw JSONL path and event window when the answer needs ground-truth evidence.", + }, +]; + +const SAMPLE_QUERIES = [ + "which session touched OpenScout search?", + "what were the unresolved decisions?", + "raw log drilldown for this area", + "what codebase area was I working in?", +]; + +const DOC_HREF = "/eng/sco-059-session-knowledge-search-exploration"; + +export default async function SessionSearchStudyPage({ + searchParams, +}: PageProps) { + const params = await searchParams; + const requestedSession = params.session ?? SESSIONS[0]!.id; + const requestedStage = params.step ?? "extract"; + const selectedSession = + SESSIONS.find((session) => session.id === requestedSession) ?? SESSIONS[0]!; + const stageId = isStageId(requestedStage) ? requestedStage : "extract"; + const stageIndex = STAGES.findIndex((s) => s.id === stageId); + const selectedStage = STAGES[stageIndex]!; + const prevStage = stageIndex > 0 ? STAGES[stageIndex - 1] : undefined; + const nextStage = stageIndex < STAGES.length - 1 ? STAGES[stageIndex + 1] : undefined; + const producedFiles = buildProducedFiles(selectedSession); + + const selectedArtifact = + producedFiles.find((file) => file.name === params.artifact) ?? + producedFiles[0]!; + + const query = params.q ?? SAMPLE_QUERIES[0]!; + const queryResult = buildQueryResult(selectedSession, query); + const selection: StudySelection = { + artifactName: selectedArtifact.name, + query, + sessionId: selectedSession.id, + stageId, + }; + + const inventoryRun = await runCommand(inventoryCommand, { since: "7d" }); + const inventory = inventoryRun.output; + const weekFootprint = inventoryRun.error + ? "scan failed" + : `${formatCount(inventory.totalFiles)} files · ${formatBytes(inventory.totalBytes)} · ${inventory.windowDays} days`; + + return ( +
+ {/* ── Header ─────────────────────────────────────────────────── */} +
+
+
+ studies / web / session-search +
+

+ Session search workbench +

+

+ Walk a real harness session through the preparation pipeline. Each + step explains itself and shows what it produces against the chosen + input. +

+
+
+ + this week · {weekFootprint} + + ({inventory.cached ? "cached" : `${inventory.durationMs} ms`}) + + + · + + SCO-059 → + +
+
+ + {/* ── Session picker row ────────────────────────────────────── */} + + + {/* ── Pipeline + active step panel ─────────────────────────── */} +
+ + + + + +
+ + {/* ── Week budget footer (compact) ─────────────────────────── */} + +
+ ); +} + +// ── Session picker (single inline row) ─────────────────────────── + +function SessionPickerRow({ + sessions, + selectedId, + selection, +}: { + sessions: SessionSample[]; + selectedId: string; + selection: StudySelection; +}) { + return ( +
+
+
+ input +
+ {sessions.map((session) => { + const active = session.id === selectedId; + return ( + + + + {session.harness.toLowerCase()}/{session.tier} + + + {formatBytes(session.sizeBytes)} + + + · {formatCount(session.events)} ev + + + ); + })} +
+ 6 representative samples +
+
+
+ ); +} + +function TierDot({ tier }: { tier: Tier }) { + const cls: Record = { + large: "bg-status-error-fg", + normal: "bg-status-info-fg", + small: "bg-status-ok-fg", + }; + return ; +} + +// ── Pipeline strip ─────────────────────────────────────────────── + +function PipelineStrip({ + stages, + activeStage, + session, + selection, +}: { + stages: Stage[]; + activeStage: StageId; + session: SessionSample; + selection: StudySelection; +}) { + return ( +
+
+
+ pipeline +
+ {stages.map((stage, index) => { + const active = stage.id === activeStage; + const isLast = index === stages.length - 1; + return ( + +
+ ); +} + +// ── Active stage: title row with prev/next ──────────────────────── + +function StageHeader({ + index, + total, + stage, + session, + selection, + prev, + next, +}: { + index: number; + total: number; + stage: Stage; + session: SessionSample; + selection: StudySelection; + prev?: Stage; + next?: Stage; +}) { + return ( +
+
+
+ step {String(index + 1).padStart(2, "0")} / {String(total).padStart(2, "0")} · {stage.verb} +
+

+ {stage.label} + + {stage.input} → {stage.output} + +

+

+ {stage.summary} +

+
+
+ {prev ? ( + + ← {prev.label} + + ) : ( + + ← start + + )} + {next ? ( + + {next.label} → + + ) : ( + + end → + + )} +
+
+ ); +} + +// ── Stage-specific panel content ───────────────────────────────── + +function StagePanel({ + stage, + session, + selection, + producedFiles, + selectedArtifact, + query, + queryResult, + inventoryRun, +}: { + stage: Stage; + session: SessionSample; + selection: StudySelection; + producedFiles: ProducedFile[]; + selectedArtifact: ProducedFile; + query: string; + queryResult: ReturnType; + inventoryRun: CommandRun; +}) { + switch (stage.id) { + case "discover": + return ; + case "normalize": + return ; + case "extract": + return ( + + ); + case "index": + return ( + + ); + case "query": + return ( + + ); + case "drilldown": + return ; + } +} + +// ── Stage panels ───────────────────────────────────────────────── + +function DiscoverPanel({ + session, + inventoryRun, +}: { + session: SessionSample; + inventoryRun: CommandRun; +}) { + const inventory = inventoryRun.output; + return ( +
+ } + footnote={ + inventory && !inventory.error ? ( + <> + Filesystem-only — size, mtime, line count. The{" "} + events column is a real{" "} + wc -l{" "} + on the {inventory.rows.length} most-recent files.{" "} + Selected for walkthrough:{" "} + + {qmdSlug(session)} + + . + + ) : null + } + /> +
+ ); +} + +function InventoryRowsBody({ + inventory, + session: _session, +}: { + inventory: InventoryResult | undefined; + session: SessionSample; +}) { + if (!inventory || inventory.error) { + return ( + 
+        {inventory?.error ?? "no inventory available"}
+
+ ); + } + const harnessLabel: Record = { + codex: "codex", + claude: "claude", + "claude-subagent": "claude/sub", + }; + const remaining = Math.max(0, inventory.totalFiles - inventory.rows.length); + const remainingBytes = Math.max( + 0, + inventory.totalBytes - inventory.rows.reduce((a, r) => a + r.sizeBytes, 0), + ); + return ( +
+
+ harness + session_id + size + events + modified +
+
    + {inventory.rows.map((row) => ( +
  • + + {harnessLabel[row.harness] ?? row.harness} + + + {row.sessionId} + + + {formatBytes(row.sizeBytes)} + + + {row.events != null ? formatCount(row.events) : "—"} + + + {row.modified} + +
    • + ))} + {remaining > 0 ? ( +
  • + … + {formatCount(remaining)} more in window + {formatBytes(remainingBytes)} + — + {inventory.windowDays}d +
    • + ) : null} +
+
+ total + + across {inventory.byHarness.filter((b) => b.files > 0).length} harnesses + + {formatBytes(inventory.totalBytes)} + — + {inventory.windowDays}d +
+
+ ); +} + +async function NormalizePanel({ session }: { session: SessionSample }) { + const limit = 14; + const run = await runCommand(parseSessionCommand, { + path: session.fullPath, + limit, + }); + const more = Math.max(0, session.events - (run.output?.records.length ?? 0)); + return ( +
+ + } + footnote={ + run.output && !run.output.error ? ( + <> + Read the first {run.output.scannedLines} lines from the real{" "} + + {session.harness}/{session.tier} + {" "} + JSONL ({formatBytes(run.output.bytesRead)}). Every downstream + stage reads this uniform record shape regardless of source + harness. + + ) : null + } + /> +
+ ); +} + +const NORMALIZED_KIND_TONE: Record = { + session_meta: "text-studio-ink-faint", + user_turn: "text-status-info-fg", + assistant_turn: "text-studio-ink", + command_or_tool: "text-status-warn-fg", + observation: "text-status-ok-fg", + system_record: "text-studio-ink-faint", + unknown: "text-status-error-fg", +}; + +function NormalizedStreamBody({ + result, + moreCount, +}: { + result: ParseSessionResult | undefined; + moreCount: number; +}) { + if (!result || result.error) { + return ( + 
+        {result?.error ?? "no parse result"}
+
+ ); + } + return ( +
+
+ idx + kind + tag + detail +
+
    + {result.records.map((r) => ( + + ))} + {moreCount > 0 ? ( +
  • + … + {formatCount(moreCount)} more + — + source-ordered +
    • + ) : null} +
+
+ ); +} + +function NormalizedRow({ record }: { record: NormalizedRecord }) { + return ( +
  • + + [{String(record.i).padStart(3, "0")}] + + {record.kind} + + {record.tag ?? record.sourceType} + + + {record.detail} + +
    • + ); +} + +function ExtractPanel({ + session, + selection, + producedFiles, + selectedArtifact, +}: { + session: SessionSample; + selection: StudySelection; + producedFiles: ProducedFile[]; + selectedArtifact: ProducedFile; +}) { + return ( + + {producedFiles.map((file) => { + const active = file.name === selectedArtifact.name; + return ( +
  • + + + + {file.name} + + + {file.kind} · {file.refs} + + + + {formatBytes(file.bytes)} + + +
    • + ); + })} + + } + rightLabel={`preview · ${selectedArtifact.name}`} + right={} + /> + ); +} + +function IndexPanel({ + session, + selectedArtifact, +}: { + session: SessionSample; + selectedArtifact: ProducedFile; +}) { + const indexBytes = estimateIndexBytes(session); + const rows = databaseRows(session, selectedArtifact, indexBytes); + return ( + + {rows.map((row) => ( +
  • +
    + + {row.table} + + + {row.ref} + +
    +
    {row.key}
    +
    + {row.value} +
    +
    • + ))} + + } + rightLabel="cli + budget" + right={ +
    + +

    + The index is rebuildable. Scout owns the rows; raw harness logs + stay observed source material. +

    +
    + } + /> + ); +} + +function QueryPanel({ + session, + selection, + query, + result, +}: { + session: SessionSample; + selection: StudySelection; + query: string; + result: ReturnType; +}) { + return ( +
    +
    + + + + +
    + +
    + {SAMPLE_QUERIES.map((sample) => { + const active = sample === query; + return ( + + {sample} + + ); + })} +
    + +
    +
    +
    + assistant synthesis +
    +

    + {result.answer} +

    +
    +
    +
    + ranked hits +
    +
    + {result.hits.map((hit) => ( +
    +
    + + {hit.title} + + + {hit.score} + +
    +

    + {hit.snippet} +

    + + {hit.source} + +
    + ))} +
    +
    +
    +
    + ); +} + +function DrilldownPanel({ + session, + selection, +}: { + session: SessionSample; + selection: StudySelection; +}) { + const slug = qmdSlug(session); + const ref = `qmd://session-search/${slug}/events-001.md:44`; + const excerpt = [ + "events-001.md (excerpt)", + "", + "## window 001", + "043 assistant_turn source_event=143", + "044 tool_call(name=apply_patch) source_event=144", + "045 tool_result(ok=true) source_event=145", + "", + "raw://" + session.fullPath, + ].join("\n"); + return ( + +
    +
    + ref +
    +
    + {ref} +
    +
    +
    +
    + resolves to +
    +
    + {session.fullPath} +
    +
    +
    +
    + purpose +
    +

    + Evidence, not bulk import. The raw transcript stays observed + source material; only the qmd ref + event window come into + Scout's view. +

    +
    + + ← back to query + +
    + } + rightLabel="excerpt" + right={} + /> + ); +} + +// ── Panel atoms ────────────────────────────────────────────────── + +function PanelGrid({ + leftLabel, + left, + rightLabel, + right, +}: { + leftLabel: string; + left: React.ReactNode; + rightLabel: string; + right: React.ReactNode; +}) { + return ( +
    +
    + {leftLabel} +
    {left}
    +
    +
    + {rightLabel} +
    {right}
    +
    +
    + ); +} + +function PanelLabel({ children }: { children: React.ReactNode }) { + return ( +
    + {children} +
    + ); +} + +function CliBlock({ lines }: { lines: string[] }) { + return ( + 
    +      {lines.join("\n")}
+
    + ); +} + +function CommandRecipe({ + cmd, + output, + run = false, +}: { + cmd: string; + output: string[]; + run?: boolean; +}) { + return ( +
    +
    + + command + + + {run ? "● ran" : "○ recipe"} + +
    + 
    +        $ {cmd}
+
    +
    + output +
    + 
    +        {output.join("\n")}
+
    +
    + ); +} + +function CodeBlock({ content, maxHeight }: { content: string; maxHeight?: number }) { + return ( + 
    +      {content}
+
    + ); +} + +// ── Week budget footer ─────────────────────────────────────────── + +function WeekBudgetFooter() { + return ( +
    +
    +
    + heavy week · expected mechanical pass +
    +
    + search starts before LLM enrichment completes +
    +
    +
    + + + + +
    +
    + ); +} + +function BudgetCell({ + label, + value, + detail, +}: { + label: string; + value: string; + detail: string; +}) { + return ( +
    +
    + {label} +
    +
    + {value} +
    +
    + {detail} +
    +
    + ); +} + +// ── Data helpers (unchanged) ────────────────────────────────────── + +function buildProducedFiles(session: SessionSample): ProducedFile[] { + const derived = estimateDerivedBytes(session); + const windows = estimateEventWindows(session); + return [ + { + name: "manifest.json", + kind: "metadata", + bytes: Math.max(1_600, Math.round(derived * 0.03)), + refs: "collection config", + }, + { + name: "overview.md", + kind: "summary", + bytes: Math.max(2_400, Math.round(derived * 0.18)), + refs: "session-level", + }, + { + name: "decisions.md", + kind: "summary", + bytes: Math.max(1_800, Math.round(derived * 0.11)), + refs: "decision refs", + }, + { + name: "files.md", + kind: "catalog", + bytes: Math.max(1_800, Math.round(derived * 0.13)), + refs: "path refs", + }, + { + name: "tool-calls.md", + kind: "catalog", + bytes: Math.max(2_000, Math.round(derived * 0.15)), + refs: "command refs", + }, + { + name: + windows === 1 + ? "events-001.md" + : `events-001..${String(windows).padStart(3, "0")}.md`, + kind: "event windows", + bytes: Math.max(3_200, Math.round(derived * 0.4)), + refs: `${windows} windows`, + }, + ]; +} + +function databaseRows( + session: SessionSample, + artifact: ProducedFile, + indexBytes: number, +) { + const slug = qmdSlug(session); + const chunks = estimateChunkCount(session); + return [ + { + table: "collections", + key: "session_search_samples", + value: "Local derived knowledge collection for selected external transcripts.", + ref: "root", + }, + { + table: "sessions", + key: slug, + value: `${session.harness} ${session.tier}, ${formatBytes(session.sizeBytes)}, ${formatCount(session.events)} source events.`, + ref: "observed", + }, + { + table: "documents", + key: artifact.name, + value: `${artifact.kind}, ${formatBytes(artifact.bytes)}, derived from ${session.displayPath}.`, + ref: "qmd doc", + }, + { + table: "chunks", + key: `${chunks} searchable chunks`, + value: "Chunk rows keep markdown offsets plus source JSONL coordinates for raw drilldown.", + ref: "fts", + }, + { + table: "terms", + key: formatBytes(indexBytes), + value: "Rebuildable lexical/fuzzy index, not a second copy of the transcript.", + ref: "local", + }, + ]; +} + +function buildQueryResult(session: SessionSample, query: string) { + const normalized = query.toLowerCase(); + const slug = qmdSlug(session); + const wantsDecision = normalized.includes("decision") || normalized.includes("unresolved"); + const wantsRaw = normalized.includes("raw") || normalized.includes("drill"); + const wantsCode = normalized.includes("code") || normalized.includes("area") || normalized.includes("search"); + const answer = wantsDecision + ? `${session.harness} ${session.tier} has a compact decision trail in decisions.md, with unresolved items linked back to event windows instead of summarized away.` + : wantsRaw + ? `The derived hit is enough to choose the session; the next hop is a source coordinate into ${session.displayPath}.` + : wantsCode + ? `This sample is indexed as work around ${session.codeArea}; files.md and overview.md are the cheap first-pass surfaces.` + : `The fuzzy layer searches the derived ${session.harness} ${session.tier} corpus first, then keeps raw transcript access one click away.`; + + const hits: SearchHit[] = [ + { + title: wantsDecision ? "decisions.md" : "overview.md", + source: `qmd://session-search/${slug}/${wantsDecision ? "decisions.md" : "overview.md"}:12`, + score: "BM25 0.91", + snippet: wantsDecision + ? "Open decisions, follow-ups, and ownerless questions extracted from the session." + : `${session.focus}; indexed as ${formatTokenEstimate(session.sizeBytes)} of raw-token equivalent source.`, + }, + { + title: wantsCode ? "files.md" : "events-001.md", + source: `qmd://session-search/${slug}/${wantsCode ? "files.md" : "events-001.md"}:44`, + score: "BM25 0.78", + snippet: wantsCode + ? `Code area signal: ${session.codeArea}.` + : "Event window preserves the original ordering and source refs for verification.", + }, + { + title: "manifest.json", + source: `qmd://session-search/${slug}/manifest.json:1`, + score: "BM25 0.62", + snippet: "Collection metadata records the harness, path, event count, byte count, and extraction recipe.", + }, + ]; + + return { answer, hits }; +} + +function artifactPreview(session: SessionSample, file: ProducedFile): string { + const slug = qmdSlug(session); + if (file.name === "manifest.json") { + return JSON.stringify( + { + id: slug, + harness: session.harness, + tier: session.tier, + source: session.fullPath, + bytes: session.sizeBytes, + events: session.events, + modified: session.modified, + extraction: { + recipe: "qmd-lite", + eventWindowSize: 350, + derivedBytes: estimateDerivedBytes(session), + indexBytes: estimateIndexBytes(session), + }, + }, + null, + 2, + ); + } + + if (file.name === "overview.md") { + return [ + `# ${session.harness} ${session.tier} session`, + "", + `source: ${session.displayPath}`, + `events: ${formatCount(session.events)}`, + `raw-token-eq: ${formatTokenEstimate(session.sizeBytes)}`, + "", + "## Working summary", + `- ${session.focus}.`, + `- Primary code area signal: ${session.codeArea}.`, + "- Store policy: derived knowledge is Scout-owned; raw harness logs stay observed source material.", + "", + "## Source refs", + `- qmd://session-search/${slug}/events-001.md:1`, + `- raw://${session.fullPath}`, + ].join("\n"); + } + + if (file.name === "decisions.md") { + return [ + "# Decisions", + "", + "- Use a two-step memory path: QMD-style extraction first, fuzzy index second.", + "- Keep the engineering doc static; put moving exploration in Studio studies.", + "- Do not import external harness transcripts as Scout-owned messages.", + "", + "## Follow-ups", + "- Decide which extraction recipe becomes the first real implementation target.", + "- Measure LLM enrichment only after the mechanical pass is useful.", + ].join("\n"); + } + + if (file.name === "files.md") { + return [ + "# File and code area signals", + "", + `primary_area: ${session.codeArea}`, + "", + "| path | signal | source |", + "| --- | --- | --- |", + "| packages/web/client | product search surface | events-001.md:44 |", + "| design/studio/app/studies | interactive study surface | events-002.md:18 |", + "| docs/eng | durable engineering record | overview.md:21 |", + ].join("\n"); + } + + if (file.name === "tool-calls.md") { + return [ + "# Tool calls", + "", + "| command | reason | source |", + "| --- | --- | --- |", + "| rg | locate routes, docs, and prior study patterns | events-001.md:9 |", + "| bun --cwd design/studio dev | preview the Studio study | events-002.md:4 |", + "| browser verification | check layout and interactions | events-002.md:36 |", + "", + "Large sessions would have many more rows; this preview keeps the shape compact.", + ].join("\n"); + } + + return [ + "# Event windows", + "", + `window_size: 350 events`, + `windows: ${estimateEventWindows(session)}`, + "", + "## events-001.md", + "001 user_request source_event=0", + "002 assistant_update source_event=1", + "003 tool_call source_event=2", + "", + "Each line keeps enough source coordinate data to reopen the raw JSONL excerpt.", + ].join("\n"); +} + +function estimateDerivedBytes(session: SessionSample): number { + return Math.max(8_192, Math.round(session.sizeBytes * 0.12)); +} + +function estimateIndexBytes(session: SessionSample): number { + return Math.round(estimateDerivedBytes(session) * 3.2); +} + +function estimateEventWindows(session: SessionSample): number { + return Math.max(1, Math.ceil(session.events / 350)); +} + +function estimateChunkCount(session: SessionSample): number { + return Math.max(3, Math.ceil(estimateDerivedBytes(session) / 2_400)); +} + +function stageTiming(session: SessionSample, stage: StageId): string { + const sizeMiB = session.sizeBytes / 1024 / 1024; + const sizeFactor = Math.max(0.2, sizeMiB); + + switch (stage) { + case "discover": + return sizeMiB > 20 ? "400–900 ms" : "80–350 ms"; + case "normalize": + return secondsRange(sizeFactor * 0.8, sizeFactor * 1.7); + case "extract": + return secondsRange(sizeFactor * 1.2, sizeFactor * 2.6); + case "index": + return secondsRange(sizeFactor * 0.35, sizeFactor * 0.9); + case "query": + return "20–100 ms"; + case "drilldown": + return "10–80 ms"; + } +} + +function secondsRange(low: number, high: number): string { + const lo = Math.max(0.1, low); + const hi = Math.max(lo + 0.1, high); + if (hi < 1) return "<1 s"; + if (hi < 10) return `${Math.round(lo)}–${Math.round(hi)} s`; + return `${Math.round(lo)}–${Math.round(hi)} s`; +} + +function isStageId(value: string): value is StageId { + return STAGES.some((stage) => stage.id === value); +} + +function studyHref( + current: StudySelection, + next: Partial, +): string { + const params = new URLSearchParams(); + params.set("session", next.sessionId ?? current.sessionId); + params.set("step", next.stageId ?? current.stageId); + params.set("artifact", next.artifactName ?? current.artifactName); + params.set("q", next.query ?? current.query); + return `/studies/session-search?${params.toString()}`; +} + +function qmdSlug(session: SessionSample): string { + return `${session.harness.toLowerCase()}-${session.tier}`; +} + +function artifactTestId(name: string): string { + return name.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase(); +} + +function formatBytes(bytes: number): string { + if (bytes >= 1024 * 1024) { + return `${(bytes / 1024 / 1024).toFixed(1)} MiB`; + } + if (bytes >= 1024) { + const value = bytes / 1024; + return `${value >= 100 ? Math.round(value) : value.toFixed(1)} KiB`; + } + return `${bytes} B`; +} + +function formatCount(value: number): string { + return new Intl.NumberFormat("en-US").format(value); +} + +function formatTokenEstimate(bytes: number): string { + const tokens = Math.round(bytes / 4); + if (tokens >= 1_000_000) return `~${(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `~${Math.round(tokens / 1_000)}k`; + return `~${tokens}`; +} diff --git a/design/studio/components/studio/CommandSurface.tsx b/design/studio/components/studio/CommandSurface.tsx new file mode 100644 index 00000000..74d913ed --- /dev/null +++ b/design/studio/components/studio/CommandSurface.tsx @@ -0,0 +1,60 @@ +import type { ReactNode } from "react"; +import type { CommandRun } from "@/lib/studio/command"; + +/** + * Chrome around a single command run. + * + * Owns: copyable shell line, ran/cached/duration/error badge, output frame, + * optional footnote. Does NOT own the output body — the consumer provides JSX + * (or uses a renderer from `lib/studio/renderers`). + */ +export function CommandSurface({ + shell, + run, + body, + footnote, +}: { + shell: string; + run: Pick, "durationMs" | "cached" | "error">; + body: ReactNode; + footnote?: ReactNode; +}) { + const badge = run.error + ? { label: "● error", tone: "text-status-error-fg" } + : run.cached + ? { label: "● cached", tone: "text-studio-ink-faint" } + : { label: `● ran ${run.durationMs} ms`, tone: "text-status-ok-fg" }; + + return ( +
    +
    + + command + + + {badge.label} + +
    + 
    +        $ {shell}
+
    + +
    + output +
    + {run.error ? ( + 
    +          {run.error}
+
    + ) : ( + body + )} + + {footnote ? ( +
    + {footnote} +
    + ) : null} +
    + ); +} diff --git a/design/studio/lib/inventory.ts b/design/studio/lib/inventory.ts new file mode 100644 index 00000000..1aa095e5 --- /dev/null +++ b/design/studio/lib/inventory.ts @@ -0,0 +1,227 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { homedir } from "node:os"; + +export type InventoryHarness = "codex" | "claude" | "claude-subagent"; + +export interface InventoryRow { + harness: InventoryHarness; + path: string; + displayPath: string; + sessionId: string; + sizeBytes: number; + mtimeMs: number; + modified: string; + events?: number; + project?: string; +} + +export interface InventorySummary { + harness: string; + files: number; + bytes: number; +} + +export interface InventoryResult { + rows: InventoryRow[]; + totalFiles: number; + totalBytes: number; + byHarness: InventorySummary[]; + displayLimit: number; + windowDays: number; + durationMs: number; + scannedAt: number; + cached: boolean; + error?: string; +} + +const SCAN_WINDOW_DAYS = 7; +const DISPLAY_LIMIT = 10; +const CACHE_TTL_MS = 60_000; + +const HOME = homedir(); +const CODEX_DIR = path.join(HOME, ".codex", "sessions"); +const CLAUDE_PROJECTS_DIR = path.join(HOME, ".claude", "projects"); + +let cache: { at: number; result: InventoryResult } | undefined; + +export async function runInventory(): Promise { + const now = Date.now(); + if (cache && now - cache.at < CACHE_TTL_MS) { + return { ...cache.result, cached: true }; + } + + const start = Date.now(); + const cutoff = start - SCAN_WINDOW_DAYS * 86_400_000; + + try { + const [codexFiles, claudeFiles] = await Promise.all([ + walkJsonl(CODEX_DIR), + walkJsonl(CLAUDE_PROJECTS_DIR), + ]); + + const rows: InventoryRow[] = []; + let totalBytes = 0; + + for (const file of codexFiles) { + const stat = await safeStat(file); + if (!stat || stat.mtimeMs < cutoff) continue; + totalBytes += stat.size; + rows.push({ + harness: "codex", + path: file, + displayPath: shrinkPath(file), + sessionId: shortSessionId(file), + sizeBytes: stat.size, + mtimeMs: stat.mtimeMs, + modified: formatMtime(stat.mtimeMs), + }); + } + + for (const file of claudeFiles) { + const stat = await safeStat(file); + if (!stat || stat.mtimeMs < cutoff) continue; + totalBytes += stat.size; + const project = path.basename(path.dirname(file)); + const isSubagent = file.includes("agent-") || project.includes("subagent"); + rows.push({ + harness: isSubagent ? "claude-subagent" : "claude", + path: file, + displayPath: shrinkPath(file), + sessionId: shortSessionId(file), + sizeBytes: stat.size, + mtimeMs: stat.mtimeMs, + modified: formatMtime(stat.mtimeMs), + project, + }); + } + + rows.sort((a, b) => b.mtimeMs - a.mtimeMs); + + const displayed = rows.slice(0, DISPLAY_LIMIT); + await Promise.all( + displayed.map(async (row) => { + row.events = await countLines(row.path); + }), + ); + + const codexRows = rows.filter((r) => r.harness === "codex"); + const claudeMainRows = rows.filter((r) => r.harness === "claude"); + const claudeSubRows = rows.filter((r) => r.harness === "claude-subagent"); + + const byHarness: InventorySummary[] = [ + { + harness: "codex", + files: codexRows.length, + bytes: codexRows.reduce((a, r) => a + r.sizeBytes, 0), + }, + { + harness: "claude", + files: claudeMainRows.length, + bytes: claudeMainRows.reduce((a, r) => a + r.sizeBytes, 0), + }, + { + harness: "claude-subagent", + files: claudeSubRows.length, + bytes: claudeSubRows.reduce((a, r) => a + r.sizeBytes, 0), + }, + ]; + + const result: InventoryResult = { + rows: displayed, + totalFiles: rows.length, + totalBytes, + byHarness, + displayLimit: DISPLAY_LIMIT, + windowDays: SCAN_WINDOW_DAYS, + durationMs: Date.now() - start, + scannedAt: start, + cached: false, + }; + + cache = { at: now, result }; + return result; + } catch (err) { + return { + rows: [], + totalFiles: 0, + totalBytes: 0, + byHarness: [], + displayLimit: DISPLAY_LIMIT, + windowDays: SCAN_WINDOW_DAYS, + durationMs: Date.now() - start, + scannedAt: start, + cached: false, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +async function walkJsonl(root: string): Promise { + const out: string[] = []; + async function recurse(dir: string) { + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch { + return; + } + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) await recurse(p); + else if (e.isFile() && e.name.endsWith(".jsonl")) out.push(p); + } + } + await recurse(root); + return out; +} + +async function safeStat(file: string) { + try { + return await fs.stat(file); + } catch { + return undefined; + } +} + +async function countLines(file: string): Promise { + let fh; + try { + fh = await fs.open(file, "r"); + const buf = Buffer.alloc(64 * 1024); + let lines = 0; + let last = -1; + while (true) { + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + for (let i = 0; i < bytesRead; i++) { + if (buf[i] === 0x0a) lines++; + last = buf[i]!; + } + } + if (last !== -1 && last !== 0x0a) lines++; + return lines; + } catch { + return 0; + } finally { + await fh?.close(); + } +} + +function shrinkPath(file: string): string { + return file.startsWith(HOME) ? "~" + file.slice(HOME.length) : file; +} + +function shortSessionId(file: string): string { + const base = path.basename(file, ".jsonl"); + // Codex rollouts look like "rollout-2026-05-29T19-06-57-019e75fd-..."; trim + // the rollout prefix + timestamp so the id reads as just the uuid-ish tail. + const stripped = base.replace(/^rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-/, ""); + return stripped.length > 28 ? stripped.slice(0, 8) + "…" + stripped.slice(-6) : stripped; +} + +function formatMtime(ms: number): string { + const d = new Date(ms); + const pad = (n: number) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}`; +} diff --git a/design/studio/lib/studio-pages.ts b/design/studio/lib/studio-pages.ts index 728e693b..dd688594 100644 --- a/design/studio/lib/studio-pages.ts +++ b/design/studio/lib/studio-pages.ts @@ -147,6 +147,19 @@ export const STUDIO_PAGES: StudioPage[] = [ source: ["packages/web/client/scout/inspector/AgentsInspector.tsx"], blurb: "Info-dense agent tile — identity · state · task · project · capabilities.", }, + { + href: "/studies/session-search", + label: "Session Search", + bucket: "studies", + surface: "web", + family: "session-search", + status: "concept", + source: [ + "design/studio/app/studies/session-search/page.tsx", + "docs/eng/sco-059-session-knowledge-search-exploration.md", + ], + blurb: "Interactive QMD-style pass over six real sessions: files, index rows, queries, and raw drilldown.", + }, { href: "/studies/tree-viewer", label: "Tree Viewer", diff --git a/design/studio/lib/studio/command.ts b/design/studio/lib/studio/command.ts new file mode 100644 index 00000000..90e261c2 --- /dev/null +++ b/design/studio/lib/studio/command.ts @@ -0,0 +1,87 @@ +/** + * Studio runnable command primitive. + * + * A `Command` packages a server-side action: a copyable shell-equivalent + * for display, a typed `run(input)` that actually executes, and an optional + * cache key + TTL. Use `runCommand` instead of calling `command.run` directly + * so caching + timing + error capture are centralized. + * + * Pair with `` (components/studio/CommandSurface.tsx) to render + * the command line, run badge, and output via the renderer registry. + */ + +export interface Command { + /** Stable id used in URLs and as the default cache namespace. */ + id: string; + label: string; + /** Copyable shell line. Display only — never executed. */ + shell: (input: Input) => string; + /** Server-side execution. Uses fs/sqlite/etc, never spawns a shell. */ + run: (input: Input) => Promise; + /** Defaults to JSON.stringify(input). */ + cacheKey?: (input: Input) => string; + /** 0 or undefined disables caching. */ + cacheTtlMs?: number; +} + +export interface CommandRun { + output: Output; + durationMs: number; + cached: boolean; + ranAt: number; + error?: string; +} + +interface CacheEntry { + at: number; + run: CommandRun; +} + +const cache = new Map(); + +function entryKey(cmd: Command, input: I): string { + const inputKey = cmd.cacheKey ? cmd.cacheKey(input) : JSON.stringify(input); + return `${cmd.id}::${inputKey}`; +} + +export async function runCommand( + cmd: Command, + input: I, +): Promise> { + const ttl = cmd.cacheTtlMs ?? 0; + const key = entryKey(cmd, input); + const now = Date.now(); + + if (ttl > 0) { + const hit = cache.get(key); + if (hit && now - hit.at < ttl) { + return { ...(hit.run as CommandRun), cached: true }; + } + } + + const start = Date.now(); + try { + const output = await cmd.run(input); + const run: CommandRun = { + output, + durationMs: Date.now() - start, + cached: false, + ranAt: start, + }; + if (ttl > 0) cache.set(key, { at: now, run: run as CommandRun }); + return run; + } catch (err) { + return { + output: undefined as O, + durationMs: Date.now() - start, + cached: false, + ranAt: start, + error: err instanceof Error ? err.message : String(err), + }; + } +} + +/** Clear the in-process cache. Test/dev only. */ +export function clearCommandCache(): void { + cache.clear(); +} diff --git a/design/studio/lib/studio/commands/inventory.ts b/design/studio/lib/studio/commands/inventory.ts new file mode 100644 index 00000000..1f1b7231 --- /dev/null +++ b/design/studio/lib/studio/commands/inventory.ts @@ -0,0 +1,32 @@ +import { runInventory, type InventoryResult } from "@/lib/inventory"; +import type { Command } from "@/lib/studio/command"; + +export interface InventoryInput { + /** Window in days, e.g. "7d". */ + since: string; +} + +export const inventoryCommand: Command = { + id: "inventory", + label: "Inventory", + shell: ({ since }) => { + const days = parseSinceDays(since); + return `find ~/.codex/sessions ~/.claude/projects -name '*.jsonl' -mtime -${days}`; + }, + run: async (_input) => { + // runInventory currently has its own 7d window + module cache; the input + // is accepted for shape parity but not threaded through yet. When we add + // a second input (project filter, harness filter), runInventory will grow + // an args object. + return runInventory(); + }, + cacheKey: ({ since }) => `since:${since}`, + // runInventory has its own 60s cache; the command-level cache is redundant + // for now but harmless. Leave at 0 to let the inner cache do the work. + cacheTtlMs: 0, +}; + +function parseSinceDays(since: string): number { + const m = since.match(/^(\d+)d$/); + return m ? Number(m[1]) : 7; +} diff --git a/design/studio/lib/studio/commands/parse-session.ts b/design/studio/lib/studio/commands/parse-session.ts new file mode 100644 index 00000000..a185a765 --- /dev/null +++ b/design/studio/lib/studio/commands/parse-session.ts @@ -0,0 +1,328 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import type { Command } from "@/lib/studio/command"; + +export interface ParseSessionInput { + /** Absolute path to a Codex or Claude JSONL session file. */ + path: string; + /** Number of records to read from the head. */ + limit: number; +} + +export type NormalizedKind = + | "session_meta" + | "user_turn" + | "assistant_turn" + | "command_or_tool" + | "observation" + | "system_record" + | "unknown"; + +export interface NormalizedRecord { + /** Index in source order (0-based). */ + i: number; + /** ISO timestamp from source if present. */ + ts?: string; + /** Coarse kind. Drives renderer tone + downstream routing. */ + kind: NormalizedKind; + /** Human label: role, tool name, snapshot, etc. */ + tag?: string; + /** Full text content, untrimmed. Empty for tool calls/results that are structured. */ + text?: string; + /** Structured tool invocation, present when kind === "command_or_tool". */ + tool?: { name: string; input: unknown }; + /** Structured observation, present when kind === "observation". */ + result?: { ok?: boolean; output: unknown }; + /** Session-level metadata, present when kind === "session_meta". */ + meta?: Record; + /** Identifiers for threading (Claude has parentUuid; Codex has session_id). */ + refs?: { id?: string; parentId?: string; sessionId?: string }; + /** Raw record type from the source schema, for debugging + drilldown. */ + sourceType: string; + /** Byte offset of this record's line within the source file. */ + sourceOffset: number; +} + +export interface ParseSessionResult { + harness: "codex" | "claude" | "unknown"; + records: NormalizedRecord[]; + /** Raw source line for each record, parallel-indexed. Kept for inspect views. */ + rawLines: string[]; + scannedLines: number; + bytesRead: number; + error?: string; +} + +export const parseSessionCommand: Command = { + id: "parse-session", + label: "Parse session", + shell: ({ path: p, limit }) => + `head -n ${limit} ${shellQuote(shrinkPath(p))} | jq -c '.'`, + run: async ({ path: filePath, limit }) => { + try { + const text = await readHeadLines(filePath, limit); + const harness = detectHarness(filePath); + const records: NormalizedRecord[] = []; + const rawLines: string[] = []; + let offset = 0; + let i = 0; + for (const line of text.split("\n")) { + if (i >= limit) break; + if (line.length === 0) { + offset += 1; // empty line + its \n + continue; + } + const lineOffset = offset; + offset += Buffer.byteLength(line, "utf8") + 1; // line + \n + try { + const obj = JSON.parse(line); + records.push(normalize(obj, i, lineOffset, harness)); + rawLines.push(line); + } catch { + records.push({ + i, + kind: "unknown", + sourceType: "unparseable", + sourceOffset: lineOffset, + }); + rawLines.push(line); + } + i++; + } + return { + harness, + records, + rawLines, + scannedLines: records.length, + bytesRead: Buffer.byteLength(text, "utf8"), + }; + } catch (err) { + return { + harness: "unknown", + records: [], + rawLines: [], + scannedLines: 0, + bytesRead: 0, + error: err instanceof Error ? err.message : String(err), + }; + } + }, + cacheKey: ({ path: p, limit }) => `${p}::${limit}`, + cacheTtlMs: 60_000, +}; + +function detectHarness(filePath: string): "codex" | "claude" | "unknown" { + if (filePath.includes("/.codex/")) return "codex"; + if (filePath.includes("/.claude/")) return "claude"; + return "unknown"; +} + +async function readHeadLines(filePath: string, limit: number): Promise { + // Read up to ~128 KB to cover N lines for typical sessions. If lines are + // unusually long, the parser will just deliver fewer records than asked + // for — fine for a stream preview. + const fh = await fs.open(filePath, "r"); + try { + const buf = Buffer.alloc(128 * 1024); + const { bytesRead } = await fh.read(buf, 0, buf.length, 0); + const text = buf.slice(0, bytesRead).toString("utf8"); + // Trim to roughly the requested number of lines so we don't carry a + // huge tail of the buffer into the parser. + let cut = 0; + let seen = 0; + for (let i = 0; i < text.length && seen <= limit; i++) { + if (text[i] === "\n") { + seen++; + cut = i + 1; + } + } + return cut > 0 ? text.slice(0, cut) : text; + } finally { + await fh.close(); + } +} + +function normalize( + obj: Record, + i: number, + sourceOffset: number, + harness: "codex" | "claude" | "unknown", +): NormalizedRecord { + if (harness === "codex") return normalizeCodex(obj, i, sourceOffset); + if (harness === "claude") return normalizeClaude(obj, i, sourceOffset); + return { + i, + kind: "unknown", + sourceType: String(obj.type ?? "?"), + sourceOffset, + }; +} + +function normalizeCodex( + obj: Record, + i: number, + sourceOffset: number, +): NormalizedRecord { + const type = String(obj.type ?? ""); + const ts = typeof obj.timestamp === "string" ? obj.timestamp : undefined; + const payload = (obj.payload as Record | undefined) ?? {}; + const base = { i, ts, sourceType: type, sourceOffset }; + + if (type === "session_meta") { + return { + ...base, + kind: "session_meta", + tag: "meta", + meta: payload, + refs: { sessionId: payload.id as string | undefined }, + }; + } + if (type === "message") { + const role = String(payload.role ?? ""); + const text = extractCodexText(payload.content); + if (role === "user") return { ...base, kind: "user_turn", tag: "user", text }; + if (role === "assistant") return { ...base, kind: "assistant_turn", tag: "assistant", text }; + return { ...base, kind: "system_record", tag: role, text }; + } + if (type === "function_call" || type === "local_shell_call") { + const name = String(payload.name ?? payload.command ?? "tool"); + const input = payload.arguments ?? payload.args ?? payload.input ?? {}; + return { ...base, kind: "command_or_tool", tag: name, tool: { name, input } }; + } + if (type === "function_call_output" || type === "local_shell_call_output") { + const output = payload.output ?? payload.content ?? ""; + return { ...base, kind: "observation", tag: "result", result: { output } }; + } + if (type === "reasoning") { + return { + ...base, + kind: "assistant_turn", + tag: "reasoning", + text: String(payload.content ?? ""), + }; + } + return { ...base, kind: "system_record", text: JSON.stringify(payload) }; +} + +function normalizeClaude( + obj: Record, + i: number, + sourceOffset: number, +): NormalizedRecord { + const type = String(obj.type ?? ""); + const ts = typeof obj.timestamp === "string" ? obj.timestamp : undefined; + const refs = { + id: typeof obj.uuid === "string" ? obj.uuid : undefined, + parentId: typeof obj.parentUuid === "string" ? obj.parentUuid : undefined, + }; + const base = { i, ts, sourceType: type, sourceOffset, refs }; + + if (type === "user") { + const msg = obj.message as Record | undefined; + const text = extractClaudeText(msg?.content); + return { ...base, kind: "user_turn", tag: "user", text }; + } + if (type === "assistant") { + const msg = obj.message as Record | undefined; + const content = msg?.content; + if (Array.isArray(content)) { + const toolUse = content.find((c) => (c as { type?: string })?.type === "tool_use") as + | { name?: string; input?: unknown } + | undefined; + if (toolUse) { + const name = String(toolUse.name ?? "tool"); + return { + ...base, + kind: "command_or_tool", + tag: name, + tool: { name, input: toolUse.input ?? {} }, + sourceType: "tool_use", + }; + } + } + return { + ...base, + kind: "assistant_turn", + tag: "assistant", + text: extractClaudeText(content), + }; + } + if (type === "tool_use") { + const name = String(obj.name ?? "tool"); + return { + ...base, + kind: "command_or_tool", + tag: name, + tool: { name, input: (obj as { input?: unknown }).input ?? {} }, + }; + } + if (type === "tool_result") { + const output = (obj as { content?: unknown }).content ?? ""; + return { ...base, kind: "observation", tag: "result", result: { output } }; + } + if (type === "system") { + return { + ...base, + kind: "system_record", + tag: "system", + text: extractClaudeText((obj as { content?: unknown }).content), + }; + } + if (type === "summary" || type === "file-history-snapshot") { + return { + ...base, + kind: "system_record", + tag: type, + meta: obj, + }; + } + return { ...base, kind: "system_record", text: JSON.stringify(obj) }; +} + +function extractCodexText(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((c) => { + if (typeof c === "string") return c; + const block = c as { type?: string; text?: string; content?: string }; + if (block?.text) return block.text; + if (block?.content) return block.content; + return ""; + }) + .filter(Boolean) + .join(" "); + } + return ""; +} + +function extractClaudeText(content: unknown): string { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((c) => { + if (typeof c === "string") return c; + const block = c as { type?: string; text?: string; content?: unknown }; + if (block?.text) return block.text; + if (typeof block?.content === "string") return block.content; + return ""; + }) + .filter(Boolean) + .join(" "); + } + return ""; +} + +function trim(s: string, n: number): string { + const oneLine = s.replace(/\s+/g, " ").trim(); + return oneLine.length > n ? oneLine.slice(0, n - 1) + "…" : oneLine; +} + +function shrinkPath(file: string): string { + const home = process.env.HOME ?? ""; + return home && file.startsWith(home) ? "~" + file.slice(home.length) : file; +} + +function shellQuote(s: string): string { + return /[^A-Za-z0-9_./~-]/.test(s) ? `'${s.replace(/'/g, `'\\''`)}'` : s; +} diff --git a/design/studio/lib/studio/renderers.ts b/design/studio/lib/studio/renderers.ts new file mode 100644 index 00000000..84d54777 --- /dev/null +++ b/design/studio/lib/studio/renderers.ts @@ -0,0 +1,42 @@ +/** + * Output renderer registry for Studio commands. + * + * A renderer turns a typed command output into JSX. Renderers are looked up by + * `RendererKind` so a `` only needs to know which kind its + * output is, not how to draw it. + * + * Registry instance is module-singleton — register at module load. JSX is kept + * out of this file so consumers can register their own renderers without + * pulling React into pure exec code paths. + */ + +import type { ReactNode } from "react"; + +export type RendererKind = + | "rows" // table of homogeneous records + | "stream" // ordered event/record list + | "files" // file list + selected-file preview + | "kv" // key-value pairs + | "ranked" // ranked list with scores + | "raw"; // single code/text block + +export interface RendererProps { + output: T; + /** Optional sub-selection: row id, file name, record index, etc. */ + focus?: string; +} + +export interface OutputRenderer { + kind: RendererKind; + render: (props: RendererProps) => ReactNode; +} + +const registry = new Map>(); + +export function registerRenderer(r: OutputRenderer): void { + registry.set(r.kind, r as OutputRenderer); +} + +export function lookupRenderer(kind: RendererKind): OutputRenderer | undefined { + return registry.get(kind); +} diff --git a/docs/eng/sco-059-session-knowledge-search-exploration.md b/docs/eng/sco-059-session-knowledge-search-exploration.md new file mode 100644 index 00000000..fa3113e0 --- /dev/null +++ b/docs/eng/sco-059-session-knowledge-search-exploration.md @@ -0,0 +1,135 @@ +# SCO-059: Session Knowledge Search Exploration + +## Status + +Concept. + +## Proposal ID + +`sco-059` + +## Intent + +Explore the shape of Scout session/history search as an idea-navigation problem +before committing to the in-product search surface. The goal is to separate +the Studio exploration space from the shipped OpenScout web UI: Studio is where +we compare concepts, vocabulary, budgets, and workflows; the product Search +surface is where a chosen model eventually becomes operational. + +## Context + +Recent discussion mixed two surfaces: + +- **Studio**: an internal exploration app for navigating ideas, product + concepts, and design decisions. +- **Search**: an in-product OpenScout surface for session knowledge, fuzzy + retrieval, and raw-log drilldown. + +The current prototype belongs to Search, not Studio. It describes an indexable +knowledge workflow over observed harness transcripts. The Studio should host +the exploration around whether that workflow is the right one, what it should +feel like, and how much work/cost it implies for a heavy local user. + +This doc anchors that exploration with a concrete seven-day sample from the +local machine. + +## Local Sample + +Seven-day observed footprint on this machine: + +| Source | Files | Raw size | +| --- | ---: | ---: | +| Codex sessions | 78 | 191 MiB | +| Claude main sessions | 72 | 228 MiB | +| Claude subagent logs | 114 | 56 MiB | +| Claude history | 1 | 13 MiB | +| **All observed** | **266** | **489 MiB** | + +Representative session spread: + +| Harness | Tier | Size | Events | Modified | +| --- | --- | ---: | ---: | --- | +| Codex | large | 13.0 MiB | 4,220 | 2026-05-29 23:29 | +| Codex | normal | 1.1 MiB | 494 | 2026-05-25 15:45 | +| Codex | small | 34 KiB | 12 | 2026-05-29 00:16 | +| Claude | large | 52.9 MiB | 12,009 | 2026-05-26 02:49 | +| Claude | normal | 745 KiB | 252 | 2026-05-23 13:11 | +| Claude | small | 2.1 KiB | 5 | 2026-05-24 21:57 | + +## Product Shape To Explore + +The strongest shape is a two-speed system: + +1. **Mechanical index first.** Discover files, parse JSONL, normalize events, + produce QMD-style markdown chunks, build FTS/fuzzy indexes, and preserve + source anchors. This should finish in minutes for a heavy week. +2. **LLM enrichment later.** Summarize selected chunks into decisions, files, + problems, unresolved threads, and useful session labels. This must be + bounded, resumable, and optional. + +The user should be able to start searching before enrichment is complete. + +## Budget Sketch + +For a week around 489 MiB raw JSONL: + +| Stage | Input | Output | Expected timing | +| --- | --- | --- | --- | +| Inventory | 266 files | session manifest | 1-5s | +| Mechanical extraction | raw JSONL | 25-100 MiB markdown | 30-120s | +| FTS/fuzzy index | derived markdown | 75-300 MiB SQLite | 30-180s | +| First useful query | local index | ranked hits + source refs | under 100ms | +| LLM enrichment | selected chunks | decisions/files/problems | 10-60m async | + +These are working estimates, not guarantees. The important architecture +constraint is that search must not wait for full LLM enrichment. + +## Search Questions + +Studio should help compare answers to questions like: + +- Which recent sessions touched this area of the codebase? +- What was I working on when this decision happened? +- Which session had the useful plan, error, or file path? +- Was this conversation a Codex thread, a Claude main session, or a Claude + subagent side quest? +- Do I need the raw transcript, or is the derived knowledge enough? + +## Surface Boundary + +The product Search surface can be action-oriented: + +- select a date range +- choose harnesses +- build or refresh the local index +- inspect freshness and cost +- search derived knowledge +- drill into raw source spans + +The Studio exploration should remain idea-oriented: + +- compare workflows +- tune vocabulary +- show sizing and cost models +- map data ownership boundaries +- decide what becomes product UI + +## Data Ownership Rule + +External harness transcripts remain observed source material. Scout should not +bulk-import Codex or Claude transcript lines as Scout-owned messages. Scout can +own derived knowledge records, index metadata, source references, user-created +collections, and broker-owned coordination records. + +## Open Decisions + +- Should the first product version index Claude subagent logs by default, or + keep them behind an advanced toggle? +- Should "normal" sessions be selected by file size, event count, duration, or + inferred task continuity? +- How much LLM enrichment should run automatically versus only after a user + marks sessions as interesting? +- Should the search index live per machine, per project, or per explicit + session collection? +- What is the smallest derived document format that still supports good + conversation and exact raw drilldown? diff --git a/packages/web/client/screens/KnowledgeSearchScreen.tsx b/packages/web/client/screens/KnowledgeSearchScreen.tsx new file mode 100644 index 00000000..1c62fd5e --- /dev/null +++ b/packages/web/client/screens/KnowledgeSearchScreen.tsx @@ -0,0 +1,659 @@ +import "./knowledge-search.css"; + +import { useMemo, useState } from "react"; +import { + Archive, + Bot, + CheckCircle2, + Clock3, + Database, + FileSearch, + GitBranch, + Layers3, + MessageSquareText, + Search, + SlidersHorizontal, + Sparkles, + Waypoints, +} from "lucide-react"; +import type { LucideIcon } from "lucide-react"; + +import type { Route } from "../lib/types.ts"; + +type ExtractionModeId = "mechanical" | "standard" | "deep"; + +type ExtractionMode = { + id: ExtractionModeId; + label: string; + summary: string; + llmUse: string; + output: string; +}; + +type PipelineStage = { + id: string; + label: string; + status: string; + detail: string; + icon: LucideIcon; +}; + +type SearchHit = { + title: string; + location: string; + score: string; + snippet: string; + source: string; +}; + +type SearchScenario = { + id: string; + query: string; + intent: string; + answer: string; + hits: SearchHit[]; +}; + +const SESSION_PATH = + "/Users/arach/.codex/sessions/2026/05/30/rollout-2026-05-30T11-25-39-019e797d-ab15-7823-b322-4434c5831317.jsonl"; + +const SIDE_CAR_ROOT = "/tmp/openscout-qmd-e2e/docs"; + +const EXTRACTION_MODES: ExtractionMode[] = [ + { + id: "mechanical", + label: "Mechanical", + summary: "Chunk, normalize, attach source refs, and index immediately.", + llmUse: "No LLM pass", + output: "raw event docs + tool-call catalog", + }, + { + id: "standard", + label: "Summary", + summary: "Add compact decision, file, problem, and next-action summaries before indexing.", + llmUse: "Small summary pass", + output: "QMD-ready docs + topic summaries", + }, + { + id: "deep", + label: "Deep", + summary: "Run focused extraction against a declared interest set before building the index.", + llmUse: "Selective LLM pass", + output: "curated knowledge pack + raw refs", + }, +]; + +const PIPELINE: PipelineStage[] = [ + { + id: "select", + label: "Session Set", + status: "curated", + detail: "The user chooses sessions worth remembering instead of importing every harness log.", + icon: Archive, + }, + { + id: "extract", + label: "Extraction", + status: "derived", + detail: "QMD-style markdown docs are produced from observed transcripts with stable source refs.", + icon: Layers3, + }, + { + id: "index", + label: "Fuzzy Index", + status: "rebuildable", + detail: "FTS/fuzzy search targets the derived corpus; vectors can be added later, not required first.", + icon: Database, + }, + { + id: "talk", + label: "LLM Conversation", + status: "assisted", + detail: "The assistant works over extracted knowledge first, keeping answers cheap and directed.", + icon: MessageSquareText, + }, + { + id: "drilldown", + label: "Raw Drilldown", + status: "anchored", + detail: "When confidence matters, jump back to the exact JSONL source and event-level context.", + icon: FileSearch, + }, +]; + +const METRICS = [ + { label: "Source sessions", value: "1", detail: "Codex JSONL" }, + { label: "Observed events", value: "310", detail: "source material" }, + { label: "Derived docs", value: "13", detail: "markdown files" }, + { label: "Index size", value: "548 KB", detail: "SQLite / FTS" }, + { label: "Vectors", value: "0", detail: "lexical first" }, +]; + +const SEARCH_SCENARIOS: SearchScenario[] = [ + { + id: "drilldown", + query: "which session discussed raw log drilldown?", + intent: "Find the strategy conversation before opening the transcript.", + answer: + "This session is about a two-step session-knowledge flow: summarize/extract into a search corpus, then use retrieval to jump back to raw logs only when needed.", + hits: [ + { + title: "events-07.md", + location: "line 133", + score: "BM25 0.91", + snippet: + "semantic conversation first, raw-log drilldown second", + source: "qmd://scout-session/events-07.md:133", + }, + { + title: "overview.md", + location: "line 18", + score: "BM25 0.77", + snippet: + "session knowledge collections, QMD store creation, and event-level lookup", + source: "qmd://scout-session/overview.md:18", + }, + ], + }, + { + id: "policy", + query: "QMD store extraction policy", + intent: "Separate QMD's store mechanics from Scout's extraction policy.", + answer: + "QMD provides the markdown store and search mechanics. Scout would own the pre-index extraction policy: decisions, files, errors, next actions, and source coordinates.", + hits: [ + { + title: "events-09.md", + location: "line 38", + score: "BM25 0.88", + snippet: + "store creation is mostly index/database setup", + source: "qmd://scout-session/events-09.md:38", + }, + { + title: "tool-calls.md", + location: "line 74", + score: "BM25 0.65", + snippet: + "collection add, context add, update, search, and get verified the store loop", + source: "qmd://scout-session/tool-calls.md:74", + }, + ], + }, + { + id: "logs", + query: "extract knowledge and make fast search from logs", + intent: "Recover the source idea and the concrete search shape.", + answer: + "The useful pattern is not bulk transcript import. It is a derived knowledge set with line-addressable docs, fuzzy search, freshness metadata, and a raw-log escape hatch.", + hits: [ + { + title: "overview.md", + location: "line 7", + score: "BM25 0.83", + snippet: + "extract knowledge from logs, build fast search, then converse over the resulting dataset", + source: "qmd://scout-session/overview.md:7", + }, + { + title: "events-05.md", + location: "line 92", + score: "BM25 0.72", + snippet: + "user-curated session sets become indexed knowledge collections", + source: "qmd://scout-session/events-05.md:92", + }, + ], + }, +]; + +const DOC_ROWS = [ + { name: "overview.md", kind: "summary", weight: "high", refs: "session + topics" }, + { name: "tool-calls.md", kind: "catalog", weight: "medium", refs: "commands + outputs" }, + { name: "events-01.md ... events-11.md", kind: "chunks", weight: "source", refs: "event windows" }, +]; + +const WEEKLY_SCOPE_ROWS = [ + { label: "Codex sessions", value: "78", detail: "191 MiB raw JSONL" }, + { label: "Claude main", value: "72", detail: "228 MiB raw JSONL" }, + { label: "Claude subagents", value: "114", detail: "56 MiB raw JSONL" }, + { label: "Claude history", value: "1", detail: "13 MiB raw JSONL" }, + { label: "All observed", value: "266", detail: "489 MiB this week" }, +]; + +const SAMPLE_SESSION_ROWS = [ + { + harness: "Codex", + tier: "large", + size: "13.0 MiB", + events: "4,220", + rawEstimate: "~3.4M raw-token eq.", + modified: "2026-05-29 23:29", + path: "~/.codex/sessions/2026/05/29/...019e75fd-a431...jsonl", + }, + { + harness: "Codex", + tier: "normal", + size: "1.1 MiB", + events: "494", + rawEstimate: "~289k raw-token eq.", + modified: "2026-05-25 15:45", + path: "~/.codex/sessions/2026/05/25/...019e609c-4389...jsonl", + }, + { + harness: "Codex", + tier: "small", + size: "34 KiB", + events: "12", + rawEstimate: "~9k raw-token eq.", + modified: "2026-05-29 00:16", + path: "~/.codex/sessions/2026/05/29/...019e71f2-958c...jsonl", + }, + { + harness: "Claude", + tier: "large", + size: "52.9 MiB", + events: "12,009", + rawEstimate: "~13.9M raw-token eq.", + modified: "2026-05-26 02:49", + path: "~/.claude/projects/-Users-arach-dev-openscout/a00198bf...jsonl", + }, + { + harness: "Claude", + tier: "normal", + size: "745 KiB", + events: "252", + rawEstimate: "~191k raw-token eq.", + modified: "2026-05-23 13:11", + path: "~/.claude/projects/-Users-arach-dev-openscout/c680a795...jsonl", + }, + { + harness: "Claude", + tier: "small", + size: "2.1 KiB", + events: "5", + rawEstimate: "~500 raw-token eq.", + modified: "2026-05-24 21:57", + path: "~/.claude/projects/-Users-arach-dev-contextual/ada6d81e...jsonl", + }, +]; + +const WEEK_PREP_ROWS = [ + { + step: "Inventory", + input: "266 files / 489 MiB", + output: "session manifest", + timing: "1-5s", + }, + { + step: "Mechanical extraction", + input: "raw JSONL", + output: "25-100 MiB markdown", + timing: "30-120s", + }, + { + step: "FTS/fuzzy index", + input: "derived markdown", + output: "75-300 MiB SQLite", + timing: "30-180s", + }, + { + step: "First useful search", + input: "local index", + output: "ranked hits + source refs", + timing: "<100ms/query", + }, + { + step: "LLM enrichment", + input: "selected chunks", + output: "decisions, files, problems", + timing: "10-60m async", + }, +]; + +function classPart(value: string): string { + return value.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "").toLowerCase(); +} + +export function KnowledgeSearchScreen({ navigate }: { navigate: (route: Route) => void }) { + const [modeId, setModeId] = useState("standard"); + const [scenarioId, setScenarioId] = useState(SEARCH_SCENARIOS[0]!.id); + + const selectedMode = useMemo( + () => EXTRACTION_MODES.find((mode) => mode.id === modeId) ?? EXTRACTION_MODES[0]!, + [modeId], + ); + const selectedScenario = useMemo( + () => SEARCH_SCENARIOS.find((scenario) => scenario.id === scenarioId) ?? SEARCH_SCENARIOS[0]!, + [scenarioId], + ); + + return ( +
    +
    +
    +
    +
    +

    QMD-style extraction, fuzzy search, then raw-log drilldown.

    +

    + A Scout-native view of the workflow: choose sessions, derive a markdown knowledge + corpus, search that corpus quickly, and only open transcript-level evidence when the + conversation needs it. +

    +
    +
    + + +
    +
    + +
    + {METRICS.map((metric) => ( +
    + {metric.label} + {metric.value} + {metric.detail} +
    + ))} +
    + +
    +
    + +
    +
    +
    + One-week run +

    Local log footprint for a heavy week

    +
    +
    +
    + {WEEKLY_SCOPE_ROWS.map((row) => ( +
    + {row.label} + {row.value} + {row.detail} +
    + ))} +
    +
    + +
    +
    +
    + Representative sample +

    Large, normal, and small sessions from Codex and Claude

    +
    +
    +
    +
    + Harness + Tier + Size + Events + Rough size + Modified + Path +
    + {SAMPLE_SESSION_ROWS.map((row) => ( +
    + {row.harness} + {row.tier} + {row.size} + {row.events} + {row.rawEstimate} + {row.modified} + {row.path} +
    + ))} +
    +
    + +
    + + +
    +
    +
    + Conversation layer +

    Ask the derived corpus first

    +
    +
    + +
    + {SEARCH_SCENARIOS.map((scenario) => ( + + ))} +
    + +
    +
    + User +

    {selectedScenario.query}

    +
    +
    + Scout +

    {selectedScenario.answer}

    +
    +
    + +
    +
    + {selectedScenario.intent} + {selectedScenario.hits.length} hits +
    + {selectedScenario.hits.map((hit) => ( +
    +
    +
    +

    {hit.snippet}

    + {hit.source} +
    + ))} +
    +
    + + +
    + +
    +
    +
    + Back-of-envelope budget +

    What it takes to index a week

    +
    +
    +
    + {WEEK_PREP_ROWS.map((row) => ( +
    + {row.step} +
    +
    +
    Input
    +
    {row.input}
    +
    +
    +
    Output
    +
    {row.output}
    +
    +
    +
    Timing
    +
    {row.timing}
    +
    +
    +
    + ))} +
    +
    + +
    +
    +
    +
    + Derived files +

    QMD-ready corpus

    +
    +
    +
    +
    + Document + Kind + Weight + Refs +
    + {DOC_ROWS.map((doc) => ( +
    + {doc.name} + {doc.kind} + {doc.weight} + {doc.refs} +
    + ))} +
    +
    + +
    +
    +
    + Evidence +

    Raw transcript drilldown

    +
    +
    +
    +
    + Derived corpus + {SIDE_CAR_ROOT} +
    +
    + Source transcript + {SESSION_PATH} +
    +
    + Drilldown command + qmd get qmd://scout-session/events-07.md:128 -l 38 --format md +
    +
    +
    +
    +
    + ); +} diff --git a/packages/web/client/screens/knowledge-search.css b/packages/web/client/screens/knowledge-search.css new file mode 100644 index 00000000..0c4dfdc1 --- /dev/null +++ b/packages/web/client/screens/knowledge-search.css @@ -0,0 +1,912 @@ +.ks-page { + flex: 1 1 auto; + min-height: 0; + height: 100%; + overflow: auto; + container-name: knowledge-search; + container-type: inline-size; + display: flex; + flex-direction: column; + gap: 14px; + padding: 22px 24px 34px; + background: + linear-gradient(180deg, color-mix(in srgb, var(--surface) 28%, transparent), transparent 220px), + var(--bg); + color: var(--ink); +} + +.ks-toolbar { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 20px; + min-width: 0; +} + +.ks-title-block { + display: flex; + flex-direction: column; + gap: 7px; + min-width: 0; + max-width: 860px; +} + +.ks-kicker, +.ks-panel-eyebrow { + display: inline-flex; + align-items: center; + gap: 7px; + color: var(--accent); + font-family: var(--font-mono); + font-size: 10.5px; + font-weight: 700; + letter-spacing: 0; + text-transform: uppercase; +} + +.ks-title-block h1 { + max-width: 760px; + margin: 0; + font-family: var(--font-accent-title); + font-size: 27px; + font-weight: 560; + line-height: 1.15; + letter-spacing: 0; + color: var(--ink); +} + +.ks-title-block p { + margin: 0; + max-width: 760px; + font-size: 13px; + line-height: 1.45; + color: var(--muted); +} + +.ks-toolbar-actions { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; + justify-content: flex-end; + flex: none; +} + +.ks-icon-button { + appearance: none; + height: 32px; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; + padding: 0 11px; + border: 1px solid color-mix(in srgb, var(--ink) 8%, transparent); + border-radius: 7px; + background: color-mix(in srgb, var(--surface) 92%, transparent); + color: var(--ink); + font-size: 12px; + font-weight: 620; + line-height: 1; + cursor: pointer; +} + +.ks-icon-button:hover { + border-color: color-mix(in srgb, var(--accent) 35%, var(--border)); + background: color-mix(in srgb, var(--accent) 8%, var(--surface)); +} + +.ks-metrics { + display: grid; + grid-template-columns: repeat(5, minmax(120px, 1fr)); + gap: 8px; +} + +.ks-metric { + min-width: 0; + display: flex; + flex-direction: column; + gap: 3px; + padding: 11px 12px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--surface) 94%, transparent); +} + +.ks-metric-label { + font-family: var(--font-mono); + font-size: 10px; + color: var(--dim); +} + +.ks-metric strong { + font-size: 19px; + line-height: 1.1; + font-weight: 660; + color: var(--ink); +} + +.ks-metric span:last-child { + font-size: 11px; + color: var(--muted); + overflow-wrap: anywhere; +} + +.ks-boundary { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + padding: 10px 12px; + border: 1px solid color-mix(in srgb, var(--amber) 20%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--amber) 8%, transparent); + color: color-mix(in srgb, var(--amber) 88%, var(--ink)); + font-size: 12px; + line-height: 1.35; +} + +.ks-boundary strong { + flex: none; +} + +.ks-boundary span { + min-width: 0; + overflow-wrap: anywhere; +} + +.ks-workbench { + display: grid; + grid-template-columns: minmax(250px, 0.85fr) minmax(440px, 1.45fr) minmax(280px, 0.95fr); + gap: 12px; + align-items: stretch; +} + +.ks-panel { + min-width: 0; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--surface) 95%, transparent); + box-shadow: 0 14px 34px color-mix(in srgb, black 18%, transparent); +} + +.ks-pipeline-panel, +.ks-store-panel, +.ks-conversation-panel, +.ks-week-panel, +.ks-sample-panel, +.ks-prep-panel, +.ks-doc-panel, +.ks-drill-panel { + display: flex; + flex-direction: column; + gap: 13px; + padding: 14px; +} + +.ks-panel-head { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 12px; + min-width: 0; +} + +.ks-panel-head h2 { + margin: 3px 0 0; + font-family: var(--font-accent-title); + font-size: 16px; + font-weight: 620; + line-height: 1.2; + letter-spacing: 0; +} + +.ks-panel-head svg { + flex: none; + color: var(--muted); +} + +.ks-pipeline { + list-style: none; + display: flex; + flex-direction: column; + gap: 8px; + padding: 0; + margin: 0; +} + +.ks-stage { + position: relative; + min-width: 0; + display: grid; + grid-template-columns: 22px 28px minmax(0, 1fr); + gap: 8px; + align-items: start; + padding: 10px 0; + border-top: 1px solid color-mix(in srgb, var(--ink) 6%, transparent); +} + +.ks-stage:first-child { + border-top: 0; + padding-top: 0; +} + +.ks-stage-index { + display: flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + border-radius: 999px; + background: color-mix(in srgb, var(--accent) 10%, transparent); + color: var(--accent); + font-family: var(--font-mono); + font-size: 10px; + font-weight: 700; +} + +.ks-stage-icon { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + border-radius: 7px; + background: color-mix(in srgb, var(--ink) 4%, transparent); + color: var(--muted); +} + +.ks-stage-body { + min-width: 0; + display: flex; + flex-direction: column; + gap: 4px; + color: var(--muted); + font-size: 12px; + line-height: 1.35; +} + +.ks-stage-main { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + min-width: 0; +} + +.ks-stage-main strong { + min-width: 0; + color: var(--ink); + font-size: 12.5px; + font-weight: 650; + overflow-wrap: anywhere; +} + +.ks-stage-main em { + flex: none; + padding: 3px 6px; + border-radius: 999px; + background: color-mix(in srgb, var(--green) 10%, transparent); + color: var(--green); + font-family: var(--font-mono); + font-size: 9.5px; + font-style: normal; +} + +.ks-query-tabs { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 7px; +} + +.ks-query-tab { + appearance: none; + min-width: 0; + min-height: 44px; + display: flex; + align-items: flex-start; + gap: 7px; + padding: 9px 10px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 7px; + background: color-mix(in srgb, var(--bg) 30%, transparent); + color: var(--muted); + font-size: 11.5px; + line-height: 1.25; + text-align: left; + cursor: pointer; +} + +.ks-query-tab svg { + flex: none; + margin-top: 1px; +} + +.ks-query-tab span { + min-width: 0; + overflow-wrap: anywhere; +} + +.ks-query-tab:hover, +.ks-query-tab--active { + color: var(--ink); + border-color: color-mix(in srgb, var(--accent) 42%, var(--border)); + background: color-mix(in srgb, var(--accent) 8%, var(--surface)); +} + +.ks-chat-surface { + display: flex; + flex-direction: column; + gap: 8px; +} + +.ks-chat-row { + display: grid; + grid-template-columns: 64px minmax(0, 1fr); + gap: 10px; + align-items: start; + padding: 10px 11px; + border-radius: 8px; + border: 1px solid color-mix(in srgb, var(--ink) 6%, transparent); +} + +.ks-chat-row--user { + background: color-mix(in srgb, var(--bg) 35%, transparent); +} + +.ks-chat-row--assistant { + border-color: color-mix(in srgb, var(--green) 18%, transparent); + background: color-mix(in srgb, var(--green) 7%, transparent); +} + +.ks-chat-label { + font-family: var(--font-mono); + font-size: 10px; + color: var(--dim); +} + +.ks-chat-row p { + min-width: 0; + margin: 0; + color: var(--ink); + font-size: 12.5px; + line-height: 1.42; + overflow-wrap: anywhere; +} + +.ks-hit-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.ks-hit-list-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + color: var(--muted); + font-size: 11.5px; +} + +.ks-hit-list-head span { + min-width: 0; + overflow-wrap: anywhere; +} + +.ks-hit-list-head strong { + flex: none; + color: var(--accent); + font-family: var(--font-mono); + font-size: 10.5px; +} + +.ks-hit { + min-width: 0; + display: flex; + flex-direction: column; + gap: 6px; + padding: 10px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--bg) 28%, transparent); +} + +.ks-hit-title { + min-width: 0; + display: flex; + align-items: center; + gap: 7px; + flex-wrap: wrap; +} + +.ks-hit-title svg { + flex: none; + color: var(--accent); +} + +.ks-hit-title strong { + color: var(--ink); + font-size: 12px; +} + +.ks-hit-title span, +.ks-hit-title em { + color: var(--dim); + font-family: var(--font-mono); + font-size: 10px; + font-style: normal; +} + +.ks-hit p { + margin: 0; + color: var(--muted); + font-size: 12px; + line-height: 1.35; +} + +.ks-hit code, +.ks-drill code { + min-width: 0; + color: var(--green); + font-family: var(--font-mono); + font-size: 10.5px; + line-height: 1.35; + overflow-wrap: anywhere; +} + +.ks-week-grid { + display: grid; + grid-template-columns: repeat(5, minmax(120px, 1fr)); + gap: 8px; +} + +.ks-week-card { + min-width: 0; + display: flex; + flex-direction: column; + gap: 4px; + padding: 11px 12px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--bg) 30%, transparent); +} + +.ks-week-card span { + color: var(--dim); + font-family: var(--font-mono); + font-size: 10px; +} + +.ks-week-card strong { + color: var(--ink); + font-size: 20px; + line-height: 1.1; + font-weight: 680; +} + +.ks-week-card em { + color: var(--muted); + font-size: 11px; + font-style: normal; + overflow-wrap: anywhere; +} + +.ks-sample-table { + min-width: 0; + display: flex; + flex-direction: column; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; +} + +.ks-sample-row { + display: grid; + grid-template-columns: + minmax(62px, 0.55fr) + minmax(58px, 0.5fr) + minmax(70px, 0.55fr) + minmax(70px, 0.55fr) + minmax(116px, 0.9fr) + minmax(118px, 0.9fr) + minmax(220px, 1.75fr); + gap: 9px; + min-width: 0; + padding: 9px 10px; + border-top: 1px solid color-mix(in srgb, var(--ink) 6%, transparent); + color: var(--muted); + font-size: 11.5px; + align-items: center; +} + +.ks-sample-row:first-child { + border-top: 0; +} + +.ks-sample-row span, +.ks-sample-row code { + min-width: 0; + overflow-wrap: anywhere; +} + +.ks-sample-row--head { + color: var(--dim); + background: color-mix(in srgb, var(--bg) 36%, transparent); + font-family: var(--font-mono); + font-size: 10px; +} + +.ks-sample-row:not(.ks-sample-row--head) span:first-child { + color: var(--ink); + font-weight: 650; +} + +.ks-sample-row code { + color: var(--green); + font-family: var(--font-mono); + font-size: 10.5px; + line-height: 1.35; +} + +.ks-tier { + display: inline-flex; + align-items: center; + width: fit-content; + padding: 3px 7px; + border-radius: 999px; + font-family: var(--font-mono); + font-size: 10px; +} + +.ks-tier--large { + color: var(--red); + background: color-mix(in srgb, var(--red) 10%, transparent); +} + +.ks-tier--normal { + color: var(--accent); + background: color-mix(in srgb, var(--accent) 10%, transparent); +} + +.ks-tier--small { + color: var(--green); + background: color-mix(in srgb, var(--green) 10%, transparent); +} + +.ks-prep-grid { + display: grid; + grid-template-columns: repeat(5, minmax(150px, 1fr)); + gap: 8px; +} + +.ks-prep-card { + min-width: 0; + display: flex; + flex-direction: column; + gap: 10px; + padding: 12px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--bg) 28%, transparent); +} + +.ks-prep-card strong { + color: var(--ink); + font-size: 12.5px; + line-height: 1.2; +} + +.ks-prep-card dl { + display: flex; + flex-direction: column; + gap: 7px; + margin: 0; +} + +.ks-prep-card div { + min-width: 0; + display: grid; + grid-template-columns: 46px minmax(0, 1fr); + gap: 7px; +} + +.ks-prep-card dt { + color: var(--dim); + font-family: var(--font-mono); + font-size: 9.5px; +} + +.ks-prep-card dd { + min-width: 0; + margin: 0; + color: var(--muted); + font-size: 11.5px; + line-height: 1.32; + overflow-wrap: anywhere; +} + +.ks-mode-switch { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 4px; + padding: 4px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--bg) 35%, transparent); +} + +.ks-mode-button { + appearance: none; + min-width: 0; + height: 29px; + padding: 0 6px; + border: 0; + border-radius: 6px; + background: transparent; + color: var(--muted); + font-size: 11.5px; + font-weight: 650; + cursor: pointer; +} + +.ks-mode-button:hover, +.ks-mode-button--active { + color: var(--ink); + background: color-mix(in srgb, var(--accent) 12%, transparent); +} + +.ks-mode-card { + display: flex; + flex-direction: column; + gap: 12px; + min-width: 0; + padding: 12px; + border: 1px solid color-mix(in srgb, var(--accent) 20%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--accent) 7%, transparent); +} + +.ks-mode-card strong { + color: var(--ink); + font-size: 12.5px; + line-height: 1.4; + font-weight: 620; +} + +.ks-mode-card dl { + display: flex; + flex-direction: column; + gap: 8px; + margin: 0; +} + +.ks-mode-card div { + display: grid; + grid-template-columns: 76px minmax(0, 1fr); + gap: 8px; +} + +.ks-mode-card dt { + color: var(--dim); + font-family: var(--font-mono); + font-size: 10px; +} + +.ks-mode-card dd { + min-width: 0; + margin: 0; + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; +} + +.ks-manifest { + min-width: 0; + display: flex; + flex-direction: column; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; +} + +.ks-manifest-head { + display: flex; + align-items: center; + gap: 8px; + padding: 9px 10px; + border-bottom: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + color: var(--muted); + font-size: 11.5px; +} + +.ks-manifest pre { + margin: 0; + padding: 11px 12px 12px; + overflow: auto; + color: var(--ink); + background: color-mix(in srgb, var(--bg) 40%, transparent); + font-family: var(--font-mono); + font-size: 11px; + line-height: 1.45; +} + +.ks-bottom-grid { + display: grid; + grid-template-columns: minmax(440px, 1.15fr) minmax(320px, 0.85fr); + gap: 12px; +} + +.ks-doc-table { + display: flex; + flex-direction: column; + min-width: 0; + overflow: hidden; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; +} + +.ks-doc-row { + display: grid; + grid-template-columns: minmax(160px, 1.4fr) minmax(70px, 0.6fr) minmax(70px, 0.55fr) minmax(120px, 1fr); + gap: 10px; + min-width: 0; + padding: 9px 10px; + border-top: 1px solid color-mix(in srgb, var(--ink) 6%, transparent); + color: var(--muted); + font-size: 12px; +} + +.ks-doc-row:first-child { + border-top: 0; +} + +.ks-doc-row span { + min-width: 0; + overflow-wrap: anywhere; +} + +.ks-doc-row--head { + color: var(--dim); + background: color-mix(in srgb, var(--bg) 36%, transparent); + font-family: var(--font-mono); + font-size: 10px; +} + +.ks-doc-row:not(.ks-doc-row--head) span:first-child { + color: var(--ink); + font-family: var(--font-mono); + font-size: 11px; +} + +.ks-drill { + min-width: 0; + display: flex; + flex-direction: column; + gap: 10px; +} + +.ks-drill > div { + min-width: 0; + display: grid; + grid-template-columns: 118px minmax(0, 1fr); + gap: 10px; + align-items: start; + padding: 10px; + border: 1px solid color-mix(in srgb, var(--ink) 7%, transparent); + border-radius: 8px; + background: color-mix(in srgb, var(--bg) 28%, transparent); +} + +.ks-drill span { + color: var(--dim); + font-family: var(--font-mono); + font-size: 10px; +} + +@media (max-width: 1240px) { + .ks-workbench, + .ks-bottom-grid { + grid-template-columns: 1fr; + } + + .ks-query-tabs, + .ks-metrics, + .ks-week-grid, + .ks-prep-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .ks-sample-table { + overflow-x: auto; + } + + .ks-sample-row { + min-width: 920px; + } +} + +@container knowledge-search (max-width: 980px) { + .ks-workbench, + .ks-bottom-grid { + grid-template-columns: 1fr; + } + + .ks-query-tabs, + .ks-metrics, + .ks-week-grid, + .ks-prep-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .ks-sample-table { + overflow-x: auto; + } + + .ks-sample-row { + min-width: 920px; + } +} + +@media (max-width: 760px) { + .ks-page { + padding: 16px 14px 28px 28px; + } + + .ks-toolbar { + flex-direction: column; + } + + .ks-toolbar-actions { + width: 100%; + justify-content: flex-start; + } + + .ks-title-block h1 { + font-size: 22px; + } + + .ks-metrics, + .ks-week-grid, + .ks-prep-grid, + .ks-query-tabs, + .ks-mode-switch { + grid-template-columns: 1fr; + } + + .ks-boundary, + .ks-chat-row, + .ks-drill > div, + .ks-prep-card div, + .ks-mode-card div { + grid-template-columns: 1fr; + } + + .ks-boundary { + align-items: flex-start; + } + + .ks-doc-row { + grid-template-columns: 1fr; + min-width: 0; + gap: 4px; + } + + .ks-doc-row--head { + display: none; + } + + .ks-sample-table { + overflow: visible; + } + + .ks-sample-row { + grid-template-columns: 1fr; + min-width: 0; + gap: 5px; + align-items: start; + } + + .ks-sample-row--head { + display: none; + } +} From 06131994df976db46188777e23667adc4dfe1deb Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 15:04:28 -0400 Subject: [PATCH 02/14] =?UTF-8?q?=F0=9F=97=91=EF=B8=8F=20Strip=20illustrat?= =?UTF-8?q?ive=20stages=20from=20session-search=20workbench?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove Extract/Index/Query/Drilldown panels and the WeekBudgetFooter: their contents were synthesized hand-coded outputs, not real runs of the described work. Per direction to "not do anything fake", the page now only shows stages with real implementations behind them: - Discover: real filesystem scan of ~/.codex/sessions and ~/.claude/projects - Normalize: real JSONL parse of the selected session, with raw → normalized inspect view Pipeline strip narrows to 2 chips. ~700 lines of dead illustration code removed (StudySelection narrows to {sessionId, stageId}, helpers gone). Also unwraps Codex response_item / event_msg / turn_context wrappers in the normalizer so the stream shows real user_turn / assistant_turn / command_or_tool / observation kinds instead of opaque system_record. --- .../app/studies/session-search/page.tsx | 957 +++--------------- .../lib/studio/commands/parse-session.ts | 133 ++- 2 files changed, 239 insertions(+), 851 deletions(-) diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index 9c6cfd1b..66f7be13 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -11,13 +11,7 @@ import { CommandSurface } from "@/components/studio/CommandSurface"; type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; -type StageId = - | "discover" - | "normalize" - | "extract" - | "index" - | "query" - | "drilldown"; +type StageId = "discover" | "normalize"; interface SessionSample { id: string; @@ -41,32 +35,14 @@ interface Stage { summary: string; } -interface ProducedFile { - name: string; - kind: string; - bytes: number; - refs: string; -} - -interface SearchHit { - title: string; - source: string; - score: string; - snippet: string; -} - interface PageProps { searchParams: Promise<{ - artifact?: string; - q?: string; session?: string; step?: string; }>; } interface StudySelection { - artifactName: string; - query: string; sessionId: string; stageId: StageId; } @@ -171,49 +147,6 @@ const STAGES: Stage[] = [ summary: "Parse JSONL into a uniform event model so downstream code sees the same record shape regardless of harness.", }, - { - id: "extract", - label: "Extract", - verb: "derive", - input: "normalized records", - output: "QMD-style markdown files", - summary: - "Write a small sidecar corpus per session: overview, decisions, files, tool calls, and event windows with source refs.", - }, - { - id: "index", - label: "Index", - verb: "compile", - input: "derived markdown", - output: "fuzzy and FTS rows", - summary: - "Compile the derived corpus into local SQLite tables. The index is rebuildable and smaller than a transcript copy.", - }, - { - id: "query", - label: "Query", - verb: "rank", - input: "operator question", - output: "answerable hits", - summary: - "Ask the derived corpus first. The LLM sees compact retrieved evidence instead of the full raw session.", - }, - { - id: "drilldown", - label: "Drilldown", - verb: "anchor", - input: "source refs", - output: "raw log coordinates", - summary: - "Follow a qmd source ref back to the raw JSONL path and event window when the answer needs ground-truth evidence.", - }, -]; - -const SAMPLE_QUERIES = [ - "which session touched OpenScout search?", - "what were the unresolved decisions?", - "raw log drilldown for this area", - "what codebase area was I working in?", ]; const DOC_HREF = "/eng/sco-059-session-knowledge-search-exploration"; @@ -223,25 +156,15 @@ export default async function SessionSearchStudyPage({ }: PageProps) { const params = await searchParams; const requestedSession = params.session ?? SESSIONS[0]!.id; - const requestedStage = params.step ?? "extract"; + const requestedStage = params.step ?? "discover"; const selectedSession = SESSIONS.find((session) => session.id === requestedSession) ?? SESSIONS[0]!; - const stageId = isStageId(requestedStage) ? requestedStage : "extract"; + const stageId = isStageId(requestedStage) ? requestedStage : "discover"; const stageIndex = STAGES.findIndex((s) => s.id === stageId); const selectedStage = STAGES[stageIndex]!; const prevStage = stageIndex > 0 ? STAGES[stageIndex - 1] : undefined; const nextStage = stageIndex < STAGES.length - 1 ? STAGES[stageIndex + 1] : undefined; - const producedFiles = buildProducedFiles(selectedSession); - - const selectedArtifact = - producedFiles.find((file) => file.name === params.artifact) ?? - producedFiles[0]!; - - const query = params.q ?? SAMPLE_QUERIES[0]!; - const queryResult = buildQueryResult(selectedSession, query); const selection: StudySelection = { - artifactName: selectedArtifact.name, - query, sessionId: selectedSession.id, stageId, }; @@ -315,17 +238,10 @@ export default async function SessionSearchStudyPage({ - {/* ── Week budget footer (compact) ─────────────────────────── */} - ); } @@ -352,10 +268,7 @@ function SessionPickerRow({ return ( {stage.label}     - - {stageTiming(session, stage.id)} - {active ? (
    ) : null} @@ -547,20 +457,10 @@ function StageHeader({ function StagePanel({ stage, session, - selection, - producedFiles, - selectedArtifact, - query, - queryResult, inventoryRun, }: { stage: Stage; session: SessionSample; - selection: StudySelection; - producedFiles: ProducedFile[]; - selectedArtifact: ProducedFile; - query: string; - queryResult: ReturnType; inventoryRun: CommandRun; }) { switch (stage.id) { @@ -568,33 +468,6 @@ function StagePanel({ return ; case "normalize": return ; - case "extract": - return ( - - ); - case "index": - return ( - - ); - case "query": - return ( - - ); - case "drilldown": - return ; } } @@ -627,7 +500,7 @@ function DiscoverPanel({ on the {inventory.rows.length} most-recent files.{" "} Selected for walkthrough:{" "} - {qmdSlug(session)} + {session.harness.toLowerCase()}/{session.tier} . @@ -723,33 +596,135 @@ async function NormalizePanel({ session }: { session: SessionSample }) { path: session.fullPath, limit, }); - const more = Math.max(0, session.events - (run.output?.records.length ?? 0)); + const records = run.output?.records ?? []; + const more = Math.max(0, session.events - records.length); + const inspectIndex = pickInspectIndex(records); return ( -
    +
    - } + body={} footnote={ run.output && !run.output.error ? ( <> - Read the first {run.output.scannedLines} lines from the real{" "} + Parsed the first {run.output.scannedLines} JSONL lines ( + {formatBytes(run.output.bytesRead)}) from{" "} - {session.harness}/{session.tier} - {" "} - JSONL ({formatBytes(run.output.bytesRead)}). Every downstream - stage reads this uniform record shape regardless of source - harness. + {run.output.harness} + + . Normalization remaps the source schema to a uniform record + shape:{" "} + kind,{" "} + text,{" "} + tool,{" "} + result,{" "} + refs,{" "} + sourceOffset. ) : null } /> + {inspectIndex != null && run.output ? ( + + ) : null}
    ); } +function pickInspectIndex(records: NormalizedRecord[]): number | undefined { + const meaningful = records.findIndex( + (r) => r.kind === "user_turn" || r.kind === "command_or_tool", + ); + if (meaningful >= 0) return meaningful; + return records.length > 0 ? 0 : undefined; +} + +function NormalizeInspect({ + raw, + record, +}: { + raw: string; + record: NormalizedRecord; +}) { + return ( +
    +
    + + inspect record [{String(record.i).padStart(3, "0")}] · {record.kind} + + + source offset {record.sourceOffset} + +
    +
    +
    +
    + raw · {record.sourceType} +
    + 
    +            {formatRaw(raw)}
+
    +
    +
    +
    + normalized record +
    + 
    +            {JSON.stringify(stripUndefined(record), null, 2)}
+
    +
    +
    +
    + ); +} + +function formatRaw(raw: string): string { + if (!raw) return ""; + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch { + return raw; + } +} + +function stripUndefined>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +function summarizeRecord(record: NormalizedRecord): string { + if (record.text) return record.text; + if (record.tool) { + const input = trimDisplay(JSON.stringify(record.tool.input ?? {}), 60); + return `name=${record.tool.name} input=${input}`; + } + if (record.result) { + const out = typeof record.result.output === "string" + ? record.result.output + : JSON.stringify(record.result.output ?? ""); + return trimDisplay(out, 100); + } + if (record.meta) { + const model = record.meta.model ?? record.meta.model_provider; + const cwd = record.meta.cwd; + if (model || cwd) return `model=${model ?? "?"} cwd=${trimDisplay(String(cwd ?? "?"), 50)}`; + return trimDisplay(JSON.stringify(record.meta), 80); + } + return ""; +} + +function trimDisplay(s: string, n: number): string { + const oneLine = s.replace(/\s+/g, " ").trim(); + return oneLine.length > n ? oneLine.slice(0, n - 1) + "…" : oneLine; +} + const NORMALIZED_KIND_TONE: Record = { session_meta: "text-studio-ink-faint", user_turn: "text-status-info-fg", @@ -800,6 +775,7 @@ function NormalizedStreamBody({ } function NormalizedRow({ record }: { record: NormalizedRecord }) { + const detail = summarizeRecord(record); return (
  • @@ -809,695 +785,13 @@ function NormalizedRow({ record }: { record: NormalizedRecord }) { {record.tag ?? record.sourceType} - - {record.detail} + + {trimDisplay(detail, 200)}
    • ); } -function ExtractPanel({ - session, - selection, - producedFiles, - selectedArtifact, -}: { - session: SessionSample; - selection: StudySelection; - producedFiles: ProducedFile[]; - selectedArtifact: ProducedFile; -}) { - return ( - - {producedFiles.map((file) => { - const active = file.name === selectedArtifact.name; - return ( -
  • - - - - {file.name} - - - {file.kind} · {file.refs} - - - - {formatBytes(file.bytes)} - - -
    • - ); - })} - - } - rightLabel={`preview · ${selectedArtifact.name}`} - right={} - /> - ); -} - -function IndexPanel({ - session, - selectedArtifact, -}: { - session: SessionSample; - selectedArtifact: ProducedFile; -}) { - const indexBytes = estimateIndexBytes(session); - const rows = databaseRows(session, selectedArtifact, indexBytes); - return ( - - {rows.map((row) => ( -
  • -
    - - {row.table} - - - {row.ref} - -
    -
    {row.key}
    -
    - {row.value} -
    -
    • - ))} - - } - rightLabel="cli + budget" - right={ -
    - -

    - The index is rebuildable. Scout owns the rows; raw harness logs - stay observed source material. -

    -
    - } - /> - ); -} - -function QueryPanel({ - session, - selection, - query, - result, -}: { - session: SessionSample; - selection: StudySelection; - query: string; - result: ReturnType; -}) { - return ( -
    -
    - - - - -
    - -
    - {SAMPLE_QUERIES.map((sample) => { - const active = sample === query; - return ( - - {sample} - - ); - })} -
    - -
    -
    -
    - assistant synthesis -
    -

    - {result.answer} -

    -
    -
    -
    - ranked hits -
    -
    - {result.hits.map((hit) => ( -
    -
    - - {hit.title} - - - {hit.score} - -
    -

    - {hit.snippet} -

    - - {hit.source} - -
    - ))} -
    -
    -
    -
    - ); -} - -function DrilldownPanel({ - session, - selection, -}: { - session: SessionSample; - selection: StudySelection; -}) { - const slug = qmdSlug(session); - const ref = `qmd://session-search/${slug}/events-001.md:44`; - const excerpt = [ - "events-001.md (excerpt)", - "", - "## window 001", - "043 assistant_turn source_event=143", - "044 tool_call(name=apply_patch) source_event=144", - "045 tool_result(ok=true) source_event=145", - "", - "raw://" + session.fullPath, - ].join("\n"); - return ( - -
    -
    - ref -
    -
    - {ref} -
    -
    -
    -
    - resolves to -
    -
    - {session.fullPath} -
    -
    -
    -
    - purpose -
    -

    - Evidence, not bulk import. The raw transcript stays observed - source material; only the qmd ref + event window come into - Scout's view. -

    -
    - - ← back to query - -
    - } - rightLabel="excerpt" - right={} - /> - ); -} - -// ── Panel atoms ────────────────────────────────────────────────── - -function PanelGrid({ - leftLabel, - left, - rightLabel, - right, -}: { - leftLabel: string; - left: React.ReactNode; - rightLabel: string; - right: React.ReactNode; -}) { - return ( -
    -
    - {leftLabel} -
    {left}
    -
    -
    - {rightLabel} -
    {right}
    -
    -
    - ); -} - -function PanelLabel({ children }: { children: React.ReactNode }) { - return ( -
    - {children} -
    - ); -} - -function CliBlock({ lines }: { lines: string[] }) { - return ( - 
    -      {lines.join("\n")}
-
    - ); -} - -function CommandRecipe({ - cmd, - output, - run = false, -}: { - cmd: string; - output: string[]; - run?: boolean; -}) { - return ( -
    -
    - - command - - - {run ? "● ran" : "○ recipe"} - -
    - 
    -        $ {cmd}
-
    -
    - output -
    - 
    -        {output.join("\n")}
-
    -
    - ); -} - -function CodeBlock({ content, maxHeight }: { content: string; maxHeight?: number }) { - return ( - 
    -      {content}
-
    - ); -} - -// ── Week budget footer ─────────────────────────────────────────── - -function WeekBudgetFooter() { - return ( -
    -
    -
    - heavy week · expected mechanical pass -
    -
    - search starts before LLM enrichment completes -
    -
    -
    - - - - -
    -
    - ); -} - -function BudgetCell({ - label, - value, - detail, -}: { - label: string; - value: string; - detail: string; -}) { - return ( -
    -
    - {label} -
    -
    - {value} -
    -
    - {detail} -
    -
    - ); -} - -// ── Data helpers (unchanged) ────────────────────────────────────── - -function buildProducedFiles(session: SessionSample): ProducedFile[] { - const derived = estimateDerivedBytes(session); - const windows = estimateEventWindows(session); - return [ - { - name: "manifest.json", - kind: "metadata", - bytes: Math.max(1_600, Math.round(derived * 0.03)), - refs: "collection config", - }, - { - name: "overview.md", - kind: "summary", - bytes: Math.max(2_400, Math.round(derived * 0.18)), - refs: "session-level", - }, - { - name: "decisions.md", - kind: "summary", - bytes: Math.max(1_800, Math.round(derived * 0.11)), - refs: "decision refs", - }, - { - name: "files.md", - kind: "catalog", - bytes: Math.max(1_800, Math.round(derived * 0.13)), - refs: "path refs", - }, - { - name: "tool-calls.md", - kind: "catalog", - bytes: Math.max(2_000, Math.round(derived * 0.15)), - refs: "command refs", - }, - { - name: - windows === 1 - ? "events-001.md" - : `events-001..${String(windows).padStart(3, "0")}.md`, - kind: "event windows", - bytes: Math.max(3_200, Math.round(derived * 0.4)), - refs: `${windows} windows`, - }, - ]; -} - -function databaseRows( - session: SessionSample, - artifact: ProducedFile, - indexBytes: number, -) { - const slug = qmdSlug(session); - const chunks = estimateChunkCount(session); - return [ - { - table: "collections", - key: "session_search_samples", - value: "Local derived knowledge collection for selected external transcripts.", - ref: "root", - }, - { - table: "sessions", - key: slug, - value: `${session.harness} ${session.tier}, ${formatBytes(session.sizeBytes)}, ${formatCount(session.events)} source events.`, - ref: "observed", - }, - { - table: "documents", - key: artifact.name, - value: `${artifact.kind}, ${formatBytes(artifact.bytes)}, derived from ${session.displayPath}.`, - ref: "qmd doc", - }, - { - table: "chunks", - key: `${chunks} searchable chunks`, - value: "Chunk rows keep markdown offsets plus source JSONL coordinates for raw drilldown.", - ref: "fts", - }, - { - table: "terms", - key: formatBytes(indexBytes), - value: "Rebuildable lexical/fuzzy index, not a second copy of the transcript.", - ref: "local", - }, - ]; -} - -function buildQueryResult(session: SessionSample, query: string) { - const normalized = query.toLowerCase(); - const slug = qmdSlug(session); - const wantsDecision = normalized.includes("decision") || normalized.includes("unresolved"); - const wantsRaw = normalized.includes("raw") || normalized.includes("drill"); - const wantsCode = normalized.includes("code") || normalized.includes("area") || normalized.includes("search"); - const answer = wantsDecision - ? `${session.harness} ${session.tier} has a compact decision trail in decisions.md, with unresolved items linked back to event windows instead of summarized away.` - : wantsRaw - ? `The derived hit is enough to choose the session; the next hop is a source coordinate into ${session.displayPath}.` - : wantsCode - ? `This sample is indexed as work around ${session.codeArea}; files.md and overview.md are the cheap first-pass surfaces.` - : `The fuzzy layer searches the derived ${session.harness} ${session.tier} corpus first, then keeps raw transcript access one click away.`; - - const hits: SearchHit[] = [ - { - title: wantsDecision ? "decisions.md" : "overview.md", - source: `qmd://session-search/${slug}/${wantsDecision ? "decisions.md" : "overview.md"}:12`, - score: "BM25 0.91", - snippet: wantsDecision - ? "Open decisions, follow-ups, and ownerless questions extracted from the session." - : `${session.focus}; indexed as ${formatTokenEstimate(session.sizeBytes)} of raw-token equivalent source.`, - }, - { - title: wantsCode ? "files.md" : "events-001.md", - source: `qmd://session-search/${slug}/${wantsCode ? "files.md" : "events-001.md"}:44`, - score: "BM25 0.78", - snippet: wantsCode - ? `Code area signal: ${session.codeArea}.` - : "Event window preserves the original ordering and source refs for verification.", - }, - { - title: "manifest.json", - source: `qmd://session-search/${slug}/manifest.json:1`, - score: "BM25 0.62", - snippet: "Collection metadata records the harness, path, event count, byte count, and extraction recipe.", - }, - ]; - - return { answer, hits }; -} - -function artifactPreview(session: SessionSample, file: ProducedFile): string { - const slug = qmdSlug(session); - if (file.name === "manifest.json") { - return JSON.stringify( - { - id: slug, - harness: session.harness, - tier: session.tier, - source: session.fullPath, - bytes: session.sizeBytes, - events: session.events, - modified: session.modified, - extraction: { - recipe: "qmd-lite", - eventWindowSize: 350, - derivedBytes: estimateDerivedBytes(session), - indexBytes: estimateIndexBytes(session), - }, - }, - null, - 2, - ); - } - - if (file.name === "overview.md") { - return [ - `# ${session.harness} ${session.tier} session`, - "", - `source: ${session.displayPath}`, - `events: ${formatCount(session.events)}`, - `raw-token-eq: ${formatTokenEstimate(session.sizeBytes)}`, - "", - "## Working summary", - `- ${session.focus}.`, - `- Primary code area signal: ${session.codeArea}.`, - "- Store policy: derived knowledge is Scout-owned; raw harness logs stay observed source material.", - "", - "## Source refs", - `- qmd://session-search/${slug}/events-001.md:1`, - `- raw://${session.fullPath}`, - ].join("\n"); - } - - if (file.name === "decisions.md") { - return [ - "# Decisions", - "", - "- Use a two-step memory path: QMD-style extraction first, fuzzy index second.", - "- Keep the engineering doc static; put moving exploration in Studio studies.", - "- Do not import external harness transcripts as Scout-owned messages.", - "", - "## Follow-ups", - "- Decide which extraction recipe becomes the first real implementation target.", - "- Measure LLM enrichment only after the mechanical pass is useful.", - ].join("\n"); - } - - if (file.name === "files.md") { - return [ - "# File and code area signals", - "", - `primary_area: ${session.codeArea}`, - "", - "| path | signal | source |", - "| --- | --- | --- |", - "| packages/web/client | product search surface | events-001.md:44 |", - "| design/studio/app/studies | interactive study surface | events-002.md:18 |", - "| docs/eng | durable engineering record | overview.md:21 |", - ].join("\n"); - } - - if (file.name === "tool-calls.md") { - return [ - "# Tool calls", - "", - "| command | reason | source |", - "| --- | --- | --- |", - "| rg | locate routes, docs, and prior study patterns | events-001.md:9 |", - "| bun --cwd design/studio dev | preview the Studio study | events-002.md:4 |", - "| browser verification | check layout and interactions | events-002.md:36 |", - "", - "Large sessions would have many more rows; this preview keeps the shape compact.", - ].join("\n"); - } - - return [ - "# Event windows", - "", - `window_size: 350 events`, - `windows: ${estimateEventWindows(session)}`, - "", - "## events-001.md", - "001 user_request source_event=0", - "002 assistant_update source_event=1", - "003 tool_call source_event=2", - "", - "Each line keeps enough source coordinate data to reopen the raw JSONL excerpt.", - ].join("\n"); -} - -function estimateDerivedBytes(session: SessionSample): number { - return Math.max(8_192, Math.round(session.sizeBytes * 0.12)); -} - -function estimateIndexBytes(session: SessionSample): number { - return Math.round(estimateDerivedBytes(session) * 3.2); -} - -function estimateEventWindows(session: SessionSample): number { - return Math.max(1, Math.ceil(session.events / 350)); -} - -function estimateChunkCount(session: SessionSample): number { - return Math.max(3, Math.ceil(estimateDerivedBytes(session) / 2_400)); -} - -function stageTiming(session: SessionSample, stage: StageId): string { - const sizeMiB = session.sizeBytes / 1024 / 1024; - const sizeFactor = Math.max(0.2, sizeMiB); - - switch (stage) { - case "discover": - return sizeMiB > 20 ? "400–900 ms" : "80–350 ms"; - case "normalize": - return secondsRange(sizeFactor * 0.8, sizeFactor * 1.7); - case "extract": - return secondsRange(sizeFactor * 1.2, sizeFactor * 2.6); - case "index": - return secondsRange(sizeFactor * 0.35, sizeFactor * 0.9); - case "query": - return "20–100 ms"; - case "drilldown": - return "10–80 ms"; - } -} - -function secondsRange(low: number, high: number): string { - const lo = Math.max(0.1, low); - const hi = Math.max(lo + 0.1, high); - if (hi < 1) return "<1 s"; - if (hi < 10) return `${Math.round(lo)}–${Math.round(hi)} s`; - return `${Math.round(lo)}–${Math.round(hi)} s`; -} function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); @@ -1510,8 +804,6 @@ function studyHref( const params = new URLSearchParams(); params.set("session", next.sessionId ?? current.sessionId); params.set("step", next.stageId ?? current.stageId); - params.set("artifact", next.artifactName ?? current.artifactName); - params.set("q", next.query ?? current.query); return `/studies/session-search?${params.toString()}`; } @@ -1537,10 +829,3 @@ function formatBytes(bytes: number): string { function formatCount(value: number): string { return new Intl.NumberFormat("en-US").format(value); } - -function formatTokenEstimate(bytes: number): string { - const tokens = Math.round(bytes / 4); - if (tokens >= 1_000_000) return `~${(tokens / 1_000_000).toFixed(1)}M`; - if (tokens >= 1_000) return `~${Math.round(tokens / 1_000)}k`; - return `~${tokens}`; -} diff --git a/design/studio/lib/studio/commands/parse-session.ts b/design/studio/lib/studio/commands/parse-session.ts index a185a765..6fd14f9e 100644 --- a/design/studio/lib/studio/commands/parse-session.ts +++ b/design/studio/lib/studio/commands/parse-session.ts @@ -177,31 +177,134 @@ function normalizeCodex( refs: { sessionId: payload.id as string | undefined }, }; } - if (type === "message") { - const role = String(payload.role ?? ""); - const text = extractCodexText(payload.content); - if (role === "user") return { ...base, kind: "user_turn", tag: "user", text }; - if (role === "assistant") return { ...base, kind: "assistant_turn", tag: "assistant", text }; - return { ...base, kind: "system_record", tag: role, text }; + // Codex wraps most events inside response_item or event_msg. Unwrap and + // dispatch on the inner payload.type so the stream reads as real turns, + // not opaque system_records. + if (type === "response_item") { + return normalizeCodexResponseItem(payload, base); } + if (type === "event_msg") { + return normalizeCodexEventMsg(payload, base); + } + if (type === "turn_context") { + return { ...base, kind: "system_record", tag: "turn_context", meta: payload }; + } + // Legacy / un-wrapped events (older sessions). + if (type === "message") return normalizeCodexMessage(payload, base); if (type === "function_call" || type === "local_shell_call") { - const name = String(payload.name ?? payload.command ?? "tool"); - const input = payload.arguments ?? payload.args ?? payload.input ?? {}; - return { ...base, kind: "command_or_tool", tag: name, tool: { name, input } }; + return normalizeCodexFunctionCall(payload, base); } if (type === "function_call_output" || type === "local_shell_call_output") { - const output = payload.output ?? payload.content ?? ""; - return { ...base, kind: "observation", tag: "result", result: { output } }; + return normalizeCodexFunctionCallOutput(payload, base); + } + if (type === "reasoning") return normalizeCodexReasoning(payload, base); + return { ...base, kind: "system_record", text: JSON.stringify(payload) }; +} + +type CodexBase = { + i: number; + ts?: string; + sourceType: string; + sourceOffset: number; +}; + +function normalizeCodexResponseItem( + payload: Record, + base: CodexBase, +): NormalizedRecord { + const inner = String(payload.type ?? ""); + if (inner === "message") return normalizeCodexMessage(payload, base); + if (inner === "reasoning") return normalizeCodexReasoning(payload, base); + if (inner === "function_call" || inner === "local_shell_call") { + return normalizeCodexFunctionCall(payload, base); + } + if (inner === "function_call_output" || inner === "local_shell_call_output") { + return normalizeCodexFunctionCallOutput(payload, base); + } + return { + ...base, + kind: "system_record", + tag: inner || "response_item", + meta: payload, + }; +} + +function normalizeCodexEventMsg( + payload: Record, + base: CodexBase, +): NormalizedRecord { + const inner = String(payload.type ?? ""); + if (inner === "user_message") { + return { + ...base, + kind: "user_turn", + tag: "user", + text: String(payload.message ?? payload.text ?? ""), + }; } - if (type === "reasoning") { + if (inner === "agent_message") { return { ...base, kind: "assistant_turn", - tag: "reasoning", - text: String(payload.content ?? ""), + tag: "assistant", + text: String(payload.message ?? payload.text ?? ""), }; } - return { ...base, kind: "system_record", text: JSON.stringify(payload) }; + return { + ...base, + kind: "system_record", + tag: inner || "event_msg", + meta: payload, + }; +} + +function normalizeCodexMessage( + payload: Record, + base: CodexBase, +): NormalizedRecord { + const role = String(payload.role ?? ""); + const text = extractCodexText(payload.content); + if (role === "user") return { ...base, kind: "user_turn", tag: "user", text }; + if (role === "assistant") return { ...base, kind: "assistant_turn", tag: "assistant", text }; + if (role === "developer") return { ...base, kind: "system_record", tag: "developer", text }; + return { ...base, kind: "system_record", tag: role || "message", text }; +} + +function normalizeCodexFunctionCall( + payload: Record, + base: CodexBase, +): NormalizedRecord { + const name = String(payload.name ?? payload.command ?? "tool"); + const input = payload.arguments ?? payload.args ?? payload.input ?? {}; + return { ...base, kind: "command_or_tool", tag: name, tool: { name, input } }; +} + +function normalizeCodexFunctionCallOutput( + payload: Record, + base: CodexBase, +): NormalizedRecord { + const output = payload.output ?? payload.content ?? ""; + return { ...base, kind: "observation", tag: "result", result: { output } }; +} + +function normalizeCodexReasoning( + payload: Record, + base: CodexBase, +): NormalizedRecord { + // Reasoning often carries a `summary` array of {type:"summary_text", text} + // and/or an `encrypted_content`. Surface the summary text when available. + let text = ""; + if (Array.isArray(payload.summary)) { + text = payload.summary + .map((s) => { + const block = s as { text?: string }; + return block?.text ?? ""; + }) + .filter(Boolean) + .join(" "); + } + if (!text && payload.content) text = String(payload.content); + return { ...base, kind: "assistant_turn", tag: "reasoning", text }; } function normalizeClaude( From 2da82a5839cb2983a9582793abce90847ebc848f Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 15:16:51 -0400 Subject: [PATCH 03/14] =?UTF-8?q?=E2=9C=A8=20Extract=20QMD=20stage:=20real?= =?UTF-8?q?=20mechanical=20pass=20+=20real=20LLM=20summary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real Extract stage on the session-search workbench. Selecting Extract on a real session runs one extraction end-to-end and writes 6 files to $TMPDIR/scout-study/qmd//: Mechanical pass (no LLM, iterates normalized records): - manifest.json — source path, harness, recordsScanned, bytesRead - files.md — paths pulled from tool args, grouped by hits + tools touched - tool-calls.md — counts per tool name + first 30 invocations - events-NNN.md — windowed event lines with source-ordered indices LLM pass (one MiniMax-M2 call per session, cached for 1h): - overview.md — what the session was about (2 paragraphs) - decisions.md — decisions made + open follow-ups - _llm-call.json — model, usage, latency for inspection Adds the studio command primitive's first real LLM consumer. Secret access via lib/secrets.ts shells to `secret get` (keychain) so no env files touched. MiniMax client in lib/llm/minimax.ts returns content + reasoning + structured usage so the panel can surface real cost. ExtractPanel renders the file list (mech / llm tagged), live preview of the selected artifact, and a footnote with real disk path + timings + token counts. Pipeline strip now shows Discover → Normalize → Extract, all real. --- .../app/studies/session-search/page.tsx | 184 ++++++- design/studio/lib/llm/minimax.ts | 101 ++++ design/studio/lib/secrets.ts | 33 ++ .../studio/lib/studio/commands/extract-qmd.ts | 449 ++++++++++++++++++ 4 files changed, 766 insertions(+), 1 deletion(-) create mode 100644 design/studio/lib/llm/minimax.ts create mode 100644 design/studio/lib/secrets.ts create mode 100644 design/studio/lib/studio/commands/extract-qmd.ts diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index 66f7be13..b679d336 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -7,11 +7,16 @@ import { type NormalizedRecord, type ParseSessionResult, } from "@/lib/studio/commands/parse-session"; +import { + extractQmdCommand, + type ExtractQmdResult, + type ExtractedFile, +} from "@/lib/studio/commands/extract-qmd"; import { CommandSurface } from "@/components/studio/CommandSurface"; type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; -type StageId = "discover" | "normalize"; +type StageId = "discover" | "normalize" | "extract"; interface SessionSample { id: string; @@ -39,12 +44,15 @@ interface PageProps { searchParams: Promise<{ session?: string; step?: string; + /** Active artifact in the Extract panel preview. */ + artifact?: string; }>; } interface StudySelection { sessionId: string; stageId: StageId; + artifact?: string; } const SESSIONS: SessionSample[] = [ @@ -147,6 +155,15 @@ const STAGES: Stage[] = [ summary: "Parse JSONL into a uniform event model so downstream code sees the same record shape regardless of harness.", }, + { + id: "extract", + label: "Extract", + verb: "derive", + input: "normalized records", + output: "QMD sidecar files", + summary: + "Mechanical pass writes files.md / tool-calls.md / events-NNN.md. LLM pass writes overview.md / decisions.md by summarizing the condensed transcript through MiniMax. Outputs land on disk under /tmp/scout-study/qmd//.", + }, ]; const DOC_HREF = "/eng/sco-059-session-knowledge-search-exploration"; @@ -167,6 +184,7 @@ export default async function SessionSearchStudyPage({ const selection: StudySelection = { sessionId: selectedSession.id, stageId, + artifact: params.artifact, }; const inventoryRun = await runCommand(inventoryCommand, { since: "7d" }); @@ -239,6 +257,7 @@ export default async function SessionSearchStudyPage({ stage={selectedStage} session={selectedSession} inventoryRun={inventoryRun} + selection={selection} />
    @@ -458,16 +477,20 @@ function StagePanel({ stage, session, inventoryRun, + selection, }: { stage: Stage; session: SessionSample; inventoryRun: CommandRun; + selection: StudySelection; }) { switch (stage.id) { case "discover": return ; case "normalize": return ; + case "extract": + return ; } } @@ -792,6 +815,163 @@ function NormalizedRow({ record }: { record: NormalizedRecord }) { ); } +// ── Extract panel ──────────────────────────────────────────────── + +async function ExtractPanel({ + session, + selection, +}: { + session: SessionSample; + selection: StudySelection; +}) { + const run = await runCommand(extractQmdCommand, { + path: session.fullPath, + sessionId: `${session.harness.toLowerCase()}-${session.tier}`, + recordLimit: 1500, + withLlm: true, + }); + + const result = run.output; + const files = result?.files ?? []; + const selectedName = files.find((f) => f.name === selection.artifact)?.name + ?? files.find((f) => f.name === "overview.md")?.name + ?? files[0]?.name; + const selected = files.find((f) => f.name === selectedName); + const previewContent = selected ? await safeReadFile(selected.path) : ""; + + return ( +
    + + } + footnote={ + result && !result.error ? ( + <> + Wrote {files.length} files to{" "} + + {shortenTmpPath(result.outDir)} + {" "} + · mechanical {result.mechanicalMs} ms · llm {result.llmMs} ms + {result.llm ? ( + <> + {" · "}model{" "} + + {result.llm.model} + {" "} + · {result.llm.usage.promptTokens}+{result.llm.usage.completionTokens}t + ({result.llm.usage.reasoningTokens} reasoning) + + ) : null} + + ) : null + } + /> +
    + ); +} + +function shortenTmpPath(p: string): string { + // macOS tmpdir is /var/folders/.../T/...; trim to a readable suffix. + const m = p.match(/\/T\/(.+)$/); + return m ? `$TMPDIR/${m[1]}` : p; +} + +async function safeReadFile(p: string): Promise { + try { + const { promises: fs } = await import("node:fs"); + const buf = await fs.readFile(p); + const text = buf.toString("utf8"); + return text.length > 20_000 ? text.slice(0, 20_000) + "\n\n… (truncated)" : text; + } catch (err) { + return `(could not read: ${err instanceof Error ? err.message : String(err)})`; + } +} + +function ExtractBody({ + result, + selection, + files, + selected, + previewContent, +}: { + result: ExtractQmdResult | undefined; + selection: StudySelection; + files: ExtractedFile[]; + selected: ExtractedFile | undefined; + previewContent: string; +}) { + if (!result || result.error) { + return ( + 
    +        {result?.error ?? "no extract result"}
+
    + ); + } + return ( +
    + +
    +
    + preview · {selected?.name ?? "—"} +
    + 
    +          {previewContent || "(empty)"}
+
    +
    +
    + ); +} + function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); @@ -804,6 +984,8 @@ function studyHref( const params = new URLSearchParams(); params.set("session", next.sessionId ?? current.sessionId); params.set("step", next.stageId ?? current.stageId); + const artifact = next.artifact ?? current.artifact; + if (artifact) params.set("artifact", artifact); return `/studies/session-search?${params.toString()}`; } diff --git a/design/studio/lib/llm/minimax.ts b/design/studio/lib/llm/minimax.ts new file mode 100644 index 00000000..1a627519 --- /dev/null +++ b/design/studio/lib/llm/minimax.ts @@ -0,0 +1,101 @@ +import { getSecret } from "@/lib/secrets"; + +export interface MinimaxCallInput { + system: string; + user: string; + model?: string; + maxTokens?: number; + temperature?: number; +} + +export interface MinimaxUsage { + promptTokens: number; + completionTokens: number; + reasoningTokens: number; + totalTokens: number; +} + +export interface MinimaxCallResult { + model: string; + content: string; + reasoning: string; + usage: MinimaxUsage; + latencyMs: number; + finishReason: string; + raw: unknown; +} + +const ENDPOINT = "https://api.minimax.io/v1/text/chatcompletion_v2"; + +/** + * Single chat completion against MiniMax. Returns content + reasoning + usage + * so callers can show what the LLM "thought" alongside what it returned. + */ +export async function callMinimax(input: MinimaxCallInput): Promise { + const key = await getSecret("MINIMAX_API_KEY"); + const model = input.model ?? "MiniMax-M2"; + const body = { + model, + messages: [ + { role: "system", content: input.system }, + { role: "user", content: input.user }, + ], + max_tokens: input.maxTokens ?? 4000, + temperature: input.temperature ?? 0.2, + }; + const start = Date.now(); + const res = await fetch(ENDPOINT, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${key}`, + }, + body: JSON.stringify(body), + }); + const json = (await res.json()) as MinimaxRawResponse; + const latencyMs = Date.now() - start; + if (json.base_resp && json.base_resp.status_code !== 0) { + throw new Error( + `MiniMax error ${json.base_resp.status_code}: ${json.base_resp.status_msg}`, + ); + } + const choice = json.choices?.[0]; + if (!choice) throw new Error("MiniMax returned no choices"); + const usage = json.usage ?? { + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + completion_tokens_details: { reasoning_tokens: 0 }, + }; + return { + model, + content: choice.message?.content ?? "", + reasoning: choice.message?.reasoning_content ?? "", + usage: { + promptTokens: usage.prompt_tokens, + completionTokens: usage.completion_tokens, + reasoningTokens: usage.completion_tokens_details?.reasoning_tokens ?? 0, + totalTokens: usage.total_tokens, + }, + latencyMs, + finishReason: choice.finish_reason ?? "unknown", + raw: json, + }; +} + +interface MinimaxRawResponse { + choices?: Array<{ + finish_reason?: string; + message?: { + content?: string; + reasoning_content?: string; + }; + }>; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + completion_tokens_details?: { reasoning_tokens?: number }; + }; + base_resp?: { status_code: number; status_msg: string }; +} diff --git a/design/studio/lib/secrets.ts b/design/studio/lib/secrets.ts new file mode 100644 index 00000000..17c1cd3b --- /dev/null +++ b/design/studio/lib/secrets.ts @@ -0,0 +1,33 @@ +/** + * Server-side secret access. + * + * Reads secrets via the `secret` CLI (which uses the macOS keychain) so we + * never touch dotenv files. Never log or echo the resolved value. + */ + +import { execFile } from "node:child_process"; +import { homedir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const SECRET_BIN = path.join(homedir(), ".local", "bin", "secret"); + +const cache = new Map(); + +export async function getSecret(name: string): Promise { + const hit = cache.get(name); + if (hit) return hit; + // Prefer ambient env if the dev server was started with `secret run … -- bun dev`. + if (process.env[name]) { + cache.set(name, process.env[name]!); + return process.env[name]!; + } + const { stdout } = await execFileAsync(SECRET_BIN, ["get", name], { + maxBuffer: 1024 * 64, + }); + const value = stdout.trim(); + if (!value) throw new Error(`secret ${name} resolved to empty string`); + cache.set(name, value); + return value; +} diff --git a/design/studio/lib/studio/commands/extract-qmd.ts b/design/studio/lib/studio/commands/extract-qmd.ts new file mode 100644 index 00000000..0adb5ac9 --- /dev/null +++ b/design/studio/lib/studio/commands/extract-qmd.ts @@ -0,0 +1,449 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { homedir, tmpdir } from "node:os"; +import type { Command } from "@/lib/studio/command"; +import { + parseSessionCommand, + type NormalizedRecord, + type ParseSessionResult, +} from "@/lib/studio/commands/parse-session"; +import { callMinimax, type MinimaxUsage } from "@/lib/llm/minimax"; + +export interface ExtractQmdInput { + /** Absolute path to the source JSONL session. */ + path: string; + /** Stable session id used for the output directory name. */ + sessionId: string; + /** Cap on records parsed/extracted. Avoids loading huge sessions. */ + recordLimit?: number; + /** Whether to run the LLM pass for overview.md / decisions.md. */ + withLlm?: boolean; +} + +export interface ExtractedFile { + name: string; + path: string; + bytes: number; + kind: "mechanical" | "llm"; +} + +export interface ExtractQmdResult { + outDir: string; + files: ExtractedFile[]; + recordsScanned: number; + mechanicalMs: number; + llmMs: number; + llm?: { + model: string; + usage: MinimaxUsage; + finishReason: string; + reasoning: string; + }; + parseResult: ParseSessionResult; + error?: string; +} + +const ROOT = path.join(tmpdir(), "scout-study", "qmd"); +const WINDOW = 350; +const DEFAULT_LIMIT = 1500; + +export const extractQmdCommand: Command = { + id: "extract-qmd", + label: "Extract QMD", + shell: ({ path: p, sessionId, withLlm }) => + [ + `scout qmd extract`, + `--source ${shellQuote(shrinkPath(p))}`, + `--out ${shellQuote(shrinkPath(path.join(ROOT, sessionId)))}`, + withLlm === false ? "--no-llm" : "", + ] + .filter(Boolean) + .join(" "), + run: async ({ path: filePath, sessionId, recordLimit, withLlm = true }) => { + const limit = recordLimit ?? DEFAULT_LIMIT; + + // Reuse the existing parser command at a higher limit. It has its own + // 60s cache, so this is cheap on re-renders against the same input. + const parseRun = await parseSessionCommand.run({ path: filePath, limit }); + + if (parseRun.error) { + return emptyResult({ outDir: outDirFor(sessionId), parseRun, error: parseRun.error }); + } + + const outDir = outDirFor(sessionId); + await fs.mkdir(outDir, { recursive: true }); + + const mechStart = Date.now(); + const mechanical = await writeMechanical(outDir, parseRun, filePath); + const mechanicalMs = Date.now() - mechStart; + + let llmFiles: ExtractedFile[] = []; + let llmInfo: ExtractQmdResult["llm"] | undefined; + let llmMs = 0; + if (withLlm) { + const t = Date.now(); + try { + const out = await writeLlm(outDir, parseRun); + llmFiles = out.files; + llmInfo = out.info; + } catch (err) { + // Don't abort the whole extraction on LLM failure — keep mechanical + // and report the LLM error inline. + llmInfo = undefined; + const errMsg = err instanceof Error ? err.message : String(err); + await fs.writeFile(path.join(outDir, "_llm-error.txt"), errMsg); + } + llmMs = Date.now() - t; + } + + return { + outDir, + files: [...mechanical, ...llmFiles].sort((a, b) => a.name.localeCompare(b.name)), + recordsScanned: parseRun.records.length, + mechanicalMs, + llmMs, + llm: llmInfo, + parseResult: parseRun, + }; + }, + cacheKey: ({ path: p, sessionId, recordLimit, withLlm }) => + `${sessionId}::${p}::${recordLimit ?? DEFAULT_LIMIT}::${withLlm === false ? "0" : "1"}`, + // Cache for an hour. LLM calls are expensive and the inputs don't change + // between page reloads. + cacheTtlMs: 60 * 60 * 1000, +}; + +function outDirFor(sessionId: string): string { + return path.join(ROOT, sessionId); +} + +function emptyResult({ + outDir, + parseRun, + error, +}: { + outDir: string; + parseRun: ParseSessionResult; + error: string; +}): ExtractQmdResult { + return { + outDir, + files: [], + recordsScanned: 0, + mechanicalMs: 0, + llmMs: 0, + parseResult: parseRun, + error, + }; +} + +// ── Mechanical pass ─────────────────────────────────────────────────────── + +async function writeMechanical( + outDir: string, + parseRun: ParseSessionResult, + sourcePath: string, +): Promise { + const files: ExtractedFile[] = []; + files.push(await writeFile(outDir, "manifest.json", buildManifest(parseRun, sourcePath), "mechanical")); + files.push(await writeFile(outDir, "files.md", buildFilesMd(parseRun), "mechanical")); + files.push(await writeFile(outDir, "tool-calls.md", buildToolCallsMd(parseRun), "mechanical")); + const windows = buildEventWindows(parseRun.records); + for (let i = 0; i < windows.length; i++) { + const idx = String(i + 1).padStart(3, "0"); + files.push(await writeFile(outDir, `events-${idx}.md`, windows[i]!, "mechanical")); + } + return files; +} + +function buildManifest(parseRun: ParseSessionResult, sourcePath: string): string { + return JSON.stringify( + { + source: sourcePath, + harness: parseRun.harness, + recordsScanned: parseRun.records.length, + bytesRead: parseRun.bytesRead, + window: WINDOW, + generatedAt: new Date().toISOString(), + }, + null, + 2, + ); +} + +function buildFilesMd(parseRun: ParseSessionResult): string { + // Pull paths out of tool call arguments. Cheap heuristics over the union of + // tool input shapes (Codex function_call.arguments is a JSON string of + // {path, file_path, cwd, command}; Claude tool_use.input has {path, + // file_path, command, ...}). + const counts = new Map(); + const tools = new Map>(); + for (const r of parseRun.records) { + if (r.kind !== "command_or_tool" || !r.tool) continue; + const paths = extractPaths(r.tool.input); + for (const p of paths) { + counts.set(p, (counts.get(p) ?? 0) + 1); + const set = tools.get(p) ?? new Set(); + set.add(r.tool.name); + tools.set(p, set); + } + } + const rows = [...counts.entries()].sort((a, b) => b[1] - a[1]); + const lines: string[] = [ + "# Files touched", + "", + `Source records scanned: ${parseRun.records.length}.`, + `Distinct file paths: ${rows.length}.`, + "", + "| path | hits | tools |", + "| --- | ---: | --- |", + ]; + for (const [p, hits] of rows) { + const toolList = [...(tools.get(p) ?? [])].sort().join(", "); + lines.push(`| \`${p}\` | ${hits} | ${toolList} |`); + } + if (rows.length === 0) lines.push("| _no paths detected_ | 0 | — |"); + return lines.join("\n") + "\n"; +} + +function buildToolCallsMd(parseRun: ParseSessionResult): string { + const calls = parseRun.records.filter((r) => r.kind === "command_or_tool" && r.tool); + const byName = new Map(); + for (const c of calls) { + const name = c.tool!.name; + byName.set(name, (byName.get(name) ?? 0) + 1); + } + const lines: string[] = [ + "# Tool calls", + "", + `Total calls: ${calls.length}.`, + "", + "## By tool", + "", + "| tool | calls |", + "| --- | ---: |", + ]; + for (const [name, n] of [...byName.entries()].sort((a, b) => b[1] - a[1])) { + lines.push(`| \`${name}\` | ${n} |`); + } + lines.push("", "## Sample (first 30)", ""); + for (const c of calls.slice(0, 30)) { + const input = oneLineInput(c.tool!.input); + lines.push(`- [\`${String(c.i).padStart(4, "0")}\`] \`${c.tool!.name}\` ${input}`); + } + return lines.join("\n") + "\n"; +} + +function buildEventWindows(records: NormalizedRecord[]): string[] { + const windows: string[] = []; + for (let start = 0; start < records.length; start += WINDOW) { + const slice = records.slice(start, start + WINDOW); + const idx = Math.floor(start / WINDOW) + 1; + const lines: string[] = [ + `# Events window ${idx}`, + "", + `Records [${slice[0]?.i ?? "?"}..${slice[slice.length - 1]?.i ?? "?"}], source-ordered.`, + "", + ]; + for (const r of slice) { + const idxStr = String(r.i).padStart(4, "0"); + const detail = summarize(r); + lines.push(`- [${idxStr}] \`${r.kind}\` (${r.tag ?? r.sourceType}) — ${detail}`); + } + windows.push(lines.join("\n") + "\n"); + } + return windows; +} + +function extractPaths(input: unknown): string[] { + if (input == null) return []; + // Codex serializes arguments as a JSON string sometimes. + if (typeof input === "string") { + try { + return extractPaths(JSON.parse(input)); + } catch { + return []; + } + } + if (typeof input !== "object") return []; + const obj = input as Record; + const paths = new Set(); + for (const k of ["path", "file_path", "filePath", "filename", "filenames"]) { + const v = obj[k]; + if (typeof v === "string") paths.add(v); + if (Array.isArray(v)) for (const x of v) if (typeof x === "string") paths.add(x); + } + // Bash-ish: try to pluck likely-path tokens out of `command` / `cmd`. + const cmd = obj.command ?? obj.cmd; + if (typeof cmd === "string") { + const matches = cmd.match(/(?:\.\/|\.\.\/|~\/|\/)[\w./~_\-+]+\.[\w]+/g); + if (matches) for (const m of matches) paths.add(m); + } + return [...paths]; +} + +function oneLineInput(input: unknown): string { + const text = typeof input === "string" ? input : JSON.stringify(input ?? {}); + const trimmed = text.replace(/\s+/g, " ").trim(); + return trimmed.length > 100 ? trimmed.slice(0, 97) + "…" : trimmed; +} + +function summarize(r: NormalizedRecord): string { + if (r.text) return trim(r.text, 120); + if (r.tool) return `name=${r.tool.name} input=${oneLineInput(r.tool.input)}`; + if (r.result) { + const out = + typeof r.result.output === "string" ? r.result.output : JSON.stringify(r.result.output ?? ""); + return trim(out, 120); + } + if (r.meta) return trim(JSON.stringify(r.meta), 120); + return ""; +} + +function trim(s: string, n: number): string { + const oneLine = s.replace(/\s+/g, " ").trim(); + return oneLine.length > n ? oneLine.slice(0, n - 1) + "…" : oneLine; +} + +// ── LLM pass ────────────────────────────────────────────────────────────── + +async function writeLlm( + outDir: string, + parseRun: ParseSessionResult, +): Promise<{ files: ExtractedFile[]; info: NonNullable }> { + const condensed = condenseForLlm(parseRun.records); + const system = [ + "You are a careful technical analyst reading a transcript of a coding-agent session.", + "Your job is to produce two short markdown documents about the session.", + "Be specific and concrete; cite paths, commands, or file types when relevant.", + "Never invent details that aren't in the transcript.", + ].join(" "); + const user = [ + "Below is a condensed transcript of a coding-agent session. Each line is one", + "normalized event, prefixed with [NNNN] kind (tag) — detail.", + "", + "Produce exactly two top-level markdown sections, in this order:", + "", + "# Overview", + "", + "Two short paragraphs describing what this session was about: the user's goal,", + "the area of the codebase touched, and what the agent ended up doing.", + "", + "# Decisions", + "", + "A bulleted list of concrete decisions made or unresolved questions. Each bullet", + "should reference a specific event or file when possible. Include a final", + '"## Follow-ups" subsection if there are loose threads.', + "", + "Transcript:", + "```", + condensed, + "```", + ].join("\n"); + + const result = await callMinimax({ system, user, maxTokens: 4000, temperature: 0.2 }); + const content = result.content || ""; + const { overview, decisions } = splitOverviewDecisions(content); + + const files: ExtractedFile[] = []; + files.push(await writeFile(outDir, "overview.md", overview, "llm")); + files.push(await writeFile(outDir, "decisions.md", decisions, "llm")); + files.push( + await writeFile( + outDir, + "_llm-call.json", + JSON.stringify( + { + model: result.model, + usage: result.usage, + finishReason: result.finishReason, + latencyMs: result.latencyMs, + promptChars: user.length, + }, + null, + 2, + ), + "llm", + ), + ); + + return { + files, + info: { + model: result.model, + usage: result.usage, + finishReason: result.finishReason, + reasoning: result.reasoning, + }, + }; +} + +/** + * Pick a smaller representative slice of the normalized stream to feed the LLM. + * Strategy: keep all user turns, the first assistant turn after each user turn, + * tool names + short args, and short observations. Aggressive trim per line. + */ +function condenseForLlm(records: NormalizedRecord[]): string { + const lines: string[] = []; + for (const r of records) { + const idx = String(r.i).padStart(4, "0"); + if (r.kind === "user_turn") { + lines.push(`[${idx}] user — ${trim(r.text ?? "", 240)}`); + } else if (r.kind === "assistant_turn") { + lines.push(`[${idx}] assistant (${r.tag ?? ""}) — ${trim(r.text ?? "", 200)}`); + } else if (r.kind === "command_or_tool" && r.tool) { + lines.push(`[${idx}] tool ${r.tool.name} ${oneLineInput(r.tool.input)}`); + } else if (r.kind === "observation" && r.result) { + const out = + typeof r.result.output === "string" ? r.result.output : JSON.stringify(r.result.output ?? ""); + lines.push(`[${idx}] result — ${trim(out, 140)}`); + } else if (r.kind === "session_meta") { + const model = r.meta?.model_provider ?? r.meta?.model ?? "?"; + const cwd = r.meta?.cwd ?? "?"; + lines.push(`[${idx}] meta model=${model} cwd=${trim(String(cwd), 80)}`); + } + // Skip system_record / unknown — usually noise. + } + // Cap total length to keep token budget reasonable. + const joined = lines.join("\n"); + const cap = 24_000; // ~6k tokens at 4 chars/token + return joined.length > cap ? joined.slice(0, cap) + "\n… (transcript truncated)" : joined; +} + +function splitOverviewDecisions(content: string): { + overview: string; + decisions: string; +} { + const decisionsIdx = content.indexOf("# Decisions"); + if (decisionsIdx === -1) { + return { overview: content.trim(), decisions: "# Decisions\n\n_not produced_\n" }; + } + const overview = content.slice(0, decisionsIdx).trim(); + const decisions = content.slice(decisionsIdx).trim(); + return { overview, decisions }; +} + +// ── File writer ─────────────────────────────────────────────────────────── + +async function writeFile( + outDir: string, + name: string, + content: string, + kind: ExtractedFile["kind"], +): Promise { + const fullPath = path.join(outDir, name); + await fs.writeFile(fullPath, content); + const stat = await fs.stat(fullPath); + return { name, path: fullPath, bytes: stat.size, kind }; +} + +// ── Display helpers ─────────────────────────────────────────────────────── + +function shrinkPath(file: string): string { + const home = homedir(); + return file.startsWith(home) ? "~" + file.slice(home.length) : file; +} + +function shellQuote(s: string): string { + return /[^A-Za-z0-9_./~-]/.test(s) ? `'${s.replace(/'/g, `'\\''`)}'` : s; +} From efacdfebc50e99157bc2f8a8ff3991d9cfabd1de Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 15:41:05 -0400 Subject: [PATCH 04/14] =?UTF-8?q?=E2=9C=A8=20Run=20trace=20footer:=20per-c?= =?UTF-8?q?ommand=20timings=20+=20LLM=20budget=20per=20request?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every page render now ends with a run trace showing what just happened underneath: one row per command (inventory, parse-session, extract-qmd), with wall time, cached status (with "saved ~Nms" callout), and the LLM model + token breakdown for the entries that ran a model. Header summarizes the request as a whole — wall ms (cached entries contribute 0), number cached and ms saved, cumulative LLM tokens, rough USD cost using a per-model rate table (MiniMax-M2 in for now). Page handler now orchestrates all commands explicitly so the run log can be built before rendering. NormalizePanel and ExtractPanel become presentational (receive their CommandRun as a prop). This is the "what's happening underneath" the workbench needed — when a fresh session triggers a real LLM call you see Extract QMD · ran · 7385 ms · MiniMax-M2 · 1002+681t, and on re-visit you see cached (saved ~Nms) with the original cost visible so you understand what the cache bought. --- .../app/studies/session-search/page.tsx | 170 ++++++++++++++++-- design/studio/lib/studio/run-log.ts | 104 +++++++++++ 2 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 design/studio/lib/studio/run-log.ts diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index b679d336..fe784327 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -13,6 +13,11 @@ import { type ExtractedFile, } from "@/lib/studio/commands/extract-qmd"; import { CommandSurface } from "@/components/studio/CommandSurface"; +import { + makeRunLogEntry, + summarizeRunLog, + type RunLogEntry, +} from "@/lib/studio/run-log"; type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; @@ -193,6 +198,43 @@ export default async function SessionSearchStudyPage({ ? "scan failed" : `${formatCount(inventory.totalFiles)} files · ${formatBytes(inventory.totalBytes)} · ${inventory.windowDays} days`; + // ── Orchestrate per-stage runs so the run summary footer can show what + // ── happened underneath the current page render. + const runLog: RunLogEntry[] = [ + makeRunLogEntry(inventoryCommand, inventoryRun), + ]; + + let normalizeRun: CommandRun | undefined; + if (stageId === "normalize") { + normalizeRun = await runCommand(parseSessionCommand, { + path: selectedSession.fullPath, + limit: 14, + }); + runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); + } + + let extractRun: CommandRun | undefined; + if (stageId === "extract") { + extractRun = await runCommand(extractQmdCommand, { + path: selectedSession.fullPath, + sessionId: `${selectedSession.harness.toLowerCase()}-${selectedSession.tier}`, + recordLimit: 1500, + withLlm: true, + }); + runLog.push( + makeRunLogEntry(extractQmdCommand, extractRun, (out) => + out?.llm + ? { + model: out.llm.model, + promptTokens: out.llm.usage.promptTokens, + completionTokens: out.llm.usage.completionTokens, + reasoningTokens: out.llm.usage.reasoningTokens, + } + : undefined, + ), + ); + } + return (
    {/* ── Header ─────────────────────────────────────────────────── */} @@ -257,10 +299,14 @@ export default async function SessionSearchStudyPage({ stage={selectedStage} session={selectedSession} inventoryRun={inventoryRun} + normalizeRun={normalizeRun} + extractRun={extractRun} selection={selection} /> + +
    ); } @@ -477,20 +523,30 @@ function StagePanel({ stage, session, inventoryRun, + normalizeRun, + extractRun, selection, }: { stage: Stage; session: SessionSample; inventoryRun: CommandRun; + normalizeRun?: CommandRun; + extractRun?: CommandRun; selection: StudySelection; }) { switch (stage.id) { case "discover": return ; case "normalize": - return ; + return ; case "extract": - return ; + return ( + + ); } } @@ -613,12 +669,14 @@ function InventoryRowsBody({ ); } -async function NormalizePanel({ session }: { session: SessionSample }) { +function NormalizePanel({ + session, + run, +}: { + session: SessionSample; + run: CommandRun; +}) { const limit = 14; - const run = await runCommand(parseSessionCommand, { - path: session.fullPath, - limit, - }); const records = run.output?.records ?? []; const more = Math.max(0, session.events - records.length); const inspectIndex = pickInspectIndex(records); @@ -820,17 +878,12 @@ function NormalizedRow({ record }: { record: NormalizedRecord }) { async function ExtractPanel({ session, selection, + run, }: { session: SessionSample; selection: StudySelection; + run: CommandRun; }) { - const run = await runCommand(extractQmdCommand, { - path: session.fullPath, - sessionId: `${session.harness.toLowerCase()}-${session.tier}`, - recordLimit: 1500, - withLlm: true, - }); - const result = run.output; const files = result?.files ?? []; const selectedName = files.find((f) => f.name === selection.artifact)?.name @@ -977,6 +1030,95 @@ function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); } +// ── Run summary footer ─────────────────────────────────────────── + +function RunSummary({ entries }: { entries: RunLogEntry[] }) { + const sum = summarizeRunLog(entries); + return ( +
    +
    +
    + run trace · what just happened +
    +
    + {entries.length} command{entries.length === 1 ? "" : "s"} ·{" "} + {sum.wallMs} ms this request + {sum.cached > 0 ? ( + <> + {" · "} + + {sum.cached} cached (saved ~{sum.uncachedMs - sum.wallMs} ms) + + + ) : null} + {sum.llm.total > 0 ? ( + <> + {" · "} + + {sum.llm.total.toLocaleString()} llm tokens + + {" · "} + + ~${sum.llm.estCostUsd.toFixed(4)} + + + ) : null} +
    +
    +
      + {entries.map((e, i) => ( +
    • + + {String(i + 1).padStart(2, "0")} + + + + {e.label} + + + {e.id} + + + + {e.error ? ( + error · {e.error} + ) : e.cached ? ( + + ● cached{" "} + + (saved ~{e.durationMs} ms) + + + ) : ( + ● ran · {e.durationMs} ms + )} + + + {e.llm ? ( + <> + {e.llm.model} ·{" "} + {e.llm.promptTokens}+{e.llm.completionTokens}t + {e.llm.reasoningTokens > 0 ? ( + + {" "} + ({e.llm.reasoningTokens} reasoning) + + ) : null} + + ) : ( + — + )} + +
      • + ))} +
    +
    + ); +} + function studyHref( current: StudySelection, next: Partial, diff --git a/design/studio/lib/studio/run-log.ts b/design/studio/lib/studio/run-log.ts new file mode 100644 index 00000000..fbface60 --- /dev/null +++ b/design/studio/lib/studio/run-log.ts @@ -0,0 +1,104 @@ +import type { Command, CommandRun } from "@/lib/studio/command"; + +/** + * Request-scoped trace of every command that ran for the current page render. + * + * The page handler builds entries explicitly (rather than via AsyncLocalStorage) + * so the run summary can be rendered AFTER all commands have resolved, with + * the full log in scope. + */ + +export interface RunLlmCost { + model: string; + promptTokens: number; + completionTokens: number; + reasoningTokens: number; +} + +export interface RunLogEntry { + id: string; + label: string; + durationMs: number; + cached: boolean; + error?: string; + llm?: RunLlmCost; +} + +export interface RunLogSummary { + /** Wall time this request actually spent in commands. Cached entries count as 0. */ + wallMs: number; + /** Sum of durations as if nothing had been cached. Useful for "what this would have cost". */ + uncachedMs: number; + ran: number; + cached: number; + errors: number; + llm: { + prompt: number; + completion: number; + reasoning: number; + total: number; + /** Rough $ estimate using a per-model rate table. May be 0 if model unknown. */ + estCostUsd: number; + }; +} + +/** + * Per-model token rates. Numbers are best-effort approximations and only used + * for a rough budget display. + */ +const RATES: Record = { + // MiniMax-M2: ~$0.30 / 1M input, $1.20 / 1M output (approximate). + "MiniMax-M2": { input: 0.30 / 1_000_000, output: 1.20 / 1_000_000 }, +}; + +export function makeRunLogEntry( + cmd: Command, + run: CommandRun, + extractLlm?: (output: O) => RunLlmCost | undefined, +): RunLogEntry { + return { + id: cmd.id, + label: cmd.label, + durationMs: run.durationMs, + cached: run.cached, + error: run.error, + llm: !run.error && extractLlm ? extractLlm(run.output) : undefined, + }; +} + +export function summarizeRunLog(entries: RunLogEntry[]): RunLogSummary { + const uncachedMs = entries.reduce((a, e) => a + e.durationMs, 0); + const wallMs = entries.reduce((a, e) => a + (e.cached ? 0 : e.durationMs), 0); + const ran = entries.filter((e) => !e.cached && !e.error).length; + const cached = entries.filter((e) => e.cached).length; + const errors = entries.filter((e) => e.error).length; + let prompt = 0; + let completion = 0; + let reasoning = 0; + let estCostUsd = 0; + for (const e of entries) { + if (!e.llm) continue; + prompt += e.llm.promptTokens; + completion += e.llm.completionTokens; + reasoning += e.llm.reasoningTokens; + const rate = RATES[e.llm.model]; + if (rate) { + estCostUsd += e.llm.promptTokens * rate.input; + estCostUsd += (e.llm.completionTokens + e.llm.reasoningTokens) * rate.output; + } + } + return { + wallMs, + uncachedMs, + ran, + cached, + errors, + llm: { + prompt, + completion, + reasoning, + total: prompt + completion + reasoning, + estCostUsd, + }, + }; +} From 47133736c586fd568235199756e408db640ad7e7 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 16:06:52 -0400 Subject: [PATCH 05/14] =?UTF-8?q?=E2=9C=A8=20Add=20Studio=20session=20sear?= =?UTF-8?q?ch=20enrichment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/studies/session-search/page.tsx | 213 ++++-- .../lib/studio/commands/enrich-session.ts | 233 +++++++ .../studio/lib/studio/commands/extract-qmd.ts | 186 +---- packages/agent-sessions/src/history.test.ts | 203 +++++- packages/agent-sessions/src/history.ts | 649 +++++++++++++++++- packages/web/client/lib/router.test.ts | 7 + packages/web/client/lib/router.ts | 3 + packages/web/client/lib/scoutbot.ts | 2 +- packages/web/client/lib/types.ts | 1 + packages/web/client/scout/Provider.tsx | 3 +- packages/web/client/scout/hooks.ts | 7 + packages/web/client/scout/index.ts | 8 + .../scout/inspector/AgentsInspector.tsx | 65 +- packages/web/client/scout/slots/Content.tsx | 3 + .../web/client/scout/slots/GlobalJumpDock.tsx | 3 +- .../web/server/create-openscout-web-server.ts | 259 +------ packages/web/server/terminal-relay-session.ts | 116 +--- 17 files changed, 1363 insertions(+), 598 deletions(-) create mode 100644 design/studio/lib/studio/commands/enrich-session.ts diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index fe784327..dcea6ffc 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -12,6 +12,11 @@ import { type ExtractQmdResult, type ExtractedFile, } from "@/lib/studio/commands/extract-qmd"; +import { + enrichSessionCommand, + type EnrichSessionResult, + type EnrichedFile, +} from "@/lib/studio/commands/enrich-session"; import { CommandSurface } from "@/components/studio/CommandSurface"; import { makeRunLogEntry, @@ -21,7 +26,7 @@ import { type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; -type StageId = "discover" | "normalize" | "extract"; +type StageId = "discover" | "normalize" | "extract" | "enrich"; interface SessionSample { id: string; @@ -165,9 +170,18 @@ const STAGES: Stage[] = [ label: "Extract", verb: "derive", input: "normalized records", - output: "QMD sidecar files", + output: "mechanical QMD files", summary: - "Mechanical pass writes files.md / tool-calls.md / events-NNN.md. LLM pass writes overview.md / decisions.md by summarizing the condensed transcript through MiniMax. Outputs land on disk under /tmp/scout-study/qmd//.", + "Mechanical pass: emit files.md / tool-calls.md / events-NNN.md / manifest.json from the normalized records. No LLM, always fast. Outputs land at $TMPDIR/scout-study/qmd//.", + }, + { + id: "enrich", + label: "Enrich", + verb: "summarize", + input: "condensed transcript", + output: "overview.md + decisions.md (LLM)", + summary: + "One MiniMax-M2 call per session, cached for an hour. Reads the parsed records, condenses to a token-bounded transcript, asks the model for an overview + decisions doc, writes them next to the mechanical files.", }, ]; @@ -204,31 +218,45 @@ export default async function SessionSearchStudyPage({ makeRunLogEntry(inventoryCommand, inventoryRun), ]; + const sessionSlug = `${selectedSession.harness.toLowerCase()}-${selectedSession.tier}`; + const needsParse = stageId === "normalize" || stageId === "extract" || stageId === "enrich"; + const needsExtract = stageId === "extract" || stageId === "enrich"; + let normalizeRun: CommandRun | undefined; - if (stageId === "normalize") { + if (needsParse) { + const limit = stageId === "normalize" ? 14 : 1500; normalizeRun = await runCommand(parseSessionCommand, { path: selectedSession.fullPath, - limit: 14, + limit, }); runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); } let extractRun: CommandRun | undefined; - if (stageId === "extract") { + if (needsExtract) { extractRun = await runCommand(extractQmdCommand, { path: selectedSession.fullPath, - sessionId: `${selectedSession.harness.toLowerCase()}-${selectedSession.tier}`, + sessionId: sessionSlug, + recordLimit: 1500, + }); + runLog.push(makeRunLogEntry(extractQmdCommand, extractRun)); + } + + let enrichRun: CommandRun | undefined; + if (stageId === "enrich") { + enrichRun = await runCommand(enrichSessionCommand, { + path: selectedSession.fullPath, + sessionId: sessionSlug, recordLimit: 1500, - withLlm: true, }); runLog.push( - makeRunLogEntry(extractQmdCommand, extractRun, (out) => - out?.llm + makeRunLogEntry(enrichSessionCommand, enrichRun, (out) => + out?.model ? { - model: out.llm.model, - promptTokens: out.llm.usage.promptTokens, - completionTokens: out.llm.usage.completionTokens, - reasoningTokens: out.llm.usage.reasoningTokens, + model: out.model, + promptTokens: out.usage.promptTokens, + completionTokens: out.usage.completionTokens, + reasoningTokens: out.usage.reasoningTokens, } : undefined, ), @@ -301,6 +329,7 @@ export default async function SessionSearchStudyPage({ inventoryRun={inventoryRun} normalizeRun={normalizeRun} extractRun={extractRun} + enrichRun={enrichRun} selection={selection} /> @@ -525,6 +554,7 @@ function StagePanel({ inventoryRun, normalizeRun, extractRun, + enrichRun, selection, }: { stage: Stage; @@ -532,6 +562,7 @@ function StagePanel({ inventoryRun: CommandRun; normalizeRun?: CommandRun; extractRun?: CommandRun; + enrichRun?: CommandRun; selection: StudySelection; }) { switch (stage.id) { @@ -547,6 +578,14 @@ function StagePanel({ run={extractRun!} /> ); + case "enrich": + return ( + + ); } } @@ -886,9 +925,11 @@ async function ExtractPanel({ }) { const result = run.output; const files = result?.files ?? []; - const selectedName = files.find((f) => f.name === selection.artifact)?.name - ?? files.find((f) => f.name === "overview.md")?.name - ?? files[0]?.name; + const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; + const selectedName = + files.find((f) => f.name === selection.artifact)?.name ?? + files.find((f) => f.name === "files.md")?.name ?? + files[0]?.name; const selected = files.find((f) => f.name === selectedName); const previewContent = selected ? await safeReadFile(selected.path) : ""; @@ -897,37 +938,26 @@ async function ExtractPanel({ } footnote={ result && !result.error ? ( <> - Wrote {files.length} files to{" "} + Wrote {files.length} mechanical files to{" "} {shortenTmpPath(result.outDir)} {" "} - · mechanical {result.mechanicalMs} ms · llm {result.llmMs} ms - {result.llm ? ( - <> - {" · "}model{" "} - - {result.llm.model} - {" "} - · {result.llm.usage.promptTokens}+{result.llm.usage.completionTokens}t - ({result.llm.usage.reasoningTokens} reasoning) - - ) : null} + · {result.mechanicalMs} ms · {result.recordsScanned} records scanned ) : null } @@ -936,6 +966,69 @@ async function ExtractPanel({ ); } +async function EnrichPanel({ + session, + selection, + run, +}: { + session: SessionSample; + selection: StudySelection; + run: CommandRun; +}) { + const result = run.output; + const files = result?.files ?? []; + const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; + const selectedName = + files.find((f) => f.name === selection.artifact)?.name ?? + files.find((f) => f.name === "overview.md")?.name ?? + files[0]?.name; + const selected = files.find((f) => f.name === selectedName); + const previewContent = selected ? await safeReadFile(selected.path) : ""; + + return ( +
    + + } + footnote={ + result && !result.error && result.model ? ( + <> + {result.model}{" "} + · prompt {result.promptChars.toLocaleString()} chars ( + {result.usage.promptTokens}t) · completion{" "} + {result.usage.completionTokens}t ({result.usage.reasoningTokens}{" "} + reasoning) · llm latency {result.llmLatencyMs} ms · finish{" "} + {result.finishReason} + + ) : null + } + /> + {result?.reasoning ? ( +
    + + model reasoning · {result.reasoning.length.toLocaleString()} chars + + 
    +            {result.reasoning}
+
    +
    + ) : null} +
    + ); +} + function shortenTmpPath(p: string): string { // macOS tmpdir is /var/folders/.../T/...; trim to a readable suffix. const m = p.match(/\/T\/(.+)$/); @@ -953,31 +1046,32 @@ async function safeReadFile(p: string): Promise { } } -function ExtractBody({ - result, - selection, +type ArtifactFile = ExtractedFile | EnrichedFile; + +function ArtifactFilesBody({ files, + selection, selected, previewContent, + emptyMessage, }: { - result: ExtractQmdResult | undefined; + files: ArtifactFile[]; selection: StudySelection; - files: ExtractedFile[]; - selected: ExtractedFile | undefined; + selected: ArtifactFile | undefined; previewContent: string; + emptyMessage: string; }) { - if (!result || result.error) { + if (files.length === 0) { return ( 
    -        {result?.error ?? "no extract result"}
+        {emptyMessage}
    ); } return (
      -
    • - kind +
    • file bytes
      • @@ -989,21 +1083,12 @@ function ExtractBody({ href={studyHref(selection, { artifact: f.name })} aria-current={active ? "true" : undefined} className={[ - "grid grid-cols-[40px_minmax(0,1fr)_64px] items-baseline gap-2 px-3 py-1.5 transition-colors", + "grid grid-cols-[minmax(0,1fr)_64px] items-baseline gap-2 px-3 py-1.5 transition-colors", active ? "bg-scout-accent-soft shadow-[inset_2px_0_0_var(--scout-accent)]" : "hover:bg-studio-canvas-alt", ].join(" ")} > - - {f.kind === "llm" ? "llm" : "mech"} - {f.name} {formatBytes(f.bytes)} @@ -1025,6 +1110,26 @@ function ExtractBody({ ); } +function ExtractedFilesBody(props: { + files: ExtractedFile[]; + selection: StudySelection; + selected: ExtractedFile | undefined; + previewContent: string; + emptyMessage: string; +}) { + return ; +} + +function EnrichedFilesBody(props: { + files: EnrichedFile[]; + selection: StudySelection; + selected: EnrichedFile | undefined; + previewContent: string; + emptyMessage: string; +}) { + return ; +} + function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); diff --git a/design/studio/lib/studio/commands/enrich-session.ts b/design/studio/lib/studio/commands/enrich-session.ts new file mode 100644 index 00000000..62dfae51 --- /dev/null +++ b/design/studio/lib/studio/commands/enrich-session.ts @@ -0,0 +1,233 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import type { Command } from "@/lib/studio/command"; +import { + parseSessionCommand, + type NormalizedRecord, + type ParseSessionResult, +} from "@/lib/studio/commands/parse-session"; +import { callMinimax, type MinimaxUsage } from "@/lib/llm/minimax"; + +export interface EnrichSessionInput { + /** Absolute path to source JSONL. */ + path: string; + /** Stable session id used for output directory. */ + sessionId: string; + /** Cap on records considered. Matches the extract step's cap. */ + recordLimit?: number; +} + +export interface EnrichedFile { + name: string; + path: string; + bytes: number; +} + +export interface EnrichSessionResult { + outDir: string; + files: EnrichedFile[]; + model: string; + usage: MinimaxUsage; + finishReason: string; + promptChars: number; + /** The reasoning content the model surfaced, for inspection. */ + reasoning: string; + /** Latency of the LLM call itself, isolated from file write. */ + llmLatencyMs: number; + error?: string; +} + +const ROOT = path.join(tmpdir(), "scout-study", "qmd"); +const DEFAULT_LIMIT = 1500; +const PROMPT_CHAR_CAP = 24_000; + +export const enrichSessionCommand: Command = { + id: "enrich-session", + label: "Enrich (LLM)", + shell: ({ path: p, sessionId }) => + `scout enrich --source ${shellQuote(shrinkPath(p))} --out ${shellQuote(shrinkPath(path.join(ROOT, sessionId)))} --model MiniMax-M2`, + run: async ({ path: filePath, sessionId, recordLimit }) => { + const limit = recordLimit ?? DEFAULT_LIMIT; + const parseRun = await parseSessionCommand.run({ path: filePath, limit }); + + if (parseRun.error) { + return emptyResult({ sessionId, parseRun, error: parseRun.error }); + } + + const outDir = path.join(ROOT, sessionId); + await fs.mkdir(outDir, { recursive: true }); + + try { + const condensed = condenseForLlm(parseRun.records); + const system = [ + "You are a careful technical analyst reading a transcript of a coding-agent session.", + "Your job is to produce two short markdown documents about the session.", + "Be specific and concrete; cite paths, commands, or file types when relevant.", + "Never invent details that aren't in the transcript.", + ].join(" "); + const user = [ + "Below is a condensed transcript of a coding-agent session. Each line is one", + "normalized event, prefixed with [NNNN] kind (tag) — detail.", + "", + "Produce exactly two top-level markdown sections, in this order:", + "", + "# Overview", + "", + "Two short paragraphs describing what this session was about: the user's goal,", + "the area of the codebase touched, and what the agent ended up doing.", + "", + "# Decisions", + "", + "A bulleted list of concrete decisions made or unresolved questions. Each bullet", + "should reference a specific event or file when possible. Include a final", + '"## Follow-ups" subsection if there are loose threads.', + "", + "Transcript:", + "```", + condensed, + "```", + ].join("\n"); + + const llm = await callMinimax({ + system, + user, + maxTokens: 4000, + temperature: 0.2, + }); + + const content = llm.content || ""; + const { overview, decisions } = splitOverviewDecisions(content); + + const files: EnrichedFile[] = []; + files.push(await writeFile(outDir, "overview.md", overview)); + files.push(await writeFile(outDir, "decisions.md", decisions)); + files.push( + await writeFile( + outDir, + "_llm-call.json", + JSON.stringify( + { + model: llm.model, + usage: llm.usage, + finishReason: llm.finishReason, + latencyMs: llm.latencyMs, + promptChars: user.length, + }, + null, + 2, + ), + ), + ); + + return { + outDir, + files, + model: llm.model, + usage: llm.usage, + finishReason: llm.finishReason, + promptChars: user.length, + reasoning: llm.reasoning, + llmLatencyMs: llm.latencyMs, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + await fs.writeFile(path.join(outDir, "_llm-error.txt"), msg); + return emptyResult({ sessionId, parseRun, error: msg }); + } + }, + cacheKey: ({ path: p, sessionId, recordLimit }) => + `${sessionId}::${p}::${recordLimit ?? DEFAULT_LIMIT}`, + cacheTtlMs: 60 * 60 * 1000, +}; + +function emptyResult({ + sessionId, + parseRun: _parseRun, + error, +}: { + sessionId: string; + parseRun: ParseSessionResult; + error: string; +}): EnrichSessionResult { + return { + outDir: path.join(ROOT, sessionId), + files: [], + model: "", + usage: { promptTokens: 0, completionTokens: 0, reasoningTokens: 0, totalTokens: 0 }, + finishReason: "skipped", + promptChars: 0, + reasoning: "", + llmLatencyMs: 0, + error, + }; +} + +function condenseForLlm(records: NormalizedRecord[]): string { + const lines: string[] = []; + for (const r of records) { + const idx = String(r.i).padStart(4, "0"); + if (r.kind === "user_turn") { + lines.push(`[${idx}] user — ${trim(r.text ?? "", 240)}`); + } else if (r.kind === "assistant_turn") { + lines.push(`[${idx}] assistant (${r.tag ?? ""}) — ${trim(r.text ?? "", 200)}`); + } else if (r.kind === "command_or_tool" && r.tool) { + lines.push(`[${idx}] tool ${r.tool.name} ${oneLineInput(r.tool.input)}`); + } else if (r.kind === "observation" && r.result) { + const out = + typeof r.result.output === "string" + ? r.result.output + : JSON.stringify(r.result.output ?? ""); + lines.push(`[${idx}] result — ${trim(out, 140)}`); + } else if (r.kind === "session_meta") { + const model = r.meta?.model_provider ?? r.meta?.model ?? "?"; + const cwd = r.meta?.cwd ?? "?"; + lines.push(`[${idx}] meta model=${model} cwd=${trim(String(cwd), 80)}`); + } + } + const joined = lines.join("\n"); + return joined.length > PROMPT_CHAR_CAP + ? joined.slice(0, PROMPT_CHAR_CAP) + "\n… (transcript truncated)" + : joined; +} + +function splitOverviewDecisions(content: string): { + overview: string; + decisions: string; +} { + const decisionsIdx = content.indexOf("# Decisions"); + if (decisionsIdx === -1) { + return { overview: content.trim(), decisions: "# Decisions\n\n_not produced_\n" }; + } + return { + overview: content.slice(0, decisionsIdx).trim(), + decisions: content.slice(decisionsIdx).trim(), + }; +} + +async function writeFile(outDir: string, name: string, content: string): Promise { + const fullPath = path.join(outDir, name); + await fs.writeFile(fullPath, content); + const stat = await fs.stat(fullPath); + return { name, path: fullPath, bytes: stat.size }; +} + +function oneLineInput(input: unknown): string { + const text = typeof input === "string" ? input : JSON.stringify(input ?? {}); + const trimmed = text.replace(/\s+/g, " ").trim(); + return trimmed.length > 100 ? trimmed.slice(0, 97) + "…" : trimmed; +} + +function trim(s: string, n: number): string { + const oneLine = s.replace(/\s+/g, " ").trim(); + return oneLine.length > n ? oneLine.slice(0, n - 1) + "…" : oneLine; +} + +function shrinkPath(file: string): string { + const home = process.env.HOME ?? ""; + return home && file.startsWith(home) ? "~" + file.slice(home.length) : file; +} + +function shellQuote(s: string): string { + return /[^A-Za-z0-9_./~-]/.test(s) ? `'${s.replace(/'/g, `'\\''`)}'` : s; +} diff --git a/design/studio/lib/studio/commands/extract-qmd.ts b/design/studio/lib/studio/commands/extract-qmd.ts index 0adb5ac9..d8b0208a 100644 --- a/design/studio/lib/studio/commands/extract-qmd.ts +++ b/design/studio/lib/studio/commands/extract-qmd.ts @@ -7,7 +7,6 @@ import { type NormalizedRecord, type ParseSessionResult, } from "@/lib/studio/commands/parse-session"; -import { callMinimax, type MinimaxUsage } from "@/lib/llm/minimax"; export interface ExtractQmdInput { /** Absolute path to the source JSONL session. */ @@ -16,15 +15,12 @@ export interface ExtractQmdInput { sessionId: string; /** Cap on records parsed/extracted. Avoids loading huge sessions. */ recordLimit?: number; - /** Whether to run the LLM pass for overview.md / decisions.md. */ - withLlm?: boolean; } export interface ExtractedFile { name: string; path: string; bytes: number; - kind: "mechanical" | "llm"; } export interface ExtractQmdResult { @@ -32,13 +28,6 @@ export interface ExtractQmdResult { files: ExtractedFile[]; recordsScanned: number; mechanicalMs: number; - llmMs: number; - llm?: { - model: string; - usage: MinimaxUsage; - finishReason: string; - reasoning: string; - }; parseResult: ParseSessionResult; error?: string; } @@ -49,21 +38,17 @@ const DEFAULT_LIMIT = 1500; export const extractQmdCommand: Command = { id: "extract-qmd", - label: "Extract QMD", - shell: ({ path: p, sessionId, withLlm }) => + label: "Extract QMD (mechanical)", + shell: ({ path: p, sessionId }) => [ `scout qmd extract`, `--source ${shellQuote(shrinkPath(p))}`, `--out ${shellQuote(shrinkPath(path.join(ROOT, sessionId)))}`, - withLlm === false ? "--no-llm" : "", - ] - .filter(Boolean) - .join(" "), - run: async ({ path: filePath, sessionId, recordLimit, withLlm = true }) => { + `--mechanical-only`, + ].join(" "), + run: async ({ path: filePath, sessionId, recordLimit }) => { const limit = recordLimit ?? DEFAULT_LIMIT; - // Reuse the existing parser command at a higher limit. It has its own - // 60s cache, so this is cheap on re-renders against the same input. const parseRun = await parseSessionCommand.run({ path: filePath, limit }); if (parseRun.error) { @@ -77,40 +62,18 @@ export const extractQmdCommand: Command = { const mechanical = await writeMechanical(outDir, parseRun, filePath); const mechanicalMs = Date.now() - mechStart; - let llmFiles: ExtractedFile[] = []; - let llmInfo: ExtractQmdResult["llm"] | undefined; - let llmMs = 0; - if (withLlm) { - const t = Date.now(); - try { - const out = await writeLlm(outDir, parseRun); - llmFiles = out.files; - llmInfo = out.info; - } catch (err) { - // Don't abort the whole extraction on LLM failure — keep mechanical - // and report the LLM error inline. - llmInfo = undefined; - const errMsg = err instanceof Error ? err.message : String(err); - await fs.writeFile(path.join(outDir, "_llm-error.txt"), errMsg); - } - llmMs = Date.now() - t; - } - return { outDir, - files: [...mechanical, ...llmFiles].sort((a, b) => a.name.localeCompare(b.name)), + files: mechanical.sort((a, b) => a.name.localeCompare(b.name)), recordsScanned: parseRun.records.length, mechanicalMs, - llmMs, - llm: llmInfo, parseResult: parseRun, }; }, - cacheKey: ({ path: p, sessionId, recordLimit, withLlm }) => - `${sessionId}::${p}::${recordLimit ?? DEFAULT_LIMIT}::${withLlm === false ? "0" : "1"}`, - // Cache for an hour. LLM calls are expensive and the inputs don't change - // between page reloads. - cacheTtlMs: 60 * 60 * 1000, + cacheKey: ({ path: p, sessionId, recordLimit }) => + `${sessionId}::${p}::${recordLimit ?? DEFAULT_LIMIT}`, + // Mechanical is cheap; short cache is fine. + cacheTtlMs: 5 * 60 * 1000, }; function outDirFor(sessionId: string): string { @@ -145,13 +108,13 @@ async function writeMechanical( sourcePath: string, ): Promise { const files: ExtractedFile[] = []; - files.push(await writeFile(outDir, "manifest.json", buildManifest(parseRun, sourcePath), "mechanical")); - files.push(await writeFile(outDir, "files.md", buildFilesMd(parseRun), "mechanical")); - files.push(await writeFile(outDir, "tool-calls.md", buildToolCallsMd(parseRun), "mechanical")); + files.push(await writeFile(outDir, "manifest.json", buildManifest(parseRun, sourcePath))); + files.push(await writeFile(outDir, "files.md", buildFilesMd(parseRun))); + files.push(await writeFile(outDir, "tool-calls.md", buildToolCallsMd(parseRun))); const windows = buildEventWindows(parseRun.records); for (let i = 0; i < windows.length; i++) { const idx = String(i + 1).padStart(3, "0"); - files.push(await writeFile(outDir, `events-${idx}.md`, windows[i]!, "mechanical")); + files.push(await writeFile(outDir, `events-${idx}.md`, windows[i]!)); } return files; } @@ -305,136 +268,17 @@ function trim(s: string, n: number): string { return oneLine.length > n ? oneLine.slice(0, n - 1) + "…" : oneLine; } -// ── LLM pass ────────────────────────────────────────────────────────────── - -async function writeLlm( - outDir: string, - parseRun: ParseSessionResult, -): Promise<{ files: ExtractedFile[]; info: NonNullable }> { - const condensed = condenseForLlm(parseRun.records); - const system = [ - "You are a careful technical analyst reading a transcript of a coding-agent session.", - "Your job is to produce two short markdown documents about the session.", - "Be specific and concrete; cite paths, commands, or file types when relevant.", - "Never invent details that aren't in the transcript.", - ].join(" "); - const user = [ - "Below is a condensed transcript of a coding-agent session. Each line is one", - "normalized event, prefixed with [NNNN] kind (tag) — detail.", - "", - "Produce exactly two top-level markdown sections, in this order:", - "", - "# Overview", - "", - "Two short paragraphs describing what this session was about: the user's goal,", - "the area of the codebase touched, and what the agent ended up doing.", - "", - "# Decisions", - "", - "A bulleted list of concrete decisions made or unresolved questions. Each bullet", - "should reference a specific event or file when possible. Include a final", - '"## Follow-ups" subsection if there are loose threads.', - "", - "Transcript:", - "```", - condensed, - "```", - ].join("\n"); - - const result = await callMinimax({ system, user, maxTokens: 4000, temperature: 0.2 }); - const content = result.content || ""; - const { overview, decisions } = splitOverviewDecisions(content); - - const files: ExtractedFile[] = []; - files.push(await writeFile(outDir, "overview.md", overview, "llm")); - files.push(await writeFile(outDir, "decisions.md", decisions, "llm")); - files.push( - await writeFile( - outDir, - "_llm-call.json", - JSON.stringify( - { - model: result.model, - usage: result.usage, - finishReason: result.finishReason, - latencyMs: result.latencyMs, - promptChars: user.length, - }, - null, - 2, - ), - "llm", - ), - ); - - return { - files, - info: { - model: result.model, - usage: result.usage, - finishReason: result.finishReason, - reasoning: result.reasoning, - }, - }; -} - -/** - * Pick a smaller representative slice of the normalized stream to feed the LLM. - * Strategy: keep all user turns, the first assistant turn after each user turn, - * tool names + short args, and short observations. Aggressive trim per line. - */ -function condenseForLlm(records: NormalizedRecord[]): string { - const lines: string[] = []; - for (const r of records) { - const idx = String(r.i).padStart(4, "0"); - if (r.kind === "user_turn") { - lines.push(`[${idx}] user — ${trim(r.text ?? "", 240)}`); - } else if (r.kind === "assistant_turn") { - lines.push(`[${idx}] assistant (${r.tag ?? ""}) — ${trim(r.text ?? "", 200)}`); - } else if (r.kind === "command_or_tool" && r.tool) { - lines.push(`[${idx}] tool ${r.tool.name} ${oneLineInput(r.tool.input)}`); - } else if (r.kind === "observation" && r.result) { - const out = - typeof r.result.output === "string" ? r.result.output : JSON.stringify(r.result.output ?? ""); - lines.push(`[${idx}] result — ${trim(out, 140)}`); - } else if (r.kind === "session_meta") { - const model = r.meta?.model_provider ?? r.meta?.model ?? "?"; - const cwd = r.meta?.cwd ?? "?"; - lines.push(`[${idx}] meta model=${model} cwd=${trim(String(cwd), 80)}`); - } - // Skip system_record / unknown — usually noise. - } - // Cap total length to keep token budget reasonable. - const joined = lines.join("\n"); - const cap = 24_000; // ~6k tokens at 4 chars/token - return joined.length > cap ? joined.slice(0, cap) + "\n… (transcript truncated)" : joined; -} - -function splitOverviewDecisions(content: string): { - overview: string; - decisions: string; -} { - const decisionsIdx = content.indexOf("# Decisions"); - if (decisionsIdx === -1) { - return { overview: content.trim(), decisions: "# Decisions\n\n_not produced_\n" }; - } - const overview = content.slice(0, decisionsIdx).trim(); - const decisions = content.slice(decisionsIdx).trim(); - return { overview, decisions }; -} - // ── File writer ─────────────────────────────────────────────────────────── async function writeFile( outDir: string, name: string, content: string, - kind: ExtractedFile["kind"], ): Promise { const fullPath = path.join(outDir, name); await fs.writeFile(fullPath, content); const stat = await fs.stat(fullPath); - return { name, path: fullPath, bytes: stat.size, kind }; + return { name, path: fullPath, bytes: stat.size }; } // ── Display helpers ─────────────────────────────────────────────────────── diff --git a/packages/agent-sessions/src/history.test.ts b/packages/agent-sessions/src/history.test.ts index 6e425ab4..b571d91b 100644 --- a/packages/agent-sessions/src/history.test.ts +++ b/packages/agent-sessions/src/history.test.ts @@ -403,16 +403,213 @@ describe("history snapshot replay", () => { ); }); + test("reconstructs a Codex snapshot from external jsonl history", () => { + const basePath = "/Users/arach/.codex/sessions/2026/05/29/rollout-2026-05-29T21-08-19-codex-session.jsonl"; + const content = [ + JSON.stringify({ + timestamp: "2026-05-30T01:08:36.827Z", + type: "session_meta", + payload: { + id: "codex-session", + cwd: "/Users/arach/dev/openscout", + originator: "Codex Desktop", + cli_version: "0.133.0-alpha.1", + source: "vscode", + thread_source: "user", + model_provider: "openai", + git: { + branch: "codex/embed-vox-transcription", + commit_hash: "abc123", + repository_url: "https://github.com/arach/openscout.git", + }, + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:08:36.827Z", + type: "event_msg", + payload: { + type: "task_started", + turn_id: "turn-codex-1", + started_at: 1780103316, + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:08:36.839Z", + type: "turn_context", + payload: { + cwd: "/Users/arach/dev/openscout", + model: "gpt-5.5", + approval_policy: "never", + timezone: "America/Toronto", + effort: "xhigh", + sandbox_policy: { type: "danger-full-access" }, + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:08:36.848Z", + type: "event_msg", + payload: { + type: "user_message", + message: "inspect the repo", + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:08:51.653Z", + type: "event_msg", + payload: { + type: "agent_message", + message: "I am checking the repo.", + phase: "commentary", + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:12.371Z", + type: "response_item", + payload: { + type: "function_call", + name: "exec_command", + arguments: JSON.stringify({ + cmd: "pwd", + workdir: "/Users/arach/dev/openscout", + }), + call_id: "call-shell-1", + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:12.782Z", + type: "response_item", + payload: { + type: "function_call_output", + call_id: "call-shell-1", + output: "Exit code: 0\nOutput:\n/Users/arach/dev/openscout\n", + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:13.000Z", + type: "response_item", + payload: { + type: "custom_tool_call", + name: "apply_patch", + input: "*** Begin Patch\n*** End Patch\n", + call_id: "call-patch-1", + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:13.100Z", + type: "event_msg", + payload: { + type: "patch_apply_end", + call_id: "call-patch-1", + stdout: "Success. Updated the following files:\nM file.ts\n", + stderr: "", + success: true, + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:14.000Z", + type: "event_msg", + payload: { + type: "token_count", + info: { + total_token_usage: { + input_tokens: 100, + cached_input_tokens: 40, + output_tokens: 20, + reasoning_output_tokens: 7, + }, + }, + }, + }), + JSON.stringify({ + timestamp: "2026-05-30T01:09:15.000Z", + type: "event_msg", + payload: { + type: "task_complete", + completed_at: 1780103355, + }, + }), + ].join("\n"); + + expect(inferHistorySessionAdapterType(basePath)).toBe("codex"); + expect(supportsHistorySessionSnapshotForPath(basePath)).toBe(true); + + const result = createHistorySessionSnapshot({ + path: basePath, + content, + }); + + expect(result.adapterType).toBe("codex"); + expect(result.lineCount).toBe(11); + expect(result.parsedLineCount).toBe(11); + expect(result.skippedLineCount).toBe(0); + + const snapshot = result.snapshot; + expect(snapshot.session.adapterType).toBe("codex"); + expect(snapshot.session.name).toBe("openscout"); + expect(snapshot.session.cwd).toBe("/Users/arach/dev/openscout"); + expect(snapshot.session.model).toBe("gpt-5.5"); + expect(snapshot.session.providerMeta).toEqual( + expect.objectContaining({ + historyPath: basePath, + historyAdapterType: "codex", + source: "external_history", + externalSessionId: "codex-session", + observeRuntime: expect.objectContaining({ + originator: "Codex Desktop", + cliVersion: "0.133.0-alpha.1", + modelProvider: "openai", + approvalPolicy: "never", + gitBranch: "codex/embed-vox-transcription", + }), + observeUsage: expect.objectContaining({ + inputTokens: 100, + outputTokens: 20, + reasoningOutputTokens: 7, + cacheReadInputTokens: 40, + tokenEvents: 1, + }), + }), + ); + expect(snapshot.currentTurnId).toBeUndefined(); + expect(snapshot.turns).toHaveLength(1); + + const turn = snapshot.turns[0]!; + expect(turn.status).toBe("completed"); + expect(turn.blocks).toHaveLength(3); + + const textBlock = turn.blocks[0]!.block; + expect(textBlock.type).toBe("text"); + if (textBlock.type === "text") { + expect(textBlock.text).toBe("I am checking the repo."); + } + + const commandBlock = turn.blocks[1]!.block; + expect(commandBlock.type).toBe("action"); + if (commandBlock.type === "action") { + expect(commandBlock.action.kind).toBe("command"); + expect(commandBlock.action.status).toBe("completed"); + expect(commandBlock.action.output).toContain("/Users/arach/dev/openscout"); + } + + const patchBlock = turn.blocks[2]!.block; + expect(patchBlock.type).toBe("action"); + if (patchBlock.type === "action") { + expect(patchBlock.action.kind).toBe("tool_call"); + expect(patchBlock.action.status).toBe("completed"); + expect(patchBlock.action.output).toContain("Success. Updated"); + } + }); + test("marks unsupported harness history clearly", () => { - const path = "/Users/arach/.codex/history.jsonl"; + const path = "/Users/arach/.unknown/history.jsonl"; - expect(inferHistorySessionAdapterType(path)).toBe("codex"); + expect(inferHistorySessionAdapterType(path)).toBe("unknown"); expect(supportsHistorySessionSnapshotForPath(path)).toBe(false); expect(() => createHistorySessionSnapshot({ path, content: `${JSON.stringify({ cwd: "/tmp/project" })}\n`, }), - ).toThrow('History snapshot is not supported for adapter type "codex".'); + ).toThrow('History snapshot is not supported for adapter type "unknown".'); }); }); diff --git a/packages/agent-sessions/src/history.ts b/packages/agent-sessions/src/history.ts index bf789712..3bffffd7 100644 --- a/packages/agent-sessions/src/history.ts +++ b/packages/agent-sessions/src/history.ts @@ -29,8 +29,8 @@ export type HistorySessionEvent = { event: PairingEvent; }; -export type SupportedHistoryAdapterType = "claude-code"; -export type HistoryAdapterType = SupportedHistoryAdapterType | "codex" | "unknown"; +export type SupportedHistoryAdapterType = "claude-code" | "codex"; +export type HistoryAdapterType = SupportedHistoryAdapterType | "unknown"; export interface HistorySessionSnapshotInput { path: string; @@ -187,6 +187,35 @@ function renderToolResultContent(content: unknown): string { return stringifyUnknown(content); } +function renderContentPartsText(content: unknown): string { + if (typeof content === "string") { + return content; + } + + if (!Array.isArray(content)) { + return ""; + } + + return content + .map((entry) => { + if (typeof entry === "string") { + return entry; + } + if (!isRecord(entry)) { + return ""; + } + if (typeof entry.text === "string") { + return entry.text; + } + if (typeof entry.content === "string") { + return entry.content; + } + return ""; + }) + .filter(Boolean) + .join("\n"); +} + function extractReasoningText(item: Record): string { const summary = Array.isArray(item.summary) ? item.summary : []; const content = Array.isArray(item.content) ? item.content : []; @@ -272,8 +301,8 @@ function inferHistoryAdapterType(path: string, adapterType?: string | null): His } export function supportsHistorySessionSnapshot(adapterType: string | null | undefined): boolean { - return inferHistoryAdapterType("", adapterType ?? undefined) === "claude-code" - || adapterType === "claude-code"; + const inferred = inferHistoryAdapterType("", adapterType ?? undefined); + return inferred === "claude-code" || inferred === "codex"; } function buildBaseHistorySession(input: HistorySessionSnapshotInput, adapterType: HistoryAdapterType): Session { @@ -1103,6 +1132,609 @@ class ClaudeCodeHistoryParser { } } +class CodexHistoryParser { + private readonly events: HistorySessionEvent[] = []; + private currentTurn: Turn | null = null; + private turnCounter = 0; + private blockIndex = 0; + private blockById = new Map(); + private toolBlockMap = new Map(); + private assistantMessageTextThisTurn = new Set(); + private inputTokens = 0; + private outputTokens = 0; + private reasoningOutputTokens = 0; + private cachedInputTokens = 0; + private tokenEventCount = 0; + + constructor( + private readonly session: Session, + private readonly baseTimestampMs: number, + ) {} + + parse(content: string): { + events: HistorySessionEvent[]; + lineCount: number; + parsedLineCount: number; + skippedLineCount: number; + } { + const lines = content.split(/\r?\n/u); + let parsedLineCount = 0; + let skippedLineCount = 0; + let lineCount = 0; + let lastCapturedAt = this.baseTimestampMs; + + for (let index = 0; index < lines.length; index += 1) { + const rawLine = lines[index]; + const trimmed = rawLine.trim(); + if (!trimmed) { + continue; + } + + lineCount += 1; + + let record: Record; + try { + const parsed = JSON.parse(trimmed) as unknown; + if (!isRecord(parsed)) { + skippedLineCount += 1; + continue; + } + record = parsed; + } catch { + skippedLineCount += 1; + continue; + } + + const capturedAt = extractRecordTimestamp(record) ?? (this.baseTimestampMs + index); + lastCapturedAt = capturedAt; + if (this.handleRecord(record, capturedAt)) { + parsedLineCount += 1; + } else { + skippedLineCount += 1; + } + } + + if (this.persistUsageMetadata()) { + this.emitSessionUpdate(lastCapturedAt); + } + + return { + events: this.events, + lineCount, + parsedLineCount, + skippedLineCount, + }; + } + + private handleRecord(record: Record, capturedAt: number): boolean { + const type = maybeString(record.type); + if (!type) { + return false; + } + + switch (type) { + case "session_meta": + this.handleSessionMeta(record, capturedAt); + return true; + case "turn_context": + this.handleTurnContext(record, capturedAt); + return true; + case "event_msg": + this.handleEventMessage(record, capturedAt); + return true; + case "response_item": + this.handleResponseItem(record, capturedAt); + return true; + case "compacted": + return true; + default: + return false; + } + } + + private handleSessionMeta(record: Record, capturedAt: number): void { + const payload = isRecord(record.payload) ? record.payload : {}; + let changed = false; + const providerMeta = this.ensureProviderMeta(); + + const externalSessionId = maybeString(payload.id); + if (externalSessionId && providerMeta.externalSessionId !== externalSessionId) { + providerMeta.externalSessionId = externalSessionId; + changed = true; + } + + const cwd = maybeString(payload.cwd); + if (cwd && this.session.cwd !== cwd) { + this.session.cwd = cwd; + if (!this.session.name || /^\d+$/u.test(this.session.name)) { + this.session.name = basename(cwd) || "Codex Session"; + } + changed = true; + } + + const git = isRecord(payload.git) ? payload.git : null; + const runtime = this.ensureObserveMetaRecord("observeRuntime"); + changed = this.assignObserveString(runtime, "originator", payload.originator) || changed; + changed = this.assignObserveString(runtime, "cliVersion", payload.cli_version) || changed; + changed = this.assignObserveString(runtime, "source", payload.source) || changed; + changed = this.assignObserveString(runtime, "threadSource", payload.thread_source) || changed; + changed = this.assignObserveString(runtime, "modelProvider", payload.model_provider) || changed; + changed = this.assignObserveString(runtime, "gitBranch", git?.branch) || changed; + changed = this.assignObserveString(runtime, "gitCommitHash", git?.commit_hash) || changed; + changed = this.assignObserveString(runtime, "repositoryUrl", git?.repository_url) || changed; + + if (changed) { + this.emitSessionUpdate(capturedAt); + } + } + + private handleTurnContext(record: Record, capturedAt: number): void { + const payload = isRecord(record.payload) ? record.payload : {}; + let changed = false; + + const cwd = maybeString(payload.cwd); + if (cwd && this.session.cwd !== cwd) { + this.session.cwd = cwd; + changed = true; + } + + const model = maybeString(payload.model); + if (model && this.session.model !== model) { + this.session.model = model; + changed = true; + } + + const runtime = this.ensureObserveMetaRecord("observeRuntime"); + changed = this.assignObserveString(runtime, "approvalPolicy", payload.approval_policy) || changed; + changed = this.assignObserveString(runtime, "currentDate", payload.current_date) || changed; + changed = this.assignObserveString(runtime, "timezone", payload.timezone) || changed; + changed = this.assignObserveString(runtime, "effort", payload.effort) || changed; + changed = this.assignObserveString(runtime, "personality", payload.personality) || changed; + + if (isRecord(payload.sandbox_policy) && runtime.sandboxPolicy !== payload.sandbox_policy) { + runtime.sandboxPolicy = payload.sandbox_policy; + changed = true; + } + + if (changed) { + this.emitSessionUpdate(capturedAt); + } + } + + private handleEventMessage(record: Record, capturedAt: number): void { + const payload = isRecord(record.payload) ? record.payload : {}; + const eventType = maybeString(payload.type); + if (!eventType) { + return; + } + + switch (eventType) { + case "task_started": { + const startedAt = normalizeTimestamp(payload.started_at) ?? capturedAt; + this.startTurn(startedAt, maybeString(payload.turn_id)); + break; + } + case "user_message": + this.ensureTurn(capturedAt); + break; + case "agent_message": { + const message = maybeString(payload.message); + if (message) { + this.appendCompletedText(capturedAt, message); + this.assistantMessageTextThisTurn.add(message); + } + break; + } + case "patch_apply_end": + this.handleToolOutput( + capturedAt, + maybeString(payload.call_id), + [maybeString(payload.stdout), maybeString(payload.stderr)].filter(Boolean).join("\n"), + payload.success === false, + ); + break; + case "token_count": + this.captureTokenUsage(payload.info); + break; + case "task_complete": + this.endTurn("completed", normalizeTimestamp(payload.completed_at) ?? capturedAt); + break; + case "context_compacted": + case "web_search_end": + break; + default: + break; + } + } + + private handleResponseItem(record: Record, capturedAt: number): void { + const payload = isRecord(record.payload) ? record.payload : {}; + const itemType = maybeString(payload.type); + if (!itemType) { + return; + } + + switch (itemType) { + case "message": + this.handleResponseMessage(payload, capturedAt); + break; + case "reasoning": + this.handleReasoning(payload, capturedAt); + break; + case "function_call": + case "custom_tool_call": + this.handleToolCall(payload, capturedAt); + break; + case "function_call_output": + case "custom_tool_call_output": + this.handleToolOutput( + capturedAt, + maybeString(payload.call_id), + renderToolResultContent(payload.output), + false, + ); + break; + case "web_search_call": + this.handleWebSearchCall(payload, capturedAt); + break; + default: + break; + } + } + + private handleResponseMessage(payload: Record, capturedAt: number): void { + const role = maybeString(payload.role); + if (role !== "assistant") { + return; + } + + const text = renderContentPartsText(payload.content).trim(); + if (!text || this.assistantMessageTextThisTurn.has(text)) { + return; + } + + this.appendCompletedText(capturedAt, text); + this.assistantMessageTextThisTurn.add(text); + } + + private handleReasoning(payload: Record, capturedAt: number): void { + const text = extractReasoningText(payload); + if (!text) { + return; + } + + const turn = this.ensureTurn(capturedAt); + const block = this.startBlock>(turn, capturedAt, { + type: "reasoning", + text, + status: "completed", + }); + this.emitBlockEnd(capturedAt, turn, block, "completed"); + } + + private handleToolCall(payload: Record, capturedAt: number): void { + const turn = this.ensureTurn(capturedAt); + const toolName = maybeString(payload.name) ?? "unknown"; + const toolCallId = maybeString(payload.call_id) ?? `${turn.id}:tool:${this.blockIndex}`; + const input = this.parseToolInput(payload.arguments ?? payload.input); + const action = this.buildAction(toolName, toolCallId, input); + + const block = this.startBlock>(turn, capturedAt, { + id: `${turn.id}:action:${toolCallId}`, + type: "action", + action, + status: "streaming", + }); + this.toolBlockMap.set(toolCallId, block.id); + } + + private handleWebSearchCall(payload: Record, capturedAt: number): void { + const turn = this.ensureTurn(capturedAt); + const toolCallId = `${turn.id}:web-search:${this.blockIndex}`; + const action: Action = { + kind: "tool_call", + toolName: "web_search", + toolCallId, + input: payload.action, + result: payload.status, + status: maybeString(payload.status) === "failed" ? "failed" : "completed", + output: "", + }; + const block = this.startBlock>(turn, capturedAt, { + id: `${turn.id}:action:${toolCallId}`, + type: "action", + action, + status: "completed", + }); + this.emitBlockEnd(capturedAt, turn, block, "completed"); + } + + private parseToolInput(value: unknown): unknown { + if (typeof value !== "string") { + return value; + } + + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + + try { + return JSON.parse(trimmed) as unknown; + } catch { + return value; + } + } + + private buildAction(toolName: string, toolCallId: string, input: unknown): Action { + const inputRecord = isRecord(input) ? input : {}; + + if (toolName === "exec_command") { + return { + kind: "command", + command: maybeString(inputRecord.cmd) ?? "", + status: "running", + output: "", + }; + } + + return { + kind: "tool_call", + toolName, + toolCallId, + input, + status: "running", + output: "", + }; + } + + private handleToolOutput( + capturedAt: number, + toolCallId: string | undefined, + output: string, + isError: boolean, + ): void { + if (!toolCallId) { + return; + } + + const blockId = this.toolBlockMap.get(toolCallId); + const turn = this.currentTurn; + if (!blockId || !turn) { + return; + } + + if (output) { + this.emitEvent(capturedAt, { + event: "block:action:output", + sessionId: this.session.id, + turnId: turn.id, + blockId, + output, + }); + } + + const exitCode = this.extractExitCode(output); + const status = isError || (exitCode != null && exitCode !== 0) ? "failed" : "completed"; + this.emitEvent(capturedAt, { + event: "block:action:status", + sessionId: this.session.id, + turnId: turn.id, + blockId, + status, + ...(exitCode != null ? { meta: { exitCode } } : {}), + }); + + const block = this.blockById.get(blockId); + if (block) { + this.emitBlockEnd(capturedAt, turn, block, status === "failed" ? "failed" : "completed"); + } + this.toolBlockMap.delete(toolCallId); + } + + private extractExitCode(output: string): number | null { + const match = output.match(/(?:exit code|exited with code):\s*(-?\d+)/iu); + if (!match) { + return null; + } + + const exitCode = Number(match[1]); + return Number.isFinite(exitCode) ? exitCode : null; + } + + private captureTokenUsage(info: unknown): void { + if (!isRecord(info)) { + return; + } + + const total = isRecord(info.total_token_usage) ? info.total_token_usage : {}; + this.inputTokens = maybeNumber(total.input_tokens) ?? this.inputTokens; + this.outputTokens = maybeNumber(total.output_tokens) ?? this.outputTokens; + this.reasoningOutputTokens = maybeNumber(total.reasoning_output_tokens) ?? this.reasoningOutputTokens; + this.cachedInputTokens = maybeNumber(total.cached_input_tokens) ?? this.cachedInputTokens; + this.tokenEventCount += 1; + } + + private persistUsageMetadata(): boolean { + if (this.tokenEventCount === 0) { + return false; + } + + const usage = this.ensureObserveMetaRecord("observeUsage"); + let changed = false; + const assignNumber = (key: string, value: number): void => { + if (value > 0 && usage[key] !== value) { + usage[key] = value; + changed = true; + } + }; + + assignNumber("inputTokens", this.inputTokens); + assignNumber("outputTokens", this.outputTokens); + assignNumber("reasoningOutputTokens", this.reasoningOutputTokens); + assignNumber("cacheReadInputTokens", this.cachedInputTokens); + assignNumber("tokenEvents", this.tokenEventCount); + return changed; + } + + private startTurn(capturedAt: number, turnId?: string): Turn { + if (this.currentTurn) { + this.endTurn("stopped", capturedAt); + } + + this.turnCounter += 1; + this.blockIndex = 0; + this.blockById.clear(); + this.toolBlockMap.clear(); + this.assistantMessageTextThisTurn.clear(); + + this.session.status = "active"; + this.emitSessionUpdate(capturedAt); + + const turn: Turn = { + id: turnId || `history-turn-${this.turnCounter}`, + sessionId: this.session.id, + status: "started", + startedAt: new Date(capturedAt).toISOString(), + blocks: [], + }; + this.currentTurn = turn; + + this.emitEvent(capturedAt, { + event: "turn:start", + sessionId: this.session.id, + turn, + }); + + return turn; + } + + private ensureTurn(capturedAt: number): Turn { + return this.currentTurn ?? this.startTurn(capturedAt); + } + + private endTurn(status: TurnStatus, capturedAt: number): void { + const turn = this.currentTurn; + if (!turn) { + return; + } + + turn.status = status; + turn.endedAt = new Date(capturedAt).toISOString(); + this.emitEvent(capturedAt, { + event: "turn:end", + sessionId: this.session.id, + turnId: turn.id, + status, + }); + + this.currentTurn = null; + this.blockById.clear(); + this.toolBlockMap.clear(); + this.assistantMessageTextThisTurn.clear(); + + this.session.status = "idle"; + this.emitSessionUpdate(capturedAt); + } + + private appendCompletedText(capturedAt: number, text: string): void { + const turn = this.ensureTurn(capturedAt); + const block = this.startBlock>(turn, capturedAt, { + type: "text", + text, + status: "completed", + }); + this.emitBlockEnd(capturedAt, turn, block, "completed"); + } + + private startBlock( + turn: Turn, + capturedAt: number, + partial: Omit & { id?: string }, + ): T { + const block = { + ...partial, + id: partial.id || `${turn.id}:block:${this.blockIndex}`, + turnId: turn.id, + index: this.blockIndex, + } as T; + this.blockIndex += 1; + turn.blocks.push(block); + this.blockById.set(block.id, block); + + this.emitEvent(capturedAt, { + event: "block:start", + sessionId: this.session.id, + turnId: turn.id, + block, + }); + + return block; + } + + private emitBlockEnd( + capturedAt: number, + turn: Turn, + block: Block, + status: Block["status"], + ): void { + block.status = status; + this.emitEvent(capturedAt, { + event: "block:end", + sessionId: this.session.id, + turnId: turn.id, + blockId: block.id, + status, + }); + } + + private ensureProviderMeta(): Record { + const providerMeta = isRecord(this.session.providerMeta) ? this.session.providerMeta : {}; + this.session.providerMeta = providerMeta; + return providerMeta; + } + + private ensureObserveMetaRecord(key: "observeRuntime" | "observeUsage"): Record { + const providerMeta = this.ensureProviderMeta(); + const existing = providerMeta[key]; + if (isRecord(existing)) { + return existing; + } + + const next: Record = {}; + providerMeta[key] = next; + return next; + } + + private assignObserveString( + target: Record, + key: string, + value: unknown, + ): boolean { + const next = maybeString(value); + if (!next || target[key] === next) { + return false; + } + + target[key] = next; + return true; + } + + private emitSessionUpdate(capturedAt: number): void { + this.emitEvent(capturedAt, { + event: "session:update", + session: { ...this.session }, + }); + } + + private emitEvent(capturedAt: number, event: PairingEvent): void { + this.events.push({ + capturedAt, + event: structuredClone(event), + }); + } +} + export function inferHistorySessionAdapterType( path: string, adapterType?: string | null, @@ -1114,20 +1746,23 @@ export function supportsHistorySessionSnapshotForPath( path: string, adapterType?: string | null, ): boolean { - return inferHistoryAdapterType(path, adapterType) === "claude-code"; + const inferred = inferHistoryAdapterType(path, adapterType); + return inferred === "claude-code" || inferred === "codex"; } export function createHistorySessionSnapshot( input: HistorySessionSnapshotInput, ): HistorySessionSnapshotResult { const adapterType = inferHistoryAdapterType(input.path, input.adapterType); - if (adapterType !== "claude-code") { + if (adapterType !== "claude-code" && adapterType !== "codex") { throw new Error(`History snapshot is not supported for adapter type "${adapterType}".`); } const session = buildBaseHistorySession(input, adapterType); const baseTimestampMs = normalizeTimestamp(input.baseTimestampMs) ?? Date.now(); - const parser = new ClaudeCodeHistoryParser(session, baseTimestampMs); + const parser = adapterType === "codex" + ? new CodexHistoryParser(session, baseTimestampMs) + : new ClaudeCodeHistoryParser(session, baseTimestampMs); const replay = parser.parse(input.content); const tracker = new StateTracker(); diff --git a/packages/web/client/lib/router.test.ts b/packages/web/client/lib/router.test.ts index 09ae1ac2..118a1d7b 100644 --- a/packages/web/client/lib/router.test.ts +++ b/packages/web/client/lib/router.test.ts @@ -165,6 +165,13 @@ describe("agents route parsing", () => { expect(routePath({ view: "messages" })).toBe("/messages"); }); + test("search route round-trips", () => { + expect(routeFromUrl("http://127.0.0.1:3200/search")).toEqual({ + view: "search", + }); + expect(routePath({ view: "search" })).toBe("/search"); + }); + test("messages route preserves conversationId, filter, and sort", () => { const route = routeFromUrl( "http://127.0.0.1:3200/messages/channel.font-studio?filter=channel&sort=unread", diff --git a/packages/web/client/lib/router.ts b/packages/web/client/lib/router.ts index 012c879d..1f556f9a 100644 --- a/packages/web/client/lib/router.ts +++ b/packages/web/client/lib/router.ts @@ -239,6 +239,7 @@ export function routeFromUrl(urlLike: string | URL): Route { return scoped(base); } if (parts[0] === "sessions") return scoped({ view: "sessions" }); + if (parts[0] === "search") return { view: "search" }; if (parts[0] === "channels" && parts[1]) { return scoped({ view: "channels", channelId: decodeURIComponent(parts[1]) }); } @@ -376,6 +377,8 @@ export function routePath(r: Route): string { return pathWithMachineScope(r.sessionId ? `/sessions/${encodeURIComponent(r.sessionId)}` : "/sessions", r); + case "search": + return "/search"; case "channels": return pathWithMachineScope(r.channelId ? `/channels/${encodeURIComponent(r.channelId)}` diff --git a/packages/web/client/lib/scoutbot.ts b/packages/web/client/lib/scoutbot.ts index d3cae422..c05f11ad 100644 --- a/packages/web/client/lib/scoutbot.ts +++ b/packages/web/client/lib/scoutbot.ts @@ -1,6 +1,6 @@ import type { Agent, OpsMode, Route } from "./types.ts"; -export const DEFAULT_SCOUTBOT_AGENT_ID = "scoutbot.main.mini"; +export const DEFAULT_SCOUTBOT_AGENT_ID = "scoutbot"; const SCOUTBOT_AGENT_IDS = new Set([ DEFAULT_SCOUTBOT_AGENT_ID, diff --git a/packages/web/client/lib/types.ts b/packages/web/client/lib/types.ts index bd524460..9dd2bf54 100644 --- a/packages/web/client/lib/types.ts +++ b/packages/web/client/lib/types.ts @@ -1021,6 +1021,7 @@ export type Route = sort?: MessagesSort; } & MachineScopedRoute) | ({ view: "sessions"; sessionId?: string } & MachineScopedRoute) + | { view: "search" } | ({ view: "channels"; channelId?: string } & MachineScopedRoute) | ({ view: "mesh" } & MachineScopedRoute) | { view: "broker" } diff --git a/packages/web/client/scout/Provider.tsx b/packages/web/client/scout/Provider.tsx index bb1fe76a..9c1ee085 100644 --- a/packages/web/client/scout/Provider.tsx +++ b/packages/web/client/scout/Provider.tsx @@ -14,7 +14,6 @@ import { api } from "../lib/api.ts"; import { useBrokerEvents } from "../lib/sse.ts"; import { isAgentOnline } from "../lib/agent-state.ts"; import { - isScoutbotAgent, scoutbotConversationId, resolveScoutbotAgentId, type ScoutbotUiAction, @@ -195,7 +194,7 @@ export function ScoutProvider({ const request = (async () => { const agentsResult = await api("/api/agents").catch(() => null); if (agentsResult) { - setAgents(agentsResult.filter((agent) => !isScoutbotAgent(agent))); + setAgents(agentsResult); } })(); diff --git a/packages/web/client/scout/hooks.ts b/packages/web/client/scout/hooks.ts index df31e2ac..6190d975 100644 --- a/packages/web/client/scout/hooks.ts +++ b/packages/web/client/scout/hooks.ts @@ -86,6 +86,11 @@ export function useScoutCommands(): CommandOption[] { label: "Go to Sessions", action: () => navigate({ view: "sessions" }), }, + { + id: "nav:search", + label: "Go to Search", + action: () => navigate({ view: "search" }), + }, { id: "nav:activity", label: "Go to Activity", @@ -275,6 +280,7 @@ const VIEW_LABELS: Record = { fleet: "Fleet", conversations: "Conversations", sessions: "Sessions", + search: "Search", channels: "Channels", activity: "Activity", mesh: "Mesh", @@ -292,6 +298,7 @@ export function useScoutNavCenter(): ReactNode | null { { label: "Agents", view: "agents" }, { label: "Messages", view: "messages" }, { label: "Sessions", view: "sessions" }, + { label: "Search", view: "search" }, { label: "Mesh", view: "mesh" }, { label: "Broker", view: "broker" }, ...(opsEnabled ? [{ label: "Ops" as const, view: "ops" as Route["view"] }] : []), diff --git a/packages/web/client/scout/index.ts b/packages/web/client/scout/index.ts index 7066605a..c50897af 100644 --- a/packages/web/client/scout/index.ts +++ b/packages/web/client/scout/index.ts @@ -59,6 +59,14 @@ const intents: AppIntent[] = [ keywords: ["sessions", "conversations", "threads"], shortcut: "Cmd+4", }, + { + commandId: "nav:search", + title: "Go to Search", + description: + "Navigate to the session knowledge search surface for extraction, search, and raw-log drilldown planning", + category: "navigation", + keywords: ["search", "knowledge", "qmd", "history"], + }, { commandId: "nav:channels", title: "Go to Channels", diff --git a/packages/web/client/scout/inspector/AgentsInspector.tsx b/packages/web/client/scout/inspector/AgentsInspector.tsx index 387f0f31..9126f388 100644 --- a/packages/web/client/scout/inspector/AgentsInspector.tsx +++ b/packages/web/client/scout/inspector/AgentsInspector.tsx @@ -664,7 +664,24 @@ function InspectorMesh({ const CX = W / 2; const CY = H / 2; const R = 68; - const others = agents.filter((a) => a.id !== focusAgent.id).slice(0, 8); + const others = useMemo(() => { + const peers = agents.filter((a) => a.id !== focusAgent.id); + const stateRank = (s: string) => { + const n = normalizeAgentState(s); + if (n === "working") return 0; + if (n === "available") return 1; + return 2; + }; + return peers + .slice() + .sort((a, b) => { + const sa = stateRank(a.state ?? ""); + const sb = stateRank(b.state ?? ""); + if (sa !== sb) return sa - sb; + return compareTimestampsDesc(a.updatedAt, b.updatedAt); + }) + .slice(0, 8); + }, [agents, focusAgent.id]); const nodes = useMemo(() => { const result: Array<{ @@ -687,11 +704,13 @@ function InspectorMesh({ }, [focusAgent, others, CX, CY, R]); return ( - +
      + @@ -799,7 +818,39 @@ function InspectorMesh({ ); })} - + + {others.length > 0 && ( +
        + {others.map((peer) => { + const nState = normalizeAgentState(peer.state); + return ( +
      • + +
        • + ); + })} +
      + )} +
      ); } diff --git a/packages/web/client/scout/slots/Content.tsx b/packages/web/client/scout/slots/Content.tsx index c1af1052..b3571e61 100644 --- a/packages/web/client/scout/slots/Content.tsx +++ b/packages/web/client/scout/slots/Content.tsx @@ -11,6 +11,7 @@ import { ConversationScreen } from "../../screens/ConversationScreen.tsx"; import { ConversationsScreen } from "../../screens/ConversationsScreen.tsx"; import { FollowScreen } from "../../screens/FollowScreen.tsx"; import { HomeScreen } from "../../screens/HomeScreen.tsx"; +import { KnowledgeSearchScreen } from "../../screens/KnowledgeSearchScreen.tsx"; import { MeshScreen } from "../../screens/MeshScreen.tsx"; import { MessagesScreen } from "../../screens/MessagesScreen.tsx"; import { SessionsScreen } from "../../screens/SessionsScreen.tsx"; @@ -99,6 +100,8 @@ function renderScreen( ); } return ; + case "search": + return ; case "channels": return ; case "mesh": diff --git a/packages/web/client/scout/slots/GlobalJumpDock.tsx b/packages/web/client/scout/slots/GlobalJumpDock.tsx index 58f318e5..1712d50f 100644 --- a/packages/web/client/scout/slots/GlobalJumpDock.tsx +++ b/packages/web/client/scout/slots/GlobalJumpDock.tsx @@ -1,5 +1,5 @@ import { useEffect, useState, type ReactNode } from "react"; -import { Activity, Compass, GitBranch, MessageSquare, ScrollText } from "lucide-react"; +import { Activity, Compass, Database, GitBranch, MessageSquare, ScrollText } from "lucide-react"; import { useScout } from "../Provider.tsx"; import { MeshCanvasMinimap } from "./MeshCanvasMinimap.tsx"; import type { Route } from "../../lib/types.ts"; @@ -89,6 +89,7 @@ export function GlobalJumpDock() { const JUMPS: { id: string; label: string; icon: ReactNode; route: Route }[] = [ { id: "sessions", label: "Sessions", icon: , route: { view: "sessions" } }, + { id: "search", label: "Search", icon: , route: { view: "search" } }, { id: "tail", label: "Tail", icon: , route: { view: "ops", mode: "tail" } }, { id: "control", label: "Control", icon: , route: { view: "ops", mode: "mission" } }, { id: "fleet", label: "Fleet", icon: , route: { view: "fleet" } }, diff --git a/packages/web/server/create-openscout-web-server.ts b/packages/web/server/create-openscout-web-server.ts index b44525a9..e09dc8ea 100644 --- a/packages/web/server/create-openscout-web-server.ts +++ b/packages/web/server/create-openscout-web-server.ts @@ -6,7 +6,6 @@ import { homedir } from "node:os"; import { Hono, type Context } from "hono"; import type { - AgentHarness, CollaborationEvent, CollaborationKind, ConversationDefinition, @@ -178,7 +177,6 @@ import { } from "@openscout/runtime/setup"; import { relayAgentRuntimeDirectory } from "@openscout/runtime/support-paths"; import { readSessionCatalogSync } from "@openscout/runtime/claude-stream-json"; -import { SUPPORTED_LOCAL_AGENT_HARNESSES } from "@openscout/runtime/local-agents"; function parseConversationKinds(value: string | undefined): ConversationKind[] | undefined { const trimmed = value?.trim(); @@ -1893,7 +1891,7 @@ async function buildAgentConfigurationSnapshot(currentDirectory: string) { workspaceRoots: settings?.discovery.workspaceRoots ?? [], hiddenProjectCount: settings?.discovery.hiddenProjectRoots.length ?? 0, defaultHarness: settings?.agents.defaultHarness ?? "claude", - defaultTransport: settings?.agents.defaultTransport ?? "claude_stream_json", + defaultTransport: settings?.agents.defaultTransport ?? "tmux", defaultCapabilities: settings?.agents.defaultCapabilities ?? [], sessionPrefix: settings?.agents.sessionPrefix ?? "relay", }, @@ -1968,177 +1966,6 @@ async function buildAgentConfigurationSnapshot(currentDirectory: string) { }; } -const RUNNER_MODEL_PRESETS = [ - { id: "sonnet", label: "Sonnet", harnesses: ["claude"] }, - { id: "opus", label: "Opus", harnesses: ["claude"] }, - { id: "haiku", label: "Haiku", harnesses: ["claude"] }, - { id: "gpt-5.4", label: "GPT-5.4", harnesses: ["codex"] }, - { id: "gpt-5.4-mini", label: "GPT-5.4 mini", harnesses: ["codex"] }, - { id: "gpt-5.5", label: "GPT-5.5", harnesses: ["codex"] }, -] as const; - -function localRunnerHarness(value: unknown): AgentHarness | undefined { - const normalized = optionalString(value)?.trim() as AgentHarness | undefined; - if (normalized && SUPPORTED_LOCAL_AGENT_HARNESSES.includes(normalized)) { - return normalized; - } - return undefined; -} - -function resolveRunnerDirectory(value: unknown, currentDirectory: string): string { - const raw = optionalString(value)?.trim(); - if (!raw) { - throw new Error("directory is required"); - } - const resolved = resolve(currentDirectory, expandHomePath(raw)); - const realPath = realpathSync(resolved); - const stats = statSync(realPath); - if (!stats.isDirectory()) { - throw new Error(`${realPath} is not a directory`); - } - return realPath; -} - -function recommendedRunnerModel( - harness: AgentHarness | undefined, - models: Array<{ id: string; harnesses: ReadonlyArray }>, -): string | undefined { - if (!harness) return undefined; - const candidates = models.filter((model) => - model.id.trim().length > 0 - && (model.harnesses.length === 0 || model.harnesses.includes(harness)) - ); - const preference = harness === "claude" - ? ["claude-opus-4-7", "opus", "sonnet", "haiku"] - : harness === "codex" - ? ["gpt-5.5", "gpt-5.4", "gpt-5.4-mini"] - : []; - for (const preferred of preference) { - const match = candidates.find((model) => model.id.toLowerCase() === preferred); - if (match) return match.id; - } - return candidates[0]?.id; -} - -async function buildAgentRunnerOptions(currentDirectory: string) { - const [settingsResult, setupResult, catalogResult] = await Promise.allSettled([ - readOpenScoutSettings({ currentDirectory }), - loadResolvedRelayAgents({ currentDirectory }), - loadHarnessCatalogSnapshot(), - ]); - const settings = settingsResult.status === "fulfilled" ? settingsResult.value : null; - const setup = setupResult.status === "fulfilled" ? setupResult.value : null; - const catalog = catalogResult.status === "fulfilled" ? catalogResult.value : null; - const agents = queryAgents(200); - const seenModels = new Set(); - const observedModels = agents - .map((agent) => ({ - id: agent.model?.trim() ?? "", - harness: agent.harness?.trim() ?? "", - })) - .filter((entry) => entry.id.length > 0) - .filter((entry) => { - const key = `${entry.harness}:${entry.id}`; - if (seenModels.has(key)) return false; - seenModels.add(key); - return true; - }) - .map((entry) => ({ - id: entry.id, - label: entry.id, - harnesses: entry.harness ? [entry.harness] : [], - source: "observed" as const, - })); - const models = [ - ...observedModels, - ...RUNNER_MODEL_PRESETS.map((model) => ({ - ...model, - harnesses: [...model.harnesses], - source: "preset" as const, - })), - ]; - const defaultHarness = localRunnerHarness(settings?.agents.defaultHarness) ?? "claude"; - - return { - generatedAt: Date.now(), - defaults: { - runner: "scout", - directory: currentDirectory, - harness: defaultHarness, - model: recommendedRunnerModel(defaultHarness, models), - persistence: "sticky" as const, - }, - runners: [ - { - id: "scout", - label: "Scout", - description: "Create or route to a Scout project agent, then deliver a broker-owned ask.", - supports: ["project_path", "harness", "model", "sticky_agent", "voice_instructions"], - }, - ], - harnesses: (catalog?.entries ?? []) - .filter((entry) => SUPPORTED_LOCAL_AGENT_HARNESSES.includes(entry.harness as AgentHarness)) - .map((entry) => ({ - id: entry.harness, - name: entry.name, - label: entry.label, - description: entry.description, - state: entry.readinessReport.state, - ready: entry.readinessReport.ready, - detail: entry.readinessReport.detail, - binaryPath: entry.readinessReport.binaryPath, - loginCommand: entry.readinessReport.loginCommand, - capabilities: entry.capabilities, - })), - models, - agents: agents.map((agent) => ({ - id: agent.id, - name: agent.name, - handle: agent.handle, - status: agent.state ?? "offline", - harness: agent.harness, - model: agent.model, - projectRoot: agent.projectRoot, - cwd: agent.cwd, - harnessSessionId: agent.harnessSessionId, - })), - projects: (setup?.projectInventory ?? []).slice(0, 120).map((project) => ({ - id: project.agentId, - title: project.displayName, - root: project.projectRoot, - source: project.source, - registrationKind: project.registrationKind, - defaultHarness: project.defaultHarness, - })), - }; -} - -function runnerRequestTargetDirectory(body: Record, currentDirectory: string): string { - const target = recordInput(body.target); - const targetKind = optionalString(target?.kind)?.trim(); - if (target && targetKind && targetKind !== "project_path") { - throw new Error(`unsupported runner target kind: ${targetKind}`); - } - return resolveRunnerDirectory( - target?.path - ?? target?.projectPath - ?? body.directory - ?? body.projectPath, - currentDirectory, - ); -} - -function runnerRequestSession(value: unknown): "new" | "existing" | "any" | undefined { - const normalized = optionalString(value)?.trim(); - return normalized === "new" || normalized === "existing" || normalized === "any" - ? normalized - : undefined; -} - -function runnerRequestPersistence(value: unknown): "one_time" | "sticky" { - return optionalString(value)?.trim() === "one_time" ? "one_time" : "sticky"; -} - export async function createOpenScoutWebServer( options: CreateOpenScoutWebServerOptions, ): Promise { @@ -2691,87 +2518,6 @@ export async function createOpenScoutWebServer( app.get("/api/agent-config/snapshot", async (c) => c.json(await buildAgentConfigurationSnapshot(currentDirectory)), ); - const runnerOptionsHandler = async (c: Context) => - c.json(await buildAgentRunnerOptions(currentDirectory)); - const runnerAskHandler = async (c: Context) => { - const body = await c.req.json().catch(() => ({})) as Record; - const runner = optionalString(body.runner)?.trim() || "scout"; - if (runner !== "scout") { - return c.json({ error: `unsupported runner: ${runner}` }, 400); - } - - let directory: string; - try { - directory = runnerRequestTargetDirectory(body, currentDirectory); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return c.json({ error: message }, 400); - } - - const instructions = (optionalString(body.instructions) ?? optionalString(body.body) ?? "").trim(); - if (!instructions) { - return c.json({ error: "instructions are required" }, 400); - } - - const execution = recordInput(body.execution) ?? {}; - const agent = recordInput(body.agent) ?? {}; - const harness = localRunnerHarness(execution.harness ?? body.harness); - const model = optionalString(execution.model ?? body.model)?.trim() || undefined; - const session = runnerRequestSession(execution.session ?? body.session); - const persistence = runnerRequestPersistence(agent.persistence ?? body.persistence); - const agentName = optionalString(agent.name ?? agent.agentName ?? body.agentName)?.trim() || undefined; - const displayName = optionalString(agent.displayName ?? body.displayName)?.trim() || undefined; - - const result = await askScoutQuestion({ - senderId: resolveOperatorName().trim() || "operator", - target: { - kind: "project_path", - projectPath: directory, - }, - targetLabel: directory, - body: instructions, - ...(harness ? { executionHarness: harness } : {}), - ...(model ? { executionModel: model } : {}), - executionSession: session ?? (persistence === "sticky" ? "any" : "new"), - projectAgent: { - persistence, - ...(agentName ? { agentName } : {}), - ...(displayName ? { displayName } : {}), - }, - currentDirectory: directory, - source: "runner-api", - }); - - if (!result.usedBroker) { - return c.json({ error: "broker unreachable" }, 502); - } - if (result.unresolvedTarget) { - return c.json( - { - error: `could not route ask for ${result.unresolvedTarget}`, - targetDiagnostic: result.targetDiagnostic ?? null, - }, - 409, - ); - } - - shellStateCache.invalidate(); - return c.json({ - ok: true, - runner: "scout", - directory, - persistence, - flight: result.flight ?? null, - conversationId: result.conversationId ?? null, - messageId: result.messageId ?? null, - targetAgentId: result.flight?.targetAgentId ?? null, - }); - }; - app.get("/api/runner/options", runnerOptionsHandler); - app.post("/api/runner/ask", runnerAskHandler); - // Back-compat for the exploration endpoint shape used by early HUD builds. - app.get("/api/agent-runner/options", runnerOptionsHandler); - app.post("/api/agent-runner/scout/ask", runnerAskHandler); app.get("/api/agents", (c) => c.json(queryAgents())); app.get("/api/agents/:id", (c) => { const agent = queryAgentById(c.req.param("id")); @@ -2820,6 +2566,7 @@ export async function createOpenScoutWebServer( systemPrompt: optionalString(body.systemPrompt) ?? existing.systemPrompt, launchArgs: stringList(body.launchArgs, existing.launchArgs), model, + channelEnabled: hasOwn(body, "channelEnabled") ? body.channelEnabled === true : existing.channelEnabled, capabilities: stringList(body.capabilities, existing.capabilities), }); if (!nextConfig) { @@ -3710,7 +3457,7 @@ export async function createOpenScoutWebServer( signal: c.req.raw.signal, })); } catch (error) { - const message = error instanceof Error ? error.message : "Vox speech failed"; + const message = error instanceof Error ? error.message : "Voice speech failed"; return c.json({ error: message }, 503); } }); diff --git a/packages/web/server/terminal-relay-session.ts b/packages/web/server/terminal-relay-session.ts index b7fae689..f4bcd739 100644 --- a/packages/web/server/terminal-relay-session.ts +++ b/packages/web/server/terminal-relay-session.ts @@ -21,7 +21,7 @@ export interface SessionInitMessage { backend?: 'pty' | 'tmux'; /** For tmux backend: the tmux session name. Required when backend is 'tmux'. */ tmuxSession?: string; - /** CLI agent to spawn. 'claude' (default), 'pi', or 'shell'. */ + /** Process to spawn. 'claude' (default), 'pi', or 'shell' for a normal login shell. */ agent?: 'claude' | 'pi' | 'shell'; /** For pi agent: provider name (e.g. 'minimax', 'github-copilot'). */ provider?: string; @@ -54,9 +54,9 @@ export type ClientMessage = | TerminalResizeMessage; import { createRequire } from 'module'; -import { execFileSync, execSync } from 'child_process'; -import { accessSync, constants, existsSync, mkdirSync, writeFileSync } from 'fs'; -import { delimiter as pathDelimiter, join, dirname as pathDirname, resolve as pathResolve } from 'path'; +import { execSync } from 'child_process'; +import { existsSync, mkdirSync, writeFileSync } from 'fs'; +import { join, dirname as pathDirname } from 'path'; import type { IPty } from 'node-pty'; const require = createRequire(import.meta.url); @@ -145,9 +145,6 @@ export function resizeSession(session: Session, cols: number, rows: number): boo if (session.exited) return false; try { session.pty.resize(cols, rows); - if (session.backend === 'tmux' && session.tmuxSession) { - resizeTmuxWindow(session.tmuxSession, cols, rows); - } session.cols = cols; session.rows = rows; return true; @@ -193,58 +190,9 @@ function findBin(name: string, envOverride?: string): string | null { } } -function expandHomePath(value: string): string { - const home = process.env.HOME || '/tmp'; - if (value === '~') return home; - if (value.startsWith('~/')) return join(home, value.slice(2)); - return value; -} - -function isExecutablePath(candidate: string | null | undefined): candidate is string { - if (!candidate) return false; - try { - accessSync(candidate, constants.X_OK); - return true; - } catch { - return false; - } -} - -function findExecutableInDirectories(name: string, directories: string[]): string | null { - const seen = new Set(); - for (const directory of directories) { - if (!directory) continue; - const normalizedDirectory = pathResolve(expandHomePath(directory)); - if (seen.has(normalizedDirectory)) continue; - seen.add(normalizedDirectory); - const candidate = join(normalizedDirectory, name); - if (isExecutablePath(candidate)) return candidate; - } - return null; -} - -function findExecutableOnPath(name: string): string | null { - return findExecutableInDirectories(name, (process.env.PATH || '').split(pathDelimiter)); -} /** Locate the claude binary, returning null if not found. */ function findClaudeBin(): string | null { - for (const envKey of ['OPENSCOUT_CLAUDE_BIN', 'SCOUT_CLAUDE_BIN', 'CLAUDE_BIN']) { - const explicit = process.env[envKey]?.trim(); - if (!explicit) continue; - const expanded = expandHomePath(explicit); - if (isExecutablePath(expanded)) return pathResolve(expanded); - const foundOnPath = findExecutableOnPath(explicit); - if (foundOnPath) return foundOnPath; - } - - const home = process.env.HOME || '/tmp'; - return findExecutableInDirectories('claude', [ - join(home, '.local', 'bin'), - join(home, '.claude', 'local'), - '/opt/homebrew/bin', - '/usr/local/bin', - join(home, '.bun', 'bin'), - ]) ?? findExecutableOnPath('claude'); + return findBin('claude', 'CLAUDE_BIN'); } /** Locate the pi binary, returning null if not found. */ @@ -252,18 +200,11 @@ function findPiBin(): string | null { return findBin('pi', 'PI_BIN'); } -/** Locate the user's shell, returning null if not found. */ +/** Locate the user's shell, falling back to common POSIX shells. */ function findShellBin(): string | null { - const candidates = [ - process.env.SHELL, - '/bin/zsh', - '/bin/bash', - '/bin/sh', - ].filter((candidate): candidate is string => Boolean(candidate)); - for (const candidate of candidates) { - if (existsSync(candidate)) return candidate; - } - return null; + const configured = process.env.SHELL; + if (configured && existsSync(configured)) return configured; + return findBin('zsh') ?? findBin('bash') ?? findBin('sh') ?? (existsSync('/bin/sh') ? '/bin/sh' : null); } /** Map Hudson-facing provider ids to the exact provider names accepted by the Pi CLI. */ @@ -283,24 +224,6 @@ function tmuxSessionExists(name: string): boolean { } } -/** Resize the tmux window behind an attached bridge PTY. */ -function resizeTmuxWindow(name: string, cols: number, rows: number): boolean { - try { - execFileSync('tmux', [ - 'resize-window', - '-t', - name, - '-x', - String(cols), - '-y', - String(rows), - ], { stdio: 'ignore' }); - return true; - } catch { - return false; - } -} - /** Bootstrap workspace files into a directory (only creates if missing). */ function bootstrapFiles(cwd: string, files: Record, sessionId: string) { for (const [relPath, content] of Object.entries(files)) { @@ -324,15 +247,15 @@ function spawnTmuxSession( cols: number, rows: number, cwd: string, - claudeBin: string, - claudeArgs: string[], + commandBin: string, + commandArgs: string[], env: Record, ): IPty { const exists = tmuxSessionExists(tmuxName); if (!exists) { - // Create the tmux session detached, running claude inside it - const shellCmd = [claudeBin, ...claudeArgs].map(a => a.includes(' ') ? `'${a}'` : a).join(' '); + // Create the tmux session detached, running the requested command inside it. + const shellCmd = [commandBin, ...commandArgs].map(a => a.includes(' ') ? `'${a}'` : a).join(' '); execSync( `tmux new-session -d -s ${tmuxName} -x ${cols} -y ${rows} -c '${cwd}' '${shellCmd}'`, { env: env as NodeJS.ProcessEnv }, @@ -340,7 +263,7 @@ function spawnTmuxSession( console.log(`[relay] Created tmux session: ${tmuxName}`); } else { // Resize existing session to match client - resizeTmuxWindow(tmuxName, cols, rows); + try { execSync(`tmux resize-window -t ${tmuxName} -x ${cols} -y ${rows} 2>/dev/null`); } catch {} console.log(`[relay] Attaching to existing tmux session: ${tmuxName}`); } @@ -363,12 +286,12 @@ export function createSession(ws: RelaySocket, msg: SessionInitMessage): Session const tmuxName = msg.tmuxSession || `hudson-${id}`; const agent = msg.agent || 'claude'; - // ---- Pre-flight: locate agent binary ---- + // ---- Pre-flight: locate command binary ---- let agentBin: string | null; if (agent === 'shell') { agentBin = findShellBin(); if (!agentBin) { - const reason = 'Shell not found. Set SHELL or install zsh/bash/sh.'; + const reason = 'No login shell found. Set SHELL or install zsh, bash, or sh.'; console.error(`[relay] Session ${id} failed: ${reason}`); send(ws, { type: 'session:error', error: reason }); return null; @@ -384,7 +307,7 @@ export function createSession(ws: RelaySocket, msg: SessionInitMessage): Session } else { agentBin = findClaudeBin(); if (!agentBin) { - const reason = 'Claude CLI not found. Install it with: curl -fsSL https://claude.ai/install.sh | bash'; + const reason = 'Claude CLI not found. Install it with: npm install -g @anthropic-ai/claude-code'; console.error(`[relay] Session ${id} failed: ${reason}`); send(ws, { type: 'session:error', error: reason }); return null; @@ -414,11 +337,12 @@ export function createSession(ws: RelaySocket, msg: SessionInitMessage): Session bootstrapFiles(cwd, msg.workspaceFiles, id); } - // ---- Build CLI arguments based on agent type ---- + // ---- Build command arguments based on process type ---- let agentArgs: string[]; if (agent === 'shell') { - agentArgs = []; + const shellName = agentBin.split('/').pop() ?? ''; + agentArgs = shellName === 'sh' ? [] : ['-l']; } else if (agent === 'pi') { agentArgs = ['--verbose']; const provider = normalizePiProviderForCli(msg.provider); From 4d4cf80a0ad297a4ac53ac956bcc451c891dd9e1 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 16:22:52 -0400 Subject: [PATCH 06/14] =?UTF-8?q?=E2=9C=A8=20Stream=20stage=20body=20via?= =?UTF-8?q?=20Suspense;=20chrome=20renders=20immediately?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page handler now only awaits the cheap inventory before returning. Everything panel-dependent (parse, extract, enrich) moves into an async wrapped in . When you click a session pill or a pipeline chip, chrome (header, picker, pipeline strip, stage header, prev/next nav) renders instantly while the stage body shows a structured in-flight skeleton. Skeleton lists the commands that will run for the active stage with expected timing — Inventory · Parse session · Extract QMD · Enrich (LLM). For Enrich the user immediately sees "first run: 5–15 s" so the wait is explained, not opaque. Suspense boundary keyed on session+stage so each click resets the boundary cleanly. RunSummary moves inside StageBody so its log is built after all panel commands resolve. This pairs with the existing run trace footer: skeleton = expected sequence, trace = what actually ran. --- .../app/studies/session-search/page.tsx | 230 +++++++++++++----- 1 file changed, 168 insertions(+), 62 deletions(-) diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index dcea6ffc..6abd3e64 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -23,6 +23,7 @@ import { summarizeRunLog, type RunLogEntry, } from "@/lib/studio/run-log"; +import { Suspense } from "react"; type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; @@ -206,63 +207,14 @@ export default async function SessionSearchStudyPage({ artifact: params.artifact, }; + // Inventory is cheap and the page header needs its totals. Always run it + // synchronously before sending chrome to the client. const inventoryRun = await runCommand(inventoryCommand, { since: "7d" }); const inventory = inventoryRun.output; const weekFootprint = inventoryRun.error ? "scan failed" : `${formatCount(inventory.totalFiles)} files · ${formatBytes(inventory.totalBytes)} · ${inventory.windowDays} days`; - // ── Orchestrate per-stage runs so the run summary footer can show what - // ── happened underneath the current page render. - const runLog: RunLogEntry[] = [ - makeRunLogEntry(inventoryCommand, inventoryRun), - ]; - - const sessionSlug = `${selectedSession.harness.toLowerCase()}-${selectedSession.tier}`; - const needsParse = stageId === "normalize" || stageId === "extract" || stageId === "enrich"; - const needsExtract = stageId === "extract" || stageId === "enrich"; - - let normalizeRun: CommandRun | undefined; - if (needsParse) { - const limit = stageId === "normalize" ? 14 : 1500; - normalizeRun = await runCommand(parseSessionCommand, { - path: selectedSession.fullPath, - limit, - }); - runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); - } - - let extractRun: CommandRun | undefined; - if (needsExtract) { - extractRun = await runCommand(extractQmdCommand, { - path: selectedSession.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }); - runLog.push(makeRunLogEntry(extractQmdCommand, extractRun)); - } - - let enrichRun: CommandRun | undefined; - if (stageId === "enrich") { - enrichRun = await runCommand(enrichSessionCommand, { - path: selectedSession.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }); - runLog.push( - makeRunLogEntry(enrichSessionCommand, enrichRun, (out) => - out?.model - ? { - model: out.model, - promptTokens: out.usage.promptTokens, - completionTokens: out.usage.completionTokens, - reasoningTokens: out.usage.reasoningTokens, - } - : undefined, - ), - ); - } - return (
      {/* ── Header ─────────────────────────────────────────────────── */} @@ -323,23 +275,177 @@ export default async function SessionSearchStudyPage({ next={nextStage} /> - + + } + > + +
    - - ); } +// ── Stage body (streams in after commands resolve) ─────────────── + +async function StageBody({ + stage, + session, + selection, + inventoryRun, +}: { + stage: Stage; + session: SessionSample; + selection: StudySelection; + inventoryRun: CommandRun; +}) { + const stageId = stage.id; + const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; + const needsParse = stageId === "normalize" || stageId === "extract" || stageId === "enrich"; + const needsExtract = stageId === "extract" || stageId === "enrich"; + + const runLog: RunLogEntry[] = [makeRunLogEntry(inventoryCommand, inventoryRun)]; + + let normalizeRun: CommandRun | undefined; + if (needsParse) { + const limit = stageId === "normalize" ? 14 : 1500; + normalizeRun = await runCommand(parseSessionCommand, { + path: session.fullPath, + limit, + }); + runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); + } + + let extractRun: CommandRun | undefined; + if (needsExtract) { + extractRun = await runCommand(extractQmdCommand, { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }); + runLog.push(makeRunLogEntry(extractQmdCommand, extractRun)); + } + + let enrichRun: CommandRun | undefined; + if (stageId === "enrich") { + enrichRun = await runCommand(enrichSessionCommand, { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }); + runLog.push( + makeRunLogEntry(enrichSessionCommand, enrichRun, (out) => + out?.model + ? { + model: out.model, + promptTokens: out.usage.promptTokens, + completionTokens: out.usage.completionTokens, + reasoningTokens: out.usage.reasoningTokens, + } + : undefined, + ), + ); + } + + return ( + <> + +
    + +
    + + ); +} + +function StageBodySkeleton({ + stageId, + stageLabel, +}: { + stageId: StageId; + stageLabel: string; +}) { + const expected = expectedCommands(stageId); + return ( +
    +
    +
    + + running {stageLabel.toLowerCase()} + + + + in flight + +
    +
      + {expected.map((cmd, i) => ( +
    • + + {String(i + 1).padStart(2, "0")} + + + {cmd.label} + {cmd.id} + + + {cmd.eta} + +
      • + ))} +
    +
    + Chrome rendered; panel and run trace stream in when commands resolve. +
    +
    +
    + ); +} + +function expectedCommands(stageId: StageId): Array<{ + id: string; + label: string; + eta: string; +}> { + const inv = { id: "inventory", label: "Inventory", eta: "cached if recent" }; + const parse = { id: "parse-session", label: "Parse session", eta: "~100 ms · cached" }; + const extract = { + id: "extract-qmd", + label: "Extract QMD (mechanical)", + eta: "~ms · cached", + }; + const enrich = { + id: "enrich-session", + label: "Enrich (LLM)", + eta: "first run: 5–15 s", + }; + if (stageId === "discover") return [inv]; + if (stageId === "normalize") return [inv, parse]; + if (stageId === "extract") return [inv, parse, extract]; + return [inv, parse, extract, enrich]; +} + // ── Session picker (single inline row) ─────────────────────────── function SessionPickerRow({ From 6bfe28f274bc9690abd489483807456bd7dc5456 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 17:01:58 -0400 Subject: [PATCH 07/14] =?UTF-8?q?=E2=9C=A8=20Smooth=20artifact=20switching?= =?UTF-8?q?=20+=20force=20re-run=20+=20drop=20mechanical=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three workbench polish moves prompted by usage: 1. ArtifactPicker client island — Extract / Enrich panels pre-read every artifact's content server-side and hand the array to a "use client" component that swaps preview body via local state. URL stays in sync via router.replace (pushState) so links are shareable, but clicking a file does not navigate, does not refetch, does not flash the chrome. Verified zero document loads + zero HTTP doc requests on click. 2. Force re-run — runCommand learns an optional { force: true } that bypasses the cache lookup. URL ?force= threads it through to the active stage; CommandSurface gains a "re-run ↻" link in the command header. Lets you see Extract / Normalize / Enrich actually run instead of always reading "cached". 3. Drop "(mechanical)" labels now that Enrich is split out — Extract's single output kind speaks for itself. Suspense boundary key includes the force param so a re-run triggers the in-flight skeleton properly. --- .../app/studies/session-search/page.tsx | 205 +++++++----------- .../components/studio/ArtifactPicker.tsx | 105 +++++++++ .../components/studio/CommandSurface.tsx | 18 +- design/studio/lib/studio/command.ts | 3 +- .../studio/lib/studio/commands/extract-qmd.ts | 3 +- 5 files changed, 201 insertions(+), 133 deletions(-) create mode 100644 design/studio/components/studio/ArtifactPicker.tsx diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index 6abd3e64..a7f718ee 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -18,6 +18,10 @@ import { type EnrichedFile, } from "@/lib/studio/commands/enrich-session"; import { CommandSurface } from "@/components/studio/CommandSurface"; +import { + ArtifactPicker, + type ArtifactPickerFile, +} from "@/components/studio/ArtifactPicker"; import { makeRunLogEntry, summarizeRunLog, @@ -57,6 +61,8 @@ interface PageProps { step?: string; /** Active artifact in the Extract panel preview. */ artifact?: string; + /** Stage id whose command should bypass the cache for this request. */ + force?: string; }>; } @@ -64,6 +70,8 @@ interface StudySelection { sessionId: string; stageId: StageId; artifact?: string; + /** Stage id whose command should be force-rerun. */ + force?: StageId; } const SESSIONS: SessionSample[] = [ @@ -171,9 +179,9 @@ const STAGES: Stage[] = [ label: "Extract", verb: "derive", input: "normalized records", - output: "mechanical QMD files", + output: "QMD sidecar files", summary: - "Mechanical pass: emit files.md / tool-calls.md / events-NNN.md / manifest.json from the normalized records. No LLM, always fast. Outputs land at $TMPDIR/scout-study/qmd//.", + "Emit files.md / tool-calls.md / events-NNN.md / manifest.json from the normalized records. Pure derivation, always fast. Outputs land at $TMPDIR/scout-study/qmd//.", }, { id: "enrich", @@ -201,10 +209,13 @@ export default async function SessionSearchStudyPage({ const selectedStage = STAGES[stageIndex]!; const prevStage = stageIndex > 0 ? STAGES[stageIndex - 1] : undefined; const nextStage = stageIndex < STAGES.length - 1 ? STAGES[stageIndex + 1] : undefined; + const forceStage = + params.force && isStageId(params.force) ? params.force : undefined; const selection: StudySelection = { sessionId: selectedSession.id, stageId, artifact: params.artifact, + force: forceStage, }; // Inventory is cheap and the page header needs its totals. Always run it @@ -276,7 +287,7 @@ export default async function SessionSearchStudyPage({ /> ; }) { const stageId = stage.id; + const force = selection.force; const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; const needsParse = stageId === "normalize" || stageId === "extract" || stageId === "enrich"; const needsExtract = stageId === "extract" || stageId === "enrich"; @@ -320,30 +332,39 @@ async function StageBody({ let normalizeRun: CommandRun | undefined; if (needsParse) { const limit = stageId === "normalize" ? 14 : 1500; - normalizeRun = await runCommand(parseSessionCommand, { - path: session.fullPath, - limit, - }); + normalizeRun = await runCommand( + parseSessionCommand, + { path: session.fullPath, limit }, + { force: force === "normalize" }, + ); runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); } let extractRun: CommandRun | undefined; if (needsExtract) { - extractRun = await runCommand(extractQmdCommand, { - path: session.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }); + extractRun = await runCommand( + extractQmdCommand, + { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }, + { force: force === "extract" }, + ); runLog.push(makeRunLogEntry(extractQmdCommand, extractRun)); } let enrichRun: CommandRun | undefined; if (stageId === "enrich") { - enrichRun = await runCommand(enrichSessionCommand, { - path: session.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }); + enrichRun = await runCommand( + enrichSessionCommand, + { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }, + { force: force === "enrich" }, + ); runLog.push( makeRunLogEntry(enrichSessionCommand, enrichRun, (out) => out?.model @@ -432,7 +453,7 @@ function expectedCommands(stageId: StageId): Array<{ const parse = { id: "parse-session", label: "Parse session", eta: "~100 ms · cached" }; const extract = { id: "extract-qmd", - label: "Extract QMD (mechanical)", + label: "Extract QMD", eta: "~ms · cached", }; const enrich = { @@ -675,7 +696,13 @@ function StagePanel({ case "discover": return ; case "normalize": - return ; + return ( + + ); case "extract": return ( ; + selection: StudySelection; }) { const limit = 14; const records = run.output?.records ?? []; @@ -830,6 +859,7 @@ function NormalizePanel({ } footnote={ run.output && !run.output.error ? ( @@ -1032,12 +1062,8 @@ async function ExtractPanel({ const result = run.output; const files = result?.files ?? []; const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; - const selectedName = - files.find((f) => f.name === selection.artifact)?.name ?? - files.find((f) => f.name === "files.md")?.name ?? - files[0]?.name; - const selected = files.find((f) => f.name === selectedName); - const previewContent = selected ? await safeReadFile(selected.path) : ""; + const filesWithContent = await loadFileContents(files); + const initialSelected = selection.artifact ?? "files.md"; return (
    @@ -1047,19 +1073,18 @@ async function ExtractPanel({ sessionId: sessionSlug, })} run={run} + rerunHref={studyHref(selection, { force: "extract" })} body={ - } footnote={ result && !result.error ? ( <> - Wrote {files.length} mechanical files to{" "} + Wrote {files.length} files to{" "} {shortenTmpPath(result.outDir)} {" "} @@ -1072,6 +1097,18 @@ async function ExtractPanel({ ); } +async function loadFileContents( + files: Array<{ name: string; path: string; bytes: number }>, +): Promise { + return Promise.all( + files.map(async (f) => ({ + name: f.name, + bytes: f.bytes, + content: await safeReadFile(f.path), + })), + ); +} + async function EnrichPanel({ session, selection, @@ -1084,12 +1121,8 @@ async function EnrichPanel({ const result = run.output; const files = result?.files ?? []; const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; - const selectedName = - files.find((f) => f.name === selection.artifact)?.name ?? - files.find((f) => f.name === "overview.md")?.name ?? - files[0]?.name; - const selected = files.find((f) => f.name === selectedName); - const previewContent = selected ? await safeReadFile(selected.path) : ""; + const filesWithContent = await loadFileContents(files); + const initialSelected = selection.artifact ?? "overview.md"; return (
    @@ -1099,12 +1132,11 @@ async function EnrichPanel({ sessionId: sessionSlug, })} run={run} + rerunHref={studyHref(selection, { force: "enrich" })} body={ - } @@ -1152,91 +1184,6 @@ async function safeReadFile(p: string): Promise { } } -type ArtifactFile = ExtractedFile | EnrichedFile; - -function ArtifactFilesBody({ - files, - selection, - selected, - previewContent, - emptyMessage, -}: { - files: ArtifactFile[]; - selection: StudySelection; - selected: ArtifactFile | undefined; - previewContent: string; - emptyMessage: string; -}) { - if (files.length === 0) { - return ( - 
    -        {emptyMessage}
-
    - ); - } - return ( -
    - -
    -
    - preview · {selected?.name ?? "—"} -
    - 
    -          {previewContent || "(empty)"}
-
    -
    -
    - ); -} - -function ExtractedFilesBody(props: { - files: ExtractedFile[]; - selection: StudySelection; - selected: ExtractedFile | undefined; - previewContent: string; - emptyMessage: string; -}) { - return ; -} - -function EnrichedFilesBody(props: { - files: EnrichedFile[]; - selection: StudySelection; - selected: EnrichedFile | undefined; - previewContent: string; - emptyMessage: string; -}) { - return ; -} - - function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); } @@ -1337,8 +1284,10 @@ function studyHref( const params = new URLSearchParams(); params.set("session", next.sessionId ?? current.sessionId); params.set("step", next.stageId ?? current.stageId); - const artifact = next.artifact ?? current.artifact; + const artifact = "artifact" in next ? next.artifact : current.artifact; if (artifact) params.set("artifact", artifact); + const force = "force" in next ? next.force : undefined; + if (force) params.set("force", force); return `/studies/session-search?${params.toString()}`; } diff --git a/design/studio/components/studio/ArtifactPicker.tsx b/design/studio/components/studio/ArtifactPicker.tsx new file mode 100644 index 00000000..aff0a29e --- /dev/null +++ b/design/studio/components/studio/ArtifactPicker.tsx @@ -0,0 +1,105 @@ +"use client"; + +import { useState, useTransition } from "react"; +import { useRouter, useSearchParams } from "next/navigation"; + +export interface ArtifactPickerFile { + name: string; + bytes: number; + content: string; +} + +/** + * Client-side artifact list + preview swap. + * + * The server pre-reads every file's content and hands the array down. Switching + * files is instant — local state flips the preview body. The URL stays in sync + * via router.replace so the link is still shareable, but no server round-trip + * happens on click. + */ +export function ArtifactPicker({ + files, + initialSelected, + emptyMessage, +}: { + files: ArtifactPickerFile[]; + initialSelected?: string; + emptyMessage?: string; +}) { + const router = useRouter(); + const searchParams = useSearchParams(); + const [, startTransition] = useTransition(); + const initial = + files.find((f) => f.name === initialSelected)?.name ?? files[0]?.name ?? ""; + const [selectedName, setSelectedName] = useState(initial); + + if (files.length === 0) { + return ( + 
    +        {emptyMessage ?? "no files"}
+
    + ); + } + + const selected = files.find((f) => f.name === selectedName) ?? files[0]!; + + function select(name: string) { + setSelectedName(name); + startTransition(() => { + const params = new URLSearchParams(searchParams?.toString() ?? ""); + params.set("artifact", name); + router.replace(`?${params.toString()}`, { scroll: false }); + }); + } + + return ( +
    +
      +
    • + file + bytes +
      • + {files.map((f) => { + const active = selected.name === f.name; + return ( +
    • + +
      • + ); + })} +
    +
    +
    + preview · {selected.name} +
    + 
    +          {selected.content || "(empty)"}
+
    +
    +
    + ); +} + +function formatBytes(bytes: number): string { + if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)} MiB`; + if (bytes >= 1024) { + const value = bytes / 1024; + return `${value >= 100 ? Math.round(value) : value.toFixed(1)} KiB`; + } + return `${bytes} B`; +} diff --git a/design/studio/components/studio/CommandSurface.tsx b/design/studio/components/studio/CommandSurface.tsx index 74d913ed..5db16321 100644 --- a/design/studio/components/studio/CommandSurface.tsx +++ b/design/studio/components/studio/CommandSurface.tsx @@ -13,11 +13,14 @@ export function CommandSurface({ run, body, footnote, + rerunHref, }: { shell: string; run: Pick, "durationMs" | "cached" | "error">; body: ReactNode; footnote?: ReactNode; + /** When provided, renders a "re-run" link that navigates to this URL. */ + rerunHref?: string; }) { const badge = run.error ? { label: "● error", tone: "text-status-error-fg" } @@ -31,8 +34,19 @@ export function CommandSurface({ command - - {badge.label} + + {rerunHref ? ( + + re-run ↻ + + ) : null} + + {badge.label} + 
    diff --git a/design/studio/lib/studio/command.ts b/design/studio/lib/studio/command.ts
index 90e261c2..20487621 100644
--- a/design/studio/lib/studio/command.ts
+++ b/design/studio/lib/studio/command.ts
@@ -47,12 +47,13 @@ function entryKey(cmd: Command, input: I): string {
 export async function runCommand(
   cmd: Command,
   input: I,
+  options?: { force?: boolean },
 ): Promise> {
   const ttl = cmd.cacheTtlMs ?? 0;
   const key = entryKey(cmd, input);
   const now = Date.now();
 
-  if (ttl > 0) {
+  if (ttl > 0 && !options?.force) {
     const hit = cache.get(key);
     if (hit && now - hit.at < ttl) {
       return { ...(hit.run as CommandRun), cached: true };
diff --git a/design/studio/lib/studio/commands/extract-qmd.ts b/design/studio/lib/studio/commands/extract-qmd.ts
index d8b0208a..01a02d7a 100644
--- a/design/studio/lib/studio/commands/extract-qmd.ts
+++ b/design/studio/lib/studio/commands/extract-qmd.ts
@@ -38,13 +38,12 @@ const DEFAULT_LIMIT = 1500;
 
 export const extractQmdCommand: Command = {
   id: "extract-qmd",
-  label: "Extract QMD (mechanical)",
+  label: "Extract QMD",
   shell: ({ path: p, sessionId }) =>
     [
       `scout qmd extract`,
       `--source ${shellQuote(shrinkPath(p))}`,
       `--out ${shellQuote(shrinkPath(path.join(ROOT, sessionId)))}`,
-      `--mechanical-only`,
     ].join(" "),
   run: async ({ path: filePath, sessionId, recordLimit }) => {
     const limit = recordLimit ?? DEFAULT_LIMIT;

From 04f06cf3a18bf8aed1134d2fa97875c698525b83 Mon Sep 17 00:00:00 2001
From: Arach 
Date: Sat, 30 May 2026 20:57:34 -0400
Subject: [PATCH 08/14] =?UTF-8?q?=E2=9C=A8=20Real=20Normalize=20timing=20+?=
 =?UTF-8?q?=20top-level=20/=20per-row=20re-run=20controls?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- parseSessionCommand reads from the head in 64 KiB chunks until it has
  `limit` newlines (or EOF). Previously it read one 128 KiB buffer, which
  capped any session at whatever fit in that window — Normalize at
  limit=14 was always ~20 ms regardless of source size.

- Lift Normalize's record limit from 14 to 1500, matching Extract /
  Enrich so all three share the parse-session cache key. Force-re-running
  Normalize now shows a meaningful workload (~150–340 ms on codex-large
  parsing 1500 records).

- NormalizedStreamBody caps the visible stream at 30 rows + a "N parsed
  but hidden · M not parsed" tail so the page stays compact.

- Force handling expands: ?force=all bypasses every command's cache in
  the active pipeline; ?force= still works for a single stage.
  Inventory honors force too via the discover stage id.

- Three new re-run affordances:
  * "re-run all ↻" link in the page header (force=all)
  * "re-run ↻" already exists in each CommandSurface header
  * per-row "re-run ↻" links in the run trace footer, mapping command id
    to its stage so you can rerun any single step from the trace

Suspense key includes the force param so the in-flight skeleton appears
on force-rerun.
---
 .../app/studies/session-search/page.tsx       | 187 ++++++++++++------
 .../lib/studio/commands/parse-session.ts      |  41 ++--
 2 files changed, 150 insertions(+), 78 deletions(-)

diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx
index a7f718ee..a23443e8 100644
--- a/design/studio/app/studies/session-search/page.tsx
+++ b/design/studio/app/studies/session-search/page.tsx
@@ -70,8 +70,18 @@ interface StudySelection {
   sessionId: string;
   stageId: StageId;
   artifact?: string;
-  /** Stage id whose command should be force-rerun. */
-  force?: StageId;
+  /**
+   * What to force-rerun (bypassing cache):
+   * - "all" — every command in the active pipeline
+   * - a stage id ("discover" | "normalize" | "extract" | "enrich") — just that one
+   */
+  force?: string;
+}
+
+const FORCE_ALL = "all";
+
+function shouldForce(force: string | undefined, stageId: StageId): boolean {
+  return force === FORCE_ALL || force === stageId;
 }
 
 const SESSIONS: SessionSample[] = [
@@ -209,18 +219,25 @@ export default async function SessionSearchStudyPage({
   const selectedStage = STAGES[stageIndex]!;
   const prevStage = stageIndex > 0 ? STAGES[stageIndex - 1] : undefined;
   const nextStage = stageIndex < STAGES.length - 1 ? STAGES[stageIndex + 1] : undefined;
-  const forceStage =
-    params.force && isStageId(params.force) ? params.force : undefined;
+  const force = params.force
+    ? params.force === FORCE_ALL || isStageId(params.force)
+      ? params.force
+      : undefined
+    : undefined;
   const selection: StudySelection = {
     sessionId: selectedSession.id,
     stageId,
     artifact: params.artifact,
-    force: forceStage,
+    force,
   };
 
   // Inventory is cheap and the page header needs its totals. Always run it
   // synchronously before sending chrome to the client.
-  const inventoryRun = await runCommand(inventoryCommand, { since: "7d" });
+  const inventoryRun = await runCommand(
+    inventoryCommand,
+    { since: "7d" },
+    { force: shouldForce(force, "discover") },
+  );
   const inventory = inventoryRun.output;
   const weekFootprint = inventoryRun.error
     ? "scan failed"
@@ -251,6 +268,14 @@ export default async function SessionSearchStudyPage({
             
           
           ·
+          
+            re-run all ↻
+          
+          ·
           
 
          | undefined;
   if (needsParse) {
-    const limit = stageId === "normalize" ? 14 : 1500;
+    // One limit across normalize / extract / enrich so the parse cache is
+    // shared. Older builds capped normalize at 14 records, which made the
+    // force-rerun report meaningless timing (~20 ms over a 128 KB head read).
     normalizeRun = await runCommand(
       parseSessionCommand,
-      { path: session.fullPath, limit },
-      { force: force === "normalize" },
+      { path: session.fullPath, limit: 1500 },
+      { force: shouldForce(force, "normalize") },
     );
     runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun));
   }
@@ -349,7 +376,7 @@ async function StageBody({
         sessionId: sessionSlug,
         recordLimit: 1500,
       },
-      { force: force === "extract" },
+      { force: shouldForce(force, "extract") },
     );
     runLog.push(makeRunLogEntry(extractQmdCommand, extractRun));
   }
@@ -363,7 +390,7 @@ async function StageBody({
         sessionId: sessionSlug,
         recordLimit: 1500,
       },
-      { force: force === "enrich" },
+      { force: shouldForce(force, "enrich") },
     );
     runLog.push(
       makeRunLogEntry(enrichSessionCommand, enrichRun, (out) =>
@@ -391,7 +418,7 @@ async function StageBody({
         selection={selection}
       />
       
    
-        
+        
       
    
     
   );
@@ -991,6 +1018,8 @@ const NORMALIZED_KIND_TONE: Record = {
   unknown: "text-status-error-fg",
 };
 
+const STREAM_DISPLAY_CAP = 30;
+
 function NormalizedStreamBody({
   result,
   moreCount,
@@ -1005,6 +1034,9 @@ function NormalizedStreamBody({
    ); } + const visible = result.records.slice(0, STREAM_DISPLAY_CAP); + const cappedHere = Math.max(0, result.records.length - visible.length); + const totalTail = cappedHere + moreCount; return (
    @@ -1014,15 +1046,19 @@ function NormalizedStreamBody({ detail
      - {result.records.map((r) => ( + {visible.map((r) => ( ))} - {moreCount > 0 ? ( + {totalTail > 0 ? (
    • … - {formatCount(moreCount)} more + {formatCount(totalTail)} more — - source-ordered + + {cappedHere > 0 + ? `${formatCount(cappedHere)} parsed but hidden · ${formatCount(moreCount)} not parsed` + : "not parsed in this run"} +
      • ) : null}
    @@ -1190,7 +1226,20 @@ function isStageId(value: string): value is StageId { // ── Run summary footer ─────────────────────────────────────────── -function RunSummary({ entries }: { entries: RunLogEntry[] }) { +const COMMAND_TO_STAGE: Record = { + inventory: "discover", + "parse-session": "normalize", + "extract-qmd": "extract", + "enrich-session": "enrich", +}; + +function RunSummary({ + entries, + selection, +}: { + entries: RunLogEntry[]; + selection: StudySelection; +}) { const sum = summarizeRunLog(entries); return (
    @@ -1224,54 +1273,66 @@ function RunSummary({ entries }: { entries: RunLogEntry[] }) {
    ); diff --git a/design/studio/lib/studio/commands/parse-session.ts b/design/studio/lib/studio/commands/parse-session.ts index 6fd14f9e..4226de5e 100644 --- a/design/studio/lib/studio/commands/parse-session.ts +++ b/design/studio/lib/studio/commands/parse-session.ts @@ -118,25 +118,36 @@ function detectHarness(filePath: string): "codex" | "claude" | "unknown" { } async function readHeadLines(filePath: string, limit: number): Promise { - // Read up to ~128 KB to cover N lines for typical sessions. If lines are - // unusually long, the parser will just deliver fewer records than asked - // for — fine for a stream preview. + // Stream from the head until we have `limit` newlines (or hit EOF). Keeps + // memory bounded while letting normalize see a real workload, not just the + // first 128 KB. const fh = await fs.open(filePath, "r"); try { - const buf = Buffer.alloc(128 * 1024); - const { bytesRead } = await fh.read(buf, 0, buf.length, 0); - const text = buf.slice(0, bytesRead).toString("utf8"); - // Trim to roughly the requested number of lines so we don't carry a - // huge tail of the buffer into the parser. - let cut = 0; - let seen = 0; - for (let i = 0; i < text.length && seen <= limit; i++) { - if (text[i] === "\n") { - seen++; - cut = i + 1; + const chunk = Buffer.alloc(64 * 1024); + const collected: Buffer[] = []; + let lines = 0; + let pos = 0; + let lastCutTotal = 0; + let runningTotal = 0; + while (lines <= limit) { + const { bytesRead } = await fh.read(chunk, 0, chunk.length, pos); + if (bytesRead === 0) break; + pos += bytesRead; + const slice = Buffer.from(chunk.subarray(0, bytesRead)); + collected.push(slice); + for (let i = 0; i < bytesRead; i++) { + if (slice[i] === 0x0a) { + lines++; + if (lines > limit) { + lastCutTotal = runningTotal + i + 1; + break; + } + } } + runningTotal += bytesRead; } - return cut > 0 ? text.slice(0, cut) : text; + const text = Buffer.concat(collected).toString("utf8"); + return lastCutTotal > 0 ? text.slice(0, lastCutTotal) : text; } finally { await fh.close(); } From 43d28409bf555437b8b07c5f7dbbea30436e65e9 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 21:06:02 -0400 Subject: [PATCH 09/14] =?UTF-8?q?=E2=9C=A8=20Trace=20inspect=20+=20run-in-?= =?UTF-8?q?place=20re-run=20with=20pending=20indicator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two responsiveness wins: 1. Run-in-place — RerunLink Client Component wraps every re-run affordance (header "re-run all", in-CommandSurface "re-run", per-row trace re-runs). It uses useTransition so the previous UI stays mounted while the navigation streams in. The link swaps its label to "running ↻", gets aria-busy=true, and pulses (animate-pulse + status-info-fg) while the request is in flight. The stage Suspense key drops the force param so force-rerun navigation no longer unmounts the panel — old data stays visible until the new render lands. Session / stage switches still show the skeleton (key still changes on those). 2. Trace inspect — each run-trace row is now a
    that expands to show the actual shell-equivalent, input summary, output summary, and resolved cache key for that command's run. makeRunLogEntry gains summarizeInput / summarizeOutput hooks so each command projects its inputs and outputs into one-line strings for the inspect drawer. --- .../app/studies/session-search/page.tsx | 247 +++++++++++------- .../components/studio/CommandSurface.tsx | 6 +- design/studio/components/studio/RerunLink.tsx | 53 ++++ design/studio/lib/studio/run-log.ts | 46 +++- 4 files changed, 260 insertions(+), 92 deletions(-) create mode 100644 design/studio/components/studio/RerunLink.tsx diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index a23443e8..8e5dc5a5 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -22,6 +22,7 @@ import { ArtifactPicker, type ArtifactPickerFile, } from "@/components/studio/ArtifactPicker"; +import { RerunLink } from "@/components/studio/RerunLink"; import { makeRunLogEntry, summarizeRunLog, @@ -268,13 +269,14 @@ export default async function SessionSearchStudyPage({ · - re-run all ↻ - + · `since=${i.since}`, + summarizeOutput: (o) => + o + ? `${o.totalFiles} files · ${o.totalBytes} bytes · ${o.rows.length} rows w/ events` + : "no result", + }), + ]; let normalizeRun: CommandRun | undefined; if (needsParse) { + const parseInput = { path: session.fullPath, limit: 1500 }; // One limit across normalize / extract / enrich so the parse cache is // shared. Older builds capped normalize at 14 records, which made the // force-rerun report meaningless timing (~20 ms over a 128 KB head read). - normalizeRun = await runCommand( - parseSessionCommand, - { path: session.fullPath, limit: 1500 }, - { force: shouldForce(force, "normalize") }, + normalizeRun = await runCommand(parseSessionCommand, parseInput, { + force: shouldForce(force, "normalize"), + }); + runLog.push( + makeRunLogEntry(parseSessionCommand, parseInput, normalizeRun, { + summarizeInput: (i) => `limit=${i.limit} · ${i.path.split("/").pop()}`, + summarizeOutput: (o) => + o + ? `harness=${o.harness} · ${o.records.length} records · ${o.bytesRead} bytes read` + : "no result", + }), ); - runLog.push(makeRunLogEntry(parseSessionCommand, normalizeRun)); } let extractRun: CommandRun | undefined; if (needsExtract) { - extractRun = await runCommand( - extractQmdCommand, - { - path: session.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }, - { force: shouldForce(force, "extract") }, + const extractInput = { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }; + extractRun = await runCommand(extractQmdCommand, extractInput, { + force: shouldForce(force, "extract"), + }); + runLog.push( + makeRunLogEntry(extractQmdCommand, extractInput, extractRun, { + summarizeInput: (i) => `sessionId=${i.sessionId} · limit=${i.recordLimit}`, + summarizeOutput: (o) => + o + ? `${o.files.length} files · ${o.recordsScanned} records · ${o.mechanicalMs} ms mech` + : "no result", + }), ); - runLog.push(makeRunLogEntry(extractQmdCommand, extractRun)); } let enrichRun: CommandRun | undefined; if (stageId === "enrich") { - enrichRun = await runCommand( - enrichSessionCommand, - { - path: session.fullPath, - sessionId: sessionSlug, - recordLimit: 1500, - }, - { force: shouldForce(force, "enrich") }, - ); + const enrichInput = { + path: session.fullPath, + sessionId: sessionSlug, + recordLimit: 1500, + }; + enrichRun = await runCommand(enrichSessionCommand, enrichInput, { + force: shouldForce(force, "enrich"), + }); runLog.push( - makeRunLogEntry(enrichSessionCommand, enrichRun, (out) => - out?.model - ? { - model: out.model, - promptTokens: out.usage.promptTokens, - completionTokens: out.usage.completionTokens, - reasoningTokens: out.usage.reasoningTokens, - } - : undefined, - ), + makeRunLogEntry(enrichSessionCommand, enrichInput, enrichRun, { + summarizeInput: (i) => + `sessionId=${i.sessionId} · limit=${i.recordLimit}`, + summarizeOutput: (o) => + o && o.model + ? `${o.model} · ${o.usage.totalTokens}t · ${o.files.length} files` + : "no result", + extractLlm: (out) => + out?.model + ? { + model: out.model, + promptTokens: out.usage.promptTokens, + completionTokens: out.usage.completionTokens, + reasoningTokens: out.usage.reasoningTokens, + } + : undefined, + }), ); } @@ -1224,6 +1255,40 @@ function isStageId(value: string): value is StageId { return STAGES.some((stage) => stage.id === value); } +function TraceRowInspect({ entry }: { entry: RunLogEntry }) { + if (!entry.trace) { + return ( +
    + no trace data + {entry.error ? <> · error: {entry.error} : null} +
    + ); + } + const { shell, input, output, cacheKey } = entry.trace; + return ( +
    +
    +
    + shell +
    +
    $ {shell}
    +
    + input +
    +
    {input}
    +
    + output +
    +
    {output}
    +
    + cache key +
    +
    {cacheKey}
    +
    +
    + ); +} + // ── Run summary footer ─────────────────────────────────────────── const COMMAND_TO_STAGE: Record = { @@ -1276,60 +1341,66 @@ function RunSummary({ {entries.map((e, i) => { const targetStage = COMMAND_TO_STAGE[e.id]; return ( -
  • - - {String(i + 1).padStart(2, "0")} - - - - {e.label} - - - {e.id} - - {targetStage ? ( - - re-run ↻ - - ) : null} - - - {e.error ? ( - error · {e.error} - ) : e.cached ? ( - - ● cached{" "} - - (saved ~{e.durationMs} ms) +
  • +
    + + + + ▶ + {String(i + 1).padStart(2, "0")} - ) : ( - ● ran · {e.durationMs} ms - )} - - - {e.llm ? ( - <> - {e.llm.model} ·{" "} - {e.llm.promptTokens}+{e.llm.completionTokens}t - {e.llm.reasoningTokens > 0 ? ( - - {" "} - ({e.llm.reasoningTokens} reasoning) - + + + {e.label} + + + {e.id} + + {targetStage ? ( + + re-run ↻ + ) : null} - - ) : ( - — - )} - + + + {e.error ? ( + error · {e.error} + ) : e.cached ? ( + + ● cached{" "} + + (saved ~{e.durationMs} ms) + + + ) : ( + ● ran · {e.durationMs} ms + )} + + + {e.llm ? ( + <> + {e.llm.model} ·{" "} + {e.llm.promptTokens}+{e.llm.completionTokens}t + {e.llm.reasoningTokens > 0 ? ( + + {" "} + ({e.llm.reasoningTokens} reasoning) + + ) : null} + + ) : ( + — + )} + + + +
    • ); })} diff --git a/design/studio/components/studio/CommandSurface.tsx b/design/studio/components/studio/CommandSurface.tsx index 5db16321..f14c775f 100644 --- a/design/studio/components/studio/CommandSurface.tsx +++ b/design/studio/components/studio/CommandSurface.tsx @@ -1,5 +1,6 @@ import type { ReactNode } from "react"; import type { CommandRun } from "@/lib/studio/command"; +import { RerunLink } from "@/components/studio/RerunLink"; /** * Chrome around a single command run. @@ -36,13 +37,14 @@ export function CommandSurface({ {rerunHref ? ( - re-run ↻ - + ) : null} {badge.label} diff --git a/design/studio/components/studio/RerunLink.tsx b/design/studio/components/studio/RerunLink.tsx new file mode 100644 index 00000000..d6706976 --- /dev/null +++ b/design/studio/components/studio/RerunLink.tsx @@ -0,0 +1,53 @@ +"use client"; + +import { useTransition, type ReactNode } from "react"; +import { useRouter } from "next/navigation"; + +/** + * Re-run link that uses a React transition so the previous UI stays + * mounted while the new render streams in, and pulses while it's pending. + * + * Used by the workbench's three re-run affordances (header "re-run all", + * in-CommandSurface re-run, per-trace-row re-run). + */ +export function RerunLink({ + href, + className, + pendingClassName = "animate-pulse text-status-info-fg", + title, + children, + pendingLabel, +}: { + href: string; + className?: string; + pendingClassName?: string; + title?: string; + children: ReactNode; + /** Optional swap-in label while the navigation is pending. */ + pendingLabel?: ReactNode; +}) { + const router = useRouter(); + const [isPending, startTransition] = useTransition(); + + function onClick(e: React.MouseEvent) { + if (e.metaKey || e.ctrlKey || e.shiftKey || e.button !== 0) return; + e.preventDefault(); + startTransition(() => { + router.push(href); + }); + } + + return ( + + {isPending && pendingLabel ? pendingLabel : children} + + ); +} diff --git a/design/studio/lib/studio/run-log.ts b/design/studio/lib/studio/run-log.ts index fbface60..d1911ded 100644 --- a/design/studio/lib/studio/run-log.ts +++ b/design/studio/lib/studio/run-log.ts @@ -15,6 +15,17 @@ export interface RunLlmCost { reasoningTokens: number; } +export interface RunTrace { + /** Copyable shell-equivalent for the run with bound input. */ + shell: string; + /** One-line summary of the bound input. */ + input: string; + /** One-line summary of the run output. */ + output: string; + /** Cache key used for this run. */ + cacheKey: string; +} + export interface RunLogEntry { id: string; label: string; @@ -22,6 +33,7 @@ export interface RunLogEntry { cached: boolean; error?: string; llm?: RunLlmCost; + trace?: RunTrace; } export interface RunLogSummary { @@ -53,19 +65,49 @@ const RATES: Record = { export function makeRunLogEntry( cmd: Command, + input: I, run: CommandRun, - extractLlm?: (output: O) => RunLlmCost | undefined, + options?: { + extractLlm?: (output: O) => RunLlmCost | undefined; + summarizeInput?: (input: I) => string; + summarizeOutput?: (output: O) => string; + }, ): RunLogEntry { + const trace: RunTrace | undefined = run.error + ? undefined + : { + shell: cmd.shell(input), + input: options?.summarizeInput + ? options.summarizeInput(input) + : safeJson(input), + output: options?.summarizeOutput + ? options.summarizeOutput(run.output) + : safeJson(run.output, 200), + cacheKey: cmd.cacheKey ? cmd.cacheKey(input) : JSON.stringify(input), + }; return { id: cmd.id, label: cmd.label, durationMs: run.durationMs, cached: run.cached, error: run.error, - llm: !run.error && extractLlm ? extractLlm(run.output) : undefined, + llm: + !run.error && options?.extractLlm ? options.extractLlm(run.output) : undefined, + trace, }; } +function safeJson(value: unknown, max = 240): string { + let text: string; + try { + text = JSON.stringify(value); + } catch { + text = String(value); + } + if (!text) text = ""; + return text.length > max ? text.slice(0, max - 1) + "…" : text; +} + export function summarizeRunLog(entries: RunLogEntry[]): RunLogSummary { const uncachedMs = entries.reduce((a, e) => a + e.durationMs, 0); const wallMs = entries.reduce((a, e) => a + (e.cached ? 0 : e.durationMs), 0); From a8622889090fde7384d6c14f0f178166e15bb6b6 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 21:13:33 -0400 Subject: [PATCH 10/14] =?UTF-8?q?=E2=9C=A8=20Drop=20the=201500-record=20tr?= =?UTF-8?q?uncation;=20parse=20the=20whole=20session?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three command inputs (parse-session, extract-qmd, enrich-session) now accept limit: number | \"all\". Defaults flip to \"all\" so every session is parsed end-to-end, not capped at a 1500-record head slice. readHeadLines streams from the file until EOF when the limit is Infinity, so memory stays bounded by the file size rather than a fixed buffer. parseSessionCommand.shell switches to \`cat …\` when limit is \"all\" so the copyable shell line stays honest. Measured on a fresh process: - codex-large (13 MiB, 4,220 events) normalize force: ~620 ms - claude-large (52 MiB, 12,009 events) normalize force: ~5.3 s The NormalizedStreamBody display cap of 30 rows still applies; tail text now reflects \"parsed but hidden\" rather than \"not parsed\" once nothing is being skipped. --- design/studio/app/studies/session-search/page.tsx | 6 +++--- .../studio/lib/studio/commands/enrich-session.ts | 6 +++--- design/studio/lib/studio/commands/extract-qmd.ts | 6 +++--- design/studio/lib/studio/commands/parse-session.ts | 14 +++++++++----- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index 8e5dc5a5..2654916d 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -367,7 +367,7 @@ async function StageBody({ let normalizeRun: CommandRun | undefined; if (needsParse) { - const parseInput = { path: session.fullPath, limit: 1500 }; + const parseInput = { path: session.fullPath, limit: "all" as const }; // One limit across normalize / extract / enrich so the parse cache is // shared. Older builds capped normalize at 14 records, which made the // force-rerun report meaningless timing (~20 ms over a 128 KB head read). @@ -390,7 +390,7 @@ async function StageBody({ const extractInput = { path: session.fullPath, sessionId: sessionSlug, - recordLimit: 1500, + recordLimit: "all" as const, }; extractRun = await runCommand(extractQmdCommand, extractInput, { force: shouldForce(force, "extract"), @@ -411,7 +411,7 @@ async function StageBody({ const enrichInput = { path: session.fullPath, sessionId: sessionSlug, - recordLimit: 1500, + recordLimit: "all" as const, }; enrichRun = await runCommand(enrichSessionCommand, enrichInput, { force: shouldForce(force, "enrich"), diff --git a/design/studio/lib/studio/commands/enrich-session.ts b/design/studio/lib/studio/commands/enrich-session.ts index 62dfae51..f19e76af 100644 --- a/design/studio/lib/studio/commands/enrich-session.ts +++ b/design/studio/lib/studio/commands/enrich-session.ts @@ -14,8 +14,8 @@ export interface EnrichSessionInput { path: string; /** Stable session id used for output directory. */ sessionId: string; - /** Cap on records considered. Matches the extract step's cap. */ - recordLimit?: number; + /** Cap on records considered, or "all" to read the entire parse result. */ + recordLimit?: number | "all"; } export interface EnrichedFile { @@ -39,7 +39,7 @@ export interface EnrichSessionResult { } const ROOT = path.join(tmpdir(), "scout-study", "qmd"); -const DEFAULT_LIMIT = 1500; +const DEFAULT_LIMIT: number | "all" = "all"; const PROMPT_CHAR_CAP = 24_000; export const enrichSessionCommand: Command = { diff --git a/design/studio/lib/studio/commands/extract-qmd.ts b/design/studio/lib/studio/commands/extract-qmd.ts index 01a02d7a..724a25f8 100644 --- a/design/studio/lib/studio/commands/extract-qmd.ts +++ b/design/studio/lib/studio/commands/extract-qmd.ts @@ -13,8 +13,8 @@ export interface ExtractQmdInput { path: string; /** Stable session id used for the output directory name. */ sessionId: string; - /** Cap on records parsed/extracted. Avoids loading huge sessions. */ - recordLimit?: number; + /** Cap on records parsed/extracted, or "all" to parse the entire file. */ + recordLimit?: number | "all"; } export interface ExtractedFile { @@ -34,7 +34,7 @@ export interface ExtractQmdResult { const ROOT = path.join(tmpdir(), "scout-study", "qmd"); const WINDOW = 350; -const DEFAULT_LIMIT = 1500; +const DEFAULT_LIMIT: number | "all" = "all"; export const extractQmdCommand: Command = { id: "extract-qmd", diff --git a/design/studio/lib/studio/commands/parse-session.ts b/design/studio/lib/studio/commands/parse-session.ts index 4226de5e..ec2fee7b 100644 --- a/design/studio/lib/studio/commands/parse-session.ts +++ b/design/studio/lib/studio/commands/parse-session.ts @@ -5,8 +5,8 @@ import type { Command } from "@/lib/studio/command"; export interface ParseSessionInput { /** Absolute path to a Codex or Claude JSONL session file. */ path: string; - /** Number of records to read from the head. */ - limit: number; + /** Number of records to read from the head, or "all" to parse the entire file. */ + limit: number | "all"; } export type NormalizedKind = @@ -57,17 +57,21 @@ export const parseSessionCommand: Command id: "parse-session", label: "Parse session", shell: ({ path: p, limit }) => - `head -n ${limit} ${shellQuote(shrinkPath(p))} | jq -c '.'`, + limit === "all" + ? `cat ${shellQuote(shrinkPath(p))} | jq -c '.'` + : `head -n ${limit} ${shellQuote(shrinkPath(p))} | jq -c '.'`, run: async ({ path: filePath, limit }) => { try { - const text = await readHeadLines(filePath, limit); + const effectiveLimit = + limit === "all" ? Number.POSITIVE_INFINITY : limit; + const text = await readHeadLines(filePath, effectiveLimit); const harness = detectHarness(filePath); const records: NormalizedRecord[] = []; const rawLines: string[] = []; let offset = 0; let i = 0; for (const line of text.split("\n")) { - if (i >= limit) break; + if (i >= effectiveLimit) break; if (line.length === 0) { offset += 1; // empty line + its \n continue; From bf8649e5545405689f9bf4cb06babee51818ce21 Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 21:24:04 -0400 Subject: [PATCH 11/14] =?UTF-8?q?=E2=9C=A8=20Index=20stage:=20real=20bette?= =?UTF-8?q?r-sqlite3=20+=20FTS5=20at=20$TMPDIR/scout-study/index.db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pipeline gains a fifth stage that turns the QMD sidecar files into a real, queryable data store. Selecting Index on a session walks every $TMPDIR/scout-study/qmd// directory, splits each markdown file into H2 sections, and writes rows into a better-sqlite3 db with FTS5. Schema: - sessions (id, harness, indexed_at) - documents (id, session_id, kind, path, bytes) - chunks (id, document_id, ordinal, source_ref, text) - chunks_fts (virtual FTS5 over chunks.text, with triggers for sync) Tried `bun --bun next dev` first for bun:sqlite; it ships but breaks the client Router with "Router action dispatched before initialization" errors, so the RerunLink useTransition pattern fails. Reverted to `next dev` + better-sqlite3 — boring, works, FTS5 included. IndexPanel shows the schema with live row counts, plus a "this session" breakdown so you can see what just landed. Run trace footer treats index-corpus as a regular command with input / output summaries and a re-run link. Verified end to end with a sqlite3 CLI snippet from outside the app: - 1.17 MB db file on disk after one session - chunks_fts MATCH 'VOX' returns real source-anchored snippets from decisions.md and events-001.md --- .../app/studies/session-search/page.tsx | 174 +++++++++++- design/studio/bun.lock | 74 +++++ .../lib/studio/commands/index-corpus.ts | 265 ++++++++++++++++++ design/studio/package.json | 2 + 4 files changed, 510 insertions(+), 5 deletions(-) create mode 100644 design/studio/lib/studio/commands/index-corpus.ts diff --git a/design/studio/app/studies/session-search/page.tsx b/design/studio/app/studies/session-search/page.tsx index 2654916d..11af1912 100644 --- a/design/studio/app/studies/session-search/page.tsx +++ b/design/studio/app/studies/session-search/page.tsx @@ -17,6 +17,10 @@ import { type EnrichSessionResult, type EnrichedFile, } from "@/lib/studio/commands/enrich-session"; +import { + indexCorpusCommand, + type IndexCorpusResult, +} from "@/lib/studio/commands/index-corpus"; import { CommandSurface } from "@/components/studio/CommandSurface"; import { ArtifactPicker, @@ -32,7 +36,7 @@ import { Suspense } from "react"; type Harness = "Codex" | "Claude"; type Tier = "large" | "normal" | "small"; -type StageId = "discover" | "normalize" | "extract" | "enrich"; +type StageId = "discover" | "normalize" | "extract" | "enrich" | "index"; interface SessionSample { id: string; @@ -203,6 +207,15 @@ const STAGES: Stage[] = [ summary: "One MiniMax-M2 call per session, cached for an hour. Reads the parsed records, condenses to a token-bounded transcript, asks the model for an overview + decisions doc, writes them next to the mechanical files.", }, + { + id: "index", + label: "Index", + verb: "store", + input: "qmd sidecar files", + output: "sqlite tables + FTS5", + summary: + "Walks every $TMPDIR/scout-study/qmd// directory, splits each markdown file into H2 sections, and writes rows into a real better-sqlite3 db at $TMPDIR/scout-study/index.db. Schema: sessions, documents, chunks, chunks_fts (FTS5 over chunks.text).", + }, ]; const DOC_HREF = "/eng/sco-059-session-knowledge-search-exploration"; @@ -351,8 +364,11 @@ async function StageBody({ const stageId = stage.id; const force = selection.force; const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; - const needsParse = stageId === "normalize" || stageId === "extract" || stageId === "enrich"; - const needsExtract = stageId === "extract" || stageId === "enrich"; + const needsParse = + stageId === "normalize" || stageId === "extract" || stageId === "enrich" || stageId === "index"; + const needsExtract = + stageId === "extract" || stageId === "enrich" || stageId === "index"; + const needsEnrich = stageId === "enrich" || stageId === "index"; const inventoryInput = { since: "7d" } as const; const runLog: RunLogEntry[] = [ @@ -407,7 +423,7 @@ async function StageBody({ } let enrichRun: CommandRun | undefined; - if (stageId === "enrich") { + if (needsEnrich) { const enrichInput = { path: session.fullPath, sessionId: sessionSlug, @@ -437,6 +453,26 @@ async function StageBody({ ); } + let indexRun: CommandRun | undefined; + if (stageId === "index") { + const indexInput = { sessionIds: [sessionSlug] }; + indexRun = await runCommand(indexCorpusCommand, indexInput, { + force: shouldForce(force, "index"), + }); + runLog.push( + makeRunLogEntry(indexCorpusCommand, indexInput, indexRun, { + summarizeInput: (i) => + i.sessionIds && i.sessionIds.length > 0 + ? `sessionIds=${i.sessionIds.join(",")}` + : "sessionIds=all", + summarizeOutput: (o) => + o + ? `${o.sessions} sessions · ${o.documents} docs · ${o.chunks} chunks · ${o.ftsRows} fts rows · ${o.dbBytes} db bytes` + : "no result", + }), + ); + } + return ( <>
    @@ -519,10 +556,16 @@ function expectedCommands(stageId: StageId): Array<{ label: "Enrich (LLM)", eta: "first run: 5–15 s", }; + const index = { + id: "index-corpus", + label: "Index corpus", + eta: "first run: ~ms · cached", + }; if (stageId === "discover") return [inv]; if (stageId === "normalize") return [inv, parse]; if (stageId === "extract") return [inv, parse, extract]; - return [inv, parse, extract, enrich]; + if (stageId === "enrich") return [inv, parse, extract, enrich]; + return [inv, parse, extract, enrich, index]; } // ── Session picker (single inline row) ─────────────────────────── @@ -740,6 +783,7 @@ function StagePanel({ normalizeRun, extractRun, enrichRun, + indexRun, selection, }: { stage: Stage; @@ -748,6 +792,7 @@ function StagePanel({ normalizeRun?: CommandRun; extractRun?: CommandRun; enrichRun?: CommandRun; + indexRun?: CommandRun; selection: StudySelection; }) { switch (stage.id) { @@ -777,6 +822,8 @@ function StagePanel({ run={enrichRun!} /> ); + case "index": + return ; } } @@ -1234,6 +1281,122 @@ async function EnrichPanel({ ); } +function IndexPanel({ + session, + selection, + run, +}: { + session: SessionSample; + selection: StudySelection; + run: CommandRun; +}) { + const result = run.output; + const sessionSlug = `${session.harness.toLowerCase()}-${session.tier}`; + const indexedHere = result?.indexedSessions.find( + (s) => s.sessionId === sessionSlug, + ); + return ( +
    + } + footnote={ + result && !run.error ? ( + <> + {result.schemaWasFresh ? "Created schema · " : "Reused schema · "} + db at{" "} + + {shortenTmpPath(result.dbPath)} + {" "} + · {formatBytes(result.dbBytes)} ·{" "} + {result.sessions} sessions · {result.documents} docs ·{" "} + {result.chunks} chunks · {result.ftsRows} fts rows + + ) : null + } + /> +
    + ); +} + +function IndexBody({ + result, + indexedHere, +}: { + result: IndexCorpusResult | undefined; + indexedHere?: IndexedSessionSummaryLocal; +}) { + if (!result) { + return ( + 
    +        no index result
+
    + ); + } + const schemaRows: Array<{ table: string; rows: number; note: string }> = [ + { table: "sessions", rows: result.sessions, note: "one per QMD directory" }, + { table: "documents", rows: result.documents, note: "one per markdown / json file" }, + { table: "chunks", rows: result.chunks, note: "one per H2 section" }, + { table: "chunks_fts", rows: result.ftsRows, note: "FTS5 virtual table" }, + ]; + return ( +
    +
    +
    + schema · row counts +
    +
      + {schemaRows.map((r) => ( +
    • + {r.table} + + {r.rows.toLocaleString()} + + {r.note} +
      • + ))} +
    +
    +
    +
    + this session +
    +
    + {indexedHere ? ( + <> +
    session_id = {indexedHere.sessionId}
    +
    documents = {indexedHere.documents}
    +
    chunks = {indexedHere.chunks}
    + + ) : ( + + this session has no QMD output yet — run Extract first + + )} +
    +
    + Chunks are one per H2 section. Walks every file in{" "} + $TMPDIR/scout-study/qmd/<session>/{" "} + (skipping _-prefixed metadata). +
    +
    +
    + ); +} + +// Local mirror of the indexed-session shape so the panel doesn't import the +// command module's interface twice. +type IndexedSessionSummaryLocal = { + sessionId: string; + documents: number; + chunks: number; +}; + function shortenTmpPath(p: string): string { // macOS tmpdir is /var/folders/.../T/...; trim to a readable suffix. const m = p.match(/\/T\/(.+)$/); @@ -1296,6 +1459,7 @@ const COMMAND_TO_STAGE: Record = { "parse-session": "normalize", "extract-qmd": "extract", "enrich-session": "enrich", + "index-corpus": "index", }; function RunSummary({ diff --git a/design/studio/bun.lock b/design/studio/bun.lock index 405f0f9a..f745ea4e 100644 --- a/design/studio/bun.lock +++ b/design/studio/bun.lock @@ -14,8 +14,10 @@ "@codemirror/legacy-modes": "^6.5.3", "@codemirror/state": "^6.6.0", "@codemirror/view": "^6.43.0", + "@types/better-sqlite3": "^7.6.13", "@uiw/codemirror-theme-github": "^4.25.10", "@uiw/react-codemirror": "^4.25.10", + "better-sqlite3": "^12.10.0", "gray-matter": "^4.0.3", "highlight.js": "^11.11.1", "marked": "^14.1.3", @@ -174,6 +176,8 @@ "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], + "@types/better-sqlite3": ["@types/better-sqlite3@7.6.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA=="], + "@types/debug": ["@types/debug@4.1.13", "", { "dependencies": { "@types/ms": "*" } }, "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw=="], "@types/estree": ["@types/estree@1.0.9", "", {}, "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg=="], @@ -216,14 +220,24 @@ "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + "baseline-browser-mapping": ["baseline-browser-mapping@2.10.32", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-wbPvpyjJPC0zdfdKXxqEL3Ea+bOMD/87X4lftiJkkaBiuG6ALQy1SLmEd7BSmVCuwCQsBrCamgBoLyfFDD1EPg=="], + "better-sqlite3": ["better-sqlite3@12.10.0", "", { "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" } }, "sha512-CyzaZRQKyHkB2ZInfTTl2nvT33EbDpjkLEbE8/Zck3Ll6O0qqvuGdrJ45HgtH+HykRg88ITY3AdreBGN70aBSQ=="], + "binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="], + "bindings": ["bindings@1.5.0", "", { "dependencies": { "file-uri-to-path": "1.0.0" } }, "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ=="], + + "bl": ["bl@4.1.0", "", { "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", "readable-stream": "^3.4.0" } }, "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w=="], + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], "browserslist": ["browserslist@4.28.2", "", { "dependencies": { "baseline-browser-mapping": "^2.10.12", "caniuse-lite": "^1.0.30001782", "electron-to-chromium": "^1.5.328", "node-releases": "^2.0.36", "update-browserslist-db": "^1.2.3" }, "bin": { "browserslist": "cli.js" } }, "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg=="], + "buffer": ["buffer@5.7.1", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" } }, "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ=="], + "camelcase-css": ["camelcase-css@2.0.1", "", {}, "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA=="], "caniuse-lite": ["caniuse-lite@1.0.30001793", "", {}, "sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA=="], @@ -240,6 +254,8 @@ "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], + "chownr": ["chownr@1.1.4", "", {}, "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="], + "client-only": ["client-only@0.0.1", "", {}, "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="], "codemirror": ["codemirror@6.0.2", "", { "dependencies": { "@codemirror/autocomplete": "^6.0.0", "@codemirror/commands": "^6.0.0", "@codemirror/language": "^6.0.0", "@codemirror/lint": "^6.0.0", "@codemirror/search": "^6.0.0", "@codemirror/state": "^6.0.0", "@codemirror/view": "^6.0.0" } }, "sha512-VhydHotNW5w1UGK0Qj96BwSk/Zqbp9WbnyK2W/eVMv4QyF41INRGpjUhFJY7/uDNuudSc33a/PKr4iDqRduvHw=="], @@ -258,6 +274,10 @@ "decode-named-character-reference": ["decode-named-character-reference@1.3.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q=="], + "decompress-response": ["decompress-response@6.0.0", "", { "dependencies": { "mimic-response": "^3.1.0" } }, "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ=="], + + "deep-extend": ["deep-extend@0.6.0", "", {}, "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="], + "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -270,6 +290,8 @@ "electron-to-chromium": ["electron-to-chromium@1.5.361", "", {}, "sha512-Q6Hts7N9FnJc5LeGRINFvLhCI9xZmNtTDe5ZbcVezQz7cU4a8Aua3GH1b8J2XY8Al9PF+OCwYqhgsOOheMdvkA=="], + "end-of-stream": ["end-of-stream@1.4.5", "", { "dependencies": { "once": "^1.4.0" } }, "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg=="], + "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], @@ -280,6 +302,8 @@ "estree-util-is-identifier-name": ["estree-util-is-identifier-name@3.0.0", "", {}, "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg=="], + "expand-template": ["expand-template@2.0.3", "", {}, "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="], + "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], "extend-shallow": ["extend-shallow@2.0.1", "", { "dependencies": { "is-extendable": "^0.1.0" } }, "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug=="], @@ -290,14 +314,20 @@ "fdir": ["fdir@6.5.0", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg=="], + "file-uri-to-path": ["file-uri-to-path@1.0.0", "", {}, "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="], + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], "fraction.js": ["fraction.js@5.3.4", "", {}, "sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ=="], + "fs-constants": ["fs-constants@1.0.0", "", {}, "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], + "github-from-package": ["github-from-package@0.0.0", "", {}, "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="], + "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], "gray-matter": ["gray-matter@4.0.3", "", { "dependencies": { "js-yaml": "^3.13.1", "kind-of": "^6.0.2", "section-matter": "^1.0.0", "strip-bom-string": "^1.0.0" } }, "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q=="], @@ -316,6 +346,12 @@ "html-url-attributes": ["html-url-attributes@3.0.1", "", {}, "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ=="], + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="], + "inline-style-parser": ["inline-style-parser@0.2.7", "", {}, "sha512-Nb2ctOyNR8DqQoR0OwRG95uNWIC0C1lCgf5Naz5H6Ji72KZ8OcFZLz2P5sNgwlyoJ8Yif11oMuYs5pBQa86csA=="], "is-alphabetical": ["is-alphabetical@2.0.1", "", {}, "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ=="], @@ -448,14 +484,24 @@ "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + "mimic-response": ["mimic-response@3.1.0", "", {}, "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="], + + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + + "mkdirp-classic": ["mkdirp-classic@0.5.3", "", {}, "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "mz": ["mz@2.7.0", "", { "dependencies": { "any-promise": "^1.0.0", "object-assign": "^4.0.1", "thenify-all": "^1.0.0" } }, "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q=="], "nanoid": ["nanoid@3.3.12", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ=="], + "napi-build-utils": ["napi-build-utils@2.0.0", "", {}, "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA=="], + "next": ["next@16.2.6", "", { "dependencies": { "@next/env": "16.2.6", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.6", "@next/swc-darwin-x64": "16.2.6", "@next/swc-linux-arm64-gnu": "16.2.6", "@next/swc-linux-arm64-musl": "16.2.6", "@next/swc-linux-x64-gnu": "16.2.6", "@next/swc-linux-x64-musl": "16.2.6", "@next/swc-win32-arm64-msvc": "16.2.6", "@next/swc-win32-x64-msvc": "16.2.6", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-qOVgKJg1+At15NpeUP+eJgCHvTCgXsogweq87Ri/Ix7PkqQHg4sdaXmSFqKlgaIXE4kW0g25LE68W87UANlHtw=="], + "node-abi": ["node-abi@3.92.0", "", { "dependencies": { "semver": "^7.3.5" } }, "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ=="], + "node-releases": ["node-releases@2.0.46", "", {}, "sha512-GYVXHE2KnrzAfsAjl4uP++evGFCrAU1jta4ubEjIG7YWt/64Gqv66a30yKwWczVjA6j3bM4nBwH7Pk1JmDHaxQ=="], "normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="], @@ -464,6 +510,8 @@ "object-hash": ["object-hash@3.0.0", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="], + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + "parse-entities": ["parse-entities@4.0.2", "", { "dependencies": { "@types/unist": "^2.0.0", "character-entities-legacy": "^3.0.0", "character-reference-invalid": "^2.0.0", "decode-named-character-reference": "^1.0.0", "is-alphanumerical": "^2.0.0", "is-decimal": "^2.0.0", "is-hexadecimal": "^2.0.0" } }, "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw=="], "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], @@ -490,10 +538,16 @@ "postcss-value-parser": ["postcss-value-parser@4.2.0", "", {}, "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="], + "prebuild-install": ["prebuild-install@7.1.3", "", { "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", "github-from-package": "0.0.0", "minimist": "^1.2.3", "mkdirp-classic": "^0.5.3", "napi-build-utils": "^2.0.0", "node-abi": "^3.3.0", "pump": "^3.0.0", "rc": "^1.2.7", "simple-get": "^4.0.0", "tar-fs": "^2.0.0", "tunnel-agent": "^0.6.0" }, "bin": { "prebuild-install": "bin.js" } }, "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug=="], + "property-information": ["property-information@7.1.0", "", {}, "sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ=="], + "pump": ["pump@3.0.4", "", { "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" } }, "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], + "rc": ["rc@1.2.8", "", { "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" }, "bin": { "rc": "./cli.js" } }, "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw=="], + "react": ["react@19.2.6", "", {}, "sha512-sfWGGfavi0xr8Pg0sVsyHMAOziVYKgPLNrS7ig+ivMNb3wbCBw3KxtflsGBAwD3gYQlE/AEZsTLgToRrSCjb0Q=="], "react-dom": ["react-dom@19.2.6", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.6" } }, "sha512-0prMI+hvBbPjsWnxDLxlCGyM8PN6UuWjEUCYmZhO67xIV9Xasa/r/vDnq+Xyq4Lo27g8QSbO5YzARu0D1Sps3g=="], @@ -502,6 +556,8 @@ "read-cache": ["read-cache@1.0.0", "", { "dependencies": { "pify": "^2.3.0" } }, "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], "rehype-highlight": ["rehype-highlight@7.0.2", "", { "dependencies": { "@types/hast": "^3.0.0", "hast-util-to-text": "^4.0.0", "lowlight": "^3.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-k158pK7wdC2qL3M5NcZROZ2tR/l7zOzjxXd5VGdcfIyoijjQqpHd3JKtYSBDpDZ38UI2WJWuFAtkMDxmx5kstA=="], @@ -520,6 +576,8 @@ "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "section-matter": ["section-matter@1.0.0", "", { "dependencies": { "extend-shallow": "^2.0.1", "kind-of": "^6.0.0" } }, "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA=="], @@ -528,16 +586,24 @@ "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "simple-concat": ["simple-concat@1.0.1", "", {}, "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="], + + "simple-get": ["simple-get@4.0.1", "", { "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", "simple-concat": "^1.0.0" } }, "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], "space-separated-tokens": ["space-separated-tokens@2.0.2", "", {}, "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q=="], "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], "strip-bom-string": ["strip-bom-string@1.0.0", "", {}, "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g=="], + "strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], + "style-mod": ["style-mod@4.1.3", "", {}, "sha512-i/n8VsZydrugj3Iuzll8+x/00GH2vnYsk1eomD8QiRrSAeW6ItbCQDtfXCeJHd0iwiNagqjQkvpvREEPtW3IoQ=="], "style-to-js": ["style-to-js@1.1.21", "", { "dependencies": { "style-to-object": "1.0.14" } }, "sha512-RjQetxJrrUJLQPHbLku6U/ocGtzyjbJMP9lCNK7Ag0CNh690nSH8woqWH9u16nMjYBAok+i7JO1NP2pOy8IsPQ=="], @@ -552,6 +618,10 @@ "tailwindcss": ["tailwindcss@3.4.19", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", "chokidar": "^3.6.0", "didyoumean": "^1.2.2", "dlv": "^1.1.3", "fast-glob": "^3.3.2", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", "jiti": "^1.21.7", "lilconfig": "^3.1.3", "micromatch": "^4.0.8", "normalize-path": "^3.0.0", "object-hash": "^3.0.0", "picocolors": "^1.1.1", "postcss": "^8.4.47", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", "postcss-nested": "^6.2.0", "postcss-selector-parser": "^6.1.2", "resolve": "^1.22.8", "sucrase": "^3.35.0" }, "bin": { "tailwind": "lib/cli.js", "tailwindcss": "lib/cli.js" } }, "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ=="], + "tar-fs": ["tar-fs@2.1.4", "", { "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", "tar-stream": "^2.1.4" } }, "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ=="], + + "tar-stream": ["tar-stream@2.2.0", "", { "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" } }, "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ=="], + "thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="], "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], @@ -568,6 +638,8 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], @@ -596,6 +668,8 @@ "w3c-keyname": ["w3c-keyname@2.2.8", "", {}, "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], "chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], diff --git a/design/studio/lib/studio/commands/index-corpus.ts b/design/studio/lib/studio/commands/index-corpus.ts new file mode 100644 index 00000000..dfd55de6 --- /dev/null +++ b/design/studio/lib/studio/commands/index-corpus.ts @@ -0,0 +1,265 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import Database from "better-sqlite3"; +import type { Command } from "@/lib/studio/command"; + +export interface IndexCorpusInput { + /** Specific session ids to (re)index. Empty/undefined = every qmd dir on disk. */ + sessionIds?: string[]; +} + +export interface IndexedSessionSummary { + sessionId: string; + documents: number; + chunks: number; +} + +export interface IndexCorpusResult { + dbPath: string; + dbBytes: number; + sessions: number; + documents: number; + chunks: number; + ftsRows: number; + indexedSessions: IndexedSessionSummary[]; + schemaWasFresh: boolean; +} + +const ROOT = path.join(tmpdir(), "scout-study"); +const QMD_ROOT = path.join(ROOT, "qmd"); +const DB_PATH = path.join(ROOT, "index.db"); + +/** + * Read QMD sidecar files for one or more sessions and index them into a real + * sqlite db at $TMPDIR/scout-study/index.db. Schema: + * + * sessions (id, harness, indexed_at) + * documents (id, session_id, kind, path, bytes) + * chunks (id, document_id, ordinal, source_ref, text) + * chunks_fts (virtual FTS5 over chunks.text) + * + * Chunks are one per H2 section in each markdown file; the "preamble" before + * the first H2 is its own chunk. tool-calls.md and files.md typically have one + * or two chunks; events-NNN.md has one chunk per `## window N` heading. + */ +export const indexCorpusCommand: Command = { + id: "index-corpus", + label: "Index corpus", + shell: ({ sessionIds }) => { + const target = sessionIds && sessionIds.length > 0 ? sessionIds.join(",") : "all"; + return `scout index --db ${shrinkPath(DB_PATH)} --sessions ${target}`; + }, + run: async ({ sessionIds }) => { + await fs.mkdir(ROOT, { recursive: true }); + + const wasFresh = !(await pathExists(DB_PATH)); + const db = new Database(DB_PATH); + db.pragma("journal_mode = WAL"); + + db.exec(` + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + harness TEXT, + indexed_at INTEGER + ); + + CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, + session_id TEXT REFERENCES sessions(id) ON DELETE CASCADE, + kind TEXT, + path TEXT, + bytes INTEGER + ); + + CREATE TABLE IF NOT EXISTS chunks ( + id INTEGER PRIMARY KEY, + document_id TEXT REFERENCES documents(id) ON DELETE CASCADE, + ordinal INTEGER, + source_ref TEXT, + text TEXT + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( + text, + content='chunks', + content_rowid='id' + ); + + CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN + INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text); + END; + + CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN + INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text); + END; + + CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN + INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES('delete', old.id, old.text); + INSERT INTO chunks_fts(rowid, text) VALUES (new.id, new.text); + END; + `); + + const targets = sessionIds && sessionIds.length > 0 + ? sessionIds + : await listQmdSessions(); + + const insertSession = db.prepare( + "INSERT OR REPLACE INTO sessions (id, harness, indexed_at) VALUES (?, ?, ?)", + ); + const deleteDocsForSession = db.prepare( + "DELETE FROM documents WHERE session_id = ?", + ); + const insertDoc = db.prepare( + "INSERT INTO documents (id, session_id, kind, path, bytes) VALUES (?, ?, ?, ?, ?)", + ); + const insertChunk = db.prepare( + "INSERT INTO chunks (document_id, ordinal, source_ref, text) VALUES (?, ?, ?, ?)", + ); + + const indexed: IndexedSessionSummary[] = []; + + const tx = db.transaction((sessionId: string) => { + const harness = inferHarness(sessionId); + insertSession.run(sessionId, harness, Date.now()); + deleteDocsForSession.run(sessionId); + }); + + for (const sessionId of targets) { + const sessionDir = path.join(QMD_ROOT, sessionId); + const dirExists = await pathExists(sessionDir); + if (!dirExists) continue; + + tx(sessionId); + + let docs = 0; + let chunks = 0; + const fileNames = await fs.readdir(sessionDir); + for (const fileName of fileNames) { + if (fileName.startsWith("_")) continue; // skip _llm-call.json etc. + const filePath = path.join(sessionDir, fileName); + const stat = await fs.stat(filePath); + if (!stat.isFile()) continue; + const kind = inferKind(fileName); + const docId = `${sessionId}/${fileName}`; + insertDoc.run(docId, sessionId, kind, filePath, stat.size); + docs++; + + const content = await fs.readFile(filePath, "utf8"); + const sectionChunks = + fileName === "manifest.json" + ? [{ text: content.trim(), sourceRef: "root" }] + : chunkByH2(content); + + sectionChunks.forEach((c, i) => { + insertChunk.run(docId, i, c.sourceRef ?? null, c.text); + chunks++; + }); + } + + indexed.push({ sessionId, documents: docs, chunks }); + } + + const sessionsCount = (db.prepare("SELECT COUNT(*) AS c FROM sessions").get() as { c: number }).c; + const documentsCount = (db.prepare("SELECT COUNT(*) AS c FROM documents").get() as { c: number }).c; + const chunksCount = (db.prepare("SELECT COUNT(*) AS c FROM chunks").get() as { c: number }).c; + const ftsRows = (db.prepare("SELECT COUNT(*) AS c FROM chunks_fts").get() as { c: number }).c; + + db.close(); + + const dbStat = await fs.stat(DB_PATH); + + return { + dbPath: DB_PATH, + dbBytes: dbStat.size, + sessions: sessionsCount, + documents: documentsCount, + chunks: chunksCount, + ftsRows, + indexedSessions: indexed, + schemaWasFresh: wasFresh, + }; + }, + cacheKey: ({ sessionIds }) => + sessionIds && sessionIds.length > 0 + ? sessionIds.slice().sort().join(",") + : "all", + cacheTtlMs: 5 * 60 * 1000, +}; + +async function listQmdSessions(): Promise { + try { + const entries = await fs.readdir(QMD_ROOT, { withFileTypes: true }); + return entries.filter((e) => e.isDirectory()).map((e) => e.name); + } catch { + return []; + } +} + +async function pathExists(p: string): Promise { + try { + await fs.stat(p); + return true; + } catch { + return false; + } +} + +function inferHarness(sessionId: string): string { + const prefix = sessionId.split("-")[0]; + return prefix ?? "unknown"; +} + +function inferKind(fileName: string): string { + if (fileName === "overview.md") return "overview"; + if (fileName === "decisions.md") return "decisions"; + if (fileName === "files.md") return "files"; + if (fileName === "tool-calls.md") return "tool-calls"; + if (fileName.startsWith("events-")) return "events"; + if (fileName === "manifest.json") return "manifest"; + return "other"; +} + +interface SectionChunk { + text: string; + sourceRef?: string; +} + +function chunkByH2(content: string): SectionChunk[] { + const lines = content.split("\n"); + const chunks: SectionChunk[] = []; + let preambleLines: string[] = []; + let current: { heading: string; lines: string[] } | undefined; + + for (const line of lines) { + if (line.startsWith("## ")) { + if (current) { + const text = current.lines.join("\n").trim(); + if (text.length > 0) chunks.push({ text, sourceRef: current.heading }); + } else if (preambleLines.length > 0) { + const text = preambleLines.join("\n").trim(); + if (text.length > 0) chunks.push({ text, sourceRef: "preamble" }); + } + current = { heading: line.slice(3).trim(), lines: [line] }; + } else if (current) { + current.lines.push(line); + } else { + preambleLines.push(line); + } + } + + if (current) { + const text = current.lines.join("\n").trim(); + if (text.length > 0) chunks.push({ text, sourceRef: current.heading }); + } else if (preambleLines.length > 0) { + const text = preambleLines.join("\n").trim(); + if (text.length > 0) chunks.push({ text, sourceRef: "preamble" }); + } + + return chunks; +} + +function shrinkPath(p: string): string { + const home = process.env.HOME ?? ""; + return home && p.startsWith(home) ? "~" + p.slice(home.length) : p; +} diff --git a/design/studio/package.json b/design/studio/package.json index dec3c23f..856da904 100644 --- a/design/studio/package.json +++ b/design/studio/package.json @@ -19,8 +19,10 @@ "@codemirror/legacy-modes": "^6.5.3", "@codemirror/state": "^6.6.0", "@codemirror/view": "^6.43.0", + "@types/better-sqlite3": "^7.6.13", "@uiw/codemirror-theme-github": "^4.25.10", "@uiw/react-codemirror": "^4.25.10", + "better-sqlite3": "^12.10.0", "gray-matter": "^4.0.3", "highlight.js": "^11.11.1", "marked": "^14.1.3", From 7974e38678a7cdf0ff490c1a275bd89d5d61d92e Mon Sep 17 00:00:00 2001 From: Arach Date: Sat, 30 May 2026 23:57:46 -0400 Subject: [PATCH 12/14] =?UTF-8?q?=E2=9C=A8=20Session=20DB=20Explorer=20+?= =?UTF-8?q?=20Step=206=20Ask=20field=20(local=20FTS5,=20no=20LLM)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New /studies/data surface that makes the shape of the session-search index legible: schema, shortcuts deck, unified MATCH/SELECT Query card with a strategy registry, and an Ask field that turns a question into FTS5 hits via local stopword stripping — sub-millisecond, no proprietary tokeniser in the loop. Same Ask surface also lands as Step 6 in the session-search pipeline workbench with schema-aware suggestion chips. --- design/studio/app/studies/data/QueryForm.tsx | 95 ++ design/studio/app/studies/data/page.tsx | 910 ++++++++++++++++++ .../app/studies/session-search/page.tsx | 279 +++++- design/studio/components/studio/RerunLink.tsx | 2 +- design/studio/lib/studio-pages.ts | 13 + .../studio/lib/studio/commands/inspect-db.ts | 557 +++++++++++ 6 files changed, 1852 insertions(+), 4 deletions(-) create mode 100644 design/studio/app/studies/data/QueryForm.tsx create mode 100644 design/studio/app/studies/data/page.tsx create mode 100644 design/studio/lib/studio/commands/inspect-db.ts diff --git a/design/studio/app/studies/data/QueryForm.tsx b/design/studio/app/studies/data/QueryForm.tsx new file mode 100644 index 00000000..f8ade332 --- /dev/null +++ b/design/studio/app/studies/data/QueryForm.tsx @@ -0,0 +1,95 @@ +"use client"; + +import { useRouter, useSearchParams } from "next/navigation"; +import { useEffect, useState, useTransition, type FormEvent } from "react"; + +export function QueryForm({ + paramName, + basePath = "/studies/data", + defaultValue, + placeholder, + multiline, + submitLabel = "run ↻", + alsoClear = ["force"], +}: { + /** Which URL search param this form writes. */ + paramName: string; + basePath?: string; + defaultValue: string; + placeholder: string; + multiline?: boolean; + submitLabel?: string; + /** Other params to clear when this form submits (default: clear `force`). */ + alsoClear?: string[]; +}) { + const router = useRouter(); + const searchParams = useSearchParams(); + const [value, setValue] = useState(defaultValue); + const [isPending, startTransition] = useTransition(); + + // Sync local state when the URL-driven defaultValue changes from the outside + // (e.g. clicking a shortcut). Does not interfere with in-flight typing — + // submits set URL = local state, so the effect is a no-op then. + useEffect(() => { + setValue(defaultValue); + }, [defaultValue]); + + function buildHref(next: string): string { + const params = new URLSearchParams(searchParams?.toString() ?? ""); + for (const k of alsoClear) params.delete(k); + if (next.trim()) params.set(paramName, next); + else params.delete(paramName); + const qs = params.toString(); + return qs ? `${basePath}?${qs}` : basePath; + } + + function submit(e: FormEvent) { + e.preventDefault(); + startTransition(() => { + router.push(buildHref(value), { scroll: false }); + }); + } + + const inputCls = + "flex-1 rounded border border-studio-edge bg-studio-canvas px-2 py-1 font-mono text-[11px] text-studio-ink placeholder:text-studio-ink-faint focus:outline-none focus:ring-1 focus:ring-studio-ink-faint"; + + return ( +
    + {multiline ? ( +