From ec154d654493b2437eff82458d888ae246385f56 Mon Sep 17 00:00:00 2001 From: sasha Date: Sat, 18 Apr 2026 20:03:04 +0300 Subject: [PATCH] feat(trace-preview): hide system messages from pretty IO view (2BB-289) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Preview tab in trace/observation detail views was rendering the full system prompt (tool catalog, skills list, operator rules — many KB of text) alongside the user/assistant turns, making it useless for scanning what the conversation actually was. Filter messages with role=system/developer/tool_definitions out of the ChatML transcript at the pretty render layer. The JSON and JSON Beta views are untouched and still show the full trace payload, so no data is hidden from reviewers who need it. - Add isSystemRole() helper (case-insensitive) in chat-message-utils - useChatMLParser accepts hideSystemMessages (defaults to true — every current caller is a trace/observation/session surface) - Tool definitions are extracted from the pre-filter combinedMessages so SectionToolDefinitions stays populated even when only a system message declared the tools - inputMessageCount tracks kept input messages so tool-call numbering stays correct after filtering Co-Authored-By: Claude Opus 4.7 --- .../chat-message-utils.clienttest.ts | 32 ++++++++++++++ .../components/chat-message-utils.ts | 19 ++++++++ .../IOPreview/hooks/useChatMLParser.ts | 43 ++++++++++++++++--- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/web/src/components/trace2/components/IOPreview/components/chat-message-utils.clienttest.ts b/web/src/components/trace2/components/IOPreview/components/chat-message-utils.clienttest.ts index 43a0e135bfe3..3bed8150e839 100644 --- a/web/src/components/trace2/components/IOPreview/components/chat-message-utils.clienttest.ts +++ b/web/src/components/trace2/components/IOPreview/components/chat-message-utils.clienttest.ts @@ -6,6 +6,7 @@ import { hasPassthroughJson, isPlaceholderMessage, isOnlyJsonMessage, + isSystemRole, shouldRenderMessage, parseToolCallsFromMessage, } from "./chat-message-utils"; @@ -313,4 +314,35 @@ describe("chat-message-utils", () => { ).toEqual(directToolCalls); }); }); + + describe("isSystemRole", () => { + it("returns true for system", () => { + expect(isSystemRole("system")).toBe(true); + }); + + it("returns true for developer", () => { + expect(isSystemRole("developer")).toBe(true); + }); + + it("returns true for tool_definitions", () => { + expect(isSystemRole("tool_definitions")).toBe(true); + }); + + it("is case-insensitive", () => { + expect(isSystemRole("System")).toBe(true); + expect(isSystemRole("DEVELOPER")).toBe(true); + }); + + it("returns false for user/assistant/tool", () => { + expect(isSystemRole("user")).toBe(false); + expect(isSystemRole("assistant")).toBe(false); + expect(isSystemRole("tool")).toBe(false); + }); + + it("returns false for empty / undefined / null", () => { + expect(isSystemRole("")).toBe(false); + expect(isSystemRole(undefined)).toBe(false); + expect(isSystemRole(null)).toBe(false); + }); + }); }); diff --git a/web/src/components/trace2/components/IOPreview/components/chat-message-utils.ts b/web/src/components/trace2/components/IOPreview/components/chat-message-utils.ts index c9572443fda6..5ffec0432341 100644 --- a/web/src/components/trace2/components/IOPreview/components/chat-message-utils.ts +++ b/web/src/components/trace2/components/IOPreview/components/chat-message-utils.ts @@ -81,6 +81,25 @@ export function parseToolCallsFromMessage( : []; } +/** + * Roles that carry system-level instructions, tool catalogs, skills, or + * operator rules rather than user/assistant turns. These messages bloat the + * Preview tab with content that is rarely what reviewers want to see. + * + * The full payload (including these messages) remains visible in the JSON and + * JSON Beta views — this is a render-layer filter only. + */ +const SYSTEM_ROLES = new Set(["system", "developer", "tool_definitions"]); + +/** + * Check if a ChatML role represents a system/developer/instruction message. + * Case-insensitive to accommodate upstream role casing variations. + */ +export function isSystemRole(role: string | undefined | null): boolean { + if (!role) return false; + return SYSTEM_ROLES.has(role.toLowerCase()); +} + /** * Check if message has thinking content. */ diff --git a/web/src/components/trace2/components/IOPreview/hooks/useChatMLParser.ts b/web/src/components/trace2/components/IOPreview/hooks/useChatMLParser.ts index 21b01e90ae85..a48d2e1c7126 100644 --- a/web/src/components/trace2/components/IOPreview/hooks/useChatMLParser.ts +++ b/web/src/components/trace2/components/IOPreview/hooks/useChatMLParser.ts @@ -9,6 +9,7 @@ import { extractAdditionalInput, } from "@/src/utils/chatml"; import type { ChatMlMessageSchema } from "@/src/components/schemas/ChatMlSchema"; +import { isSystemRole } from "../components/chat-message-utils"; // ChatML message type from schema export type ChatMlMessage = z.infer; @@ -46,6 +47,19 @@ function parseToolCallsFromMessage( : []; } +export interface UseChatMLParserOptions { + /** + * When true, drop messages whose role is system/developer/tool_definitions + * from the rendered chat transcript. The underlying trace payload is + * untouched — JSON views still display the full input/output. + * + * Defaults to true: every current caller is a trace/observation detail or + * preview surface where leaking multi-KB system prompts makes Preview + * useless for scanning conversations (see 2BB-289). + */ + hideSystemMessages?: boolean; +} + /** * Hook to parse input/output into ChatML format and extract tool information. * @@ -67,7 +81,9 @@ export function useChatMLParser( preParsedInput?: unknown, preParsedOutput?: unknown, preParsedMetadata?: unknown, + options: UseChatMLParserOptions = {}, ): ChatMLParserResult { + const { hideSystemMessages = true } = options; // Use pre-parsed data if available (from Web Worker), otherwise parse synchronously // This eliminates ~100ms of duplicate parsing when data comes from useParsedObservation const parsedInput = @@ -95,16 +111,22 @@ export function useChatMLParser( const outputClean = cleanLegacyOutput(parsedOutput, parsedOutput); // Combine messages - const messages = combineInputOutputMessages( + const combinedMessages = combineInputOutputMessages( inResult, outResult, outputClean, ); - // Extract all unique tools from messages (no numbering yet) + // Optionally hide system/developer/tool_definitions messages from the + // pretty render (Preview tab). Tools are extracted from ALL messages + // (pre-filter) so the SectionToolDefinitions catalog stays populated even + // when the only carrier of `tools` was a system message. + const rawInputMessageCount = inResult.success ? inResult.data.length : 0; const toolsMap = new Map(); + const messages: typeof combinedMessages = []; + let inputMessageCount = 0; - for (const message of messages) { + combinedMessages.forEach((message, i) => { if (message.tools && Array.isArray(message.tools)) { for (const tool of message.tools) { if (!toolsMap.has(tool.name)) { @@ -112,11 +134,14 @@ export function useChatMLParser( } } } - } + + if (hideSystemMessages && isSystemRole(message.role)) return; + messages.push(message); + if (i < rawInputMessageCount) inputMessageCount++; + }); // Count tool call invocations // Only number tool calls from OUTPUT messages (current invocation), not input (history) - const inputMessageCount = inResult.success ? inResult.data.length : 0; let toolCallCounter = 0; const messageToToolCallNumbers = new Map(); const toolCallCounts = new Map(); @@ -184,7 +209,13 @@ export function useChatMLParser( toolNameToDefinitionNumber, inputMessageCount, }; - }, [parsedInput, parsedOutput, parsedMetadata, observationName]); + }, [ + parsedInput, + parsedOutput, + parsedMetadata, + observationName, + hideSystemMessages, + ]); } // Re-export for use in ChatMessage