diff --git a/e2e/helpers/google-genai-trace-contract.ts b/e2e/helpers/google-genai-trace-contract.ts deleted file mode 100644 index 138f3588b..000000000 --- a/e2e/helpers/google-genai-trace-contract.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { expect } from "vitest"; -import { normalizeForSnapshot, type Json } from "./normalize"; -import type { - CapturedLogEvent, - CapturedLogPayload, -} from "./mock-braintrust-server"; -import { findChildSpans, findLatestSpan } from "./trace-selectors"; -import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "./wrapper-contract"; - -function findNamedChildSpan( - capturedEvents: CapturedLogEvent[], - names: string[], - parentId: string | undefined, -) { - for (const name of names) { - const span = findChildSpans(capturedEvents, name, parentId)[0]; - if (span) { - return span; - } - } - - return undefined; -} - -function normalizeGooglePayloads(payloadRows: unknown[]): unknown[] { - return payloadRows.map((payload) => { - if (!payload || typeof payload !== "object") { - return payload; - } - - const row = structuredClone(payload) as { - output?: { - usageMetadata?: { - promptTokensDetails?: Array<{ modality?: string }>; - }; - }; - }; - const promptTokensDetails = row.output?.usageMetadata?.promptTokensDetails; - if (promptTokensDetails) { - promptTokensDetails.sort((left, right) => - String(left.modality ?? "").localeCompare(String(right.modality ?? "")), - ); - } - return row; - }); -} - -export function assertGoogleGenAITraceContract(options: { - capturedEvents: CapturedLogEvent[]; - payloads: CapturedLogPayload[]; - rootName: string; - scenarioName: string; -}): { payloadSummary: Json; spanSummary: Json } { - const root = findLatestSpan(options.capturedEvents, options.rootName); - const generateOperation = findLatestSpan( - options.capturedEvents, - "google-generate-operation", - ); - const streamOperation = findLatestSpan( - options.capturedEvents, - "google-stream-operation", - ); - const streamReturnOperation = findLatestSpan( - options.capturedEvents, - "google-stream-return-operation", - ); - const toolOperation = findLatestSpan( - options.capturedEvents, - "google-tool-operation", - ); - const attachmentOperation = findLatestSpan( - options.capturedEvents, - "google-attachment-operation", - ); - - expect(root).toBeDefined(); - expect(generateOperation).toBeDefined(); - expect(streamOperation).toBeDefined(); - expect(streamReturnOperation).toBeDefined(); - expect(toolOperation).toBeDefined(); - expect(attachmentOperation).toBeDefined(); - - expect(root?.row.metadata).toMatchObject({ - scenario: options.scenarioName, - }); - expect(generateOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(streamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(streamReturnOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(toolOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(attachmentOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - - const generateSpan = findNamedChildSpan( - options.capturedEvents, - ["generate_content", "google-genai.generateContent"], - generateOperation?.span.id, - ); - const streamSpan = findNamedChildSpan( - options.capturedEvents, - ["generate_content_stream", "google-genai.generateContentStream"], - streamOperation?.span.id, - ); - const streamReturnSpan = findNamedChildSpan( - options.capturedEvents, - ["generate_content_stream", "google-genai.generateContentStream"], - streamReturnOperation?.span.id, - ); - const toolSpan = findNamedChildSpan( - options.capturedEvents, - ["generate_content", "google-genai.generateContent"], - toolOperation?.span.id, - ); - const attachmentSpan = findNamedChildSpan( - options.capturedEvents, - ["generate_content", "google-genai.generateContent"], - attachmentOperation?.span.id, - ); - - for (const wrapperSpan of [ - generateSpan, - streamSpan, - streamReturnSpan, - toolSpan, - attachmentSpan, - ]) { - expect(wrapperSpan).toBeDefined(); - expect(wrapperSpan?.row.metadata).toMatchObject({ - model: "gemini-2.5-flash-lite", - }); - } - - expect(streamSpan?.metrics).toMatchObject({ - time_to_first_token: expect.any(Number), - prompt_tokens: expect.any(Number), - completion_tokens: expect.any(Number), - }); - expect(streamReturnSpan?.metrics).toMatchObject({ - time_to_first_token: expect.any(Number), - prompt_tokens: expect.any(Number), - }); - - expect(JSON.stringify(attachmentSpan?.input)).toContain("file.png"); - - const toolMetadata = toolSpan?.row.metadata as - | { - tools?: Array<{ - functionDeclarations?: Array<{ name?: string }>; - }>; - } - | undefined; - expect( - toolMetadata?.tools?.some((tool) => - tool.functionDeclarations?.some( - (declaration) => declaration.name === "get_weather", - ), - ) || JSON.stringify(toolSpan?.input).includes("get_weather"), - ).toBe(true); - - const toolInput = toolSpan?.input as - | { - config?: { - tools?: Array; - }; - } - | undefined; - expect( - toolInput?.config?.tools === undefined || - JSON.stringify(toolInput.config.tools).includes("get_weather"), - ).toBe(true); - - const toolOutput = toolSpan?.output as - | { - candidates?: Array<{ - content?: { - parts?: Array<{ - functionCall?: { name?: string }; - }>; - }; - }>; - functionCalls?: Array<{ name?: string }>; - } - | undefined; - expect( - toolOutput?.functionCalls?.some((call) => call.name === "get_weather") || - toolOutput?.candidates?.some((candidate) => - candidate.content?.parts?.some( - (part) => part.functionCall?.name === "get_weather", - ), - ), - ).toBe(true); - - return { - spanSummary: normalizeForSnapshot( - [ - root, - generateOperation, - generateSpan, - attachmentOperation, - attachmentSpan, - streamOperation, - streamSpan, - streamReturnOperation, - streamReturnSpan, - toolOperation, - toolSpan, - ].map((event) => - summarizeWrapperContract(event!, ["model", "operation", "scenario"]), - ) as Json, - ), - payloadSummary: normalizeForSnapshot( - normalizeGooglePayloads( - payloadRowsForRootSpan(options.payloads, root?.span.id), - ) as Json, - ), - }; -} diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/log-payloads.json b/e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/log-payloads.json deleted file mode 100644 index 6667fda98..000000000 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/log-payloads.json +++ /dev/null @@ -1,893 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "google-genai-auto-instrumentation-node-hook", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "google-genai-auto-hook-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "generate", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "google-generate-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 16, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 16, - "model": "gemini-2.5-flash-lite", - "provider": "google-genai", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "google-genai.generateContent", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 2, - "duration": 0, - "prompt_tokens": 6, - "tokens": 8 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 2, - "promptTokenCount": 6, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 6 - } - ], - "totalTokenCount": 8 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "attachment", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "google-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 24, - "model": "gemini-2.5-flash-lite", - "provider": "google-genai", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "google-genai.generateContent", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 14, - "duration": 0, - "prompt_tokens": 268, - "tokens": 282 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 14, - "promptTokenCount": 268, - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": 258 - }, - { - "modality": "TEXT", - "tokenCount": 10 - } - ], - "totalTokenCount": 282 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "google-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 32, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 32, - "model": "gemini-2.5-flash-lite", - "provider": "google-genai", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "google-genai.generateContentStream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 28, - "duration": 0, - "prompt_tokens": 16, - "time_to_first_token": 0, - "tokens": 44 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": 28, - "promptTokenCount": 16, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 16 - } - ], - "totalTokenCount": 44 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-return", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "google-stream-return-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": { - "text": "Write a short poem about Paris." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 48, - "model": "gemini-2.5-flash-lite", - "provider": "google-genai", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "google-genai.generateContentStream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "duration": 0, - "prompt_tokens": 8, - "time_to_first_token": 0, - "tokens": 9 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "City" - } - ], - "role": "model" - } - } - ], - "text": "City", - "usageMetadata": { - "candidatesTokenCount": 1, - "promptTokenCount": 8, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 8 - } - ], - "totalTokenCount": 9 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "google-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - }, - "tools": [ - { - "functionDeclarations": [ - { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parametersJsonSchema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - } - ] - } - ] - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 128, - "model": "gemini-2.5-flash-lite", - "provider": "google-genai", - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "google-genai.generateContent", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 17, - "duration": 0, - "prompt_tokens": 65, - "tokens": 82 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 17, - "promptTokenCount": 65, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 65 - } - ], - "totalTokenCount": 82 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/package.json b/e2e/scenarios/google-genai-auto-instrumentation-node-hook/package.json deleted file mode 100644 index 512358ec5..000000000 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@braintrust/e2e-google-genai-auto-instrumentation-node-hook", - "private": true, - "braintrustScenario": { - "canary": { - "dependencies": { - "@google/genai": "latest" - } - } - }, - "dependencies": { - "@google/genai": "1.25.0" - } -} diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/pnpm-lock.yaml b/e2e/scenarios/google-genai-auto-instrumentation-node-hook/pnpm-lock.yaml deleted file mode 100644 index e1533d4ee..000000000 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/pnpm-lock.yaml +++ /dev/null @@ -1,246 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - '@google/genai': - specifier: 1.25.0 - version: 1.25.0 - -packages: - - '@google/genai@1.25.0': - resolution: {integrity: sha512-IBNyel/umavam98SQUfvQSvh/Rp6Ql2fysQLqPyWZr5K8d768X9AO+JZU4o+3qvFDUBA0dVYUSkxyYonVcICvA==} - engines: {node: '>=20.0.0'} - peerDependencies: - '@modelcontextprotocol/sdk': ^1.11.4 - peerDependenciesMeta: - '@modelcontextprotocol/sdk': - optional: true - - agent-base@7.1.4: - resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} - engines: {node: '>= 14'} - - base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - - bignumber.js@9.3.1: - resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} - - buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - - debug@4.4.3: - resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - - extend@3.0.2: - resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - - gaxios@6.7.1: - resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} - engines: {node: '>=14'} - - gcp-metadata@6.1.1: - resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} - engines: {node: '>=14'} - - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} - - google-logging-utils@0.0.2: - resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} - engines: {node: '>=14'} - - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} - - https-proxy-agent@7.0.6: - resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} - engines: {node: '>= 14'} - - is-stream@2.0.1: - resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} - engines: {node: '>=8'} - - json-bigint@1.0.0: - resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} - - jwa@2.0.1: - resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - - jws@4.0.1: - resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - node-fetch@2.7.0: - resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - - safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - - tr46@0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - - uuid@9.0.1: - resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} - hasBin: true - - webidl-conversions@3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - - whatwg-url@5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - - ws@8.19.0: - resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - -snapshots: - - '@google/genai@1.25.0': - dependencies: - google-auth-library: 9.15.1 - ws: 8.19.0 - transitivePeerDependencies: - - bufferutil - - encoding - - supports-color - - utf-8-validate - - agent-base@7.1.4: {} - - base64-js@1.5.1: {} - - bignumber.js@9.3.1: {} - - buffer-equal-constant-time@1.0.1: {} - - debug@4.4.3: - dependencies: - ms: 2.1.3 - - ecdsa-sig-formatter@1.0.11: - dependencies: - safe-buffer: 5.2.1 - - extend@3.0.2: {} - - gaxios@6.7.1: - dependencies: - extend: 3.0.2 - https-proxy-agent: 7.0.6 - is-stream: 2.0.1 - node-fetch: 2.7.0 - uuid: 9.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - gcp-metadata@6.1.1: - dependencies: - gaxios: 6.7.1 - google-logging-utils: 0.0.2 - json-bigint: 1.0.0 - transitivePeerDependencies: - - encoding - - supports-color - - google-auth-library@9.15.1: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 - jws: 4.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - google-logging-utils@0.0.2: {} - - gtoken@7.1.0: - dependencies: - gaxios: 6.7.1 - jws: 4.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - https-proxy-agent@7.0.6: - dependencies: - agent-base: 7.1.4 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - - is-stream@2.0.1: {} - - json-bigint@1.0.0: - dependencies: - bignumber.js: 9.3.1 - - jwa@2.0.1: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - - jws@4.0.1: - dependencies: - jwa: 2.0.1 - safe-buffer: 5.2.1 - - ms@2.1.3: {} - - node-fetch@2.7.0: - dependencies: - whatwg-url: 5.0.0 - - safe-buffer@5.2.1: {} - - tr46@0.0.3: {} - - uuid@9.0.1: {} - - webidl-conversions@3.0.1: {} - - whatwg-url@5.0.0: - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - - ws@8.19.0: {} diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.mjs b/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.mjs deleted file mode 100644 index 0c98012d6..000000000 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.mjs +++ /dev/null @@ -1,13 +0,0 @@ -import * as googleGenAI from "@google/genai"; -import { runGoogleGenAIScenario } from "../../helpers/google-genai-scenario.mjs"; -import { runMain } from "../../helpers/provider-runtime.mjs"; - -runMain(async () => - runGoogleGenAIScenario({ - projectNameBase: "e2e-google-genai-auto-instrumentation-hook", - rootName: "google-genai-auto-hook-root", - scenarioName: "google-genai-auto-instrumentation-node-hook", - sdk: googleGenAI, - testImageUrl: new URL("./test-image.png", import.meta.url), - }), -); diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.test.ts b/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.test.ts deleted file mode 100644 index d01bf4eec..000000000 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/scenario.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { expect, test } from "vitest"; -import { assertGoogleGenAITraceContract } from "../../helpers/google-genai-trace-contract"; -import { - formatJsonFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; -import { - prepareScenarioDir, - resolveScenarioDir, - withScenarioHarness, -} from "../../helpers/scenario-harness"; -import { E2E_TAGS } from "../../helpers/tags"; - -const scenarioDir = await prepareScenarioDir({ - scenarioDir: resolveScenarioDir(import.meta.url), -}); -const TIMEOUT_MS = 90_000; - -test( - "google genai auto-instrumentation via node hook collects the shared google genai trace contract", - { - tags: [E2E_TAGS.externalApi], - timeout: TIMEOUT_MS, - }, - async () => { - await withScenarioHarness( - async ({ events, payloads, runNodeScenarioDir }) => { - await runNodeScenarioDir({ - nodeArgs: ["--import", "braintrust/hook.mjs"], - scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - - const contract = assertGoogleGenAITraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "google-genai-auto-hook-root", - scenarioName: "google-genai-auto-instrumentation-node-hook", - }); - - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }, - ); - }, -); diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json new file mode 100644 index 000000000..38006e222 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-genai-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "generate" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-generate-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 6, + "start": 0, + "tokens": 8 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "totalTokenCount": 8 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-attachment-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "end": 0, + "prompt_tokens": 268, + "start": 0, + "tokens": 282 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "IMAGE", + "tokenCount": 258 + }, + { + "modality": "TEXT", + "tokenCount": 10 + } + ], + "totalTokenCount": 282 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 32, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "end": 0, + "prompt_tokens": 16, + "start": 0, + "time_to_first_token": 0, + "tokens": 44 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "totalTokenCount": 44 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream-return" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-return-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 7, + "start": 0, + "time_to_first_token": 0, + "tokens": 9 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "totalTokenCount": 9 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-tool-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "end": 0, + "prompt_tokens": 65, + "start": 0, + "tokens": 82 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "totalTokenCount": 82 + } + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json similarity index 91% rename from e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/span-events.json rename to e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json index 653609978..dacc75405 100644 --- a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/__snapshots__/span-events.json +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1300.span-events.json @@ -3,10 +3,10 @@ "has_input": false, "has_output": false, "metadata": { - "scenario": "google-genai-auto-instrumentation-node-hook" + "scenario": "google-genai-instrumentation" }, "metric_keys": [], - "name": "google-genai-auto-hook-root", + "name": "google-genai-instrumentation-root", "root_span_id": "", "span_id": "", "span_parents": [], @@ -39,7 +39,7 @@ "prompt_tokens", "tokens" ], - "name": "google-genai.generateContent", + "name": "generate_content", "root_span_id": "", "span_id": "", "span_parents": [ @@ -74,7 +74,7 @@ "prompt_tokens", "tokens" ], - "name": "google-genai.generateContent", + "name": "generate_content", "root_span_id": "", "span_id": "", "span_parents": [ @@ -110,7 +110,7 @@ "time_to_first_token", "tokens" ], - "name": "google-genai.generateContentStream", + "name": "generate_content_stream", "root_span_id": "", "span_id": "", "span_parents": [ @@ -146,7 +146,7 @@ "time_to_first_token", "tokens" ], - "name": "google-genai.generateContentStream", + "name": "generate_content_stream", "root_span_id": "", "span_id": "", "span_parents": [ @@ -181,7 +181,7 @@ "prompt_tokens", "tokens" ], - "name": "google-genai.generateContent", + "name": "generate_content", "root_span_id": "", "span_id": "", "span_parents": [ diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json new file mode 100644 index 000000000..38006e222 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-genai-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "generate" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-generate-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 6, + "start": 0, + "tokens": 8 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "totalTokenCount": 8 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-attachment-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "end": 0, + "prompt_tokens": 268, + "start": 0, + "tokens": 282 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "IMAGE", + "tokenCount": 258 + }, + { + "modality": "TEXT", + "tokenCount": 10 + } + ], + "totalTokenCount": 282 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 32, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "end": 0, + "prompt_tokens": 16, + "start": 0, + "time_to_first_token": 0, + "tokens": 44 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "totalTokenCount": 44 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream-return" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-return-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 7, + "start": 0, + "time_to_first_token": 0, + "tokens": 9 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "totalTokenCount": 9 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-tool-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "end": 0, + "prompt_tokens": 65, + "start": 0, + "tokens": 82 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "totalTokenCount": 82 + } + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json new file mode 100644 index 000000000..dacc75405 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1440.span-events.json @@ -0,0 +1,192 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metric_keys": [], + "name": "google-genai-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "generate" + }, + "metric_keys": [], + "name": "google-generate-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "google-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "google-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-return" + }, + "metric_keys": [], + "name": "google-stream-return-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "google-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json new file mode 100644 index 000000000..38006e222 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-genai-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "generate" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-generate-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 6, + "start": 0, + "tokens": 8 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "totalTokenCount": 8 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-attachment-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "end": 0, + "prompt_tokens": 268, + "start": 0, + "tokens": 282 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "IMAGE", + "tokenCount": 258 + }, + { + "modality": "TEXT", + "tokenCount": 10 + } + ], + "totalTokenCount": 282 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 32, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "end": 0, + "prompt_tokens": 16, + "start": 0, + "time_to_first_token": 0, + "tokens": 44 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "totalTokenCount": 44 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream-return" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-return-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 7, + "start": 0, + "time_to_first_token": 0, + "tokens": 9 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "totalTokenCount": 9 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-tool-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "end": 0, + "prompt_tokens": 65, + "start": 0, + "tokens": 82 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "totalTokenCount": 82 + } + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json new file mode 100644 index 000000000..dacc75405 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1450.span-events.json @@ -0,0 +1,192 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metric_keys": [], + "name": "google-genai-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "generate" + }, + "metric_keys": [], + "name": "google-generate-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "google-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "google-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-return" + }, + "metric_keys": [], + "name": "google-stream-return-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "google-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json new file mode 100644 index 000000000..38006e222 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-genai-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "generate" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-generate-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 6, + "start": 0, + "tokens": 8 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "totalTokenCount": 8 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-attachment-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "end": 0, + "prompt_tokens": 268, + "start": 0, + "tokens": 282 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "IMAGE", + "tokenCount": 258 + }, + { + "modality": "TEXT", + "tokenCount": 10 + } + ], + "totalTokenCount": 282 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 32, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "end": 0, + "prompt_tokens": 16, + "start": 0, + "time_to_first_token": 0, + "tokens": 44 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "totalTokenCount": 44 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream-return" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-return-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 7, + "start": 0, + "time_to_first_token": 0, + "tokens": 9 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "totalTokenCount": 9 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-tool-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "end": 0, + "prompt_tokens": 65, + "start": 0, + "tokens": 82 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "totalTokenCount": 82 + } + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json new file mode 100644 index 000000000..dacc75405 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/google-genai-v1460.span-events.json @@ -0,0 +1,192 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metric_keys": [], + "name": "google-genai-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "generate" + }, + "metric_keys": [], + "name": "google-generate-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "google-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "google-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-return" + }, + "metric_keys": [], + "name": "google-stream-return-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "generate_content_stream", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "google-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metric_keys": [ + "completion_tokens", + "duration", + "prompt_tokens", + "tokens" + ], + "name": "generate_content", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json new file mode 100644 index 000000000..38006e222 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/log-payloads.json @@ -0,0 +1,418 @@ +[ + { + "metadata": { + "scenario": "google-genai-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-genai-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "generate" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-generate-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly PARIS." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 6, + "start": 0, + "tokens": 8 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "PARIS" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 6, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 6 + } + ], + "totalTokenCount": 8 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-attachment-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 24, + "temperature": 0 + }, + "contents": [ + { + "parts": [ + { + "image_url": { + "url": { + "content_type": "image/png", + "filename": "file.png", + "key": "", + "type": "braintrust_attachment" + } + } + }, + { + "text": "Describe the attached image in one short sentence." + } + ], + "role": "user" + } + ], + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 14, + "duration": 0, + "end": 0, + "prompt_tokens": 268, + "start": 0, + "tokens": 282 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 14, + "promptTokenCount": 268, + "promptTokensDetails": [ + { + "modality": "IMAGE", + "tokenCount": 258 + }, + { + "modality": "TEXT", + "tokenCount": 10 + } + ], + "totalTokenCount": 282 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 32, + "temperature": 0 + }, + "contents": { + "text": "Count from 1 to 3 and include the words one two three." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 28, + "duration": 0, + "end": 0, + "prompt_tokens": 16, + "start": 0, + "time_to_first_token": 0, + "tokens": 44 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", + "usageMetadata": { + "candidatesTokenCount": 28, + "promptTokenCount": 16, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 16 + } + ], + "totalTokenCount": 44 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream-return" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-stream-return-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 16, + "temperature": 0 + }, + "contents": { + "text": "Reply with exactly BONJOUR." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 2, + "duration": 0, + "end": 0, + "prompt_tokens": 7, + "start": 0, + "time_to_first_token": 0, + "tokens": 9 + }, + "name": "generate_content_stream", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "BONJOUR" + } + ], + "role": "model" + }, + "finishReason": "STOP" + } + ], + "text": "BONJOUR", + "usageMetadata": { + "candidatesTokenCount": 2, + "promptTokenCount": 7, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ], + "totalTokenCount": 9 + } + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "google-tool-operation", + "type": null + }, + { + "input": { + "config": { + "maxOutputTokens": 128, + "temperature": 0, + "toolConfig": { + "functionCallingConfig": { + "allowedFunctionNames": [ + "get_weather" + ], + "mode": "ANY" + } + } + }, + "contents": { + "text": "Use the get_weather function for Paris, France. Do not answer from memory." + }, + "model": "gemini-2.5-flash-lite" + }, + "metadata": { + "model": "gemini-2.5-flash-lite" + }, + "metrics": { + "completion_tokens": 17, + "duration": 0, + "end": 0, + "prompt_tokens": 65, + "start": 0, + "tokens": 82 + }, + "name": "generate_content", + "output": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "args": { + "location": "Paris, France" + }, + "name": "get_weather" + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "modelVersion": "gemini-2.5-flash-lite", + "responseId": "", + "sdkHttpResponse": { + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "", + "server": "scaffolding on HTTPServer2", + "server-timing": "", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-gemini-service-tier": "", + "x-xss-protection": "0" + } + }, + "usageMetadata": { + "candidatesTokenCount": 17, + "promptTokenCount": 65, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 65 + } + ], + "totalTokenCount": 82 + } + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/span-events.json b/e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json similarity index 97% rename from e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/span-events.json rename to e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json index e0d6f070e..dacc75405 100644 --- a/e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/span-events.json +++ b/e2e/scenarios/google-genai-instrumentation/__snapshots__/span-events.json @@ -3,10 +3,10 @@ "has_input": false, "has_output": false, "metadata": { - "scenario": "wrap-google-genai-content-traces" + "scenario": "google-genai-instrumentation" }, "metric_keys": [], - "name": "google-genai-wrapper-root", + "name": "google-genai-instrumentation-root", "root_span_id": "", "span_id": "", "span_parents": [], diff --git a/e2e/scenarios/google-genai-instrumentation/assertions.ts b/e2e/scenarios/google-genai-instrumentation/assertions.ts new file mode 100644 index 000000000..dd57d1ea2 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/assertions.ts @@ -0,0 +1,408 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; +import { E2E_TAGS } from "../../helpers/tags"; +import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; + +type RunGoogleGenAIScenario = (harness: { + runNodeScenarioDir: (options: { + entry: string; + nodeArgs: string[]; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + entry: string; + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findGoogleSpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: string[], +) { + for (const name of names) { + const span = findChildSpans(events, name, parentId)[0]; + if (span) { + return span; + } + } + + return undefined; +} + +function pickMetadata( + metadata: Record | undefined, + keys: string[], +): Json { + if (!metadata) { + return null; + } + + const picked = Object.fromEntries( + keys.flatMap((key) => + key in metadata ? [[key, metadata[key] as Json]] : [], + ), + ); + + return Object.keys(picked).length > 0 ? (picked as Json) : null; +} + +function isRecord(value: Json | undefined): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeGoogleMetrics(metrics: Json): Json { + if (!isRecord(metrics)) { + return metrics; + } + + const normalized = structuredClone(metrics); + delete normalized.prompt_cached_tokens; + return normalized; +} + +function normalizeGoogleOutput(event: CapturedLogEvent): Json { + const output = event.output as Json; + if (!isRecord(output)) { + return output; + } + + const normalized = structuredClone(output); + const usageMetadata = normalized.usageMetadata; + if (isRecord(usageMetadata)) { + delete usageMetadata.cachedContentTokenCount; + delete usageMetadata.cacheTokensDetails; + + const promptTokensDetails = usageMetadata.promptTokensDetails; + if (Array.isArray(promptTokensDetails)) { + promptTokensDetails.sort((left, right) => + String( + isRecord(left as Json) ? (left.modality ?? "") : "", + ).localeCompare( + String(isRecord(right as Json) ? (right.modality ?? "") : ""), + ), + ); + } + } + + const input = event.input as Json; + const hasAttachmentInput = + Array.isArray(input) && + input.some( + (message) => + isRecord(message as Json) && + Array.isArray(message.content) && + message.content.some( + (block) => + isRecord(block as Json) && + isRecord(block.inlineData) && + block.inlineData.mimeType === "image/png", + ), + ); + + if (!hasAttachmentInput) { + return normalized; + } + + const candidates = normalized.candidates; + if (Array.isArray(candidates)) { + for (const candidate of candidates) { + if (!isRecord(candidate as Json) || !isRecord(candidate.content)) { + continue; + } + + const parts = candidate.content.parts; + if (!Array.isArray(parts)) { + continue; + } + + for (const part of parts) { + if (isRecord(part as Json) && typeof part.text === "string") { + part.text = ""; + } + } + } + } + + if (typeof normalized.text === "string") { + normalized.text = ""; + } + + return normalized; +} + +function normalizeGoogleSummary(summary: Json): Json { + if (!isRecord(summary) || !Array.isArray(summary.metric_keys)) { + return summary; + } + + return { + ...summary, + metric_keys: summary.metric_keys.filter( + (metric): metric is string => metric !== "prompt_cached_tokens", + ), + } satisfies Json; +} + +function summarizeGooglePayload(event: CapturedLogEvent): Json { + return { + input: event.input as Json, + metadata: pickMetadata( + event.row.metadata as Record | undefined, + ["model", "operation", "scenario"], + ), + metrics: normalizeGoogleMetrics(event.metrics as Json), + name: event.span.name ?? null, + output: normalizeGoogleOutput(event), + type: event.span.type ?? null, + } satisfies Json; +} + +function buildRelevantEvents(events: CapturedLogEvent[]): CapturedLogEvent[] { + const generateOperation = findLatestSpan(events, "google-generate-operation"); + const attachmentOperation = findLatestSpan( + events, + "google-attachment-operation", + ); + const streamOperation = findLatestSpan(events, "google-stream-operation"); + const streamReturnOperation = findLatestSpan( + events, + "google-stream-return-operation", + ); + const toolOperation = findLatestSpan(events, "google-tool-operation"); + + return [ + findLatestSpan(events, ROOT_NAME), + generateOperation, + findGoogleSpan(events, generateOperation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]), + attachmentOperation, + findGoogleSpan(events, attachmentOperation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]), + streamOperation, + findGoogleSpan(events, streamOperation?.span.id, [ + "generate_content_stream", + "google-genai.generateContentStream", + ]), + streamReturnOperation, + findGoogleSpan(events, streamReturnOperation?.span.id, [ + "generate_content_stream", + "google-genai.generateContentStream", + ]), + toolOperation, + findGoogleSpan(events, toolOperation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]), + ].map((event) => event!); +} + +function buildSpanSummary(events: CapturedLogEvent[]): Json { + return normalizeForSnapshot( + buildRelevantEvents(events).map((event) => + normalizeGoogleSummary( + summarizeWrapperContract(event, ["model", "operation", "scenario"]), + ), + ) as Json, + ); +} + +function buildPayloadSummary(events: CapturedLogEvent[]): Json { + return normalizeForSnapshot( + buildRelevantEvents(events).map((event) => + summarizeGooglePayload(event), + ) as Json, + ); +} + +export function defineGoogleGenAIInstrumentationAssertions(options: { + name: string; + runScenario: RunGoogleGenAIScenario; + snapshotName: string; + testFileUrl: string; + timeoutMs: number; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-events.json`, + ); + const payloadSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.log-payloads.json`, + ); + const testConfig = { + tags: [E2E_TAGS.externalApi], + timeout: options.timeoutMs, + }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures the root trace for the scenario", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + }); + + test( + "captures trace for client.models.generateContent()", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "google-generate-operation"); + const span = findGoogleSpan(events, operation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + model: "gemini-2.5-flash-lite", + }); + }, + ); + + test("captures trace for sending an attachment", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "google-attachment-operation"); + const span = findGoogleSpan(events, operation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + model: "gemini-2.5-flash-lite", + }); + expect(JSON.stringify(span?.input)).toContain("file.png"); + }); + + test( + "captures trace for client.models.generateContentStream()", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "google-stream-operation"); + const span = findGoogleSpan(events, operation?.span.id, [ + "generate_content_stream", + "google-genai.generateContentStream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + model: "gemini-2.5-flash-lite", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + }, + ); + + test( + "captures trace for the early-return streaming path", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "google-stream-return-operation", + ); + const span = findGoogleSpan(events, operation?.span.id, [ + "generate_content_stream", + "google-genai.generateContentStream", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + model: "gemini-2.5-flash-lite", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + }); + }, + ); + + test("captures trace for tool calling", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "google-tool-operation"); + const span = findGoogleSpan(events, operation?.span.id, [ + "generate_content", + "google-genai.generateContent", + ]); + const output = span?.output as + | { + candidates?: Array<{ + content?: { + parts?: Array<{ + functionCall?: { name?: string }; + }>; + }; + }>; + functionCalls?: Array<{ name?: string }>; + } + | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + model: "gemini-2.5-flash-lite", + }); + expect( + output?.functionCalls?.some((call) => call.name === "get_weather") || + output?.candidates?.some((candidate) => + candidate.content?.parts?.some( + (part) => part.functionCall?.name === "get_weather", + ), + ), + ).toBe(true); + }); + + test("matches the shared span snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot(buildSpanSummary(events)), + ).toMatchFileSnapshot(spanSnapshotPath); + }); + + test("matches the shared payload snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot(buildPayloadSummary(events)), + ).toMatchFileSnapshot(payloadSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/google-genai-instrumentation/package.json b/e2e/scenarios/google-genai-instrumentation/package.json new file mode 100644 index 000000000..2f731025f --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/package.json @@ -0,0 +1,17 @@ +{ + "name": "@braintrust/e2e-google-genai-instrumentation", + "private": true, + "braintrustScenario": { + "canary": { + "dependencies": { + "@google/genai": "latest" + } + } + }, + "dependencies": { + "@google/genai": "1.46.0", + "google-genai-sdk-v1300": "npm:@google/genai@1.30.0", + "google-genai-sdk-v1440": "npm:@google/genai@1.44.0", + "google-genai-sdk-v1450": "npm:@google/genai@1.45.0" + } +} diff --git a/e2e/scenarios/google-genai-instrumentation/pnpm-lock.yaml b/e2e/scenarios/google-genai-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..c1c689452 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/pnpm-lock.yaml @@ -0,0 +1,409 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@google/genai': + specifier: 1.46.0 + version: 1.46.0 + google-genai-sdk-v1300: + specifier: npm:@google/genai@1.30.0 + version: '@google/genai@1.30.0' + google-genai-sdk-v1440: + specifier: npm:@google/genai@1.44.0 + version: '@google/genai@1.44.0' + google-genai-sdk-v1450: + specifier: npm:@google/genai@1.45.0 + version: '@google/genai@1.45.0' + +packages: + + '@google/genai@1.30.0': + resolution: {integrity: sha512-3MRcgczBFbUat1wIlZoLJ0vCCfXgm7Qxjh59cZi2X08RgWLtm9hKOspzp7TOg1TV2e26/MLxR2GR5yD5GmBV2w==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.20.1 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + + '@google/genai@1.44.0': + resolution: {integrity: sha512-kRt9ZtuXmz+tLlcNntN/VV4LRdpl6ZOu5B1KbfNgfR65db15O6sUQcwnwLka8sT/V6qysD93fWrgJHF2L7dA9A==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.25.2 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + + '@google/genai@1.45.0': + resolution: {integrity: sha512-+sNRWhKiRibVgc4OKi7aBJJ0A7RcoVD8tGG+eFkqxAWRjASDW+ktS9lLwTDnAxZICzCVoeAdu8dYLJVTX60N9w==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.25.2 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + + '@google/genai@1.46.0': + resolution: {integrity: sha512-ewPMN5JkKfgU5/kdco9ZhXBHDPhVqZpMQqIFQhwsHLf8kyZfx1cNpw1pHo1eV6PGEW7EhIBFi3aYZraFndAXqg==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.25.2 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + + '@types/node@25.5.0': + resolution: {integrity: sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==} + + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + gaxios@7.1.4: + resolution: {integrity: sha512-bTIgTsM2bWn3XklZISBTQX7ZSddGW+IO3bMdGaemHZ3tbqExMENHLx6kKZ/KlejgrMtj8q7wBItt51yegqalrA==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + google-auth-library@10.6.2: + resolution: {integrity: sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + undici-types@7.18.2: + resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + ws@8.19.0: + resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + +snapshots: + + '@google/genai@1.30.0': + dependencies: + google-auth-library: 10.6.2 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@google/genai@1.44.0': + dependencies: + google-auth-library: 10.6.2 + p-retry: 4.6.2 + protobufjs: 7.5.4 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@google/genai@1.45.0': + dependencies: + google-auth-library: 10.6.2 + p-retry: 4.6.2 + protobufjs: 7.5.4 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@google/genai@1.46.0': + dependencies: + google-auth-library: 10.6.2 + p-retry: 4.6.2 + protobufjs: 7.5.4 + ws: 8.19.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + + '@types/node@25.5.0': + dependencies: + undici-types: 7.18.2 + + '@types/retry@0.12.0': {} + + agent-base@7.1.4: {} + + base64-js@1.5.1: {} + + bignumber.js@9.3.1: {} + + buffer-equal-constant-time@1.0.1: {} + + data-uri-to-buffer@4.0.1: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + extend@3.0.2: {} + + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + gaxios@7.1.4: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.1.4 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + google-auth-library@10.6.2: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.1.4 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + + long@5.3.2: {} + + ms@2.1.3: {} + + node-domexception@1.0.0: {} + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 25.5.0 + long: 5.3.2 + + retry@0.13.1: {} + + safe-buffer@5.2.1: {} + + undici-types@7.18.2: {} + + web-streams-polyfill@3.3.3: {} + + ws@8.19.0: {} diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.mjs new file mode 100644 index 000000000..14d0fe379 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.mjs @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1300"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.ts b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.ts new file mode 100644 index 000000000..04d7a9f90 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1300.ts @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1300"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.mjs new file mode 100644 index 000000000..aeac1e938 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.mjs @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1440"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.ts b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.ts new file mode 100644 index 000000000..e5a7ae73e --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1440.ts @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1440"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.mjs new file mode 100644 index 000000000..82a25cff8 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.mjs @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1450"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.ts b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.ts new file mode 100644 index 000000000..31acbebb4 --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.google-genai-v1450.ts @@ -0,0 +1,5 @@ +import * as googleGenAI from "google-genai-sdk-v1450"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/helpers/google-genai-scenario.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs similarity index 64% rename from e2e/helpers/google-genai-scenario.mjs rename to e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs index 080bafd41..31ce210dd 100644 --- a/e2e/helpers/google-genai-scenario.mjs +++ b/e2e/scenarios/google-genai-instrumentation/scenario.impl.mjs @@ -1,18 +1,39 @@ import { readFile } from "node:fs/promises"; +import { wrapGoogleGenAI } from "braintrust"; import { collectAsync, runOperation, runTracedScenario, -} from "./provider-runtime.mjs"; +} from "../../helpers/provider-runtime.mjs"; const GOOGLE_MODEL = "gemini-2.5-flash-lite"; +const ROOT_NAME = "google-genai-instrumentation-root"; +const SCENARIO_NAME = "google-genai-instrumentation"; +const WEATHER_TOOL = { + functionDeclarations: [ + { + name: "get_weather", + description: "Get the current weather in a given location", + parametersJsonSchema: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state or city and country", + }, + }, + required: ["location"], + }, + }, + ], +}; -export async function runGoogleGenAIScenario(options) { - const imageBase64 = (await readFile(options.testImageUrl)).toString("base64"); - const sdk = options.decorateSDK - ? options.decorateSDK(options.sdk) - : options.sdk; - const { GoogleGenAI } = sdk; +async function runGoogleGenAIInstrumentationScenario(sdk, options = {}) { + const imageBase64 = ( + await readFile(new URL("./test-image.png", import.meta.url)) + ).toString("base64"); + const decoratedSDK = options.decorateSDK ? options.decorateSDK(sdk) : sdk; + const { GoogleGenAI } = decoratedSDK; const client = new GoogleGenAI({ apiKey: process.env.GOOGLE_API_KEY || process.env.GEMINI_API_KEY, }); @@ -78,9 +99,9 @@ export async function runGoogleGenAIScenario(options) { async () => { const stream = await client.models.generateContentStream({ model: GOOGLE_MODEL, - contents: "Write a short poem about Paris.", + contents: "Reply with exactly BONJOUR.", config: { - maxOutputTokens: 48, + maxOutputTokens: 16, temperature: 0, }, }); @@ -99,30 +120,11 @@ export async function runGoogleGenAIScenario(options) { config: { maxOutputTokens: 128, temperature: 0, - tools: [ - { - functionDeclarations: [ - { - name: "get_weather", - description: "Get the current weather in a given location", - parametersJsonSchema: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state or city and country", - }, - }, - required: ["location"], - }, - }, - ], - }, - ], + tools: [WEATHER_TOOL], toolConfig: { functionCallingConfig: { allowedFunctionNames: ["get_weather"], - mode: options.sdk.FunctionCallingConfigMode.ANY, + mode: sdk.FunctionCallingConfigMode.ANY, }, }, }, @@ -130,9 +132,21 @@ export async function runGoogleGenAIScenario(options) { }); }, metadata: { - scenario: options.scenarioName, + scenario: SCENARIO_NAME, }, - projectNameBase: options.projectNameBase, - rootName: options.rootName, + projectNameBase: "e2e-google-genai-instrumentation", + rootName: ROOT_NAME, + }); +} + +export async function runWrappedGoogleGenAIInstrumentation(sdk) { + await runGoogleGenAIInstrumentationScenario(sdk, { + decorateSDK: wrapGoogleGenAI, }); } + +export async function runAutoGoogleGenAIInstrumentation(sdk) { + await runGoogleGenAIInstrumentationScenario(sdk); +} + +export { ROOT_NAME, SCENARIO_NAME }; diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.mjs b/e2e/scenarios/google-genai-instrumentation/scenario.mjs new file mode 100644 index 000000000..15d612a2f --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.mjs @@ -0,0 +1,5 @@ +import * as googleGenAI from "@google/genai"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.test.ts b/e2e/scenarios/google-genai-instrumentation/scenario.test.ts new file mode 100644 index 000000000..03a3b170a --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.test.ts @@ -0,0 +1,79 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineGoogleGenAIInstrumentationAssertions } from "./assertions"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); +const TIMEOUT_MS = 90_000; +const googleGenAIScenarios = await Promise.all( + [ + { + autoEntry: "scenario.google-genai-v1300.mjs", + dependencyName: "google-genai-sdk-v1300", + snapshotName: "google-genai-v1300", + wrapperEntry: "scenario.google-genai-v1300.ts", + }, + { + autoEntry: "scenario.google-genai-v1440.mjs", + dependencyName: "google-genai-sdk-v1440", + snapshotName: "google-genai-v1440", + wrapperEntry: "scenario.google-genai-v1440.ts", + }, + { + autoEntry: "scenario.google-genai-v1450.mjs", + dependencyName: "google-genai-sdk-v1450", + snapshotName: "google-genai-v1450", + wrapperEntry: "scenario.google-genai-v1450.ts", + }, + { + autoEntry: "scenario.mjs", + dependencyName: "@google/genai", + snapshotName: "google-genai-v1460", + wrapperEntry: "scenario.ts", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +for (const scenario of googleGenAIScenarios) { + describe(`google genai sdk ${scenario.version}`, () => { + defineGoogleGenAIInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: scenario.wrapperEntry, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + + defineGoogleGenAIInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + }); +} diff --git a/e2e/scenarios/google-genai-instrumentation/scenario.ts b/e2e/scenarios/google-genai-instrumentation/scenario.ts new file mode 100644 index 000000000..e5429fe1f --- /dev/null +++ b/e2e/scenarios/google-genai-instrumentation/scenario.ts @@ -0,0 +1,5 @@ +import * as googleGenAI from "@google/genai"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedGoogleGenAIInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedGoogleGenAIInstrumentation(googleGenAI)); diff --git a/e2e/scenarios/google-genai-auto-instrumentation-node-hook/test-image.png b/e2e/scenarios/google-genai-instrumentation/test-image.png similarity index 100% rename from e2e/scenarios/google-genai-auto-instrumentation-node-hook/test-image.png rename to e2e/scenarios/google-genai-instrumentation/test-image.png diff --git a/e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/log-payloads.json deleted file mode 100644 index ccf016306..000000000 --- a/e2e/scenarios/wrap-google-genai-content-traces/__snapshots__/log-payloads.json +++ /dev/null @@ -1,951 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "wrap-google-genai-content-traces", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "google-genai-wrapper-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "generate", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "google-generate-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 16, - "temperature": 0 - }, - "contents": { - "text": "Reply with exactly PARIS." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 16, - "model": "gemini-2.5-flash-lite", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "generate_content", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 2, - "duration": 0, - "prompt_tokens": 6, - "start": 0, - "tokens": 8 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "PARIS" - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 2, - "promptTokenCount": 6, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 6 - } - ], - "totalTokenCount": 8 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "attachment", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "google-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 24, - "temperature": 0 - }, - "contents": [ - { - "parts": [ - { - "image_url": { - "url": { - "content_type": "image/png", - "filename": "file.png", - "key": "", - "type": "braintrust_attachment" - } - } - }, - { - "text": "Describe the attached image in one short sentence." - } - ], - "role": "user" - } - ], - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 24, - "model": "gemini-2.5-flash-lite", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "generate_content", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 14, - "duration": 0, - "prompt_tokens": 268, - "start": 0, - "tokens": 282 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "A majestic sailing ship battles a fierce storm with towering waves and lightning." - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 14, - "promptTokenCount": 268, - "promptTokensDetails": [ - { - "modality": "IMAGE", - "tokenCount": 258 - }, - { - "modality": "TEXT", - "tokenCount": 10 - } - ], - "totalTokenCount": 282 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "google-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "collectAsync", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 32, - "temperature": 0 - }, - "contents": { - "text": "Count from 1 to 3 and include the words one two three." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 32, - "model": "gemini-2.5-flash-lite", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "generate_content_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 28, - "duration": 0, - "prompt_tokens": 16, - "start": 0, - "time_to_first_token": 0, - "tokens": 44 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three" - } - ], - "role": "model" - }, - "finishReason": "STOP" - } - ], - "text": "Here's the count from 1 to 3, including the words:\n\n1. One\n2. Two\n3. Three", - "usageMetadata": { - "candidatesTokenCount": 28, - "promptTokenCount": 16, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 16 - } - ], - "totalTokenCount": 44 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-return", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "google-stream-return-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 48, - "temperature": 0 - }, - "contents": { - "text": "Write a short poem about Paris." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 48, - "model": "gemini-2.5-flash-lite", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "generate_content_stream", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "duration": 0, - "prompt_tokens": 8, - "start": 0, - "time_to_first_token": 0, - "tokens": 9 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "text": "City" - } - ], - "role": "model" - } - } - ], - "text": "City", - "usageMetadata": { - "candidatesTokenCount": 1, - "promptTokenCount": 8, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 8 - } - ], - "totalTokenCount": 9 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "google-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": { - "config": { - "maxOutputTokens": 128, - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - } - }, - "contents": { - "text": "Use the get_weather function for Paris, France. Do not answer from memory." - }, - "model": "gemini-2.5-flash-lite" - }, - "log_id": "g", - "metadata": { - "maxOutputTokens": 128, - "model": "gemini-2.5-flash-lite", - "temperature": 0, - "toolConfig": { - "functionCallingConfig": { - "allowedFunctionNames": [ - "get_weather" - ], - "mode": "ANY" - } - }, - "tools": [ - { - "functionDeclarations": [ - { - "description": "Get the current weather in a given location", - "name": "get_weather", - "parametersJsonSchema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - } - } - ] - } - ] - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "generate_content", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 17, - "duration": 0, - "prompt_tokens": 65, - "start": 0, - "tokens": 82 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": { - "candidates": [ - { - "content": { - "parts": [ - { - "functionCall": { - "args": { - "location": "Paris, France" - }, - "name": "get_weather" - } - } - ], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "modelVersion": "gemini-2.5-flash-lite", - "responseId": "", - "sdkHttpResponse": { - "headers": { - "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", - "content-encoding": "gzip", - "content-type": "application/json; charset=UTF-8", - "date": "", - "server": "scaffolding on HTTPServer2", - "server-timing": "", - "transfer-encoding": "chunked", - "vary": "Origin, X-Origin, Referer", - "x-content-type-options": "nosniff", - "x-frame-options": "SAMEORIGIN", - "x-gemini-service-tier": "", - "x-xss-protection": "0" - } - }, - "usageMetadata": { - "candidatesTokenCount": 17, - "promptTokenCount": 65, - "promptTokensDetails": [ - { - "modality": "TEXT", - "tokenCount": 65 - } - ], - "totalTokenCount": 82 - } - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/wrap-google-genai-content-traces/package.json b/e2e/scenarios/wrap-google-genai-content-traces/package.json deleted file mode 100644 index e0ffaadab..000000000 --- a/e2e/scenarios/wrap-google-genai-content-traces/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@braintrust/e2e-wrap-google-genai-content-traces", - "private": true, - "braintrustScenario": { - "canary": { - "dependencies": { - "@google/genai": "latest" - } - } - }, - "dependencies": { - "@google/genai": "1.25.0" - } -} diff --git a/e2e/scenarios/wrap-google-genai-content-traces/pnpm-lock.yaml b/e2e/scenarios/wrap-google-genai-content-traces/pnpm-lock.yaml deleted file mode 100644 index e1533d4ee..000000000 --- a/e2e/scenarios/wrap-google-genai-content-traces/pnpm-lock.yaml +++ /dev/null @@ -1,246 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - '@google/genai': - specifier: 1.25.0 - version: 1.25.0 - -packages: - - '@google/genai@1.25.0': - resolution: {integrity: sha512-IBNyel/umavam98SQUfvQSvh/Rp6Ql2fysQLqPyWZr5K8d768X9AO+JZU4o+3qvFDUBA0dVYUSkxyYonVcICvA==} - engines: {node: '>=20.0.0'} - peerDependencies: - '@modelcontextprotocol/sdk': ^1.11.4 - peerDependenciesMeta: - '@modelcontextprotocol/sdk': - optional: true - - agent-base@7.1.4: - resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} - engines: {node: '>= 14'} - - base64-js@1.5.1: - resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - - bignumber.js@9.3.1: - resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} - - buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - - debug@4.4.3: - resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - - extend@3.0.2: - resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - - gaxios@6.7.1: - resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==} - engines: {node: '>=14'} - - gcp-metadata@6.1.1: - resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==} - engines: {node: '>=14'} - - google-auth-library@9.15.1: - resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==} - engines: {node: '>=14'} - - google-logging-utils@0.0.2: - resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==} - engines: {node: '>=14'} - - gtoken@7.1.0: - resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==} - engines: {node: '>=14.0.0'} - - https-proxy-agent@7.0.6: - resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} - engines: {node: '>= 14'} - - is-stream@2.0.1: - resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} - engines: {node: '>=8'} - - json-bigint@1.0.0: - resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} - - jwa@2.0.1: - resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - - jws@4.0.1: - resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - node-fetch@2.7.0: - resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} - engines: {node: 4.x || >=6.0.0} - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - - safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - - tr46@0.0.3: - resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} - - uuid@9.0.1: - resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} - hasBin: true - - webidl-conversions@3.0.1: - resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} - - whatwg-url@5.0.0: - resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} - - ws@8.19.0: - resolution: {integrity: sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - -snapshots: - - '@google/genai@1.25.0': - dependencies: - google-auth-library: 9.15.1 - ws: 8.19.0 - transitivePeerDependencies: - - bufferutil - - encoding - - supports-color - - utf-8-validate - - agent-base@7.1.4: {} - - base64-js@1.5.1: {} - - bignumber.js@9.3.1: {} - - buffer-equal-constant-time@1.0.1: {} - - debug@4.4.3: - dependencies: - ms: 2.1.3 - - ecdsa-sig-formatter@1.0.11: - dependencies: - safe-buffer: 5.2.1 - - extend@3.0.2: {} - - gaxios@6.7.1: - dependencies: - extend: 3.0.2 - https-proxy-agent: 7.0.6 - is-stream: 2.0.1 - node-fetch: 2.7.0 - uuid: 9.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - gcp-metadata@6.1.1: - dependencies: - gaxios: 6.7.1 - google-logging-utils: 0.0.2 - json-bigint: 1.0.0 - transitivePeerDependencies: - - encoding - - supports-color - - google-auth-library@9.15.1: - dependencies: - base64-js: 1.5.1 - ecdsa-sig-formatter: 1.0.11 - gaxios: 6.7.1 - gcp-metadata: 6.1.1 - gtoken: 7.1.0 - jws: 4.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - google-logging-utils@0.0.2: {} - - gtoken@7.1.0: - dependencies: - gaxios: 6.7.1 - jws: 4.0.1 - transitivePeerDependencies: - - encoding - - supports-color - - https-proxy-agent@7.0.6: - dependencies: - agent-base: 7.1.4 - debug: 4.4.3 - transitivePeerDependencies: - - supports-color - - is-stream@2.0.1: {} - - json-bigint@1.0.0: - dependencies: - bignumber.js: 9.3.1 - - jwa@2.0.1: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - - jws@4.0.1: - dependencies: - jwa: 2.0.1 - safe-buffer: 5.2.1 - - ms@2.1.3: {} - - node-fetch@2.7.0: - dependencies: - whatwg-url: 5.0.0 - - safe-buffer@5.2.1: {} - - tr46@0.0.3: {} - - uuid@9.0.1: {} - - webidl-conversions@3.0.1: {} - - whatwg-url@5.0.0: - dependencies: - tr46: 0.0.3 - webidl-conversions: 3.0.1 - - ws@8.19.0: {} diff --git a/e2e/scenarios/wrap-google-genai-content-traces/scenario.test.ts b/e2e/scenarios/wrap-google-genai-content-traces/scenario.test.ts deleted file mode 100644 index 1b1916187..000000000 --- a/e2e/scenarios/wrap-google-genai-content-traces/scenario.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { expect, test } from "vitest"; -import { assertGoogleGenAITraceContract } from "../../helpers/google-genai-trace-contract"; -import { - formatJsonFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; -import { - prepareScenarioDir, - resolveScenarioDir, - withScenarioHarness, -} from "../../helpers/scenario-harness"; -import { E2E_TAGS } from "../../helpers/tags"; - -const scenarioDir = await prepareScenarioDir({ - scenarioDir: resolveScenarioDir(import.meta.url), -}); -const TIMEOUT_MS = 90_000; - -test( - "wrap-google-genai-content-traces captures generate, attachment, stream, early-return, and tool spans", - { - tags: [E2E_TAGS.externalApi], - timeout: TIMEOUT_MS, - }, - async () => { - await withScenarioHarness(async ({ events, payloads, runScenarioDir }) => { - await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); - - const contract = assertGoogleGenAITraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "google-genai-wrapper-root", - scenarioName: "wrap-google-genai-content-traces", - }); - - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }); - }, -); diff --git a/e2e/scenarios/wrap-google-genai-content-traces/scenario.ts b/e2e/scenarios/wrap-google-genai-content-traces/scenario.ts deleted file mode 100644 index fe5835fbc..000000000 --- a/e2e/scenarios/wrap-google-genai-content-traces/scenario.ts +++ /dev/null @@ -1,15 +0,0 @@ -import * as googleGenAI from "@google/genai"; -import { wrapGoogleGenAI } from "braintrust"; -import { runGoogleGenAIScenario } from "../../helpers/google-genai-scenario.mjs"; -import { runMain } from "../../helpers/scenario-runtime"; - -runMain(async () => - runGoogleGenAIScenario({ - decorateSDK: wrapGoogleGenAI, - projectNameBase: "e2e-wrap-google-genai", - rootName: "google-genai-wrapper-root", - scenarioName: "wrap-google-genai-content-traces", - sdk: googleGenAI, - testImageUrl: new URL("./test-image.png", import.meta.url), - }), -); diff --git a/e2e/scenarios/wrap-google-genai-content-traces/test-image.png b/e2e/scenarios/wrap-google-genai-content-traces/test-image.png deleted file mode 100644 index d59518eb3..000000000 Binary files a/e2e/scenarios/wrap-google-genai-content-traces/test-image.png and /dev/null differ diff --git a/e2e/scripts/run-canary-tests.mjs b/e2e/scripts/run-canary-tests.mjs index 419838ced..2c346b413 100644 --- a/e2e/scripts/run-canary-tests.mjs +++ b/e2e/scripts/run-canary-tests.mjs @@ -19,7 +19,7 @@ async function fileExists(filePath) { async function getCanaryTestFiles() { const entries = await readdir(SCENARIOS_DIR, { withFileTypes: true }); - const testFiles = []; + const testFiles = new Set(); for (const entry of entries) { if (!entry.isDirectory()) { @@ -44,17 +44,24 @@ async function getCanaryTestFiles() { continue; } - const testPath = path.join(scenarioDir, "scenario.test.ts"); + const configuredTestFile = manifest?.braintrustScenario?.canary?.testFile; + const testPath = + typeof configuredTestFile === "string" && configuredTestFile.length > 0 + ? path.resolve(scenarioDir, configuredTestFile) + : path.join(scenarioDir, "scenario.test.ts"); if (!(await fileExists(testPath))) { throw new Error( - `Canary scenario ${entry.name} is missing scenario.test.ts`, + `Canary scenario ${entry.name} is missing test file ${path.relative( + E2E_ROOT, + testPath, + )}`, ); } - testFiles.push(path.relative(E2E_ROOT, testPath)); + testFiles.add(path.relative(E2E_ROOT, testPath)); } - return testFiles.sort(); + return [...testFiles].sort(); } async function runVitest(testFiles) { diff --git a/js/src/instrumentation/plugins/google-genai-plugin.ts b/js/src/instrumentation/plugins/google-genai-plugin.ts index 21a65525e..55cb70f6c 100644 --- a/js/src/instrumentation/plugins/google-genai-plugin.ts +++ b/js/src/instrumentation/plugins/google-genai-plugin.ts @@ -1,10 +1,23 @@ import { BasePlugin } from "../core"; +import { unsubscribeAll } from "../core/channel-tracing"; +import type { + ChannelMessage, + ErrorOf, + StartOf, +} from "../core/channel-definitions"; +import type { + IsoAsyncLocalStorage, + IsoChannelHandlers, + IsoTracingChannel, +} from "../../isomorph"; import { - traceAsyncChannel, - traceStreamingChannel, - unsubscribeAll, -} from "../core/channel-tracing"; -import { Attachment } from "../../logger"; + _internalGetGlobalState, + Attachment, + BRAINTRUST_CURRENT_SPAN_STORE, + startSpan, + type StartSpanArgs, + type Span, +} from "../../logger"; import { SpanTypeAttribute } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import { googleGenAIChannels } from "./google-genai-channels"; @@ -16,6 +29,37 @@ import type { GoogleGenAIUsageMetadata, } from "../../vendor-sdk-types/google-genai"; +type GenerateContentChannel = typeof googleGenAIChannels.generateContent; +type GenerateContentStreamChannel = + typeof googleGenAIChannels.generateContentStream; +type GenerateContentStreamEvent = + ChannelMessage & { + googleGenAIInput?: Record; + googleGenAIMetadata?: Record; + }; + +type SpanState = { + span: Span; + startTime: number; +}; + +const GOOGLE_GENAI_INTERNAL_CONTEXT = { + caller_filename: "", + caller_functionname: "", + caller_lineno: 0, +}; + +function createWrapperParityEvent(args: { + input: Record; + metadata: Record; +}): StartSpanArgs["event"] { + return { + context: GOOGLE_GENAI_INTERNAL_CONTEXT, + input: args.input, + metadata: args.metadata, + } as StartSpanArgs["event"]; +} + /** * Auto-instrumentation plugin for the Google GenAI SDK. * @@ -40,53 +84,414 @@ export class GoogleGenAIPlugin extends BasePlugin { } private subscribeToGoogleGenAIChannels(): void { - // GenerativeModel.generateContent (non-streaming) - this.unsubscribers.push( - traceAsyncChannel(googleGenAIChannels.generateContent, { - name: "google-genai.generateContent", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const input = serializeInput(params); - const metadata = extractMetadata(params); - return { - input, - metadata: { ...metadata, provider: "google-genai" }, - }; - }, - extractOutput: (result) => { - return result; - }, - extractMetrics: (result, startTime) => { - return extractGenerateContentMetrics(result, startTime); - }, - }), + this.subscribeToGenerateContentChannel(); + this.subscribeToGenerateContentStreamChannel(); + } + + private subscribeToGenerateContentChannel(): void { + const tracingChannel = + googleGenAIChannels.generateContent.tracingChannel() as IsoTracingChannel< + ChannelMessage + >; + const states = new WeakMap(); + const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + tracingChannel, + states, + (event) => { + const params = event.arguments[0]; + const input = serializeInput(params); + const metadata = extractMetadata(params); + const span = startSpan({ + name: "generate_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, + }, + event: createWrapperParityEvent({ input, metadata }), + }); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }, ); - // GenerativeModel.generateContentStream (streaming) - this.unsubscribers.push( - traceStreamingChannel(googleGenAIChannels.generateContentStream, { - name: "google-genai.generateContentStream", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const input = serializeInput(params); - const metadata = extractMetadata(params); - return { - input, - metadata: { ...metadata, provider: "google-genai" }, - }; + const handlers: IsoChannelHandlers> = + { + start: (event) => { + ensureSpanState(states, event, () => { + const params = event.arguments[0]; + const input = serializeInput(params); + const metadata = extractMetadata(params); + const span = startSpan({ + name: "generate_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, + }, + event: createWrapperParityEvent({ input, metadata }), + }); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }); }, - extractOutput: (result) => { - return result; - }, - extractMetrics: () => { - return {}; + asyncEnd: (event) => { + const spanState = states.get(event as object); + if (!spanState) { + return; + } + + try { + spanState.span.log({ + metrics: cleanMetrics( + extractGenerateContentMetrics( + event.result, + spanState.startTime, + ), + ), + output: event.result, + }); + } finally { + spanState.span.end(); + states.delete(event as object); + } }, - aggregateChunks: (chunks, _result, _endEvent, startTime) => { - return aggregateGenerateContentChunks(chunks, startTime); + error: (event) => { + logErrorAndEndSpan(states, event as ErrorOf); }, - }), - ); + }; + + tracingChannel.subscribe(handlers); + this.unsubscribers.push(() => { + unbindCurrentSpanStore?.(); + tracingChannel.unsubscribe(handlers); + }); + } + + private subscribeToGenerateContentStreamChannel(): void { + const tracingChannel = + googleGenAIChannels.generateContentStream.tracingChannel() as IsoTracingChannel< + ChannelMessage + >; + + const handlers: IsoChannelHandlers< + ChannelMessage + > = { + start: (event) => { + const streamEvent = event as GenerateContentStreamEvent; + const params = event.arguments[0]; + streamEvent.googleGenAIInput = serializeInput(params); + streamEvent.googleGenAIMetadata = extractMetadata(params); + }, + asyncEnd: (event) => { + const streamEvent = event as GenerateContentStreamEvent; + patchGoogleGenAIStreamingResult({ + input: streamEvent.googleGenAIInput, + metadata: streamEvent.googleGenAIMetadata, + result: streamEvent.result, + }); + }, + error: () => {}, + }; + + tracingChannel.subscribe(handlers); + this.unsubscribers.push(() => { + tracingChannel.unsubscribe(handlers); + }); + } +} + +function ensureSpanState( + states: WeakMap, + event: TEvent, + create: () => SpanState, +): SpanState { + const existing = states.get(event); + if (existing) { + return existing; } + + const created = create(); + states.set(event, created); + return created; +} + +function bindCurrentSpanStoreToStart( + tracingChannel: IsoTracingChannel>, + states: WeakMap, + create: (event: StartOf) => SpanState, +): (() => void) | undefined { + const state = _internalGetGlobalState(); + const startChannel = tracingChannel.start as + | ({ + bindStore?: ( + store: IsoAsyncLocalStorage, + callback: (event: ChannelMessage) => Span, + ) => void; + unbindStore?: (store: IsoAsyncLocalStorage) => void; + } & object) + | undefined; + const currentSpanStore = state?.contextManager + ? ( + state.contextManager as { + [BRAINTRUST_CURRENT_SPAN_STORE]?: IsoAsyncLocalStorage; + } + )[BRAINTRUST_CURRENT_SPAN_STORE] + : undefined; + + if (!startChannel?.bindStore || !currentSpanStore) { + return undefined; + } + + startChannel.bindStore( + currentSpanStore, + (event) => + ensureSpanState(states, event as object, () => + create(event as StartOf), + ).span, + ); + + return () => { + startChannel.unbindStore?.(currentSpanStore); + }; +} + +function logErrorAndEndSpan( + states: WeakMap, + event: ErrorOf, +): void { + const spanState = states.get(event as object); + if (!spanState) { + return; + } + + spanState.span.log({ + error: event.error.message, + }); + spanState.span.end(); + states.delete(event as object); +} + +function patchGoogleGenAIStreamingResult(args: { + input: Record | undefined; + metadata: Record | undefined; + result: unknown; +}): boolean { + const { input, metadata, result } = args; + + if ( + !input || + !metadata || + !result || + typeof result !== "object" || + typeof (result as AsyncIterator) + .next !== "function" + ) { + return false; + } + + const chunks: GoogleGenAIGenerateContentResponse[] = []; + let firstTokenTime: number | null = null; + let finalized = false; + let span: Span | null = null; + let startTime: number | null = null; + + const ensureSpan = () => { + if (!span) { + span = startSpan({ + name: "generate_content_stream", + spanAttributes: { + type: SpanTypeAttribute.LLM, + }, + event: { + input, + metadata, + }, + }); + startTime = getCurrentUnixTimestamp(); + } + + return span; + }; + + const finalize = (options: { + error?: unknown; + result?: { + aggregated: Record; + metrics: Record; + }; + }) => { + if (finalized || !span) { + return; + } + + finalized = true; + + if (options.result) { + const { end, ...metricsWithoutEnd } = options.result.metrics; + span.log({ + metrics: cleanMetrics(metricsWithoutEnd), + output: options.result.aggregated, + }); + span.end(typeof end === "number" ? { endTime: end } : undefined); + return; + } + + if (options.error !== undefined) { + span.log({ + error: + options.error instanceof Error + ? options.error.message + : String(options.error), + }); + } + + span.end(); + }; + + const patchIterator = ( + iterator: AsyncIterator, + ): AsyncIterator => { + if ( + typeof iterator !== "object" || + iterator === null || + "__braintrustGoogleGenAIPatched" in (iterator as object) + ) { + return iterator; + } + + const iteratorRecord = + iterator as AsyncIterator & + Record; + const originalNext = + typeof iteratorRecord.next === "function" + ? ( + iteratorRecord.next as ( + ...args: [] | [undefined] + ) => Promise> + ).bind(iterator) + : undefined; + const originalReturn = + typeof iteratorRecord.return === "function" + ? ( + iteratorRecord.return as ( + ...args: [] | [unknown] + ) => Promise> + ).bind(iterator) + : undefined; + const originalThrow = + typeof iteratorRecord.throw === "function" + ? ( + iteratorRecord.throw as ( + ...args: [] | [unknown] + ) => Promise> + ).bind(iterator) + : undefined; + const asyncIteratorMethod = iteratorRecord[Symbol.asyncIterator]; + const originalAsyncIterator = + typeof asyncIteratorMethod === "function" + ? ( + asyncIteratorMethod as () => AsyncIterator + ).bind(iterator) + : undefined; + + Object.defineProperty(iteratorRecord, "__braintrustGoogleGenAIPatched", { + configurable: true, + enumerable: false, + value: true, + writable: false, + }); + + if (originalNext) { + iteratorRecord.next = async (...nextArgs: [] | [undefined]) => { + ensureSpan(); + + try { + const nextResult = (await originalNext( + ...nextArgs, + )) as IteratorResult; + + if (!nextResult.done && nextResult.value) { + if (firstTokenTime === null) { + firstTokenTime = getCurrentUnixTimestamp(); + } + chunks.push(nextResult.value); + } + + if (nextResult.done && startTime !== null) { + finalize({ + result: aggregateGenerateContentChunks( + chunks, + startTime, + firstTokenTime, + ), + }); + } + + return nextResult; + } catch (error) { + finalize({ error }); + throw error; + } + }; + } + + if (originalReturn) { + iteratorRecord.return = async (...returnArgs: [] | [unknown]) => { + ensureSpan(); + + try { + return (await originalReturn( + ...returnArgs, + )) as IteratorResult; + } finally { + if (startTime !== null) { + finalize({ + result: + chunks.length > 0 + ? aggregateGenerateContentChunks( + chunks, + startTime, + firstTokenTime, + ) + : undefined, + }); + } else { + finalize({}); + } + } + }; + } + + if (originalThrow) { + iteratorRecord.throw = async (...throwArgs: [] | [unknown]) => { + ensureSpan(); + + try { + return (await originalThrow( + ...throwArgs, + )) as IteratorResult; + } catch (error) { + finalize({ error }); + throw error; + } + }; + } + + iteratorRecord[Symbol.asyncIterator] = () => { + const asyncIterator = originalAsyncIterator + ? (originalAsyncIterator() as AsyncIterator) + : iterator; + return patchIterator(asyncIterator); + }; + + return iterator; + }; + + patchIterator(result as AsyncIterator); + return true; } /** @@ -103,11 +508,13 @@ function serializeInput( if (params.config) { const config = tryToDict(params.config); if (config) { - const tools = serializeTools(params); - if (tools) { - config.tools = tools; - } - input.config = config; + const filteredConfig: Record = {}; + Object.keys(config).forEach((key) => { + if (key !== "tools") { + filteredConfig[key] = config[key]; + } + }); + input.config = filteredConfig; } } @@ -255,6 +662,11 @@ function extractMetadata( } } + const tools = serializeTools(params); + if (tools) { + metadata.tools = tools; + } + return metadata; } @@ -267,8 +679,10 @@ function extractGenerateContentMetrics( ): Record { const metrics: Record = {}; - if (startTime) { + if (startTime !== undefined) { const end = getCurrentUnixTimestamp(); + metrics.start = startTime; + metrics.end = end; metrics.duration = end - startTime; } @@ -305,27 +719,25 @@ function populateUsageMetrics( */ function aggregateGenerateContentChunks( chunks: GoogleGenAIGenerateContentResponse[], - startTime?: number, + startTime: number, + firstTokenTime: number | null, ): { - output: Record; + aggregated: Record; metrics: Record; } { - const metrics: Record = {}; - - if (startTime !== undefined) { - const end = getCurrentUnixTimestamp(); - metrics.duration = end - startTime; - } - - let firstTokenTime: number | null = null; + const end = getCurrentUnixTimestamp(); + const metrics: Record = { + start: startTime, + end, + duration: end - startTime, + }; - if (chunks.length > 0 && firstTokenTime === null && startTime !== undefined) { - firstTokenTime = getCurrentUnixTimestamp(); + if (firstTokenTime !== null) { metrics.time_to_first_token = firstTokenTime - startTime; } if (chunks.length === 0) { - return { output: {}, metrics }; + return { aggregated: {}, metrics }; } let text = ""; @@ -366,7 +778,7 @@ function aggregateGenerateContentChunks( } } - const output: Record = {}; + const aggregated: Record = {}; const parts: Record[] = []; if (thoughtText) { @@ -396,19 +808,29 @@ function aggregateGenerateContentChunks( candidates.push(candidateDict); } - output.candidates = candidates; + aggregated.candidates = candidates; } if (usageMetadata) { - output.usageMetadata = usageMetadata; + aggregated.usageMetadata = usageMetadata; populateUsageMetrics(metrics, usageMetadata); } if (text) { - output.text = text; + aggregated.text = text; } - return { output, metrics }; + return { aggregated, metrics }; +} + +function cleanMetrics(metrics: Record): Record { + const cleaned: Record = {}; + for (const [key, value] of Object.entries(metrics)) { + if (value !== null && value !== undefined) { + cleaned[key] = value; + } + } + return cleaned; } /**