From ab0f7185e73fa3c134ba95263b4a5d350190627e Mon Sep 17 00:00:00 2001 From: Pratik Bodkhe Date: Sat, 6 Jun 2026 10:21:03 +0900 Subject: [PATCH] fix(ai): tolerate fenced provider JSON in suggestions, harden prompt Suggestions route carried a divergent getTextFromOpenRouter that skipped fence stripping, so gemini/minimax markdown-fenced JSON failed parsing and returned 502 "AI provider returned invalid suggestions". Reuse the shared fence-aware helper from @/lib/ai/ask-series-answer. Add output-contract guardrails to the suggestions prompt (JSON-only, no fences, empty array when nothing qualifies, verbatim evidence, no invented owners/dates). Dedupe enhance-notes onto the same shared helper. Lock both behaviors into the AI notes contract verifier. --- scripts/verify-ai-notes-contract.mjs | 35 +++++++++++- .../[meetingId]/enhance-notes/route.ts | 24 ++------- .../meetings/[meetingId]/suggestions/route.ts | 53 +++++++++---------- 3 files changed, 60 insertions(+), 52 deletions(-) diff --git a/scripts/verify-ai-notes-contract.mjs b/scripts/verify-ai-notes-contract.mjs index 672654e..f6b4301 100644 --- a/scripts/verify-ai-notes-contract.mjs +++ b/scripts/verify-ai-notes-contract.mjs @@ -96,7 +96,10 @@ assert(route.includes("response_format: { type: \"json_object\" }"), "Enhance ro assert(route.includes("Return only the JSON object"), "Enhance prompt must forbid non-JSON wrapper text"); assert(route.includes("Do not invent owners, dates, or decisions"), "Enhance prompt must forbid invented accountability details"); assert(route.includes("ai_notes: parsed"), "Enhance route must return structured AI notes JSON"); -assert(route.includes("stripJsonFences"), "Enhance route must strip markdown code fences before parsing provider JSON"); +assert( + route.includes('from "@/lib/ai/ask-series-answer"') && !/function getTextFromOpenRouter/.test(route), + "Enhance route must reuse the shared fence-aware getTextFromOpenRouter so markdown-fenced provider JSON still parses" +); const suggestionsRoute = read("src/app/api/meetings/[meetingId]/suggestions/route.ts"); assert(suggestionsRoute.includes("getAiModel"), "Suggestions route must resolve the model from config (getAiModel)"); @@ -104,6 +107,26 @@ assert(!suggestionsRoute.includes('"minimax/minimax-m3"'), "Suggestions route mu assert(suggestionsRoute.includes("OPENROUTER_API_KEY"), "Suggestions route must read OPENROUTER_API_KEY"); assert(suggestionsRoute.includes("AI_API_KEY"), "Suggestions route must support AI_API_KEY fallback"); assert(suggestionsRoute.includes("https://openrouter.ai/api/v1/chat/completions"), "Suggestions route must call OpenRouter chat completions"); +assert( + suggestionsRoute.includes('from "@/lib/ai/ask-series-answer"'), + "Suggestions route must reuse the shared fence-aware OpenRouter text helper" +); +assert( + !/function getTextFromOpenRouter/.test(suggestionsRoute), + "Suggestions route must not redefine a non-fence-aware getTextFromOpenRouter; reuse the shared one so markdown-fenced provider JSON still parses" +); +assert( + suggestionsRoute.includes("Do not wrap it in markdown fences"), + "Suggestions prompt must forbid markdown-fenced output" +); +assert( + suggestionsRoute.includes('return {"suggestions": []}') || suggestionsRoute.includes('{\\"suggestions\\": []}'), + "Suggestions prompt must instruct an empty array when nothing qualifies" +); +assert( + suggestionsRoute.includes("verbatim") && suggestionsRoute.includes("Never guess"), + "Suggestions prompt must forbid invented owners and require verbatim evidence" +); const askSeriesRoute = read("src/app/api/series/[seriesId]/ask/route.ts"); assert(askSeriesRoute.includes("getAiModel"), "Ask series route must resolve the model from config (getAiModel)"); @@ -127,7 +150,15 @@ await esbuild.build({ logLevel: "silent", }); -const { parseAskSeriesAnswer, stripJsonFences } = await import(pathToFileURL(bundledParser).href); +const { parseAskSeriesAnswer, stripJsonFences, getTextFromOpenRouter } = await import(pathToFileURL(bundledParser).href); + +// The suggestions route shares this extractor; fenced provider JSON must round-trip to parseable text. +assert( + getTextFromOpenRouter({ + choices: [{ message: { content: "```json\n{\"suggestions\":[]}\n```" } }], + }) === '{"suggestions":[]}', + "Shared getTextFromOpenRouter must unwrap fenced suggestions JSON" +); assert( stripJsonFences("```json\n{\"a\":1}\n```") === '{"a":1}', diff --git a/src/app/api/meetings/[meetingId]/enhance-notes/route.ts b/src/app/api/meetings/[meetingId]/enhance-notes/route.ts index 2f7b569..9a971da 100644 --- a/src/app/api/meetings/[meetingId]/enhance-notes/route.ts +++ b/src/app/api/meetings/[meetingId]/enhance-notes/route.ts @@ -1,7 +1,7 @@ import { NextResponse, type NextRequest } from "next/server"; import { z } from "zod"; import { createClient } from "@/lib/supabase/server"; -import { stripJsonFences } from "@/lib/ai/ask-series-answer"; +import { getTextFromOpenRouter } from "@/lib/ai/ask-series-answer"; import { getAiModel } from "@/lib/ai/model"; const OPENROUTER_MODEL = getAiModel(); @@ -69,27 +69,10 @@ function buildPrompt(input: { input.transcript || "(not provided)", "", "Existing open context:", - JSON.stringify({ issues: input.issues, decisions: input.decisions }, null, 2), + JSON.stringify({ issues: input.issues, decisions: input.decisions }), ].join("\n"); } -function getTextFromOpenRouter(data: unknown) { - const content = (data as { - choices?: { message?: { content?: unknown } }[]; - })?.choices?.[0]?.message?.content; - if (typeof content === "string") return content; - if (Array.isArray(content)) { - return content - .map((part) => { - if (!part || typeof part !== "object" || !("text" in part)) return ""; - return typeof part.text === "string" ? part.text : ""; - }) - .filter(Boolean) - .join("\n"); - } - return ""; -} - async function getOpenRouterData(prompt: string, apiKey: string) { const providerResponse = await fetch(OPENROUTER_URL, { method: "POST", @@ -185,8 +168,7 @@ export async function POST( let parsed: AiNotes; try { - const text = stripJsonFences(getTextFromOpenRouter(providerData)); - parsed = notesSchema.parse(JSON.parse(text)); + parsed = notesSchema.parse(JSON.parse(getTextFromOpenRouter(providerData))); } catch { return NextResponse.json( { error: "AI provider returned invalid notes.", request_id: requestId }, diff --git a/src/app/api/meetings/[meetingId]/suggestions/route.ts b/src/app/api/meetings/[meetingId]/suggestions/route.ts index 6d7ee17..636ec8e 100644 --- a/src/app/api/meetings/[meetingId]/suggestions/route.ts +++ b/src/app/api/meetings/[meetingId]/suggestions/route.ts @@ -1,6 +1,7 @@ import { NextResponse, type NextRequest } from "next/server"; import { z } from "zod"; import { createClient } from "@/lib/supabase/server"; +import { getTextFromOpenRouter } from "@/lib/ai/ask-series-answer"; import { getAiModel } from "@/lib/ai/model"; const OPENROUTER_MODEL = getAiModel(); @@ -25,29 +26,6 @@ const suggestionsSchema = z.object({ suggestions: z.array(suggestionSchema).default([]), }); -function getTextFromOpenRouter(data: unknown) { - const parsed = z - .object({ - choices: z.array( - z.object({ - message: z.object({ - content: z.union([ - z.string(), - z.array(z.object({ text: z.string().optional() }).passthrough()), - ]), - }), - }).passthrough() - ).min(1), - }) - .passthrough() - .safeParse(data); - - if (!parsed.success) return ""; - const content = parsed.data.choices[0].message.content; - if (typeof content === "string") return content; - return content.map((part) => part.text ?? "").filter(Boolean).join("\n"); -} - function buildPrompt(input: { title: string; seriesName: string; @@ -56,12 +34,29 @@ function buildPrompt(input: { transcript: string | null; }) { return [ - "Extract reviewable accountability suggestions for Minutia, an Outstanding Issues Log.", - "Return strict JSON with a suggestions array.", - "Each suggestion must include category, title, details, owner_name, due_date, confidence, and source_excerpt.", - "Allowed categories: action, decision, info, risk, blocker.", - "Only suggest durable records that a facilitator should review. Do not invent owners or due dates.", - "Use source_excerpt to quote the smallest supporting phrase from the notes or transcript.", + "You extract reviewable accountability suggestions for Minutia, an Outstanding Issues Log.", + "A facilitator reviews every suggestion before it enters a permanent record, so omitting a weak item is always better than inventing one.", + "", + "OUTPUT CONTRACT", + 'Return only a single JSON object of the form {"suggestions": [ ... ]}.', + "Do not wrap it in markdown fences. Do not add commentary, explanations, or text before or after the JSON.", + "If nothing in the notes or transcript qualifies, return {\"suggestions\": []}.", + "", + "Each suggestion object must have exactly these fields:", + "- category: one of action, decision, info, risk, blocker.", + " action = a task someone must do. decision = a choice that was made.", + " risk = a possible future problem. blocker = something currently preventing progress. info = a durable fact worth tracking.", + "- title: concise imperative summary, max 120 characters, no trailing punctuation.", + "- details: one or two sentences of supporting context, or \"\" if none.", + "- owner_name: copy a person's name verbatim only if the source explicitly assigns them. Never guess. Use \"\" when unassigned.", + "- due_date: an explicit calendar date as YYYY-MM-DD, only if the source states one. Never infer from relative phrasing. Use null otherwise.", + "- confidence: 0 to 1. Use 0.9+ when explicitly stated and owned, 0.5 to 0.8 when implied, and omit any item you would score below 0.4.", + "- source_excerpt: a verbatim quote copied from the notes or transcript that supports this item. Do not paraphrase. Keep it under 160 characters.", + "", + "RULES", + "Only surface durable items a facilitator should track. Skip greetings, small talk, and resolved chatter.", + "Emit at most one suggestion per distinct item; do not duplicate.", + "Prefer fewer, high-signal suggestions over many speculative ones.", "", `Series: ${input.seriesName}`, `Meeting: ${input.title}`,