From 327b8d7b57e96414f0091f7b21b0da93e6ee62b6 Mon Sep 17 00:00:00 2001 From: Kinfe123 Date: Mon, 25 May 2026 00:33:43 +0300 Subject: [PATCH 1/4] feat: codeblock verifier via planner and runner on sandbox --- packages/docs/package.json | 1 + packages/docs/src/cli/codeblocks.ts | 221 +++++ packages/docs/src/cli/index.ts | 19 + packages/docs/src/code-blocks.test.ts | 172 ++++ packages/docs/src/code-blocks.ts | 1044 ++++++++++++++++++++ packages/docs/src/define-docs.ts | 1 + packages/docs/src/index.ts | 8 + packages/docs/src/mcp.ts | 61 ++ packages/docs/src/types.ts | 127 +++ pnpm-lock.yaml | 139 ++- skills/farming-labs/README.md | 4 +- skills/farming-labs/cli/SKILL.md | 50 +- skills/farming-labs/configuration/SKILL.md | 47 +- website/app/docs/cli/page.mdx | 47 +- website/app/docs/configuration/agent.md | 4 +- website/app/docs/configuration/page.mdx | 51 + 16 files changed, 1982 insertions(+), 14 deletions(-) create mode 100644 packages/docs/src/cli/codeblocks.ts create mode 100644 packages/docs/src/code-blocks.test.ts create mode 100644 packages/docs/src/code-blocks.ts diff --git a/packages/docs/package.json b/packages/docs/package.json index 2149f4a7..75b07fab 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -47,6 +47,7 @@ "@clack/prompts": "^0.9.1", "@modelcontextprotocol/sdk": "1.29.0", "@scalar/core": "^0.4.3", + "@vercel/sandbox": "^2.0.0", "gray-matter": "^4.0.3", "jiti": "^2.6.1", "picocolors": "^1.1.1", diff --git a/packages/docs/src/cli/codeblocks.ts b/packages/docs/src/cli/codeblocks.ts new file mode 100644 index 00000000..65f3a545 --- /dev/null +++ b/packages/docs/src/cli/codeblocks.ts @@ -0,0 +1,221 @@ +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import pc from "picocolors"; +import { + resolveDocsCodeBlocksValidateConfig, + validateCodeBlocks, + type DocsCodeBlocksValidationReport, +} from "../code-blocks.js"; +import type { DocsCodeBlocksValidateConfig } from "../types.js"; +import { + extractNestedObjectLiteral, + loadDocsConfigModule, + readTopLevelStringProperty, + resolveDocsConfigPath, + resolveDocsContentDir, +} from "./config.js"; + +export interface CodeBlocksValidateOptions { + configPath?: string; + json?: boolean; + plan?: boolean; + run?: boolean; +} + +export interface ParsedCodeBlocksValidateArgs extends CodeBlocksValidateOptions { + help?: boolean; +} + +function parseInlineFlag(arg: string): { key: string; value?: string } { + const [rawKey, value] = arg.slice(2).split("=", 2); + return { key: rawKey.trim(), value }; +} + +export function parseCodeBlocksValidateArgs(argv: string[]): ParsedCodeBlocksValidateArgs { + const parsed: ParsedCodeBlocksValidateArgs = {}; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + + if (arg === "--help" || arg === "-h") { + parsed.help = true; + continue; + } + + if (arg === "--json") { + parsed.json = true; + continue; + } + + if (arg === "--plan") { + parsed.plan = true; + continue; + } + + if (arg === "--run") { + parsed.run = true; + continue; + } + + if (arg.startsWith("--config=")) { + const value = parseInlineFlag(arg).value; + if (!value) throw new Error("Missing value for --config."); + parsed.configPath = value; + continue; + } + + if (arg === "--config") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("Missing value for --config."); + parsed.configPath = value; + index += 1; + continue; + } + + throw new Error(`Unknown codeblocks validate flag: ${arg}.`); + } + + return parsed; +} + +export function printCodeBlocksValidateHelp() { + console.log(` +${pc.bold("@farming-labs/docs codeblocks validate")} + +${pc.dim("Usage:")} + pnpm exec docs codeblocks validate + pnpm exec docs codeblocks validate --plan + pnpm exec docs codeblocks validate --json + pnpm exec docs codeblocks validate --config docs.config.ts + +${pc.dim("Options:")} + ${pc.cyan("--plan")} Build the execution plan without running code + ${pc.cyan("--run")} Force execution even when config mode is ${pc.dim('"plan"')} + ${pc.cyan("--json")} Print machine-readable output + ${pc.cyan("--config ")} Use a custom docs config path instead of ${pc.dim("docs.config.ts[x]")} + ${pc.cyan("-h, --help")} Show this help message +`); +} + +export async function runCodeBlocksValidate(options: CodeBlocksValidateOptions = {}) { + const rootDir = process.cwd(); + const loaded = await loadDocsConfigModule(rootDir, options.configPath); + const configPath = loaded?.path ?? resolveDocsConfigPath(rootDir, options.configPath); + const configContent = existsSync(configPath) ? readFileSync(configPath, "utf-8") : ""; + const entry = + loaded?.config.entry ?? readTopLevelStringProperty(configContent, "entry") ?? "docs"; + const contentDir = + loaded?.config.contentDir ?? resolveDocsContentDir(rootDir, configContent, entry); + const validateInput = + loaded?.config.codeBlocks?.validate ?? readStaticCodeBlocksValidateConfig(configContent); + const config = resolveDocsCodeBlocksValidateConfig(validateInput); + + if (!config.enabled) { + const disabledReport: DocsCodeBlocksValidationReport = { + summary: { total: 0, planned: 0, pass: 0, skip: 0, fail: 0 }, + config, + targets: [], + plans: [], + results: [], + }; + if (options.json) { + console.log(JSON.stringify(disabledReport, null, 2)); + } else { + console.log( + pc.yellow( + "codeBlocks.validate is disabled. Add `codeBlocks: { validate: true }` to docs.config.ts.", + ), + ); + } + return disabledReport; + } + + const report = await validateCodeBlocks({ + rootDir, + contentDir, + config: { + ...config, + mode: options.run ? "report" : config.mode, + }, + planOnly: options.plan, + }); + + if (options.json) { + console.log(JSON.stringify(redactReport(report), null, 2)); + } else { + printCodeBlocksReport(report, options.plan === true || config.mode === "plan"); + } + + if (!options.plan && report.summary.fail > 0) { + process.exitCode = 1; + } + + return report; +} + +function readStaticCodeBlocksValidateConfig( + content: string, +): boolean | DocsCodeBlocksValidateConfig | undefined { + const block = extractNestedObjectLiteral(content, ["codeBlocks"]); + if (!block) return undefined; + if (/\bvalidate\s*:\s*true\b/.test(block)) return true; + if (/\bvalidate\s*:\s*false\b/.test(block)) return false; + if (/\bvalidate\s*:\s*\{/.test(block)) { + return true; + } + return undefined; +} + +function printCodeBlocksReport(report: DocsCodeBlocksValidationReport, planOnly: boolean) { + const label = planOnly ? "Code block plan" : "Code block validation"; + console.log(pc.bold(label)); + console.log( + [ + ...(report.summary.planned > 0 ? [`${pc.cyan(`${report.summary.planned} planned`)}`] : []), + `${pc.green(`${report.summary.pass} pass`)}`, + `${pc.yellow(`${report.summary.skip} skip`)}`, + `${report.summary.fail > 0 ? pc.red(`${report.summary.fail} fail`) : pc.dim("0 fail")}`, + `${report.targets.length} code blocks`, + ].join(pc.dim(" • ")), + ); + + if (report.results.length === 0) return; + console.log(); + + for (const result of report.results) { + const status = + result.status === "PLAN" + ? pc.cyan(result.status) + : result.status === "PASS" + ? pc.green(result.status) + : result.status === "FAIL" + ? pc.red(result.status) + : pc.yellow(result.status); + const location = `${result.target.relativePath}:${result.target.lineStart}`; + const detail = result.reason ?? result.plan.reason ?? result.plan.template; + console.log(`${status} ${pc.cyan(location)} ${pc.dim(detail)}`); + } +} + +function redactReport(report: DocsCodeBlocksValidationReport): DocsCodeBlocksValidationReport { + return { + ...report, + config: { + ...report.config, + planner: { + ...report.config.planner, + apiKey: report.config.planner.apiKey ? "[REDACTED]" : undefined, + }, + }, + results: report.results.map((result) => ({ + ...result, + stdout: trimOutput(result.stdout), + stderr: trimOutput(result.stderr), + })), + }; +} + +function trimOutput(value?: string): string | undefined { + if (!value) return value; + return value.length > 4000 ? `${value.slice(0, 4000)}...` : value; +} diff --git a/packages/docs/src/cli/index.ts b/packages/docs/src/cli/index.ts index 19625e71..39a87fe3 100644 --- a/packages/docs/src/cli/index.ts +++ b/packages/docs/src/cli/index.ts @@ -148,6 +148,24 @@ async function main() { return; } await runReview(reviewOptions); + } else if ( + (parsedCommand.command === "codeblocks" || parsedCommand.command === "code-blocks") && + subcommand === "validate" + ) { + const { parseCodeBlocksValidateArgs, printCodeBlocksValidateHelp, runCodeBlocksValidate } = + await import("./codeblocks.js"); + const codeBlocksOptions = parseCodeBlocksValidateArgs(args.slice(2)); + if (codeBlocksOptions.help) { + printCodeBlocksValidateHelp(); + return; + } + await runCodeBlocksValidate(codeBlocksOptions); + } else if (parsedCommand.command === "codeblocks" || parsedCommand.command === "code-blocks") { + console.error(pc.red(`Unknown codeblocks subcommand: ${subcommand ?? "(missing)"}`)); + console.error(); + const { printCodeBlocksValidateHelp } = await import("./codeblocks.js"); + printCodeBlocksValidateHelp(); + process.exit(1); } else if (parsedCommand.command === "search" && subcommand === "sync") { const { syncSearch } = await import("./search.js"); await syncSearch(searchSyncOptions); @@ -240,6 +258,7 @@ ${pc.dim("Commands:")} ${pc.cyan("agents")} AGENTS.md utilities (${pc.dim("generate")} for static agent instructions) ${pc.cyan("doctor")} Inspect and score agent or reader-facing docs quality ${pc.cyan("review")} Review changed docs files and wire Docs Review CI + ${pc.cyan("codeblocks")} Validate fenced MDX code blocks (${pc.dim("validate")}) ${pc.cyan("mcp")} Run the built-in docs MCP server over stdio ${pc.cyan("robots")} Robots.txt utilities (${pc.dim("generate")} for agent access policy) ${pc.cyan("search")} Search utilities (${pc.dim("sync")} for external indexes) diff --git a/packages/docs/src/code-blocks.test.ts b/packages/docs/src/code-blocks.test.ts new file mode 100644 index 00000000..93960fd2 --- /dev/null +++ b/packages/docs/src/code-blocks.test.ts @@ -0,0 +1,172 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + extractCodeBlocksFromMarkdown, + resolveDocsCodeBlocksValidateConfig, + validateCodeBlocks, +} from "./code-blocks.js"; + +describe("code block validation", () => { + it("resolves disabled and enabled validate config", () => { + expect(resolveDocsCodeBlocksValidateConfig().enabled).toBe(false); + + const config = resolveDocsCodeBlocksValidateConfig({ + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + }); + + expect(config.enabled).toBe(true); + expect(config.planner).toMatchObject({ + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }); + expect(config.runner).toMatchObject({ + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }); + expect(config.env).toEqual({ + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }); + }); + + it("extracts code fence metadata used by agents and validators", () => { + const blocks = extractCodeBlocksFromMarkdown({ + filePath: "/repo/docs/page.mdx", + relativePath: "docs/page.mdx", + source: [ + "Intro", + "", + '```ts title="app/api/chat/route.ts" framework="nextjs" packageManager="pnpm" env="OPENAI_API_KEY" runnable', + "console.log(process.env.OPENAI_API_KEY)", + "```", + ].join("\n"), + }); + + expect(blocks).toHaveLength(1); + expect(blocks[0]).toMatchObject({ + id: "docs/page.mdx#code-1", + lineStart: 3, + lineEnd: 5, + language: "ts", + title: "app/api/chat/route.ts", + framework: "nextjs", + packageManager: "pnpm", + runnable: true, + env: ["OPENAI_API_KEY"], + code: "console.log(process.env.OPENAI_API_KEY)", + }); + }); + + it("builds a metadata execution plan without running when plan mode is requested", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-plan-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + ['```js title="hello.js" runnable', 'console.log("hello")', "```"].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + planOnly: true, + config: resolveDocsCodeBlocksValidateConfig(true), + }); + + expect(report.targets).toHaveLength(1); + expect(report.plans[0]).toMatchObject({ + action: "execute", + template: "node", + runtime: "node", + command: { + cmd: "node", + }, + }); + expect(report.summary).toEqual({ + total: 1, + planned: 1, + pass: 0, + skip: 0, + fail: 0, + }); + expect(report.results[0]).toMatchObject({ + status: "PLAN", + reason: "planned", + }); + }); + + it("runs local executable and syntax-validation plans", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-run-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + [ + '```js title="hello.js" runnable', + 'console.log("hello")', + "```", + "", + '```json title="payload.json" runnable', + '{"ok": true}', + "```", + ].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig(true), + }); + + expect(report.summary).toEqual({ + total: 2, + planned: 0, + pass: 2, + skip: 0, + fail: 0, + }); + expect(report.results.map((result) => result.status)).toEqual(["PASS", "PASS"]); + }); + + it("skips runnable blocks when mapped env is missing", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-env-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + [ + '```js title="needs-env.js" env="OPENAI_API_KEY" runnable', + "console.log(Boolean(process.env.OPENAI_API_KEY))", + "```", + ].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig({ + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + }), + }); + + expect(report.summary).toMatchObject({ + total: 1, + pass: 0, + skip: 1, + fail: 0, + }); + expect(report.results[0]?.reason).toBe("missing env: OPENAI_API_KEY"); + }); +}); diff --git a/packages/docs/src/code-blocks.ts b/packages/docs/src/code-blocks.ts new file mode 100644 index 00000000..d992549e --- /dev/null +++ b/packages/docs/src/code-blocks.ts @@ -0,0 +1,1044 @@ +import { execFile } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import type { + DocsCodeBlocksPlannerConfig, + DocsCodeBlocksPlannerProvider, + DocsCodeBlocksRunnerConfig, + DocsCodeBlocksRunnerProvider, + DocsCodeBlocksValidateConfig, + DocsCodeBlocksValidationMode, + DocsCodeBlocksValidationPolicy, +} from "./types.js"; + +const execFileAsync = promisify(execFile); + +const DEFAULT_ENV_FILES = [".env.local", ".env.test", ".env"]; +const DEFAULT_COMMAND_TIMEOUT_MS = 60_000; + +export interface DocsCodeBlockMeta { + language?: string; + meta: Record; +} + +export interface DocsCodeBlockTarget { + id: string; + filePath: string; + relativePath: string; + lineStart: number; + lineEnd: number; + language?: string; + title?: string; + framework?: string; + packageManager?: string; + runnable: boolean; + env: string[]; + meta: Record; + code: string; +} + +export interface DocsCodeBlocksResolvedPlannerConfig { + provider: DocsCodeBlocksPlannerProvider; + model?: string; + baseUrl?: string; + baseUrlEnv?: string; + apiKey?: string; + apiKeyEnv?: string; +} + +export interface DocsCodeBlocksResolvedRunnerConfig { + provider: DocsCodeBlocksRunnerProvider; + tokenEnv: string; + runtime: "node24" | "node22" | "python3.13"; + timeoutMs: number; +} + +export interface DocsCodeBlocksResolvedValidateConfig { + enabled: boolean; + planner: DocsCodeBlocksResolvedPlannerConfig; + runner: DocsCodeBlocksResolvedRunnerConfig; + envFile: string[]; + env: Record; + missingEnv: DocsCodeBlocksValidationPolicy; + unsupportedLanguage: DocsCodeBlocksValidationPolicy; + mode: DocsCodeBlocksValidationMode; +} + +export interface DocsCodeBlockCommand { + cmd: string; + args: string[]; +} + +export interface DocsCodeBlockExecutionPlan { + id: string; + target: DocsCodeBlockTarget; + action: "execute" | "validate-syntax" | "skip"; + template: string; + runtime?: string; + filePath?: string; + command?: DocsCodeBlockCommand; + requiredEnv: string[]; + reason?: string; + planner: DocsCodeBlocksPlannerProvider; +} + +export type DocsCodeBlockValidationStatus = "PLAN" | "PASS" | "SKIP" | "FAIL"; + +export interface DocsCodeBlockValidationResult { + id: string; + target: DocsCodeBlockTarget; + plan: DocsCodeBlockExecutionPlan; + status: DocsCodeBlockValidationStatus; + stdout?: string; + stderr?: string; + exitCode?: number | null; + reason?: string; +} + +export interface DocsCodeBlocksValidationReport { + summary: { + total: number; + planned: number; + pass: number; + skip: number; + fail: number; + }; + config: DocsCodeBlocksResolvedValidateConfig; + targets: DocsCodeBlockTarget[]; + plans: DocsCodeBlockExecutionPlan[]; + results: DocsCodeBlockValidationResult[]; +} + +interface CollectTargetsOptions { + rootDir: string; + contentDir: string; +} + +interface LoadedValidationEnv { + env: Record; + missing: string[]; +} + +export function resolveDocsCodeBlocksValidateConfig( + input?: boolean | DocsCodeBlocksValidateConfig, +): DocsCodeBlocksResolvedValidateConfig { + if (input === false || input === undefined) { + return { + enabled: false, + planner: { provider: "metadata" }, + runner: { + provider: "local", + tokenEnv: "VERCEL_TOKEN", + runtime: "node24", + timeoutMs: DEFAULT_COMMAND_TIMEOUT_MS, + }, + envFile: DEFAULT_ENV_FILES, + env: {}, + missingEnv: "skip", + unsupportedLanguage: "skip", + mode: "report", + }; + } + + const config = input === true ? {} : input; + const planner = normalizePlannerConfig(config.planner); + const runner = normalizeRunnerConfig(config.runner); + const envFile = Array.isArray(config.envFile) + ? config.envFile + : typeof config.envFile === "string" + ? [config.envFile] + : DEFAULT_ENV_FILES; + + return { + enabled: config.enabled ?? true, + planner, + runner, + envFile, + env: config.env ?? {}, + missingEnv: config.missingEnv ?? "skip", + unsupportedLanguage: config.unsupportedLanguage ?? "skip", + mode: config.mode ?? "report", + }; +} + +export function parseCodeFenceInfo(info: string): DocsCodeBlockMeta { + const trimmed = info.trim(); + if (!trimmed) return { meta: {} }; + + const firstTokenMatch = trimmed.match(/^(\S+)/); + const firstToken = firstTokenMatch?.[1] ?? ""; + const language = firstToken && !firstToken.includes("=") ? firstToken : undefined; + const attributeSource = language ? trimmed.slice(firstToken.length).trim() : trimmed; + const meta: Record = {}; + const attributePattern = /([A-Za-z_:][\w:.-]*)(?:=(?:"([^"]*)"|'([^']*)'|([^\s"']+)))?/g; + + let match: RegExpExecArray | null; + while ((match = attributePattern.exec(attributeSource))) { + const key = match[1]; + const value = match[2] ?? match[3] ?? match[4]; + meta[key] = value ?? true; + } + + return { language, meta }; +} + +export function extractCodeBlocksFromMarkdown(input: { + source: string; + filePath: string; + relativePath?: string; +}): DocsCodeBlockTarget[] { + const blocks: DocsCodeBlockTarget[] = []; + const lines = input.source.split("\n"); + let openFence: + | { + marker: string; + info: string; + code: string[]; + lineStart: number; + } + | undefined; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index] ?? ""; + const trimmed = line.trim(); + + if (!openFence) { + const openMatch = trimmed.match(/^(`{3,}|~{3,})(.*)$/); + if (!openMatch) continue; + + openFence = { + marker: openMatch[1], + info: openMatch[2]?.trim() ?? "", + code: [], + lineStart: index + 1, + }; + continue; + } + + if (isClosingFence(trimmed, openFence.marker)) { + const parsed = parseCodeFenceInfo(openFence.info); + const meta = parsed.meta; + const blockIndex = blocks.length + 1; + const relativePath = input.relativePath ?? input.filePath; + + blocks.push({ + id: `${relativePath}#code-${blockIndex}`, + filePath: input.filePath, + relativePath, + lineStart: openFence.lineStart, + lineEnd: index + 1, + language: parsed.language, + title: readStringMeta(meta, "title"), + framework: readStringMeta(meta, "framework"), + packageManager: readStringMeta(meta, "packageManager"), + runnable: readBooleanMeta(meta, "runnable") ?? false, + env: readEnvMeta(meta), + meta, + code: openFence.code.join("\n"), + }); + openFence = undefined; + continue; + } + + openFence.code.push(line); + } + + return blocks; +} + +export function collectCodeBlockTargets(options: CollectTargetsOptions): DocsCodeBlockTarget[] { + const root = path.resolve(options.rootDir); + const contentRoot = path.resolve(root, options.contentDir); + if (!existsSync(contentRoot)) return []; + + const files = walkMarkdownFiles(contentRoot); + return files.flatMap((filePath) => { + const source = readFileSync(filePath, "utf-8"); + const relativePath = path.relative(root, filePath).replace(/\\/g, "/"); + return extractCodeBlocksFromMarkdown({ source, filePath, relativePath }); + }); +} + +export async function planCodeBlockTargets( + targets: DocsCodeBlockTarget[], + config: DocsCodeBlocksResolvedValidateConfig, +): Promise { + if (config.planner.provider === "metadata") { + return targets.map((target) => buildMetadataExecutionPlan(target, config)); + } + + if (config.planner.provider === "cloud") { + throw new Error("Hosted cloud code block planning is not available in this package yet."); + } + + return planWithOpenAICompatibleProvider(targets, config); +} + +export async function validateCodeBlockPlans(input: { + plans: DocsCodeBlockExecutionPlan[]; + rootDir: string; + config: DocsCodeBlocksResolvedValidateConfig; +}): Promise { + const validationEnv = loadValidationEnv(input.rootDir, input.config); + const preflight = input.plans.map((plan) => preflightPlan(plan, input.config, validationEnv)); + const runnable = preflight.filter((result) => result.status !== "SKIP" && !result.reason); + + const skippedOrFailed = preflight.filter((result) => result.status === "SKIP" || result.reason); + const plansToRun = runnable.map((result) => result.plan); + + if (plansToRun.length === 0) { + return skippedOrFailed.map((result) => + result.reason + ? { + ...result, + status: input.config.missingEnv === "error" ? "FAIL" : result.status, + } + : result, + ); + } + + const runResults = + input.config.runner.provider === "vercel-sandbox" + ? await runPlansInVercelSandbox(plansToRun, input.config, validationEnv.env) + : await runPlansLocally(plansToRun, input.config, validationEnv.env); + + return [...skippedOrFailed, ...runResults].sort((a, b) => a.id.localeCompare(b.id)); +} + +export async function validateCodeBlocks(input: { + rootDir: string; + contentDir: string; + config: DocsCodeBlocksResolvedValidateConfig; + planOnly?: boolean; +}): Promise { + const targets = collectCodeBlockTargets({ + rootDir: input.rootDir, + contentDir: input.contentDir, + }); + const plans = await planCodeBlockTargets(targets, input.config); + const results = + input.planOnly || input.config.mode === "plan" + ? plans.map((plan) => ({ + id: plan.id, + target: plan.target, + plan, + status: "PLAN" as const, + reason: plan.reason ?? (plan.action === "skip" ? "planned skip" : "planned"), + })) + : await validateCodeBlockPlans({ + plans, + rootDir: input.rootDir, + config: input.config, + }); + + const summary = results.reduce( + (acc, result) => { + acc.total += 1; + if (result.status === "PLAN") acc.planned += 1; + if (result.status === "PASS") acc.pass += 1; + if (result.status === "SKIP") acc.skip += 1; + if (result.status === "FAIL") acc.fail += 1; + return acc; + }, + { total: 0, planned: 0, pass: 0, skip: 0, fail: 0 }, + ); + + return { + summary, + config: input.config, + targets, + plans, + results, + }; +} + +function normalizePlannerConfig( + input?: DocsCodeBlocksPlannerProvider | DocsCodeBlocksPlannerConfig, +): DocsCodeBlocksResolvedPlannerConfig { + if (!input) return { provider: "metadata" }; + if (typeof input === "string") return { provider: input }; + return { provider: input.provider ?? "metadata", ...input }; +} + +function normalizeRunnerConfig( + input?: DocsCodeBlocksRunnerProvider | DocsCodeBlocksRunnerConfig, +): DocsCodeBlocksResolvedRunnerConfig { + const config = typeof input === "string" ? { provider: input } : (input ?? {}); + return { + provider: config.provider ?? "local", + tokenEnv: config.tokenEnv ?? "VERCEL_TOKEN", + runtime: config.runtime ?? "node24", + timeoutMs: config.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, + }; +} + +function buildMetadataExecutionPlan( + target: DocsCodeBlockTarget, + config: DocsCodeBlocksResolvedValidateConfig, +): DocsCodeBlockExecutionPlan { + const language = normalizeLanguage(target.language); + const template = target.framework ?? templateFromLanguage(language); + const requiredEnv = target.env; + + if (readBooleanMeta(target.meta, "partial") || looksPartial(target.code)) { + return skipPlan(target, template, requiredEnv, "partial fragment", config.planner.provider); + } + + if (!language) { + if (looksLikeShellCommand(target.code)) { + return shellPlan(target, "shell", requiredEnv, config.planner.provider); + } + return skipPlan( + target, + "unknown", + requiredEnv, + "missing language and not obviously runnable", + config.planner.provider, + ); + } + + if (isShellLanguage(language)) + return shellPlan(target, template, requiredEnv, config.planner.provider); + if (language === "json") + return syntaxPlan(target, "json", "node", requiredEnv, config.planner.provider); + + if (language === "javascript" || language === "js" || language === "jsx") { + return executePlan( + target, + template, + "node", + "js", + { cmd: "node", args: [] }, + requiredEnv, + config.planner.provider, + ); + } + + if (language === "typescript" || language === "ts" || language === "tsx") { + return executePlan( + target, + template, + "tsx", + "ts", + { cmd: "npx", args: ["--yes", "tsx"] }, + requiredEnv, + config.planner.provider, + ); + } + + if (language === "python" || language === "py") { + return executePlan( + target, + template, + "python3", + "py", + { cmd: "python3", args: [] }, + requiredEnv, + config.planner.provider, + ); + } + + if (language === "ruby" || language === "rb") { + return executePlan( + target, + template, + "ruby", + "rb", + { cmd: "ruby", args: [] }, + requiredEnv, + config.planner.provider, + ); + } + + if (language === "elixir" || language === "ex" || language === "exs") { + return executePlan( + target, + template, + "elixir", + "exs", + { cmd: "elixir", args: [] }, + requiredEnv, + config.planner.provider, + ); + } + + if (language === "yaml" || language === "yml") { + return skipPlan( + target, + "yaml", + requiredEnv, + "YAML syntax validation requires a YAML parser", + config.planner.provider, + ); + } + + if (!target.runnable) { + return skipPlan( + target, + template, + requiredEnv, + "code block is not marked runnable", + config.planner.provider, + ); + } + + return skipPlan( + target, + template, + requiredEnv, + `unsupported language: ${target.language}`, + config.planner.provider, + ); +} + +function executePlan( + target: DocsCodeBlockTarget, + template: string, + runtime: string, + extension: string, + command: DocsCodeBlockCommand, + requiredEnv: string[], + planner: DocsCodeBlocksPlannerProvider, +): DocsCodeBlockExecutionPlan { + const filePath = `snippet-${slugify(target.id)}.${extension}`; + return { + id: target.id, + target, + action: "execute", + template, + runtime, + filePath, + command: { cmd: command.cmd, args: [...command.args, filePath] }, + requiredEnv, + planner, + }; +} + +function shellPlan( + target: DocsCodeBlockTarget, + template: string, + requiredEnv: string[], + planner: DocsCodeBlocksPlannerProvider, +): DocsCodeBlockExecutionPlan { + const filePath = `snippet-${slugify(target.id)}.sh`; + return { + id: target.id, + target, + action: "execute", + template, + runtime: "bash", + filePath, + command: { cmd: "bash", args: [filePath] }, + requiredEnv, + planner, + }; +} + +function syntaxPlan( + target: DocsCodeBlockTarget, + template: string, + runtime: string, + requiredEnv: string[], + planner: DocsCodeBlocksPlannerProvider, +): DocsCodeBlockExecutionPlan { + const filePath = `snippet-${slugify(target.id)}.json`; + return { + id: target.id, + target, + action: "validate-syntax", + template, + runtime, + filePath, + command: { + cmd: "node", + args: [ + "-e", + `JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8"))`, + filePath, + ], + }, + requiredEnv, + planner, + }; +} + +function skipPlan( + target: DocsCodeBlockTarget, + template: string, + requiredEnv: string[], + reason: string, + planner: DocsCodeBlocksPlannerProvider, +): DocsCodeBlockExecutionPlan { + return { + id: target.id, + target, + action: "skip", + template, + requiredEnv, + reason, + planner, + }; +} + +async function planWithOpenAICompatibleProvider( + targets: DocsCodeBlockTarget[], + config: DocsCodeBlocksResolvedValidateConfig, +): Promise { + const baseUrl = + config.planner.baseUrl ?? + (config.planner.baseUrlEnv ? process.env[config.planner.baseUrlEnv] : undefined) ?? + (config.planner.provider === "openai" ? "https://api.openai.com/v1" : undefined); + const apiKey = + config.planner.apiKey ?? + (config.planner.apiKeyEnv ? process.env[config.planner.apiKeyEnv] : undefined); + + if (!baseUrl) { + throw new Error("codeBlocks.validate planner requires baseUrl or baseUrlEnv."); + } + if (!apiKey) { + throw new Error( + `codeBlocks.validate planner requires an API key. Set ${config.planner.apiKeyEnv ?? "apiKeyEnv"} in your environment.`, + ); + } + + const metadataPlans = targets.map((target) => buildMetadataExecutionPlan(target, config)); + const response = await fetch(`${baseUrl.replace(/\/+$/, "")}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: config.planner.model ?? "gpt-4.1-mini", + temperature: 0, + response_format: { type: "json_object" }, + messages: [ + { + role: "system", + content: + "You produce JSON execution plans for documentation code blocks. Do not rewrite snippets. If the snippet is partial, missing setup, or unsafe, choose action skip. Return only JSON.", + }, + { + role: "user", + content: JSON.stringify({ + contract: { + plans: + "array of {id, action: execute|validate-syntax|skip, template, runtime?, reason?, requiredEnv?: string[]}", + }, + codeBlocks: targets.map((target) => ({ + id: target.id, + language: target.language, + title: target.title, + framework: target.framework, + packageManager: target.packageManager, + runnable: target.runnable, + env: target.env, + meta: target.meta, + code: target.code.slice(0, 8000), + })), + }), + }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`Planner request failed with ${response.status} ${response.statusText}.`); + } + + const payload = (await response.json()) as { + choices?: Array<{ message?: { content?: string } }>; + }; + const content = payload.choices?.[0]?.message?.content ?? ""; + + let parsed: { plans?: Array & { id?: string }> }; + try { + parsed = JSON.parse(content); + } catch { + throw new Error("Planner returned non-JSON content."); + } + + const byId = new Map(metadataPlans.map((plan) => [plan.id, plan])); + for (const plan of parsed.plans ?? []) { + if (!plan.id || !byId.has(plan.id)) continue; + const fallback = byId.get(plan.id)!; + byId.set(plan.id, { + ...fallback, + action: isPlanAction(plan.action) ? plan.action : fallback.action, + template: + typeof plan.template === "string" && plan.template ? plan.template : fallback.template, + runtime: typeof plan.runtime === "string" && plan.runtime ? plan.runtime : fallback.runtime, + requiredEnv: Array.isArray(plan.requiredEnv) + ? plan.requiredEnv.filter((value): value is string => typeof value === "string") + : fallback.requiredEnv, + reason: typeof plan.reason === "string" ? plan.reason : fallback.reason, + planner: config.planner.provider, + }); + } + + return metadataPlans.map((plan) => byId.get(plan.id)!); +} + +function preflightPlan( + plan: DocsCodeBlockExecutionPlan, + config: DocsCodeBlocksResolvedValidateConfig, + validationEnv: LoadedValidationEnv, +): DocsCodeBlockValidationResult { + if (plan.action === "skip") { + return { + id: plan.id, + target: plan.target, + plan, + status: "SKIP", + reason: plan.reason, + }; + } + + const missingEnv = plan.requiredEnv.filter((key) => !validationEnv.env[key]); + if (missingEnv.length > 0) { + const reason = `missing env: ${missingEnv.join(", ")}`; + return { + id: plan.id, + target: plan.target, + plan, + status: config.missingEnv === "error" ? "FAIL" : "SKIP", + reason, + }; + } + + if (!plan.command || !plan.filePath) { + return { + id: plan.id, + target: plan.target, + plan, + status: config.unsupportedLanguage === "error" ? "FAIL" : "SKIP", + reason: "no executable command in plan", + }; + } + + return { + id: plan.id, + target: plan.target, + plan, + status: "PASS", + }; +} + +async function runPlansLocally( + plans: DocsCodeBlockExecutionPlan[], + config: DocsCodeBlocksResolvedValidateConfig, + env: Record, +): Promise { + const tempDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-")); + try { + return await Promise.all( + plans.map(async (plan) => { + if (!plan.command || !plan.filePath) { + return skippedResult(plan, "no executable command in plan"); + } + + writeFileSync(path.join(tempDir, plan.filePath), plan.target.code, "utf-8"); + try { + const result = await execFileAsync(plan.command.cmd, plan.command.args, { + cwd: tempDir, + env: { ...process.env, ...env }, + timeout: config.runner.timeoutMs, + maxBuffer: 1024 * 1024, + }); + return { + id: plan.id, + target: plan.target, + plan, + status: "PASS" as const, + stdout: result.stdout, + stderr: result.stderr, + exitCode: 0, + }; + } catch (error) { + const err = error as { + stdout?: string; + stderr?: string; + code?: number; + signal?: string; + message?: string; + }; + return { + id: plan.id, + target: plan.target, + plan, + status: "FAIL" as const, + stdout: err.stdout, + stderr: err.stderr, + exitCode: typeof err.code === "number" ? err.code : null, + reason: err.signal ? `terminated by ${err.signal}` : err.message, + }; + } + }), + ); + } finally { + rmSync(tempDir, { force: true, recursive: true }); + } +} + +async function runPlansInVercelSandbox( + plans: DocsCodeBlockExecutionPlan[], + config: DocsCodeBlocksResolvedValidateConfig, + env: Record, +): Promise { + const token = process.env[config.runner.tokenEnv]; + if (!token) { + return plans.map((plan) => skippedResult(plan, `missing ${config.runner.tokenEnv}`)); + } + + const previousToken = process.env.VERCEL_TOKEN; + process.env.VERCEL_TOKEN = token; + + try { + const { Sandbox } = (await import("@vercel/sandbox")) as unknown as { + Sandbox: { + create(input: { + runtime?: string; + timeout?: number; + env?: Record; + }): Promise<{ + writeFiles(files: Array<{ path: string; content: Buffer }>): Promise; + runCommand(input: { + cmd: string; + args?: string[]; + cwd?: string; + env?: Record; + }): Promise<{ + exitCode: number | null; + stdout(): Promise; + stderr(): Promise; + }>; + stop(): Promise; + }>; + }; + }; + + const sandbox = await Sandbox.create({ + runtime: config.runner.runtime, + timeout: Math.max( + config.runner.timeoutMs * Math.max(1, plans.length), + config.runner.timeoutMs, + ), + env, + }); + + try { + return await Promise.all( + plans.map(async (plan) => { + if (!plan.command || !plan.filePath) { + return skippedResult(plan, "no executable command in plan"); + } + + await sandbox.writeFiles([ + { + path: plan.filePath, + content: Buffer.from(plan.target.code), + }, + ]); + + const command = await sandbox.runCommand({ + cmd: plan.command.cmd, + args: plan.command.args, + cwd: "/vercel/sandbox", + env, + }); + const [stdout, stderr] = await Promise.all([command.stdout(), command.stderr()]); + + return { + id: plan.id, + target: plan.target, + plan, + status: command.exitCode === 0 ? ("PASS" as const) : ("FAIL" as const), + stdout, + stderr, + exitCode: command.exitCode, + reason: command.exitCode === 0 ? undefined : `exit code ${command.exitCode}`, + }; + }), + ); + } finally { + await sandbox.stop().catch(() => {}); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return plans.map((plan) => skippedResult(plan, `vercel-sandbox unavailable: ${message}`)); + } finally { + if (previousToken === undefined) { + delete process.env.VERCEL_TOKEN; + } else { + process.env.VERCEL_TOKEN = previousToken; + } + } +} + +function loadValidationEnv( + rootDir: string, + config: DocsCodeBlocksResolvedValidateConfig, +): LoadedValidationEnv { + const fileEnv: Record = {}; + for (const file of config.envFile) { + const fullPath = path.resolve(rootDir, file); + if (!existsSync(fullPath)) continue; + Object.assign(fileEnv, parseEnvFile(readFileSync(fullPath, "utf-8"))); + } + + const resolved: Record = {}; + const missing: string[] = []; + for (const [runtimeKey, sourceKey] of Object.entries(config.env)) { + const value = process.env[sourceKey] ?? fileEnv[sourceKey]; + if (value === undefined) { + missing.push(runtimeKey); + continue; + } + resolved[runtimeKey] = value; + } + + return { env: resolved, missing }; +} + +function parseEnvFile(source: string): Record { + const env: Record = {}; + for (const line of source.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const equals = trimmed.indexOf("="); + if (equals <= 0) continue; + const key = trimmed.slice(0, equals).trim(); + const value = trimmed + .slice(equals + 1) + .trim() + .replace(/^['"]|['"]$/g, ""); + env[key] = value; + } + return env; +} + +function readStringMeta(meta: Record, key: string): string | undefined { + const value = meta[key]; + return typeof value === "string" && value.trim().length > 0 ? value : undefined; +} + +function readBooleanMeta(meta: Record, key: string): boolean | undefined { + const value = meta[key]; + if (typeof value === "boolean") return value; + if (typeof value !== "string") return undefined; + + const normalized = value.trim().toLowerCase(); + if (!normalized || normalized === "true" || normalized === "1" || normalized === "yes") + return true; + if (normalized === "false" || normalized === "0" || normalized === "no") return false; + return true; +} + +function readEnvMeta(meta: Record): string[] { + const direct = meta.env; + const values = new Set(); + if (typeof direct === "string") { + for (const item of direct.split(/[,\s]+/)) { + const trimmed = item.trim(); + if (trimmed) values.add(trimmed); + } + } + + for (const [key, value] of Object.entries(meta)) { + if (!key.startsWith("env:") && !key.startsWith("env.")) continue; + const envName = key.slice(4).trim(); + if (envName && value !== false) values.add(envName); + } + + return [...values]; +} + +function walkMarkdownFiles(root: string): string[] { + const files: string[] = []; + const ignored = new Set([".git", ".next", ".nuxt", ".svelte-kit", "dist", "node_modules", "out"]); + + function visit(dir: string) { + for (const entry of readdirSync(dir)) { + if (ignored.has(entry)) continue; + const fullPath = path.join(dir, entry); + const stat = statSync(fullPath); + if (stat.isDirectory()) { + visit(fullPath); + } else if (/\.(?:md|mdx)$/i.test(entry)) { + files.push(fullPath); + } + } + } + + visit(root); + return files.sort(); +} + +function normalizeLanguage(language?: string): string | undefined { + return language?.trim().toLowerCase(); +} + +function isClosingFence(trimmedLine: string, marker: string): boolean { + if (!trimmedLine.startsWith(marker)) return false; + return trimmedLine.slice(marker.length).trim().length === 0; +} + +function isShellLanguage(language: string): boolean { + return ["bash", "sh", "shell", "zsh", "curl"].includes(language); +} + +function templateFromLanguage(language?: string): string { + if (!language) return "unknown"; + if (isShellLanguage(language)) return "shell"; + if (language === "js" || language === "javascript" || language === "jsx") return "node"; + if (language === "ts" || language === "typescript" || language === "tsx") return "typescript"; + if (language === "py" || language === "python") return "python"; + return language; +} + +function looksLikeShellCommand(code: string): boolean { + return /^(?:\s*(?:npm|pnpm|yarn|bun|npx|curl|git|node|python3?|deno|uv|pip)\b)/m.test(code); +} + +function looksPartial(code: string): boolean { + const trimmed = code.trim(); + if (!trimmed) return true; + return /^\.\.\.$/m.test(trimmed) || /\/\/\s*\.\.\.|#\s*\.\.\./.test(trimmed); +} + +function slugify(value: string): string { + return ( + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 80) || "snippet" + ); +} + +function isPlanAction(value: unknown): value is DocsCodeBlockExecutionPlan["action"] { + return value === "execute" || value === "validate-syntax" || value === "skip"; +} + +function skippedResult( + plan: DocsCodeBlockExecutionPlan, + reason: string, +): DocsCodeBlockValidationResult { + return { + id: plan.id, + target: plan.target, + plan, + status: "SKIP", + reason, + }; +} diff --git a/packages/docs/src/define-docs.ts b/packages/docs/src/define-docs.ts index 948a2ca7..b7fd72c1 100644 --- a/packages/docs/src/define-docs.ts +++ b/packages/docs/src/define-docs.ts @@ -19,6 +19,7 @@ export function defineDocs(config: DocsConfig): DocsConfig { analytics: config.analytics, observability: config.observability, onCopyClick: config.onCopyClick, + codeBlocks: config.codeBlocks, feedback: config.feedback, search: config.search, mcp: config.mcp, diff --git a/packages/docs/src/index.ts b/packages/docs/src/index.ts index e3a769e8..8a8ca6e7 100644 --- a/packages/docs/src/index.ts +++ b/packages/docs/src/index.ts @@ -287,6 +287,14 @@ export type { DocsAnalyticsEventType, DocsAnalyticsInput, DocsAnalyticsSource, + DocsCodeBlocksConfig, + DocsCodeBlocksPlannerConfig, + DocsCodeBlocksPlannerProvider, + DocsCodeBlocksRunnerConfig, + DocsCodeBlocksRunnerProvider, + DocsCodeBlocksValidateConfig, + DocsCodeBlocksValidationMode, + DocsCodeBlocksValidationPolicy, DocsObservabilityConfig, DocsObservabilityEvent, DocsObservabilityEventInput, diff --git a/packages/docs/src/mcp.ts b/packages/docs/src/mcp.ts index e3ad9d23..4dcbe355 100644 --- a/packages/docs/src/mcp.ts +++ b/packages/docs/src/mcp.ts @@ -579,6 +579,45 @@ const DOCS_CONFIG_SCHEMA_OPTIONS: DocsMcpConfigSchemaOption[] = [ }, ], }, + { + path: "codeBlocks", + name: "codeBlocks", + type: "{ validate?: boolean | DocsCodeBlocksValidateConfig }", + default: false, + description: + "Code block intelligence for MD/MDX fences, including execution planning and optional sandboxed validation.", + docs: "/docs/configuration#code-block-validation", + children: [ + { + path: "codeBlocks.validate", + name: "validate", + type: "boolean | DocsCodeBlocksValidateConfig", + description: "Enable `docs codeblocks validate` for fenced code examples.", + }, + { + path: "codeBlocks.validate.planner", + name: "planner", + type: '"metadata" | "openai" | "openai-compatible" | "cloud" | DocsCodeBlocksPlannerConfig', + default: "metadata", + description: + "Planner that turns code fence metadata into an execution plan. Use OpenAI-compatible providers when metadata alone is not enough.", + }, + { + path: "codeBlocks.validate.runner", + name: "runner", + type: '"local" | "vercel-sandbox" | "cloud" | DocsCodeBlocksRunnerConfig', + default: "local", + description: "Runner used to execute planned snippets.", + }, + { + path: "codeBlocks.validate.env", + name: "env", + type: "Record", + description: + 'Runtime env mapping, for example `{ OPENAI_API_KEY: "OPENAI_TEST_API_KEY" }`.', + }, + ], + }, { path: "sitemap", name: "sitemap", @@ -631,6 +670,28 @@ export default defineDocs({ getCodeExamples: true, }, }, +});`, + }, + { + title: "Code block validation", + code: `export default defineDocs({ + entry: "docs", + codeBlocks: { + validate: { + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + }, + }, });`, }, ]; diff --git a/packages/docs/src/types.ts b/packages/docs/src/types.ts index 26c79c84..c8859d0a 100644 --- a/packages/docs/src/types.ts +++ b/packages/docs/src/types.ts @@ -2327,6 +2327,128 @@ export interface DocsReviewConfig { rules?: DocsReviewRulesConfig; } +export type DocsCodeBlocksPlannerProvider = "metadata" | "openai" | "openai-compatible" | "cloud"; + +export type DocsCodeBlocksRunnerProvider = "local" | "vercel-sandbox" | "cloud"; + +export type DocsCodeBlocksValidationMode = "plan" | "report"; + +export type DocsCodeBlocksValidationPolicy = "skip" | "warn" | "error"; + +export interface DocsCodeBlocksPlannerConfig { + /** + * Planner used to turn code fence metadata into an execution plan. + * + * - `"metadata"` reads the fence language and metadata locally. + * - `"openai"` calls OpenAI's chat completions API. + * - `"openai-compatible"` calls an OpenAI-compatible chat completions endpoint. + * - `"cloud"` is reserved for the hosted Farming Labs planner. + * + * @default "metadata" + */ + provider?: DocsCodeBlocksPlannerProvider; + /** Model name for LLM-backed planners. */ + model?: string; + /** OpenAI-compatible base URL. Defaults to `https://api.openai.com/v1` for `provider: "openai"`. */ + baseUrl?: string; + /** Environment variable containing the OpenAI-compatible base URL. */ + baseUrlEnv?: string; + /** API key value. Prefer `apiKeyEnv` so secrets stay out of docs.config. */ + apiKey?: string; + /** Environment variable containing the planner API key. */ + apiKeyEnv?: string; +} + +export interface DocsCodeBlocksRunnerConfig { + /** + * Runner used to execute planned code blocks. + * + * @default "local" + */ + provider?: DocsCodeBlocksRunnerProvider; + /** Environment variable containing the Vercel token for `provider: "vercel-sandbox"`. */ + tokenEnv?: string; + /** Vercel Sandbox runtime. */ + runtime?: "node24" | "node22" | "python3.13"; + /** Per-command timeout in milliseconds. */ + timeoutMs?: number; +} + +export interface DocsCodeBlocksValidateConfig { + /** + * Enable code block validation. + * + * @default true when `codeBlocks.validate` is an object or `true` + */ + enabled?: boolean; + /** Planner config. Use `"metadata"` for local deterministic planning. */ + planner?: DocsCodeBlocksPlannerProvider | DocsCodeBlocksPlannerConfig; + /** Runner config. Use `"vercel-sandbox"` for isolated runtime checks. */ + runner?: DocsCodeBlocksRunnerProvider | DocsCodeBlocksRunnerConfig; + /** + * Env files loaded for validation. These are read locally and never committed. + * + * @default [".env.local", ".env.test", ".env"] + */ + envFile?: string | string[]; + /** + * Runtime env mapping. + * + * The key is the env var used by the docs code block. The value is the local + * env var to read from. For example, `{ OPENAI_API_KEY: "OPENAI_TEST_API_KEY" }` + * injects `OPENAI_API_KEY` into the runner from `OPENAI_TEST_API_KEY`. + */ + env?: Record; + /** + * Behavior when a runnable block declares an env var that cannot be resolved. + * + * @default "skip" + */ + missingEnv?: DocsCodeBlocksValidationPolicy; + /** + * Behavior when a language cannot be executed by the selected runner. + * + * @default "skip" + */ + unsupportedLanguage?: DocsCodeBlocksValidationPolicy; + /** + * Default command mode. + * + * - `"plan"` builds execution plans without running them. + * - `"report"` runs executable plans and reports pass/skip/fail. + * + * @default "report" + */ + mode?: DocsCodeBlocksValidationMode; +} + +export interface DocsCodeBlocksConfig { + /** + * Validate fenced code blocks from MD/MDX docs. + * + * @example + * ```ts + * codeBlocks: { + * validate: { + * planner: { + * provider: "openai", + * model: "gpt-4.1-mini", + * apiKeyEnv: "OPENAI_API_KEY", + * }, + * runner: { + * provider: "vercel-sandbox", + * tokenEnv: "VERCEL_TOKEN", + * }, + * env: { + * OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + * }, + * }, + * } + * ``` + */ + validate?: boolean | DocsCodeBlocksValidateConfig; +} + export interface DocsConfig { /** Entry folder for docs (e.g. "docs" → /docs) */ entry: string; @@ -2478,6 +2600,11 @@ export interface DocsConfig { * ``` */ onCopyClick?: (data: CodeBlockCopyData) => void; + /** + * Code block intelligence for MD/MDX fences, including validation planning + * and optional sandboxed execution. + */ + codeBlocks?: DocsCodeBlocksConfig; /** * Built-in page feedback prompt shown at the end of a docs page. * diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 31515436..d6454a02 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -73,7 +73,7 @@ importers: dependencies: '@astrojs/vercel': specifier: ^9.0.4 - version: 9.0.4(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(astro@5.17.3(@types/node@22.19.11)(@vercel/functions@2.2.13)(db0@0.3.4)(ioredis@5.9.3)(jiti@2.6.1)(lightningcss@1.31.1)(rollup@4.59.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(rollup@4.59.0)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3)) + version: 9.0.4(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(astro@5.17.3(@types/node@22.19.11)(@vercel/functions@2.2.13)(db0@0.3.4)(ioredis@5.9.3)(jiti@2.6.1)(lightningcss@1.31.1)(rollup@4.59.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(rollup@4.59.0)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3)) '@farming-labs/astro': specifier: 0.1.123 version: link:../../packages/astro @@ -307,6 +307,9 @@ importers: '@scalar/core': specifier: ^0.4.3 version: 0.4.3 + '@vercel/sandbox': + specifier: ^2.0.0 + version: 2.0.0 gray-matter: specifier: ^4.0.3 version: 4.0.3 @@ -3639,9 +3642,16 @@ packages: resolution: {integrity: sha512-59PBFx3T+k5hLTEWa3ggiMpGRz1OVvl9eN8SUai+A43IsqiOuAe7qPBf+cray/Fj6mkgnxm/D7IAtjc8zSHi7g==} engines: {node: '>= 18'} + '@vercel/oidc@3.2.0': + resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} + engines: {node: '>= 20'} + '@vercel/routing-utils@5.3.3': resolution: {integrity: sha512-KYm2sLNUD48gDScv8ob4ejc3Gww2jcJyW80hTdYlenAPz/5BQar1Gyh38xrUuZ532TUwSb5mV1uRbAuiykq0EQ==} + '@vercel/sandbox@2.0.0': + resolution: {integrity: sha512-fmg6lNfDJdhQ43Njc0jUIXTQP2wKPY6tJszeVq5C25v1XsDK3wwnlFBQfSKzZfkMkh1N269pPgOopjTEqJJamA==} + '@vitejs/plugin-react@5.2.0': resolution: {integrity: sha512-YmKkfhOAi3wsB1PhJq5Scj3GXMn3WvtQ/JC0xoopuHoXSdmtdStOpFrYaT1kie2YgFBcIe64ROzMYRjCrYOdYw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3768,6 +3778,9 @@ packages: '@vue/shared@3.5.28': resolution: {integrity: sha512-cfWa1fCGBxrvaHRhvV3Is0MgmrbSCxYTXCSCau2I0a1Xw1N1pHAvkWCiXPRAqjvToILvguNyEwjevUqAuBQWvQ==} + '@workflow/serde@4.1.0-beta.2': + resolution: {integrity: sha512-8kkeoQKLDaKXefjV5dbhBj2aErfKp1Mc4pb6tj8144cF+Em5SPbyMbyLCHp+BVrFfFVCBluCtMx+jjvaFVZGww==} + abbrev@3.0.1: resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} engines: {node: ^18.17.0 || >=20.5.0} @@ -3909,6 +3922,9 @@ packages: engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true + async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + async-sema@3.1.1: resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==} @@ -5362,6 +5378,9 @@ packages: jose@6.2.2: resolution: {integrity: sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==} + jose@6.2.3: + resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -5401,6 +5420,9 @@ packages: jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + jsonlines@0.1.1: + resolution: {integrity: sha512-ekDrAGso79Cvf+dtm+mL8OBI2bmAOt3gssYs833De/C9NmIpWDWyUO4zPgB5x2/OhY366dkhgfPMYfwZF7yOZA==} + kind-of@6.0.3: resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} engines: {node: '>=0.10.0'} @@ -6102,6 +6124,10 @@ packages: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} + os-paths@4.4.0: + resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} + engines: {node: '>= 6.0'} + oxc-minify@0.112.0: resolution: {integrity: sha512-rkVSeeIRSt+RYI9uX6xonBpLUpvZyegxIg0UL87ev7YAfUqp7IIZlRjkgQN5Us1lyXD//TOo0Dcuuro/TYOWoQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -6732,6 +6758,10 @@ packages: retext@9.0.0: resolution: {integrity: sha512-sbMDcpHCNjvlheSgMfEcVrZko3cDzdbe1x/e7G66dFp0Ff7Mldvi2uv6JkJQzdRcvLYE8CA8Oe8siQx8ZOgTcA==} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -7840,6 +7870,14 @@ packages: resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} engines: {node: '>=18'} + xdg-app-paths@5.1.0: + resolution: {integrity: sha512-RAQ3WkPf4KTU1A8RtFx3gWywzVKe00tfOPFfl2NDGqbIFENQO4kqAJp7mhQjNj/33W5x5hiWWUdyfPq/5SU3QA==} + engines: {node: '>=6'} + + xdg-portable@7.3.0: + resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} + engines: {node: '>= 6.0'} + xml-js@1.6.11: resolution: {integrity: sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==} hasBin: true @@ -7922,6 +7960,9 @@ packages: typescript: ^4.9.4 || ^5.0.2 zod: ^3 + zod@3.24.4: + resolution: {integrity: sha512-OdqJE9UDRPwWsrHjLN2F8bPxvwJBK22EHLWtanu0LSYr5YqzsaaW3RMgmjwr8Rypg5k+meEJdSPXJZXE/yqOMg==} + zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} @@ -7981,10 +8022,10 @@ snapshots: transitivePeerDependencies: - supports-color - '@astrojs/vercel@9.0.4(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(astro@5.17.3(@types/node@22.19.11)(@vercel/functions@2.2.13)(db0@0.3.4)(ioredis@5.9.3)(jiti@2.6.1)(lightningcss@1.31.1)(rollup@4.59.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(rollup@4.59.0)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3))': + '@astrojs/vercel@9.0.4(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(astro@5.17.3(@types/node@22.19.11)(@vercel/functions@2.2.13)(db0@0.3.4)(ioredis@5.9.3)(jiti@2.6.1)(lightningcss@1.31.1)(rollup@4.59.0)(terser@5.46.0)(tsx@4.21.0)(typescript@5.9.3)(yaml@2.8.2))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(rollup@4.59.0)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3))': dependencies: '@astrojs/internal-helpers': 0.7.5 - '@vercel/analytics': 1.6.1(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3)) + '@vercel/analytics': 1.6.1(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3)) '@vercel/functions': 2.2.13 '@vercel/nft': 0.30.4(rollup@4.59.0) '@vercel/routing-utils': 5.3.3 @@ -10248,6 +10289,27 @@ snapshots: optionalDependencies: typescript: 5.9.3 + '@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@standard-schema/spec': 1.1.0 + '@sveltejs/acorn-typescript': 1.0.9(acorn@8.15.0) + '@sveltejs/vite-plugin-svelte': 5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + '@types/cookie': 0.6.0 + acorn: 8.15.0 + cookie: 0.6.0 + devalue: 5.8.1 + esm-env: 1.2.2 + kleur: 4.1.5 + magic-string: 0.30.21 + mrmime: 2.0.1 + set-cookie-parser: 3.0.1 + sirv: 3.0.2 + svelte: 5.51.3 + vite: 7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + optionalDependencies: + typescript: 5.9.3 + optional: true + '@sveltejs/vite-plugin-svelte-inspector@4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@sveltejs/vite-plugin-svelte': 5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) @@ -10257,6 +10319,16 @@ snapshots: transitivePeerDependencies: - supports-color + '@sveltejs/vite-plugin-svelte-inspector@4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@sveltejs/vite-plugin-svelte': 5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + debug: 4.4.3 + svelte: 5.51.3 + vite: 7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + transitivePeerDependencies: + - supports-color + optional: true + '@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@sveltejs/vite-plugin-svelte-inspector': 4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) @@ -10270,6 +10342,20 @@ snapshots: transitivePeerDependencies: - supports-color + '@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': + dependencies: + '@sveltejs/vite-plugin-svelte-inspector': 4.0.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + debug: 4.4.3 + deepmerge: 4.3.1 + kleur: 4.1.5 + magic-string: 0.30.21 + svelte: 5.51.3 + vite: 7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2) + vitefu: 1.1.1(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + transitivePeerDependencies: + - supports-color + optional: true + '@swc/helpers@0.5.15': dependencies: tslib: 2.8.1 @@ -10692,9 +10778,9 @@ snapshots: unhead: 2.1.4 vue: 3.5.28(typescript@5.9.3) - '@vercel/analytics@1.6.1(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3))': + '@vercel/analytics@1.6.1(@sveltejs/kit@2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(next@16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4)(svelte@5.51.3)(vue-router@4.6.4(vue@3.5.28(typescript@5.9.3)))(vue@3.5.28(typescript@5.9.3))': optionalDependencies: - '@sveltejs/kit': 2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@6.4.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) + '@sveltejs/kit': 2.57.1(@sveltejs/vite-plugin-svelte@5.1.1(svelte@5.51.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)))(svelte@5.51.3)(typescript@5.9.3)(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2)) next: 16.2.6(@playwright/test@1.59.1)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react: 19.2.4 svelte: 5.51.3 @@ -10748,6 +10834,8 @@ snapshots: '@types/ms': 2.1.0 ms: 2.1.3 + '@vercel/oidc@3.2.0': {} + '@vercel/routing-utils@5.3.3': dependencies: path-to-regexp: 6.3.0 @@ -10755,6 +10843,23 @@ snapshots: optionalDependencies: ajv: 6.12.6 + '@vercel/sandbox@2.0.0': + dependencies: + '@vercel/oidc': 3.2.0 + '@workflow/serde': 4.1.0-beta.2 + async-retry: 1.3.3 + jose: 6.2.3 + jsonlines: 0.1.1 + ms: 2.1.3 + picocolors: 1.1.1 + tar-stream: 3.1.7 + undici: 7.24.3 + xdg-app-paths: 5.1.0 + zod: 3.24.4 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + '@vitejs/plugin-react@5.2.0(vite@7.3.2(@types/node@22.19.11)(jiti@2.6.1)(lightningcss@1.31.1)(terser@5.46.0)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@babel/core': 7.29.0 @@ -10964,6 +11069,8 @@ snapshots: '@vue/shared@3.5.28': {} + '@workflow/serde@4.1.0-beta.2': {} + abbrev@3.0.1: {} abort-controller@3.0.0: @@ -11208,6 +11315,10 @@ snapshots: - uploadthing - yaml + async-retry@1.3.3: + dependencies: + retry: 0.13.1 + async-sema@3.1.1: {} async@3.2.6: {} @@ -12789,6 +12900,8 @@ snapshots: jose@6.2.2: {} + jose@6.2.3: {} + js-tokens@4.0.0: {} js-tokens@9.0.1: {} @@ -12817,6 +12930,8 @@ snapshots: jsonc-parser@3.3.1: {} + jsonlines@0.1.1: {} + kind-of@6.0.3: {} kleur@3.0.3: {} @@ -13928,6 +14043,8 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 + os-paths@4.4.0: {} + oxc-minify@0.112.0: optionalDependencies: '@oxc-minify/binding-android-arm-eabi': 0.112.0 @@ -14755,6 +14872,8 @@ snapshots: retext-stringify: 4.0.0 unified: 11.0.5 + retry@0.13.1: {} + reusify@1.1.0: {} rfdc@1.4.1: {} @@ -15944,6 +16063,14 @@ snapshots: dependencies: is-wsl: 3.1.1 + xdg-app-paths@5.1.0: + dependencies: + xdg-portable: 7.3.0 + + xdg-portable@7.3.0: + dependencies: + os-paths: 4.4.0 + xml-js@1.6.11: dependencies: sax: 1.5.0 @@ -16031,6 +16158,8 @@ snapshots: typescript: 5.9.3 zod: 3.25.76 + zod@3.24.4: {} + zod@3.25.76: {} zod@4.3.6: {} diff --git a/skills/farming-labs/README.md b/skills/farming-labs/README.md index b21b618d..11c2a454 100644 --- a/skills/farming-labs/README.md +++ b/skills/farming-labs/README.md @@ -53,11 +53,11 @@ first-run setup. | Skill | Path | When to use | | ----- | ---- | ----------- | | **Getting started** | [getting-started](./getting-started/SKILL.md) | Setting up docs, init, manual install, theme CSS, docs.config, packages by framework, generated changelog pages in Next.js, machine-readable markdown routes with `Agent` blocks or `agent.md` overrides, and API reference wiring from local routes or a hosted OpenAPI JSON. | -| **CLI** | [cli](./cli/SKILL.md) | Scaffolding and commands: init flow (existing vs fresh), Create your own theme, optional defaults (Enter to accept), `init` / `upgrade` / `doctor` / `agent compact` / `sitemap generate` / `robots generate` / `mcp`, hosted `doctor --agent --url`, `--template`, `--name`, `--theme`, `--entry`, `--api-reference`, `--framework`, `--config`, package manager commands. | +| **CLI** | [cli](./cli/SKILL.md) | Scaffolding and commands: init flow (existing vs fresh), Create your own theme, optional defaults (Enter to accept), `init` / `upgrade` / `doctor` / `agent compact` / `codeblocks validate` / `sitemap generate` / `robots generate` / `mcp`, hosted `doctor --agent --url`, `--template`, `--name`, `--theme`, `--entry`, `--api-reference`, `--framework`, `--config`, package manager commands. | | **Creating themes** | [creating-themes](./creating-themes/SKILL.md) | Building a custom theme with `createTheme()`, `extendTheme()`, `ui.components` defaults like `HoverLink`, publishing as npm, CSS overrides. | | **Ask AI** | [ask-ai](./ask-ai/SKILL.md) | Enabling and configuring the RAG-powered AI chat: mode, floatingStyle, providers, models, suggestedQuestions, apiKey. | | **Page actions** | [page-actions](./page-actions/SKILL.md) | Copy Markdown and Open in LLM buttons: copyMarkdown, openDocs, providers, urlTemplate, `{url}.md` markdown route patterns, position, alignment, and provider defaults. | -| **Configuration** | [configuration](./configuration/SKILL.md) | docs.config.ts options: entry, theme, staticExport, sidebar, breadcrumb, github, components, `search`, `changelog`, `agent.compact`, human page feedback, agent feedback endpoints, metadata, og, `mcp`, `sitemap`, `robots`, built-in markdown routes with `Agent` blocks or `agent.md`, and `apiReference` including remote `specUrl` support. | +| **Configuration** | [configuration](./configuration/SKILL.md) | docs.config.ts options: entry, theme, staticExport, sidebar, breadcrumb, github, components, `search`, `changelog`, `agent.compact`, `codeBlocks.validate`, human page feedback, agent feedback endpoints, metadata, og, `mcp`, `sitemap`, `robots`, built-in markdown routes with `Agent` blocks or `agent.md`, and `apiReference` including remote `specUrl` support. | --- diff --git a/skills/farming-labs/cli/SKILL.md b/skills/farming-labs/cli/SKILL.md index c019eaeb..c6e000b0 100644 --- a/skills/farming-labs/cli/SKILL.md +++ b/skills/farming-labs/cli/SKILL.md @@ -1,14 +1,14 @@ --- name: cli -description: @farming-labs/docs CLI — scaffold, upgrade, downgrade, run doctor audits, compact agent docs, generate AGENTS.md, generate sitemaps, generate robots.txt, sync external search indexes, and run MCP for docs. Use when running init, upgrade, downgrade, doctor, agent compact, agents generate, sitemap generate, robots generate, search sync, mcp, or flags like --template, --name, --theme, --entry, --api-reference, --api-route-root, --framework, --latest, --beta, --version, --config, --url, --page, --all, --api-key, or --dry-run. Covers init flow, Create your own theme, optional defaults, npm/pnpm/yarn/bun, and framework detection. +description: @farming-labs/docs CLI — scaffold, upgrade, downgrade, run doctor audits, compact agent docs, validate code blocks, generate AGENTS.md, generate sitemaps, generate robots.txt, sync external search indexes, and run MCP for docs. Use when running init, upgrade, downgrade, doctor, agent compact, codeblocks validate, agents generate, sitemap generate, robots generate, search sync, mcp, or flags like --template, --name, --theme, --entry, --api-reference, --api-route-root, --framework, --latest, --beta, --version, --config, --url, --page, --all, --api-key, or --dry-run. Covers init flow, Create your own theme, optional defaults, npm/pnpm/yarn/bun, and framework detection. --- # @farming-labs/docs — CLI The `@farming-labs/docs` CLI scaffolds, upgrades, downgrades, audits agent and reader readiness, compacts -page-level agent docs, reviews docs PR changes, generates `AGENTS.md`, syncs external search indexes, generates robots.txt policy files, and can +page-level agent docs, validates fenced code blocks, reviews docs PR changes, generates `AGENTS.md`, syncs external search indexes, generates robots.txt policy files, and can run the built-in MCP server for documentation projects. Use this skill when the user asks about CLI -commands, init, upgrade, downgrade, `doctor`, `agent compact`, `agents generate`, `sitemap generate`, `robots generate`, search +commands, init, upgrade, downgrade, `doctor`, `agent compact`, `codeblocks validate`, `agents generate`, `sitemap generate`, `robots generate`, search sync, mcp, review, or scaffolding. --- @@ -202,6 +202,50 @@ example. This metadata is for markdown/MCP consumers and does not require a UI c Use the docs config `mcp` block when you also want the HTTP route version at `/mcp` or `/.well-known/mcp`. +## Code block validation + +Use `docs codeblocks validate` to scan MD/MDX fences, build execution plans from code fence metadata, +and run executable snippets when `codeBlocks.validate` is enabled in `docs.config.ts`. + +```bash +pnpm exec docs codeblocks validate --plan +pnpm exec docs codeblocks validate +pnpm exec docs codeblocks validate --json +``` + +Config example: + +```ts +codeBlocks: { + validate: { + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + envFile: [".env.local", ".env.test", ".env"], + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + }, +} +``` + +Fence metadata example: + +````md +```ts title="app/api/chat/route.ts" framework="nextjs" packageManager="pnpm" env="OPENAI_API_KEY" runnable +const apiKey = process.env.OPENAI_API_KEY; +``` +```` + +The `env` map injects runtime env names from local test env vars. Do not put actual secrets in +`docs.config.ts`. + ## Docs Review Use `docs review` to score changed docs files, check broken internal links, required frontmatter, diff --git a/skills/farming-labs/configuration/SKILL.md b/skills/farming-labs/configuration/SKILL.md index c395ed96..c9e2c386 100644 --- a/skills/farming-labs/configuration/SKILL.md +++ b/skills/farming-labs/configuration/SKILL.md @@ -1,6 +1,6 @@ --- name: configuration -description: docs.config.ts options for @farming-labs/docs. Use when configuring entry, contentDir, theme, staticExport, nav, github, themeToggle, breadcrumb, sidebar, icons, components, search, changelog, feedback, readingTime, agent.compact, metadata, og, apiReference, MCP, llmsTxt, sitemap, robots, onCopyClick, pageActions, or ai. Covers Next.js, TanStack Start, SvelteKit, Astro, Nuxt config file location. +description: docs.config.ts options for @farming-labs/docs. Use when configuring entry, contentDir, theme, staticExport, nav, github, themeToggle, breadcrumb, sidebar, icons, components, search, changelog, feedback, readingTime, agent.compact, metadata, og, apiReference, MCP, llmsTxt, sitemap, robots, codeBlocks.validate, onCopyClick, pageActions, or ai. Covers Next.js, TanStack Start, SvelteKit, Astro, Nuxt config file location. --- # @farming-labs/docs — Configuration @@ -41,6 +41,7 @@ TanStack Start, SvelteKit, Astro, and Nuxt require `contentDir` (path to markdow | `icons` | `Record` | — | Shared icon registry for frontmatter `icon` fields and built-ins like `Prompt` | | `components` | `Record` | — | Custom MDX components and built-in overrides like `HoverLink` and `Prompt` | | `onCopyClick` | `(data: CodeBlockCopyData) => void` | — | Callback when user copies a code block (title, content, url, language) | +| `codeBlocks` | `DocsCodeBlocksConfig` | — | Validate fenced MDX code blocks with metadata planning and optional sandbox execution | | `feedback` | `boolean \| FeedbackConfig` | `false` for UI | Human page feedback UI; agent feedback endpoints are default-on unless opted out | | `readingTime` | `boolean \| ReadingTimeConfig` | `false` | Opt-in estimated read-time label with per-page overrides | | `agent` | `DocsAgentConfig` | — | Defaults for `docs agent compact` | @@ -76,6 +77,50 @@ review: { --- +## Code block validation + +Use `codeBlocks.validate` when docs code fences should be planned and checked by the CLI. + +```ts +codeBlocks: { + validate: { + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + envFile: [".env.local", ".env.test", ".env"], + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + missingEnv: "skip", + }, +} +``` + +Fence metadata example: + +````md +```ts title="app/api/chat/route.ts" framework="nextjs" packageManager="pnpm" env="OPENAI_API_KEY" runnable +const apiKey = process.env.OPENAI_API_KEY; +``` +```` + +Useful commands: + +```bash +pnpm exec docs codeblocks validate --plan +pnpm exec docs codeblocks validate +``` + +Do not put actual API keys in `docs.config.ts`. Use env variable names and map runtime names to test keys with `env`. + +--- + ## Static export For fully static builds (e.g. Cloudflare Pages, no server): diff --git a/website/app/docs/cli/page.mdx b/website/app/docs/cli/page.mdx index 3f7bdbc5..0638b0d8 100644 --- a/website/app/docs/cli/page.mdx +++ b/website/app/docs/cli/page.mdx @@ -1,6 +1,6 @@ --- title: "CLI" -description: "Scaffold, upgrade, compact agent docs, generate sitemaps and robots.txt, sync search, and run MCP" +description: "Scaffold, upgrade, compact agent docs, validate code blocks, generate sitemaps and robots.txt, sync search, and run MCP" icon: "terminal" order: 2 related: @@ -13,7 +13,7 @@ related: # CLI -Use this page when the user asks about this topic: Scaffold, upgrade, compact agent docs, generate sitemaps and robots.txt, sync search, and run MCP. +Use this page when the user asks about this topic: Scaffold, upgrade, compact agent docs, validate code blocks, generate sitemaps and robots.txt, sync search, and run MCP. Keep answers grounded in the exact options, routes, commands, and examples documented here. If the request moves beyond this page, point to the closest related docs instead of inventing config. @@ -26,6 +26,7 @@ The `@farming-labs/docs` CLI has a few main jobs: - **`agents generate`** — write root and static `AGENTS.md` instructions for coding agents - **`doctor`** — audit local docs readiness and optional hosted agent routes - **`review`** — review docs changes locally or in GitHub Actions +- **`codeblocks validate`** — plan and validate runnable MDX code fences - **`mcp`** — run the built-in docs MCP server over stdio for local clients and IDE agents - **`search sync`** — push docs content into an external search index like Typesense or Algolia - **`sitemap generate`** — write sitemap metadata plus static `sitemap.xml`, `sitemap.md`, and `docs/sitemap.md` files @@ -946,6 +947,48 @@ returns structured `docs.config.ts` option metadata. For the HTTP endpoint version, see [MCP Server](/docs/customization/mcp). +## Code Blocks + +Use `docs codeblocks validate` to plan and validate fenced code blocks from your docs content. + +```bash title="terminal" +pnpm exec docs codeblocks validate --plan +pnpm exec docs codeblocks validate +pnpm exec docs codeblocks validate --json +``` + +The command reads `codeBlocks.validate` from `docs.config.ts[x]`. `--plan` shows what would run +without executing snippets. + +```ts title="docs.config.ts" +codeBlocks: { + validate: { + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + }, +} +``` + +Code fence metadata gives the planner enough context: + +````md title="page.mdx" +```ts title="app/api/chat/route.ts" framework="nextjs" packageManager="pnpm" env="OPENAI_API_KEY" runnable +const apiKey = process.env.OPENAI_API_KEY; +``` +```` + +See [Configuration](/docs/configuration#code-block-validation) for the config fields. + ## Docs Review Use `docs review` to check changed docs files before a PR is merged. It scores the changed docs, diff --git a/website/app/docs/configuration/agent.md b/website/app/docs/configuration/agent.md index 87b0ee2f..32cd532e 100644 --- a/website/app/docs/configuration/agent.md +++ b/website/app/docs/configuration/agent.md @@ -13,6 +13,7 @@ Use this machine-oriented page when the user needs implementation guidance for ` - `components` - `pageActions` - `agent` + - `codeBlocks` - `search` - `ai` - `mcp` @@ -35,6 +36,7 @@ Use this machine-oriented page when the user needs implementation guidance for ` - When they want AI-facing behavior, distinguish between: - `ai` for Ask AI / chat - `agent.compact` for defaults used by `docs agent compact` + - `codeBlocks.validate` for planning and validating fenced MDX code blocks - `mcp` for the built-in MCP server, including default tools like `list_docs`, `search_docs`, `read_page`, `get_code_examples`, and `get_config_schema` - `llmsTxt` for crawler-friendly site summaries @@ -55,7 +57,7 @@ Use this machine-oriented page when the user needs implementation guidance for ` ## Follow-up pages - Use [/docs/installation](/docs/installation) when the user is still wiring the framework into an app or has not created the docs route yet. -- Use [/docs/cli](/docs/cli) when they want scaffolding, upgrades, sitemap generation, robots generation, search sync, or MCP commands instead of manual setup. +- Use [/docs/cli](/docs/cli) when they want scaffolding, upgrades, code block validation, sitemap generation, robots generation, search sync, or MCP commands instead of manual setup. - Use [/docs/reference](/docs/reference) when they need the full typed `defineDocs()` surface or nested option details. - Use [/docs/customization](/docs/customization) when the question moves from config into layout, sidebar, colors, or page-level polish. - Use [/docs/themes](/docs/themes) when they are choosing a preset theme or building their own. diff --git a/website/app/docs/configuration/page.mdx b/website/app/docs/configuration/page.mdx index ec8bc7b3..4cf52f33 100644 --- a/website/app/docs/configuration/page.mdx +++ b/website/app/docs/configuration/page.mdx @@ -133,6 +133,7 @@ All configuration lives in a single `docs.config.ts` file. | `icons` | `Record` | — | Shared icon registry for frontmatter `icon` fields and built-ins like `Prompt` | | `components` | `Record` | — | Custom MDX components and built-in overrides like `HoverLink` and `Prompt` | | `onCopyClick` | `(data: CodeBlockCopyData) => void` | — | Callback when the user clicks the copy button on a code block | +| `codeBlocks` | `DocsCodeBlocksConfig` | — | Validate fenced MDX code blocks with metadata planning and optional sandbox execution | | `feedback` | `boolean \| FeedbackConfig` | `false` for UI | Human page feedback UI; agent feedback endpoints are default-on unless opted out | | `agent` | `DocsAgentConfig` | — | Defaults for `docs agent compact` | | `review` | `boolean \| DocsReviewConfig` | `true` | Docs Review CI workflow generation, scoring, and rule severity | @@ -150,6 +151,56 @@ All configuration lives in a single `docs.config.ts` file. | `metadata` | `DocsMetadata` | — | SEO metadata template and JSON-LD page inputs | | `og` | `OGConfig` | — | Dynamic Open Graph images (see [API Reference](/docs/reference#ogconfig)) | +## Code block validation + +`codeBlocks.validate` lets the CLI inspect fenced MDX code blocks, build an execution plan from +metadata, and optionally run executable snippets in a sandbox. + +```ts title="docs.config.ts" +import { defineDocs } from "@farming-labs/docs"; + +export default defineDocs({ + entry: "docs", + codeBlocks: { + validate: { + planner: { + provider: "openai", + model: "gpt-4.1-mini", + apiKeyEnv: "OPENAI_API_KEY", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + envFile: [".env.local", ".env.test", ".env"], + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + missingEnv: "skip", + }, + }, +}); +``` + +Use metadata on the code fence to tell agents and validators how a snippet should be treated: + +````md title="page.mdx" +```ts title="app/api/chat/route.ts" framework="nextjs" packageManager="pnpm" env="OPENAI_API_KEY" runnable +const apiKey = process.env.OPENAI_API_KEY; +``` +```` + +Run the validator locally: + +```bash +pnpm exec docs codeblocks validate --plan +pnpm exec docs codeblocks validate +``` + +The planner never writes secrets into docs. The `env` map means the snippet can keep using +`process.env.OPENAI_API_KEY`, while validation injects that runtime variable from +`OPENAI_TEST_API_KEY`. + ## Public docs path In Next.js, `entry` controls where the docs live in your app directory. `docsPath` controls the From 5a269e50323f308a5cc5a0fe0d3ba1d0541b9092 Mon Sep 17 00:00:00 2001 From: Kinfe123 Date: Mon, 25 May 2026 01:06:01 +0300 Subject: [PATCH 2/4] chore: vercel sandbox --- packages/docs/src/cli/codeblocks.ts | 135 ++++++++++++++++++++++- packages/docs/src/cli/config.ts | 3 +- packages/docs/src/code-blocks.test.ts | 108 +++++++++++++++++- packages/docs/src/code-blocks.ts | 139 +++++++++++++++++++++--- packages/docs/src/mcp.ts | 3 +- packages/docs/src/types.ts | 13 +++ website/app/docs/configuration/page.mdx | 29 ++++- website/docs.config.tsx | 13 +++ 8 files changed, 422 insertions(+), 21 deletions(-) diff --git a/packages/docs/src/cli/codeblocks.ts b/packages/docs/src/cli/codeblocks.ts index 65f3a545..ff995f16 100644 --- a/packages/docs/src/cli/codeblocks.ts +++ b/packages/docs/src/cli/codeblocks.ts @@ -10,6 +10,9 @@ import type { DocsCodeBlocksValidateConfig } from "../types.js"; import { extractNestedObjectLiteral, loadDocsConfigModule, + readBooleanProperty, + readNumberProperty, + readStringProperty, readTopLevelStringProperty, resolveDocsConfigPath, resolveDocsContentDir, @@ -99,7 +102,9 @@ ${pc.dim("Options:")} export async function runCodeBlocksValidate(options: CodeBlocksValidateOptions = {}) { const rootDir = process.cwd(); - const loaded = await loadDocsConfigModule(rootDir, options.configPath); + const loaded = await loadDocsConfigModule(rootDir, options.configPath, { + silent: options.json, + }); const configPath = loaded?.path ?? resolveDocsConfigPath(rootDir, options.configPath); const configContent = existsSync(configPath) ? readFileSync(configPath, "utf-8") : ""; const entry = @@ -160,12 +165,136 @@ function readStaticCodeBlocksValidateConfig( if (!block) return undefined; if (/\bvalidate\s*:\s*true\b/.test(block)) return true; if (/\bvalidate\s*:\s*false\b/.test(block)) return false; - if (/\bvalidate\s*:\s*\{/.test(block)) { - return true; + + const validateBlock = extractNestedObjectLiteral(content, ["codeBlocks", "validate"]); + if (validateBlock) { + const config: DocsCodeBlocksValidateConfig = {}; + const enabled = readBooleanProperty(validateBlock, "enabled"); + const mode = readStringProperty(validateBlock, "mode"); + const missingEnv = readStringProperty(validateBlock, "missingEnv"); + const unsupportedLanguage = readStringProperty(validateBlock, "unsupportedLanguage"); + const envFile = readStringArrayProperty(validateBlock, "envFile"); + + if (enabled !== undefined) config.enabled = enabled; + if (mode === "plan" || mode === "report") config.mode = mode; + if (missingEnv === "skip" || missingEnv === "warn" || missingEnv === "error") { + config.missingEnv = missingEnv; + } + if ( + unsupportedLanguage === "skip" || + unsupportedLanguage === "warn" || + unsupportedLanguage === "error" + ) { + config.unsupportedLanguage = unsupportedLanguage; + } + if (envFile) config.envFile = envFile; + + const plannerBlock = extractNestedObjectLiteral(content, ["codeBlocks", "validate", "planner"]); + const planner = readStaticPlannerConfig(plannerBlock); + if (planner) config.planner = planner; + + const runnerBlock = extractNestedObjectLiteral(content, ["codeBlocks", "validate", "runner"]); + const runner = readStaticRunnerConfig(runnerBlock); + if (runner) config.runner = runner; + + const envBlock = extractNestedObjectLiteral(content, ["codeBlocks", "validate", "env"]); + const env = readStringRecord(envBlock); + if (env && Object.keys(env).length > 0) config.env = env; + + return config; } + + if (/\bvalidate\s*:\s*\{/.test(block)) return true; return undefined; } +function readStaticPlannerConfig( + block?: string, +): DocsCodeBlocksValidateConfig["planner"] | undefined { + if (!block) return undefined; + const provider = readStringProperty(block, "provider"); + const model = readStringProperty(block, "model"); + const baseUrl = readStringProperty(block, "baseUrl"); + const baseUrlEnv = readStringProperty(block, "baseUrlEnv"); + const apiKeyEnv = readStringProperty(block, "apiKeyEnv"); + + if ( + provider !== "metadata" && + provider !== "openai" && + provider !== "openai-compatible" && + provider !== "cloud" + ) { + return undefined; + } + + return { + provider, + ...(model ? { model } : {}), + ...(baseUrl ? { baseUrl } : {}), + ...(baseUrlEnv ? { baseUrlEnv } : {}), + ...(apiKeyEnv ? { apiKeyEnv } : {}), + }; +} + +function readStaticRunnerConfig( + block?: string, +): DocsCodeBlocksValidateConfig["runner"] | undefined { + if (!block) return undefined; + const provider = readStringProperty(block, "provider"); + const tokenEnv = readStringProperty(block, "tokenEnv"); + const projectIdEnv = readStringProperty(block, "projectIdEnv"); + const teamIdEnv = readStringProperty(block, "teamIdEnv"); + const projectJson = readStringProperty(block, "projectJson"); + const runtime = readStringProperty(block, "runtime"); + const timeoutMs = readNumberProperty(block, "timeoutMs"); + + if (provider !== "local" && provider !== "vercel-sandbox" && provider !== "cloud") { + return undefined; + } + + return { + provider, + ...(tokenEnv ? { tokenEnv } : {}), + ...(projectIdEnv ? { projectIdEnv } : {}), + ...(teamIdEnv ? { teamIdEnv } : {}), + ...(projectJson ? { projectJson } : {}), + ...(runtime === "node24" || runtime === "node22" || runtime === "python3.13" + ? { runtime } + : {}), + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + }; +} + +function readStringArrayProperty(content: string, key: string): string[] | undefined { + const single = readStringProperty(content, key); + if (single) return [single]; + + const match = content.match(new RegExp(`\\b${key}\\b\\s*:\\s*\\[([\\s\\S]*?)\\]`)); + if (!match) return undefined; + + const values = [...match[1].matchAll(/["']([^"']+)["']/g)].map((item) => item[1]); + return values.length > 0 ? values : undefined; +} + +function readStringRecord(block?: string): Record | undefined { + if (!block) return undefined; + + const record: Record = {}; + const patterns = [ + /(?:^|,)\s*([A-Za-z_$][\w$]*)\s*:\s*["']([^"']+)["']/g, + /(?:^|,)\s*["']([^"']+)["']\s*:\s*["']([^"']+)["']/g, + ]; + + for (const pattern of patterns) { + let match: RegExpExecArray | null; + while ((match = pattern.exec(block))) { + record[match[1]] = match[2]; + } + } + + return record; +} + function printCodeBlocksReport(report: DocsCodeBlocksValidationReport, planOnly: boolean) { const label = planOnly ? "Code block plan" : "Code block validation"; console.log(pc.bold(label)); diff --git a/packages/docs/src/cli/config.ts b/packages/docs/src/cli/config.ts index b0a055c9..1dc82e90 100644 --- a/packages/docs/src/cli/config.ts +++ b/packages/docs/src/cli/config.ts @@ -367,6 +367,7 @@ export function loadProjectEnv(rootDir: string): Record { export async function loadDocsConfigModule( rootDir: string, explicitPath?: string, + options: { silent?: boolean } = {}, ): Promise<{ path: string; config: DocsConfig } | null> { const configPath = resolveDocsConfigPath(rootDir, explicitPath); @@ -384,7 +385,7 @@ export async function loadDocsConfigModule( return { path: configPath, config }; } catch (error) { - if (process.env.NODE_ENV !== "test") { + if (!options.silent && process.env.NODE_ENV !== "test") { const message = error instanceof Error ? error.message : String(error); console.warn( `[docs] Could not evaluate ${configPath} as a module; falling back to static parsing. ${message}`, diff --git a/packages/docs/src/code-blocks.test.ts b/packages/docs/src/code-blocks.test.ts index 93960fd2..687e2c23 100644 --- a/packages/docs/src/code-blocks.test.ts +++ b/packages/docs/src/code-blocks.test.ts @@ -1,14 +1,32 @@ import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { extractCodeBlocksFromMarkdown, resolveDocsCodeBlocksValidateConfig, validateCodeBlocks, } from "./code-blocks.js"; +const sandboxMock = vi.hoisted(() => ({ + create: vi.fn(), +})); + +vi.mock("@vercel/sandbox", () => ({ + Sandbox: { + create: sandboxMock.create, + }, +})); + describe("code block validation", () => { + afterEach(() => { + sandboxMock.create.mockReset(); + vi.unstubAllGlobals(); + delete process.env.VERCEL_TOKEN; + delete process.env.VERCEL_PROJECT_ID; + delete process.env.VERCEL_TEAM_ID; + }); + it("resolves disabled and enabled validate config", () => { expect(resolveDocsCodeBlocksValidateConfig().enabled).toBe(false); @@ -36,12 +54,38 @@ describe("code block validation", () => { expect(config.runner).toMatchObject({ provider: "vercel-sandbox", tokenEnv: "VERCEL_TOKEN", + projectIdEnv: "VERCEL_PROJECT_ID", + teamIdEnv: "VERCEL_TEAM_ID", + projectJson: ".vercel/project.json", }); expect(config.env).toEqual({ OPENAI_API_KEY: "OPENAI_TEST_API_KEY", }); }); + it("does not execute code blocks unless they are marked runnable", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-not-runnable-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + ['```js title="example.js"', "throw new Error('should not run')", "```"].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig(true), + }); + + expect(report.summary).toMatchObject({ + total: 1, + pass: 0, + skip: 1, + fail: 0, + }); + expect(report.results[0]?.reason).toBe("code block is not marked runnable"); + }); + it("extracts code fence metadata used by agents and validators", () => { const blocks = extractCodeBlocksFromMarkdown({ filePath: "/repo/docs/page.mdx", @@ -169,4 +213,66 @@ describe("code block validation", () => { }); expect(report.results[0]?.reason).toBe("missing env: OPENAI_API_KEY"); }); + + it("auto-discovers Vercel Sandbox project credentials from the token", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-vercel-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + ['```js title="sandbox.js" runnable', 'console.log("sandbox")', "```"].join("\n"), + "utf-8", + ); + + process.env.VERCEL_TOKEN = "test-token"; + const runCommand = vi.fn(async () => ({ + exitCode: 0, + stdout: async () => "sandbox\n", + stderr: async () => "", + })); + sandboxMock.create.mockImplementation(async () => ({ + writeFiles: vi.fn(async () => {}), + runCommand, + stop: vi.fn(async () => {}), + })); + const fetchMock = vi.fn(async () => { + return new Response( + JSON.stringify({ + projects: [{ id: "prj_test", accountId: "team_test" }], + }), + { status: 200 }, + ); + }); + vi.stubGlobal("fetch", fetchMock); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig({ + runner: { + provider: "vercel-sandbox", + }, + }), + }); + + expect(fetchMock).toHaveBeenCalledWith( + "https://api.vercel.com/v9/projects?limit=1", + expect.objectContaining({ + headers: { + Authorization: "Bearer test-token", + }, + }), + ); + expect(sandboxMock.create).toHaveBeenCalledWith( + expect.objectContaining({ + token: "test-token", + projectId: "prj_test", + teamId: "team_test", + }), + ); + expect(report.summary).toMatchObject({ + pass: 1, + skip: 0, + fail: 0, + }); + expect(report.results[0]?.stdout).toBe("sandbox\n"); + }); }); diff --git a/packages/docs/src/code-blocks.ts b/packages/docs/src/code-blocks.ts index d992549e..f9de4555 100644 --- a/packages/docs/src/code-blocks.ts +++ b/packages/docs/src/code-blocks.ts @@ -59,6 +59,9 @@ export interface DocsCodeBlocksResolvedPlannerConfig { export interface DocsCodeBlocksResolvedRunnerConfig { provider: DocsCodeBlocksRunnerProvider; tokenEnv: string; + projectIdEnv: string; + teamIdEnv: string; + projectJson: string | false; runtime: "node24" | "node22" | "python3.13"; timeoutMs: number; } @@ -129,6 +132,13 @@ interface LoadedValidationEnv { missing: string[]; } +interface VercelSandboxCredentials { + token?: string; + projectId?: string; + teamId?: string; + missing: string[]; +} + export function resolveDocsCodeBlocksValidateConfig( input?: boolean | DocsCodeBlocksValidateConfig, ): DocsCodeBlocksResolvedValidateConfig { @@ -139,6 +149,9 @@ export function resolveDocsCodeBlocksValidateConfig( runner: { provider: "local", tokenEnv: "VERCEL_TOKEN", + projectIdEnv: "VERCEL_PROJECT_ID", + teamIdEnv: "VERCEL_TEAM_ID", + projectJson: ".vercel/project.json", runtime: "node24", timeoutMs: DEFAULT_COMMAND_TIMEOUT_MS, }, @@ -309,7 +322,7 @@ export async function validateCodeBlockPlans(input: { const runResults = input.config.runner.provider === "vercel-sandbox" - ? await runPlansInVercelSandbox(plansToRun, input.config, validationEnv.env) + ? await runPlansInVercelSandbox(plansToRun, input.rootDir, input.config, validationEnv.env) : await runPlansLocally(plansToRun, input.config, validationEnv.env); return [...skippedOrFailed, ...runResults].sort((a, b) => a.id.localeCompare(b.id)); @@ -332,7 +345,7 @@ export async function validateCodeBlocks(input: { id: plan.id, target: plan.target, plan, - status: "PLAN" as const, + status: plan.action === "skip" ? ("SKIP" as const) : ("PLAN" as const), reason: plan.reason ?? (plan.action === "skip" ? "planned skip" : "planned"), })) : await validateCodeBlockPlans({ @@ -377,6 +390,9 @@ function normalizeRunnerConfig( return { provider: config.provider ?? "local", tokenEnv: config.tokenEnv ?? "VERCEL_TOKEN", + projectIdEnv: config.projectIdEnv ?? "VERCEL_PROJECT_ID", + teamIdEnv: config.teamIdEnv ?? "VERCEL_TEAM_ID", + projectJson: config.projectJson === undefined ? ".vercel/project.json" : config.projectJson, runtime: config.runtime ?? "node24", timeoutMs: config.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS, }; @@ -395,6 +411,15 @@ function buildMetadataExecutionPlan( } if (!language) { + if (!target.runnable) { + return skipPlan( + target, + "unknown", + requiredEnv, + "code block is not marked runnable", + config.planner.provider, + ); + } if (looksLikeShellCommand(target.code)) { return shellPlan(target, "shell", requiredEnv, config.planner.provider); } @@ -407,6 +432,16 @@ function buildMetadataExecutionPlan( ); } + if (!target.runnable) { + return skipPlan( + target, + template, + requiredEnv, + "code block is not marked runnable", + config.planner.provider, + ); + } + if (isShellLanguage(language)) return shellPlan(target, template, requiredEnv, config.planner.provider); if (language === "json") @@ -792,17 +827,15 @@ async function runPlansLocally( async function runPlansInVercelSandbox( plans: DocsCodeBlockExecutionPlan[], + rootDir: string, config: DocsCodeBlocksResolvedValidateConfig, env: Record, ): Promise { - const token = process.env[config.runner.tokenEnv]; - if (!token) { - return plans.map((plan) => skippedResult(plan, `missing ${config.runner.tokenEnv}`)); + const credentials = await resolveVercelSandboxCredentials(rootDir, config); + if (credentials.missing.length > 0) { + return plans.map((plan) => skippedResult(plan, `missing ${credentials.missing.join(", ")}`)); } - const previousToken = process.env.VERCEL_TOKEN; - process.env.VERCEL_TOKEN = token; - try { const { Sandbox } = (await import("@vercel/sandbox")) as unknown as { Sandbox: { @@ -810,6 +843,9 @@ async function runPlansInVercelSandbox( runtime?: string; timeout?: number; env?: Record; + token?: string; + projectId?: string; + teamId?: string; }): Promise<{ writeFiles(files: Array<{ path: string; content: Buffer }>): Promise; runCommand(input: { @@ -834,6 +870,9 @@ async function runPlansInVercelSandbox( config.runner.timeoutMs, ), env, + token: credentials.token, + projectId: credentials.projectId, + teamId: credentials.teamId, }); try { @@ -876,12 +915,84 @@ async function runPlansInVercelSandbox( } catch (error) { const message = error instanceof Error ? error.message : String(error); return plans.map((plan) => skippedResult(plan, `vercel-sandbox unavailable: ${message}`)); - } finally { - if (previousToken === undefined) { - delete process.env.VERCEL_TOKEN; - } else { - process.env.VERCEL_TOKEN = previousToken; - } + } +} + +async function resolveVercelSandboxCredentials( + rootDir: string, + config: DocsCodeBlocksResolvedValidateConfig, +): Promise { + const projectJson = readVercelProjectJson(rootDir, config.runner.projectJson); + const token = process.env[config.runner.tokenEnv]; + + if (!token && process.env.VERCEL_OIDC_TOKEN) { + return { missing: [] }; + } + + if (!token) { + return { missing: [config.runner.tokenEnv] }; + } + + const envProjectId = process.env[config.runner.projectIdEnv]; + const envTeamId = process.env[config.runner.teamIdEnv]; + const needsDiscovery = + !(envProjectId && envTeamId) && !(projectJson?.projectId && projectJson?.orgId); + const discoveredProject = needsDiscovery ? await discoverVercelSandboxProject(token) : undefined; + const projectId = envProjectId ?? projectJson?.projectId ?? discoveredProject?.projectId; + const teamId = envTeamId ?? projectJson?.orgId ?? discoveredProject?.teamId; + const missing = [ + projectId ? undefined : "Vercel project id", + teamId ? undefined : "Vercel team id", + ].filter((value): value is string => Boolean(value)); + + return { token, projectId, teamId, missing }; +} + +async function discoverVercelSandboxProject( + token: string, +): Promise<{ projectId?: string; teamId?: string } | undefined> { + try { + const response = await fetch("https://api.vercel.com/v9/projects?limit=1", { + headers: { + Authorization: `Bearer ${token}`, + }, + }); + if (!response.ok) return undefined; + + const payload = (await response.json()) as { + projects?: Array<{ id?: unknown; accountId?: unknown }>; + }; + const project = payload.projects?.[0]; + if (!project) return undefined; + + return { + projectId: typeof project.id === "string" ? project.id : undefined, + teamId: typeof project.accountId === "string" ? project.accountId : undefined, + }; + } catch { + return undefined; + } +} + +function readVercelProjectJson( + rootDir: string, + projectJson: string | false, +): { projectId?: string; orgId?: string } | undefined { + if (projectJson === false) return undefined; + const fullPath = path.resolve(rootDir, projectJson); + if (!existsSync(fullPath)) return undefined; + + try { + const parsed = JSON.parse(readFileSync(fullPath, "utf-8")) as { + projectId?: unknown; + orgId?: unknown; + }; + return { + projectId: typeof parsed.projectId === "string" ? parsed.projectId : undefined, + orgId: typeof parsed.orgId === "string" ? parsed.orgId : undefined, + }; + } catch { + return undefined; } } diff --git a/packages/docs/src/mcp.ts b/packages/docs/src/mcp.ts index 4dcbe355..8f7ec589 100644 --- a/packages/docs/src/mcp.ts +++ b/packages/docs/src/mcp.ts @@ -607,7 +607,8 @@ const DOCS_CONFIG_SCHEMA_OPTIONS: DocsMcpConfigSchemaOption[] = [ name: "runner", type: '"local" | "vercel-sandbox" | "cloud" | DocsCodeBlocksRunnerConfig', default: "local", - description: "Runner used to execute planned snippets.", + description: + "Runner used to execute planned snippets. The Vercel Sandbox runner can work from `VERCEL_TOKEN` alone by auto-discovering an accessible project.", }, { path: "codeBlocks.validate.env", diff --git a/packages/docs/src/types.ts b/packages/docs/src/types.ts index c8859d0a..22ad7cc8 100644 --- a/packages/docs/src/types.ts +++ b/packages/docs/src/types.ts @@ -2368,6 +2368,19 @@ export interface DocsCodeBlocksRunnerConfig { provider?: DocsCodeBlocksRunnerProvider; /** Environment variable containing the Vercel token for `provider: "vercel-sandbox"`. */ tokenEnv?: string; + /** Advanced override for the Vercel project id env var used by `provider: "vercel-sandbox"`. */ + projectIdEnv?: string; + /** Advanced override for the Vercel team/org id env var used by `provider: "vercel-sandbox"`. */ + teamIdEnv?: string; + /** + * Path to a Vercel project metadata file. When enabled, the runner reads + * `projectId` and `orgId` from `.vercel/project.json`. If those are not + * available, the runner can auto-discover an accessible project from + * `VERCEL_TOKEN`. + * + * @default ".vercel/project.json" + */ + projectJson?: string | false; /** Vercel Sandbox runtime. */ runtime?: "node24" | "node22" | "python3.13"; /** Per-command timeout in milliseconds. */ diff --git a/website/app/docs/configuration/page.mdx b/website/app/docs/configuration/page.mdx index 4cf52f33..b84ee089 100644 --- a/website/app/docs/configuration/page.mdx +++ b/website/app/docs/configuration/page.mdx @@ -190,6 +190,31 @@ const apiKey = process.env.OPENAI_API_KEY; ``` ```` +Small runnable blocks make the check easy to dogfood: + +```json title="agent-context.json" runnable +{ + "framework": "nextjs", + "packageManager": "pnpm", + "agentReady": true +} +``` + +```js title="codeblocks-smoke.js" runnable +const metadata = { framework: "nextjs", runnable: true }; + +if (!metadata.runnable) { + throw new Error("Expected runnable metadata"); +} + +console.log("metadata ok"); +``` + +```bash title="codeblocks-smoke.sh" runnable +test "docs" = "docs" +echo "shell ok" +``` + Run the validator locally: ```bash @@ -199,7 +224,9 @@ pnpm exec docs codeblocks validate The planner never writes secrets into docs. The `env` map means the snippet can keep using `process.env.OPENAI_API_KEY`, while validation injects that runtime variable from -`OPENAI_TEST_API_KEY`. +`OPENAI_TEST_API_KEY`. For Vercel Sandbox, `VERCEL_TOKEN` is enough in most projects; the runner +uses `.vercel/project.json` when present and can auto-discover an accessible Vercel project from +the token. ## Public docs path diff --git a/website/docs.config.tsx b/website/docs.config.tsx index 6287c5cf..e99ff2f8 100644 --- a/website/docs.config.tsx +++ b/website/docs.config.tsx @@ -68,6 +68,19 @@ export default defineDocs({ observability: { console: "info", }, + codeBlocks: { + validate: { + planner: { + provider: "metadata", + }, + runner: { + provider: "vercel-sandbox", + tokenEnv: "VERCEL_TOKEN", + }, + missingEnv: "skip", + unsupportedLanguage: "skip", + }, + }, theme: pixelBorder({ ui: { layout: { toc: { enabled: true, depth: 3, style: "directional" }, sidebarWidth: 320 }, From 66d8e185c1603c7ac25626632ccfa3de0ff134a6 Mon Sep 17 00:00:00 2001 From: Kinfe123 Date: Mon, 25 May 2026 01:12:42 +0300 Subject: [PATCH 3/4] chore: review --- packages/docs/src/cli/codeblocks.test.ts | 80 ++++++++++++++++++++++++ packages/docs/src/cli/codeblocks.ts | 14 +++-- packages/docs/src/code-blocks.test.ts | 80 ++++++++++++++++++++++++ packages/docs/src/code-blocks.ts | 12 +--- 4 files changed, 170 insertions(+), 16 deletions(-) create mode 100644 packages/docs/src/cli/codeblocks.test.ts diff --git a/packages/docs/src/cli/codeblocks.test.ts b/packages/docs/src/cli/codeblocks.test.ts new file mode 100644 index 00000000..c65df316 --- /dev/null +++ b/packages/docs/src/cli/codeblocks.test.ts @@ -0,0 +1,80 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { runCodeBlocksValidate } from "./codeblocks.js"; + +describe("codeblocks validate cli", () => { + let tmpDir: string; + let previousCwd: string; + let logs: string[]; + + beforeEach(() => { + tmpDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-cli-")); + previousCwd = process.cwd(); + logs = []; + process.chdir(tmpDir); + vi.spyOn(console, "log").mockImplementation((value?: unknown) => { + logs.push(String(value)); + }); + }); + + afterEach(() => { + process.chdir(previousCwd); + rmSync(tmpDir, { recursive: true, force: true }); + vi.restoreAllMocks(); + process.exitCode = undefined; + }); + + it("redacts planner api keys from disabled JSON output", async () => { + writeFileSync( + path.join(tmpDir, "docs.config.ts"), + [ + "export default {", + ' entry: "docs",', + " codeBlocks: {", + " validate: {", + " enabled: false,", + " planner: { provider: 'openai', apiKey: 'secret-key' },", + " },", + " },", + "};", + ].join("\n"), + "utf-8", + ); + + await runCodeBlocksValidate({ json: true }); + + const output = logs.join("\n"); + expect(output).not.toContain("secret-key"); + expect(JSON.parse(output).config.planner.apiKey).toBe("[REDACTED]"); + }); + + it("labels --run output as validation when config mode is plan", async () => { + mkdirSync(path.join(tmpDir, "docs")); + writeFileSync( + path.join(tmpDir, "docs.config.ts"), + [ + "export default {", + ' entry: "docs",', + " codeBlocks: {", + " validate: {", + ' mode: "plan",', + " },", + " },", + "};", + ].join("\n"), + "utf-8", + ); + writeFileSync( + path.join(tmpDir, "docs", "page.mdx"), + ['```js title="hello.js" runnable', 'console.log("hello")', "```"].join("\n"), + "utf-8", + ); + + await runCodeBlocksValidate({ run: true }); + + expect(logs[0]).toBe("Code block validation"); + expect(logs.join("\n")).toContain("1 pass"); + }); +}); diff --git a/packages/docs/src/cli/codeblocks.ts b/packages/docs/src/cli/codeblocks.ts index ff995f16..f861bfc2 100644 --- a/packages/docs/src/cli/codeblocks.ts +++ b/packages/docs/src/cli/codeblocks.ts @@ -124,7 +124,7 @@ export async function runCodeBlocksValidate(options: CodeBlocksValidateOptions = results: [], }; if (options.json) { - console.log(JSON.stringify(disabledReport, null, 2)); + console.log(JSON.stringify(redactReport(disabledReport), null, 2)); } else { console.log( pc.yellow( @@ -135,20 +135,22 @@ export async function runCodeBlocksValidate(options: CodeBlocksValidateOptions = return disabledReport; } + const effectiveConfig = { + ...config, + mode: options.run ? ("report" as const) : config.mode, + }; + const planOnly = options.plan === true || effectiveConfig.mode === "plan"; const report = await validateCodeBlocks({ rootDir, contentDir, - config: { - ...config, - mode: options.run ? "report" : config.mode, - }, + config: effectiveConfig, planOnly: options.plan, }); if (options.json) { console.log(JSON.stringify(redactReport(report), null, 2)); } else { - printCodeBlocksReport(report, options.plan === true || config.mode === "plan"); + printCodeBlocksReport(report, planOnly); } if (!options.plan && report.summary.fail > 0) { diff --git a/packages/docs/src/code-blocks.test.ts b/packages/docs/src/code-blocks.test.ts index 687e2c23..8d757679 100644 --- a/packages/docs/src/code-blocks.test.ts +++ b/packages/docs/src/code-blocks.test.ts @@ -214,6 +214,86 @@ describe("code block validation", () => { expect(report.results[0]?.reason).toBe("missing env: OPENAI_API_KEY"); }); + it("does not convert non-env skips to failures when missingEnv is error", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-non-env-skip-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + ['```mermaid title="flow.mmd" runnable', "graph TD; A-->B;", "```"].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig({ + missingEnv: "error", + }), + }); + + expect(report.summary).toMatchObject({ + pass: 0, + skip: 1, + fail: 0, + }); + expect(report.results[0]).toMatchObject({ + status: "SKIP", + reason: "unsupported language: mermaid", + }); + }); + + it("keeps missing env failures when missingEnv is error", async () => { + const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-env-error-")); + writeFileSync( + path.join(rootDir, "page.mdx"), + [ + '```js title="needs-env.js" env="OPENAI_API_KEY" runnable', + "console.log(Boolean(process.env.OPENAI_API_KEY))", + "```", + ].join("\n"), + "utf-8", + ); + + const report = await validateCodeBlocks({ + rootDir, + contentDir: ".", + config: resolveDocsCodeBlocksValidateConfig({ + env: { + OPENAI_API_KEY: "OPENAI_TEST_API_KEY", + }, + missingEnv: "error", + }), + }); + + expect(report.summary).toMatchObject({ + pass: 0, + skip: 0, + fail: 1, + }); + expect(report.results[0]).toMatchObject({ + status: "FAIL", + reason: "missing env: OPENAI_API_KEY", + }); + }); + + it("accepts closing fences longer than the opening fence", () => { + const blocks = extractCodeBlocksFromMarkdown({ + filePath: "/repo/docs/page.mdx", + relativePath: "docs/page.mdx", + source: [ + '```js title="longer-close.js" runnable', + 'console.log("ok")', + "````", + "", + '~~~json title="tilde.json" runnable', + '{"ok":true}', + "~~~~", + ].join("\n"), + }); + + expect(blocks).toHaveLength(2); + expect(blocks.map((block) => block.code)).toEqual(['console.log("ok")', '{"ok":true}']); + }); + it("auto-discovers Vercel Sandbox project credentials from the token", async () => { const rootDir = mkdtempSync(path.join(tmpdir(), "docs-codeblocks-vercel-")); writeFileSync( diff --git a/packages/docs/src/code-blocks.ts b/packages/docs/src/code-blocks.ts index f9de4555..7b559755 100644 --- a/packages/docs/src/code-blocks.ts +++ b/packages/docs/src/code-blocks.ts @@ -310,14 +310,7 @@ export async function validateCodeBlockPlans(input: { const plansToRun = runnable.map((result) => result.plan); if (plansToRun.length === 0) { - return skippedOrFailed.map((result) => - result.reason - ? { - ...result, - status: input.config.missingEnv === "error" ? "FAIL" : result.status, - } - : result, - ); + return skippedOrFailed; } const runResults = @@ -1100,8 +1093,7 @@ function normalizeLanguage(language?: string): string | undefined { } function isClosingFence(trimmedLine: string, marker: string): boolean { - if (!trimmedLine.startsWith(marker)) return false; - return trimmedLine.slice(marker.length).trim().length === 0; + return new RegExp(`^${marker[0]}{${marker.length},}[ \\t]*$`).test(trimmedLine); } function isShellLanguage(language: string): boolean { From 7d95bceaf6956a2ffe4c009339d92fff6029b660 Mon Sep 17 00:00:00 2001 From: Kinfe123 Date: Mon, 25 May 2026 01:23:44 +0300 Subject: [PATCH 4/4] chore: review --- packages/docs/src/cli/codeblocks.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/docs/src/cli/codeblocks.test.ts b/packages/docs/src/cli/codeblocks.test.ts index c65df316..2df1d9ff 100644 --- a/packages/docs/src/cli/codeblocks.test.ts +++ b/packages/docs/src/cli/codeblocks.test.ts @@ -4,6 +4,10 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { runCodeBlocksValidate } from "./codeblocks.js"; +function stripAnsi(value: string): string { + return value.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, ""); +} + describe("codeblocks validate cli", () => { let tmpDir: string; let previousCwd: string; @@ -74,7 +78,7 @@ describe("codeblocks validate cli", () => { await runCodeBlocksValidate({ run: true }); - expect(logs[0]).toBe("Code block validation"); + expect(stripAnsi(logs[0] ?? "")).toBe("Code block validation"); expect(logs.join("\n")).toContain("1 pass"); }); });