diff --git a/docs/tool-api.md b/docs/tool-api.md index b86af31..6d11040 100644 --- a/docs/tool-api.md +++ b/docs/tool-api.md @@ -29,3 +29,47 @@ Failed tools return: Built-in error codes should remain stable. Third-party tools should namespace custom codes to avoid collisions. + +## `web_fetch` + +`web_fetch` reads text from a caller-provided HTTP or HTTPS URL. It only works +when `ToolContext.permissions.network` is granted. It is not a search, page +discovery, crawler, recursive fetch, or sitemap tool; callers must already know +the exact URL. + +Input: + +```ts +{ + reason: string; + url: string; + maxLength?: number; // default 20_000, max 200_000 + timeoutMs?: number; // default 15_000, max 60_000 +} +``` + +Successful responses include HTTP metadata and capped text: + +```ts +{ + requestedUrl: string; + finalUrl: string; + status: number; + statusText: string; + contentType: string | null; + contentLength: number | null; + text: string; + truncated: boolean; + maxLength: number; +} +``` + +`requestedUrl` is the normalized input URL. `finalUrl` records the response URL +after redirects. Long text is truncated to `maxLength`, and `truncated` reports +whether truncation happened. + +The tool rejects non-HTTP(S) URLs, missing network permission, HTTP 4xx/5xx +responses, timeout/abort failures, and non-text responses. Text responses +include `text/*`, JSON, XML, XHTML, JavaScript, and SVG content types. Binary +responses such as images, archives, audio, and video are rejected before their +bodies are read so binary data is not inserted into agent context. diff --git a/src/tool/index.ts b/src/tool/index.ts index eec6f50..4547833 100644 --- a/src/tool/index.ts +++ b/src/tool/index.ts @@ -22,7 +22,13 @@ export {ToolError, toToolError} from "./tool-error.js"; export {ToolRegistry} from "./tool-registry.js"; export {errorToolResult, okToolResult} from "./tool-result.js"; export {ToolRunner} from "./tool-runner.js"; -export {webFetchTool} from "./web/index.js"; +export { + DEFAULT_WEB_FETCH_MAX_LENGTH, + DEFAULT_WEB_FETCH_TIMEOUT_MS, + MAX_WEB_FETCH_MAX_LENGTH, + MAX_WEB_FETCH_TIMEOUT_MS, + webFetchTool, +} from "./web/index.js"; export type {ToolErrorCode, ToolErrorOptions} from "./tool-error.js"; export type {LLMToolParametersSchema} from "./tool-parameters.js"; export type { diff --git a/src/tool/web/index.ts b/src/tool/web/index.ts index 12223b8..95ceabe 100644 --- a/src/tool/web/index.ts +++ b/src/tool/web/index.ts @@ -1 +1,7 @@ -export {webFetchTool} from "./web-fetch.tool.js"; +export { + DEFAULT_WEB_FETCH_MAX_LENGTH, + DEFAULT_WEB_FETCH_TIMEOUT_MS, + MAX_WEB_FETCH_MAX_LENGTH, + MAX_WEB_FETCH_TIMEOUT_MS, + webFetchTool, +} from "./web-fetch.tool.js"; diff --git a/src/tool/web/web-fetch.tool.ts b/src/tool/web/web-fetch.tool.ts index 2909312..326499b 100644 --- a/src/tool/web/web-fetch.tool.ts +++ b/src/tool/web/web-fetch.tool.ts @@ -4,6 +4,30 @@ import {ToolError} from "../tool-error.js"; import {okToolResult} from "../tool-result.js"; import type {Tool, ToolContext} from "../types.js"; +/** Successful data returned by the built-in web_fetch tool. */ +export type WebFetchResultData = { + requestedUrl: string; + finalUrl: string; + status: number; + statusText: string; + contentType: string | null; + contentLength: number | null; + text: string; + truncated: boolean; + maxLength: number; +}; + +/** Structured failure details returned by the built-in web_fetch tool. */ +export type WebFetchFailureDetails = Omit< + WebFetchResultData, + "text" | "truncated" | "maxLength" +>; + +export const DEFAULT_WEB_FETCH_MAX_LENGTH = 20_000; +export const MAX_WEB_FETCH_MAX_LENGTH = 200_000; +export const DEFAULT_WEB_FETCH_TIMEOUT_MS = 15_000; +export const MAX_WEB_FETCH_TIMEOUT_MS = 60_000; + const webFetchParameters = z.object({ reason: z .string() @@ -18,47 +42,78 @@ const webFetchParameters = z.object({ ), maxLength: z .number() + .int() .positive() + .max(MAX_WEB_FETCH_MAX_LENGTH) .optional() .describe("Maximum number of characters of response text to return."), + timeoutMs: z + .number() + .int() + .positive() + .max(MAX_WEB_FETCH_TIMEOUT_MS) + .optional() + .describe("Maximum time in milliseconds to wait for the HTTP request."), }); /** Tool that fetches text from a known HTTP(S) URL when network permission is granted. */ -export const webFetchTool: Tool = - { - definition: { - name: "web_fetch", - description: - "Fetch text from a specific known URL. Use this when you already have the exact webpage URL and need its page text. This tool does not discover unknown pages. Returns the final URL string and capped response text. This is a network tool and does not access workspace files.", - parameters: webFetchParameters, - }, - async execute(input, context) { - requireNetworkPermission(context, "web_fetch"); +export const webFetchTool: Tool = { + definition: { + name: "web_fetch", + description: + "Read text from a specific known HTTP or HTTPS URL. Use this only when the caller already has the exact URL. This tool does not search, discover pages, crawl, or recursively fetch links. Returns HTTP metadata and capped response text. Requires network permission.", + parameters: webFetchParameters, + }, + async execute(input, context) { + requireNetworkPermission(context, "web_fetch"); + + const requestedUrl = parseHttpUrl(input.url, "web_fetch"); + const maxLength = input.maxLength ?? DEFAULT_WEB_FETCH_MAX_LENGTH; + const timeoutMs = input.timeoutMs ?? DEFAULT_WEB_FETCH_TIMEOUT_MS; + const control = createFetchControl(context.signal, timeoutMs); - const url = parseHttpUrl(input.url, "web_fetch"); - const maxLength = - typeof input.maxLength === "number" && input.maxLength > 0 - ? Math.floor(input.maxLength) - : 20_000; - const response = await fetch(url, {signal: context.signal}); + try { + const response = await fetch(requestedUrl, {signal: control.signal}); + const details = getResponseDetails(response, requestedUrl); if (!response.ok) { throw new ToolError({ code: "TOOL_EXECUTION_FAILED", message: `web_fetch failed with HTTP ${response.status}.`, toolName: "web_fetch", - details: {status: response.status, url}, + details, }); } - const text = await response.text(); + if (!isTextContentType(details.contentType)) { + throw new ToolError({ + code: "TOOL_EXECUTION_FAILED", + message: contentTypeErrorMessage(details.contentType), + toolName: "web_fetch", + details, + }); + } + + const {text, truncated} = await readResponseTextWithLimit(response, maxLength); return okToolResult("Fetched webpage text.", { - url, - text: text.slice(0, maxLength), + ...details, + text, + truncated, + maxLength, }); - }, - }; + } catch (error) { + throw normalizeFetchError(error, { + requestedUrl, + timedOut: control.timedOut, + contextSignal: context.signal, + timeoutMs, + }); + } finally { + control.cleanup(); + } + }, +}; /** Ensures the current run granted network access before issuing a fetch. */ function requireNetworkPermission(context: ToolContext, toolName: string): void { @@ -90,3 +145,180 @@ function parseHttpUrl(url: string, toolName: string): string { }); } } + +function getResponseDetails( + response: Response, + requestedUrl: string, +): WebFetchFailureDetails { + const contentType = response.headers.get("content-type"); + + return { + requestedUrl, + finalUrl: response.url || requestedUrl, + status: response.status, + statusText: response.statusText, + contentType, + contentLength: parseContentLength(response.headers.get("content-length")), + }; +} + +function parseContentLength(value: string | null): number | null { + if (value === null) { + return null; + } + + const parsed = Number.parseInt(value, 10); + + return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; +} + +function isTextContentType(contentType: string | null): boolean { + if (contentType === null) { + return false; + } + + const mediaType = contentType.split(";", 1)[0]?.trim().toLowerCase(); + + return ( + mediaType.startsWith("text/") || + mediaType === "application/json" || + mediaType === "application/xml" || + mediaType === "application/xhtml+xml" || + mediaType === "application/javascript" || + mediaType === "image/svg+xml" + ); +} + +function contentTypeErrorMessage(contentType: string | null): string { + return contentType === null + ? "web_fetch refused to read a response without a text content type." + : `web_fetch refused to read non-text response content type "${contentType}".`; +} + +function createFetchControl( + contextSignal: AbortSignal | undefined, + timeoutMs: number, +): { + signal: AbortSignal; + timedOut: () => boolean; + cleanup: () => void; +} { + const controller = new AbortController(); + let didTimeOut = false; + + const abortFromContext = (): void => controller.abort(); + + if (contextSignal?.aborted) { + controller.abort(); + } else { + contextSignal?.addEventListener("abort", abortFromContext, {once: true}); + } + + const timeout = setTimeout(() => { + didTimeOut = true; + controller.abort(); + }, timeoutMs); + + return { + signal: controller.signal, + timedOut: () => didTimeOut, + cleanup: () => { + if (timeout) { + clearTimeout(timeout); + } + + contextSignal?.removeEventListener("abort", abortFromContext); + }, + }; +} + +async function readResponseTextWithLimit( + response: Response, + maxLength: number, +): Promise<{text: string; truncated: boolean}> { + const reader = response.body?.getReader(); + + if (!reader) { + // Compatibility path for fetch mocks or runtimes without stream readers. + const text = await response.text(); + + return { + text: text.slice(0, maxLength), + truncated: text.length > maxLength, + }; + } + + const decoder = new TextDecoder(); + let text = ""; + let truncated = false; + + try { + while (true) { + const {done, value} = await reader.read(); + + if (done) { + text += decoder.decode(); + break; + } + + text += decoder.decode(value, {stream: true}); + + if (text.length > maxLength) { + text = text.slice(0, maxLength); + truncated = true; + await reader.cancel(); + break; + } + } + } finally { + reader.releaseLock(); + } + + return {text, truncated}; +} + +function normalizeFetchError( + error: unknown, + input: { + requestedUrl: string; + timedOut: () => boolean; + contextSignal?: AbortSignal; + timeoutMs: number; + }, +): unknown { + if (error instanceof ToolError) { + return error; + } + + if (isAbortLikeError(error) || input.timedOut() || input.contextSignal?.aborted) { + if (input.timedOut()) { + return new ToolError({ + code: "TOOL_TIMEOUT", + message: `web_fetch timed out after ${input.timeoutMs}ms.`, + toolName: "web_fetch", + details: {requestedUrl: input.requestedUrl, timeoutMs: input.timeoutMs}, + cause: error, + }); + } + + return new ToolError({ + code: "TOOL_ABORTED", + message: "web_fetch was aborted.", + toolName: "web_fetch", + details: {requestedUrl: input.requestedUrl}, + cause: error, + }); + } + + return new ToolError({ + code: "TOOL_EXECUTION_FAILED", + message: "web_fetch network request failed.", + toolName: "web_fetch", + details: {requestedUrl: input.requestedUrl}, + cause: error, + }); +} + +function isAbortLikeError(error: unknown): boolean { + return error instanceof Error && error.name === "AbortError"; +} diff --git a/tests/tool/web-fetch.tool.test.ts b/tests/tool/web-fetch.tool.test.ts new file mode 100644 index 0000000..52f0d88 --- /dev/null +++ b/tests/tool/web-fetch.tool.test.ts @@ -0,0 +1,356 @@ +import {afterEach, describe, expect, it, vi} from "vitest"; + +import { + DEFAULT_WEB_FETCH_MAX_LENGTH, + DEFAULT_WEB_FETCH_TIMEOUT_MS, + ToolRegistry, + ToolRunner, + webFetchTool, +} from "../../src/tool/index.js"; + +const originalFetch = globalThis.fetch; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + + if (originalFetch) { + vi.stubGlobal("fetch", originalFetch); + } +}); + +describe("webFetchTool", () => { + it("returns text and HTTP metadata for a 2xx text response", async () => { + const fetchMock = stubFetch( + new Response("hello", { + status: 200, + statusText: "OK", + headers: { + "content-length": "5", + "content-type": "text/plain; charset=utf-8", + }, + }), + ); + + const result = await runWebFetch({ + reason: "read known page", + url: "https://example.com/path", + }); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + ok: true, + data: { + requestedUrl: "https://example.com/path", + finalUrl: "https://example.com/path", + status: 200, + statusText: "OK", + contentType: "text/plain; charset=utf-8", + contentLength: 5, + text: "hello", + truncated: false, + maxLength: DEFAULT_WEB_FETCH_MAX_LENGTH, + }, + }); + }); + + it("records the final URL after redirects", async () => { + const response = new Response("redirected", { + status: 200, + headers: {"content-type": "text/plain"}, + }); + Object.defineProperty(response, "url", { + value: "https://example.com/final", + }); + stubFetch(response); + + const result = await runWebFetch({ + reason: "read redirected page", + url: "https://example.com/start", + }); + + expect(result).toMatchObject({ + ok: true, + data: { + requestedUrl: "https://example.com/start", + finalUrl: "https://example.com/final", + }, + }); + }); + + it("truncates oversized text at maxLength", async () => { + stubFetch( + new Response("abcdef", { + status: 200, + headers: {"content-type": "application/json"}, + }), + ); + + const result = await runWebFetch({ + reason: "read json", + url: "https://example.com/data.json", + maxLength: 3, + }); + + expect(result).toMatchObject({ + ok: true, + data: { + text: "abc", + truncated: true, + maxLength: 3, + }, + }); + }); + + it("does not mark short text as truncated", async () => { + stubFetch( + new Response("abc", { + status: 200, + headers: {"content-type": "application/xml"}, + }), + ); + + const result = await runWebFetch({ + reason: "read xml", + url: "https://example.com/data.xml", + maxLength: 10, + }); + + expect(result).toMatchObject({ + ok: true, + data: { + text: "abc", + truncated: false, + }, + }); + }); + + it("returns structured failures for 4xx and 5xx responses", async () => { + const response = new Response("not found", { + status: 404, + statusText: "Not Found", + headers: { + "content-length": "9", + "content-type": "text/plain", + }, + }); + Object.defineProperty(response, "url", { + value: "https://example.com/missing", + }); + stubFetch(response); + + const result = await runWebFetch({ + reason: "read missing page", + url: "https://example.com/missing", + }); + + expect(result).toMatchObject({ + ok: false, + code: "TOOL_EXECUTION_FAILED", + message: "web_fetch failed with HTTP 404.", + data: { + status: 404, + statusText: "Not Found", + requestedUrl: "https://example.com/missing", + finalUrl: "https://example.com/missing", + contentType: "text/plain", + contentLength: 9, + }, + }); + }); + + it("rejects non-HTTP URLs before calling fetch", async () => { + const fetchMock = stubFetch( + new Response("never", { + status: 200, + headers: {"content-type": "text/plain"}, + }), + ); + + const result = await runWebFetch({ + reason: "read local file", + url: "file:///tmp/example.txt", + }); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + ok: false, + code: "TOOL_INVALID_INPUT", + }); + }); + + it("rejects when network permission is disabled", async () => { + const fetchMock = stubFetch( + new Response("never", { + status: 200, + headers: {"content-type": "text/plain"}, + }), + ); + + const result = await runWebFetch( + { + reason: "read page", + url: "https://example.com", + }, + {network: false}, + ); + + expect(fetchMock).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + ok: false, + code: "TOOL_PERMISSION_DENIED", + }); + }); + + it("rejects non-text content types before reading the body", async () => { + stubFetch( + new Response("binary bytes", { + status: 200, + statusText: "OK", + headers: { + "content-length": "12", + "content-type": "image/png", + }, + }), + ); + + const result = await runWebFetch({ + reason: "read image", + url: "https://example.com/image.png", + }); + + expect(result).toMatchObject({ + ok: false, + code: "TOOL_EXECUTION_FAILED", + data: { + status: 200, + statusText: "OK", + requestedUrl: "https://example.com/image.png", + finalUrl: "https://example.com/image.png", + contentType: "image/png", + contentLength: 12, + }, + }); + }); + + it("normalizes fetch timeout failures", async () => { + vi.useFakeTimers(); + stubFetchWithAbort(); + + const resultPromise = runWebFetch({ + reason: "read slow page", + url: "https://example.com/slow", + timeoutMs: 10, + }); + + await vi.advanceTimersByTimeAsync(10); + + await expect(resultPromise).resolves.toMatchObject({ + ok: false, + code: "TOOL_TIMEOUT", + data: { + requestedUrl: "https://example.com/slow", + timeoutMs: 10, + }, + }); + }); + + it("normalizes external abort failures", async () => { + const controller = new AbortController(); + stubFetchWithAbort(); + + const resultPromise = runWebFetch( + { + reason: "read aborting page", + url: "https://example.com/abort", + }, + {network: true}, + controller.signal, + ); + + controller.abort(); + + await expect(resultPromise).resolves.toMatchObject({ + ok: false, + code: "TOOL_ABORTED", + }); + }); + + it("uses default maxLength and timeoutMs when omitted", async () => { + const fetchMock = vi.fn(async (_url: URL | RequestInfo, init?: RequestInit) => { + expect(init?.signal).toBeInstanceOf(AbortSignal); + + return new Response("default", { + status: 200, + headers: {"content-type": "text/plain"}, + }); + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await runWebFetch({ + reason: "read defaults", + url: "https://example.com/default", + }); + + expect(result).toMatchObject({ + ok: true, + data: { + text: "default", + maxLength: DEFAULT_WEB_FETCH_MAX_LENGTH, + truncated: false, + }, + }); + expect(DEFAULT_WEB_FETCH_TIMEOUT_MS).toBe(15_000); + }); +}); + +function stubFetch(response: Response) { + const fetchMock = vi.fn(async () => response); + vi.stubGlobal("fetch", fetchMock); + + return fetchMock; +} + +function stubFetchWithAbort() { + const fetchMock = vi.fn((_url: URL | RequestInfo, init?: RequestInit) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + "abort", + () => { + const error = new Error("aborted"); + error.name = "AbortError"; + reject(error); + }, + {once: true}, + ); + }); + }); + vi.stubGlobal("fetch", fetchMock); + + return fetchMock; +} + +async function runWebFetch( + input: { + reason: string; + url: string; + maxLength?: number; + timeoutMs?: number; + }, + permissions: {network?: boolean} = {network: true}, + signal?: AbortSignal, +) { + const registry = new ToolRegistry(); + registry.register(webFetchTool); + + return await new ToolRunner(registry).run( + "web_fetch", + input, + { + workspaceRoot: process.cwd(), + permissions, + }, + {signal, timeoutMs: false}, + ); +}