From ca3af412b069dfd6b833715906f6dba55bcd0c8c Mon Sep 17 00:00:00 2001 From: ribomo <7699050+ribomo@users.noreply.github.com> Date: Thu, 2 Jul 2026 01:46:20 +0000 Subject: [PATCH] Adding function to fetch website --- .env.example | 2 +- README.md | 13 +- plain_agent/tools/registry.py | 9 +- plain_agent/tools/web/__init__.py | 9 ++ plain_agent/tools/web/fetch.py | 101 +++++++++++++ plain_agent/tools/web/providers/__init__.py | 15 ++ .../exa_mcp.py => web/providers/exa.py} | 97 +++++++++---- .../{web_search/tool.py => web/search.py} | 4 +- plain_agent/tools/web_search/__init__.py | 10 -- tests/test_agent.py | 6 +- tests/test_tools.py | 5 +- tests/test_web_search.py | 135 +++++++++++++++++- 12 files changed, 357 insertions(+), 49 deletions(-) create mode 100644 plain_agent/tools/web/__init__.py create mode 100644 plain_agent/tools/web/fetch.py create mode 100644 plain_agent/tools/web/providers/__init__.py rename plain_agent/tools/{web_search/exa_mcp.py => web/providers/exa.py} (59%) rename plain_agent/tools/{web_search/tool.py => web/search.py} (96%) delete mode 100644 plain_agent/tools/web_search/__init__.py diff --git a/.env.example b/.env.example index 6ab22d0..fe3de73 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,7 @@ LLM_TIMEOUT=60 # Optional override for any OpenAI-compatible provider. # LLM_BASE_URL=https://api.openai.com/v1 -# Web search is enabled by default. Every query still requires confirmation. +# Web search and fetch are enabled by default. Every request still requires confirmation. # PLAIN_AGENT_ENABLE_NETWORK=false # Optional Linux sandbox toolchain roots. Every path becomes readable by commands. diff --git a/README.md b/README.md index 0dd00d2..d48a1e3 100644 --- a/README.md +++ b/README.md @@ -76,19 +76,20 @@ Context compaction runs automatically when the estimated conversation history re Set `LLM_COMPACTION_AUTO_MAX_TOKENS` to change that threshold, or set it to `0` to disable automatic compaction. You can also run `/compact` in the terminal to compact manually. -### Web search +### Web access -The Exa-backed `web_search` tool is enabled by default and every query requires explicit approval -before Plain Agent connects to `mcp.exa.ai`. Search results include links and bounded relevant -excerpts. No Exa API key is required. +The Exa-backed `web_search` and `web_fetch` tools are enabled by default. Every query or URL +requires explicit approval before Plain Agent connects to `mcp.exa.ai`. Search returns links and +bounded excerpts; fetch returns bounded Markdown content for one HTTP or HTTPS URL. No Exa API key +is required. -To disable web search: +To disable both web tools: ```bash export PLAIN_AGENT_ENABLE_NETWORK="false" ``` -This setting controls only the built-in search tool. The `run_command` sandbox remains offline. +This setting controls only the built-in web tools. The `run_command` sandbox remains offline. ## Run diff --git a/plain_agent/tools/registry.py b/plain_agent/tools/registry.py index 0b69ac9..aaddd41 100644 --- a/plain_agent/tools/registry.py +++ b/plain_agent/tools/registry.py @@ -11,7 +11,7 @@ from plain_agent.tools.run_command import RunCommandTool from plain_agent.tools.search_text import SearchTextTool from plain_agent.tools.utils import error -from plain_agent.tools.web_search import WebSearchTool +from plain_agent.tools.web import WebFetchTool, WebSearchTool from plain_agent.tools.write_file import WriteFileTool @@ -42,7 +42,12 @@ def __init__( ] self.startup_warnings: list[str] = [] if enable_network: - registered_tools.append(WebSearchTool(self.permission_controller)) + registered_tools.extend( + [ + WebSearchTool(self.permission_controller), + WebFetchTool(self.permission_controller), + ] + ) if enable_commands: discovery = discover_linux_sandbox() if discovery.warning is not None: diff --git a/plain_agent/tools/web/__init__.py b/plain_agent/tools/web/__init__.py new file mode 100644 index 0000000..33a2de5 --- /dev/null +++ b/plain_agent/tools/web/__init__.py @@ -0,0 +1,9 @@ +"""Approved web tools.""" + +from plain_agent.tools.web.fetch import WebFetchTool +from plain_agent.tools.web.search import WebSearchTool + +__all__ = [ + "WebFetchTool", + "WebSearchTool", +] diff --git a/plain_agent/tools/web/fetch.py b/plain_agent/tools/web/fetch.py new file mode 100644 index 0000000..fcac073 --- /dev/null +++ b/plain_agent/tools/web/fetch.py @@ -0,0 +1,101 @@ +"""Model-facing webpage fetch tool.""" + +from pathlib import Path +from urllib.parse import urlsplit + +from plain_agent.tools.base_tool import BaseTool +from plain_agent.tools.permissions.controller import ( + ApprovalDeniedError, + PermissionController, +) +from plain_agent.tools.permissions.network_permission import NetworkPermissionRequest +from plain_agent.tools.utils import error, ok +from plain_agent.tools.web.providers.exa import ( + EXA_DESTINATION, + ExaFetchClient, + ExaFetchError, +) + +MAX_URL_CHARS = 2_000 + + +class WebFetchTool(BaseTool): + """Fetch a webpage through Exa after explicit user approval.""" + + name = "web_fetch" + description = "Fetch a webpage and return its clean, bounded Markdown content." + parameters = { + "type": "object", + "properties": { + "url": { + "type": "string", + "minLength": 1, + "maxLength": MAX_URL_CHARS, + "description": "Absolute HTTP or HTTPS URL to fetch.", + }, + }, + "required": ["url"], + "additionalProperties": False, + } + + def __init__( + self, + permission_controller: PermissionController | None = None, + client: ExaFetchClient | None = None, + ) -> None: + self.permission_controller = ( + permission_controller + if permission_controller is not None + else PermissionController() + ) + self.client = client if client is not None else ExaFetchClient() + + def run(self, root: Path, arguments: dict[str, object]) -> str: + url = arguments.get("url") + validation_error = _validate_url(url) + if validation_error is not None: + return error(validation_error) + assert isinstance(url, str) + url = url.strip() + + request = NetworkPermissionRequest( + tool=self.name, + destination=EXA_DESTINATION, + target=url, + ) + try: + self.permission_controller.require_approval(request) + content = self.client.fetch(url) + except ApprovalDeniedError: + return error("web_fetch was not approved") + except ExaFetchError as exc: + return error(str(exc)) + + return ok({"url": url, "content": content}) + + +def _validate_url(value: object) -> str | None: + if not isinstance(value, str) or not value.strip(): + return "url is required" + + url = value.strip() + if len(url) > MAX_URL_CHARS: + return f"url must not exceed {MAX_URL_CHARS} characters" + if any(character.isspace() or not character.isprintable() for character in url): + return "url must not contain whitespace or control characters" + + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError: + return "url is invalid" + + if parsed.scheme not in {"http", "https"}: + return "url must use http or https" + if parsed.hostname is None: + return "url must include a hostname" + if parsed.username is not None or parsed.password is not None: + return "url must not include credentials" + if port is not None and not 1 <= port <= 65535: + return "url is invalid" + return None diff --git a/plain_agent/tools/web/providers/__init__.py b/plain_agent/tools/web/providers/__init__.py new file mode 100644 index 0000000..5b2cb74 --- /dev/null +++ b/plain_agent/tools/web/providers/__init__.py @@ -0,0 +1,15 @@ +"""Provider clients used by web tools.""" + +from plain_agent.tools.web.providers.exa import ( + ExaFetchClient, + ExaFetchError, + ExaSearchClient, + ExaSearchError, +) + +__all__ = [ + "ExaFetchClient", + "ExaFetchError", + "ExaSearchClient", + "ExaSearchError", +] diff --git a/plain_agent/tools/web_search/exa_mcp.py b/plain_agent/tools/web/providers/exa.py similarity index 59% rename from plain_agent/tools/web_search/exa_mcp.py rename to plain_agent/tools/web/providers/exa.py index fb13301..5afef71 100644 --- a/plain_agent/tools/web_search/exa_mcp.py +++ b/plain_agent/tools/web/providers/exa.py @@ -1,4 +1,4 @@ -"""Client for Exa's public MCP web-search endpoint.""" +"""Exa MCP clients for web tools.""" import json @@ -7,8 +7,10 @@ EXA_MCP_URL = "https://mcp.exa.ai/mcp" EXA_DESTINATION = "mcp.exa.ai" EXA_SEARCH_TOOL = "web_search_exa" +EXA_FETCH_TOOL = "web_fetch_exa" MAX_RESULTS = 5 MAX_CONTENT_CHARS = 5_000 +MAX_FETCH_CONTENT_CHARS = 12_000 MAX_RESPONSE_BYTES = 1024 * 1024 @@ -16,8 +18,15 @@ class ExaSearchError(RuntimeError): """Raised when Exa cannot provide a safe, valid search response.""" -class ExaSearchClient: - """Small synchronous client for Exa's fixed MCP endpoint.""" +class ExaFetchError(RuntimeError): + """Raised when Exa cannot provide safe, valid webpage content.""" + + +class _ExaMcpClient: + """Bounded synchronous client for Exa's fixed MCP endpoint.""" + + action = "Exa request" + error_type: type[RuntimeError] = RuntimeError def __init__( self, @@ -34,20 +43,19 @@ def __init__( self.max_response_bytes = max_response_bytes self.transport = transport - def search(self, query: str) -> str: + def _call_tool( + self, + tool: str, + arguments: dict[str, object], + max_content_chars: int, + ) -> str: payload = { "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { - "name": EXA_SEARCH_TOOL, - "arguments": { - "query": query, - "type": "auto", - "numResults": MAX_RESULTS, - "livecrawl": "fallback", - "contextMaxCharacters": MAX_CONTENT_CHARS, - }, + "name": tool, + "arguments": arguments, }, } @@ -66,15 +74,15 @@ def search(self, query: str) -> str: response.raise_for_status() body = self._read_response(response) except httpx.TimeoutException as exc: - raise ExaSearchError("web search timed out") from exc + raise self.error_type(f"{self.action} timed out") from exc except httpx.HTTPStatusError as exc: - raise ExaSearchError( - f"web search failed with HTTP status {exc.response.status_code}" + raise self.error_type( + f"{self.action} failed with HTTP status {exc.response.status_code}" ) from exc except httpx.HTTPError as exc: - raise ExaSearchError("web search request failed") from exc + raise self.error_type(f"{self.action} request failed") from exc - return self._parse_response(body) + return self._parse_response(body, max_content_chars) def _read_response(self, response: httpx.Response) -> bytes: content_length = response.headers.get("content-length") @@ -87,20 +95,24 @@ def _read_response(self, response: httpx.Response) -> bytes: declared_length is not None and declared_length > self.max_response_bytes ): - raise ExaSearchError("web search response exceeded the size limit") + raise self.error_type( + f"{self.action} response exceeded the size limit" + ) body = bytearray() for chunk in response.iter_bytes(): if len(body) + len(chunk) > self.max_response_bytes: - raise ExaSearchError("web search response exceeded the size limit") + raise self.error_type( + f"{self.action} response exceeded the size limit" + ) body.extend(chunk) return bytes(body) - def _parse_response(self, body: bytes) -> str: + def _parse_response(self, body: bytes, max_content_chars: int) -> str: try: text = body.decode("utf-8") except UnicodeDecodeError as exc: - raise ExaSearchError("web search returned invalid JSON") from exc + raise self.error_type(f"{self.action} returned invalid JSON") from exc candidates = [text.strip()] candidates.extend( @@ -117,11 +129,48 @@ def _parse_response(self, body: bytes) -> str: parsed_payload = True content = _text_content(payload) if content is not None: - return content[:MAX_CONTENT_CHARS] + return content[:max_content_chars] if not parsed_payload: - raise ExaSearchError("web search returned invalid JSON") - raise ExaSearchError("web search returned an invalid response") + raise self.error_type(f"{self.action} returned invalid JSON") + raise self.error_type(f"{self.action} returned an invalid response") + + +class ExaSearchClient(_ExaMcpClient): + """Client for Exa's web-search MCP tool.""" + + action = "web search" + error_type = ExaSearchError + + def search(self, query: str) -> str: + return self._call_tool( + EXA_SEARCH_TOOL, + { + "query": query, + "type": "auto", + "numResults": MAX_RESULTS, + "livecrawl": "fallback", + "contextMaxCharacters": MAX_CONTENT_CHARS, + }, + MAX_CONTENT_CHARS, + ) + + +class ExaFetchClient(_ExaMcpClient): + """Client for Exa's webpage-fetch MCP tool.""" + + action = "website fetch" + error_type = ExaFetchError + + def fetch(self, url: str) -> str: + return self._call_tool( + EXA_FETCH_TOOL, + { + "urls": [url], + "maxCharacters": MAX_FETCH_CONTENT_CHARS, + }, + MAX_FETCH_CONTENT_CHARS, + ) def _text_content(payload: object) -> str | None: diff --git a/plain_agent/tools/web_search/tool.py b/plain_agent/tools/web/search.py similarity index 96% rename from plain_agent/tools/web_search/tool.py rename to plain_agent/tools/web/search.py index 4cbeca5..fe777c6 100644 --- a/plain_agent/tools/web_search/tool.py +++ b/plain_agent/tools/web/search.py @@ -1,4 +1,4 @@ -"""Model-facing web-search tool.""" +"""Model-facing web search tool.""" from pathlib import Path @@ -9,7 +9,7 @@ ) from plain_agent.tools.permissions.network_permission import NetworkPermissionRequest from plain_agent.tools.utils import error, ok -from plain_agent.tools.web_search.exa_mcp import ( +from plain_agent.tools.web.providers.exa import ( EXA_DESTINATION, ExaSearchClient, ExaSearchError, diff --git a/plain_agent/tools/web_search/__init__.py b/plain_agent/tools/web_search/__init__.py deleted file mode 100644 index ada5b4f..0000000 --- a/plain_agent/tools/web_search/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Approved web search tools and provider clients.""" - -from plain_agent.tools.web_search.exa_mcp import ExaSearchClient, ExaSearchError -from plain_agent.tools.web_search.tool import WebSearchTool - -__all__ = [ - "ExaSearchClient", - "ExaSearchError", - "WebSearchTool", -] diff --git a/tests/test_agent.py b/tests/test_agent.py index 7e3cbde..8021842 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -152,12 +152,13 @@ def test_respond_stream_yields_text_deltas(self) -> None: self.assertEqual([item["role"] for item in agent.conversation_history], ["system", "user", "assistant"]) self.assertEqual(agent.conversation_history[-1]["content"], "Hello there") - def test_exposes_web_search_by_default(self) -> None: + def test_exposes_web_tools_by_default(self) -> None: agent = SimpleAgent(llm_client=FakeLLMClient([]), model="test-model") self.assertTrue(agent.tool_registry.has("web_search")) + self.assertTrue(agent.tool_registry.has("web_fetch")) - def test_can_disable_web_search(self) -> None: + def test_can_disable_web_tools(self) -> None: agent = SimpleAgent( llm_client=FakeLLMClient([]), model="test-model", @@ -165,6 +166,7 @@ def test_can_disable_web_search(self) -> None: ) self.assertFalse(agent.tool_registry.has("web_search")) + self.assertFalse(agent.tool_registry.has("web_fetch")) def test_respond_stream_does_not_auto_compact_history(self) -> None: llm_client = FakeLLMClient([stream_response(["Hello"])]) diff --git a/tests/test_tools.py b/tests/test_tools.py index 16a56dd..f0bf38f 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -238,6 +238,7 @@ def test_tools_expose_definitions(self) -> None: "write_file", "edit_file", "web_search", + "web_fetch", "run_command", ], ) @@ -250,7 +251,7 @@ def test_tools_expose_definitions(self) -> None: ["read-only", "workspace-write"], ) - def test_registry_exposes_web_search_by_default_with_opt_out(self) -> None: + def test_registry_exposes_web_tools_by_default_with_opt_out(self) -> None: with_network = ToolRegistry(enable_commands=False) without_network = ToolRegistry( enable_commands=False, @@ -258,7 +259,9 @@ def test_registry_exposes_web_search_by_default_with_opt_out(self) -> None: ) self.assertFalse(without_network.has("web_search")) + self.assertFalse(without_network.has("web_fetch")) self.assertTrue(with_network.has("web_search")) + self.assertTrue(with_network.has("web_fetch")) def test_run_command_runs_argv_in_workspace(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: diff --git a/tests/test_web_search.py b/tests/test_web_search.py index 4852290..2ba54ba 100644 --- a/tests/test_web_search.py +++ b/tests/test_web_search.py @@ -6,7 +6,11 @@ from plain_agent.tools.permissions.controller import PermissionController from plain_agent.tools.permissions.request import ApprovalDecision -from plain_agent.tools.web_search import ExaSearchClient, WebSearchTool +from plain_agent.tools.web import WebFetchTool, WebSearchTool +from plain_agent.tools.web.providers import ( + ExaFetchClient, + ExaSearchClient, +) def permission_controller(decision: ApprovalDecision) -> PermissionController: @@ -205,5 +209,134 @@ def approve(request): self.assertEqual(requests, []) +class ExaFetchClientTest(unittest.TestCase): + def test_fetch_sends_mcp_request_and_returns_content(self) -> None: + captured_requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + captured_requests.append(request) + return httpx.Response(200, json=mcp_payload("# Example\nContent")) + + client = ExaFetchClient(transport=httpx.MockTransport(handler)) + + content = client.fetch("https://example.com/page") + + request = captured_requests[0] + self.assertEqual(str(request.url), "https://mcp.exa.ai/mcp") + self.assertNotIn("x-api-key", request.headers) + self.assertEqual( + json.loads(request.content), + { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": "web_fetch_exa", + "arguments": { + "urls": ["https://example.com/page"], + "maxCharacters": 12_000, + }, + }, + }, + ) + self.assertEqual(content, "# Example\nContent") + + def test_fetch_bounds_returned_content(self) -> None: + client = ExaFetchClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + json=mcp_payload("x" * 13_000), + ) + ) + ) + + self.assertEqual(len(client.fetch("https://example.com")), 12_000) + + def test_fetch_reports_http_status_without_response_body(self) -> None: + client = ExaFetchClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(500, text="secret response body") + ) + ) + + with self.assertRaisesRegex(RuntimeError, "HTTP status 500") as raised: + client.fetch("https://example.com") + + self.assertNotIn("secret response body", str(raised.exception)) + + +class WebFetchToolTest(unittest.TestCase): + def test_denial_does_not_send_request(self) -> None: + calls = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal calls + calls += 1 + return httpx.Response(200, json=mcp_payload("result")) + + tool = WebFetchTool( + permission_controller(ApprovalDecision.REJECT), + ExaFetchClient(transport=httpx.MockTransport(handler)), + ) + + result = json.loads(tool.run(Path.cwd(), {"url": "https://example.com"})) + + self.assertFalse(result["ok"]) + self.assertIn("not approved", result["error"]) + self.assertEqual(calls, 0) + + def test_approved_fetch_returns_tool_result_and_approval_target(self) -> None: + approval_requests = [] + + def approve(request): + approval_requests.append(request) + return ApprovalDecision.ALLOW_ONCE + + client = ExaFetchClient( + transport=httpx.MockTransport( + lambda request: httpx.Response(200, json=mcp_payload("content")) + ) + ) + tool = WebFetchTool(PermissionController(approve), client) + + result = json.loads( + tool.run(Path.cwd(), {"url": " https://example.com/page "}) + ) + + self.assertTrue(result["ok"]) + self.assertEqual(result["url"], "https://example.com/page") + self.assertEqual(result["content"], "content") + self.assertEqual(approval_requests[0].tool, "web_fetch") + self.assertEqual(approval_requests[0].destination, "mcp.exa.ai") + self.assertEqual(approval_requests[0].target, "https://example.com/page") + + def test_invalid_urls_are_rejected_before_approval(self) -> None: + approval_requests = [] + + def approve(request): + approval_requests.append(request) + return ApprovalDecision.ALLOW_ONCE + + tool = WebFetchTool(PermissionController(approve)) + invalid_urls = [ + "", + "example.com", + "file:///etc/passwd", + "https://user:password@example.com", + "https://example.com/a b", + "https://[invalid", + "https://example.com:99999", + "https://example.com/" + "x" * 2_000, + ] + + for url in invalid_urls: + with self.subTest(url=url): + result = json.loads(tool.run(Path.cwd(), {"url": url})) + self.assertFalse(result["ok"]) + + self.assertEqual(approval_requests, []) + + if __name__ == "__main__": unittest.main()