Adding function to fetch website

.env.example

@@ -13,7 +13,7 @@ LLM_TIMEOUT=60
 # Optional override for any OpenAI-compatible provider.
 # LLM_BASE_URL=https://api.openai.com/v1
 
-# Web search is enabled by default. Every query still requires confirmation.
+# Web search and fetch are enabled by default. Every request still requires confirmation.
 # PLAIN_AGENT_ENABLE_NETWORK=false
 
 # Optional Linux sandbox toolchain roots. Every path becomes readable by commands.

README.md

@@ -76,19 +76,20 @@ Context compaction runs automatically when the estimated conversation history re
 Set `LLM_COMPACTION_AUTO_MAX_TOKENS` to change that threshold, or set it to `0` to disable automatic compaction.
 You can also run `/compact` in the terminal to compact manually.
 
-### Web search
+### Web access
 
-The Exa-backed `web_search` tool is enabled by default and every query requires explicit approval
-before Plain Agent connects to `mcp.exa.ai`. Search results include links and bounded relevant
-excerpts. No Exa API key is required.
+The Exa-backed `web_search` and `web_fetch` tools are enabled by default. Every query or URL
+requires explicit approval before Plain Agent connects to `mcp.exa.ai`. Search returns links and
+bounded excerpts; fetch returns bounded Markdown content for one HTTP or HTTPS URL. No Exa API key
+is required.
 
-To disable web search:
+To disable both web tools:
 
 ```bash
 export PLAIN_AGENT_ENABLE_NETWORK="false"
 ```
 
-This setting controls only the built-in search tool. The `run_command` sandbox remains offline.
+This setting controls only the built-in web tools. The `run_command` sandbox remains offline.
 
 ## Run
 

plain_agent/tools/registry.py

@@ -11,7 +11,7 @@
 from plain_agent.tools.run_command import RunCommandTool
 from plain_agent.tools.search_text import SearchTextTool
 from plain_agent.tools.utils import error
-from plain_agent.tools.web_search import WebSearchTool
+from plain_agent.tools.web import WebFetchTool, WebSearchTool
 from plain_agent.tools.write_file import WriteFileTool
 
 
@@ -42,7 +42,12 @@ def __init__(
         ]
         self.startup_warnings: list[str] = []
         if enable_network:
-            registered_tools.append(WebSearchTool(self.permission_controller))
+            registered_tools.extend(
+                [
+                    WebSearchTool(self.permission_controller),
+                    WebFetchTool(self.permission_controller),
+                ]
+            )
         if enable_commands:
             discovery = discover_linux_sandbox()
             if discovery.warning is not None:

plain_agent/tools/web/__init__.py

@@ -0,0 +1,9 @@
+"""Approved web tools."""
+
+from plain_agent.tools.web.fetch import WebFetchTool
+from plain_agent.tools.web.search import WebSearchTool
+
+__all__ = [
+    "WebFetchTool",
+    "WebSearchTool",
+]

plain_agent/tools/web/fetch.py

@@ -0,0 +1,101 @@
+"""Model-facing webpage fetch tool."""
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from plain_agent.tools.base_tool import BaseTool
+from plain_agent.tools.permissions.controller import (
+    ApprovalDeniedError,
+    PermissionController,
+)
+from plain_agent.tools.permissions.network_permission import NetworkPermissionRequest
+from plain_agent.tools.utils import error, ok
+from plain_agent.tools.web.providers.exa import (
+    EXA_DESTINATION,
+    ExaFetchClient,
+    ExaFetchError,
+)
+
+MAX_URL_CHARS = 2_000
+
+
+class WebFetchTool(BaseTool):
+    """Fetch a webpage through Exa after explicit user approval."""
+
+    name = "web_fetch"
+    description = "Fetch a webpage and return its clean, bounded Markdown content."
+    parameters = {
+        "type": "object",
+        "properties": {
+            "url": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": MAX_URL_CHARS,
+                "description": "Absolute HTTP or HTTPS URL to fetch.",
+            },
+        },
+        "required": ["url"],
+        "additionalProperties": False,
+    }
+
+    def __init__(
+        self,
+        permission_controller: PermissionController | None = None,
+        client: ExaFetchClient | None = None,
+    ) -> None:
+        self.permission_controller = (
+            permission_controller
+            if permission_controller is not None
+            else PermissionController()
+        )
+        self.client = client if client is not None else ExaFetchClient()
+
+    def run(self, root: Path, arguments: dict[str, object]) -> str:
+        url = arguments.get("url")
+        validation_error = _validate_url(url)
+        if validation_error is not None:
+            return error(validation_error)
+        assert isinstance(url, str)
+        url = url.strip()
+
+        request = NetworkPermissionRequest(
+            tool=self.name,
+            destination=EXA_DESTINATION,
+            target=url,
+        )
+        try:
+            self.permission_controller.require_approval(request)
+            content = self.client.fetch(url)
+        except ApprovalDeniedError:
+            return error("web_fetch was not approved")
+        except ExaFetchError as exc:
+            return error(str(exc))
+
+        return ok({"url": url, "content": content})
+
+
+def _validate_url(value: object) -> str | None:
+    if not isinstance(value, str) or not value.strip():
+        return "url is required"
+
+    url = value.strip()
+    if len(url) > MAX_URL_CHARS:
+        return f"url must not exceed {MAX_URL_CHARS} characters"
+    if any(character.isspace() or not character.isprintable() for character in url):
+        return "url must not contain whitespace or control characters"
+
+    try:
+        parsed = urlsplit(url)
+        port = parsed.port
+    except ValueError:
+        return "url is invalid"
+
+    if parsed.scheme not in {"http", "https"}:
+        return "url must use http or https"
+    if parsed.hostname is None:
+        return "url must include a hostname"
+    if parsed.username is not None or parsed.password is not None:
+        return "url must not include credentials"
+    if port is not None and not 1 <= port <= 65535:
+        return "url is invalid"
+    return None

plain_agent/tools/web/providers/__init__.py

@@ -0,0 +1,15 @@
+"""Provider clients used by web tools."""
+
+from plain_agent.tools.web.providers.exa import (
+    ExaFetchClient,
+    ExaFetchError,
+    ExaSearchClient,
+    ExaSearchError,
+)
+
+__all__ = [
+    "ExaFetchClient",
+    "ExaFetchError",
+    "ExaSearchClient",
+    "ExaSearchError",
+]

plain_agent/tools/web_search/exa_mcp.py → plain_agent/tools/web/providers/exa.py

@@ -1,4 +1,4 @@
-"""Client for Exa's public MCP web-search endpoint."""
+"""Exa MCP clients for web tools."""
 
 import json
 
@@ -7,17 +7,26 @@
 EXA_MCP_URL = "https://mcp.exa.ai/mcp"
 EXA_DESTINATION = "mcp.exa.ai"
 EXA_SEARCH_TOOL = "web_search_exa"
+EXA_FETCH_TOOL = "web_fetch_exa"
 MAX_RESULTS = 5
 MAX_CONTENT_CHARS = 5_000
+MAX_FETCH_CONTENT_CHARS = 12_000
 MAX_RESPONSE_BYTES = 1024 * 1024
 
 
 class ExaSearchError(RuntimeError):
     """Raised when Exa cannot provide a safe, valid search response."""
 
 
-class ExaSearchClient:
-    """Small synchronous client for Exa's fixed MCP endpoint."""
+class ExaFetchError(RuntimeError):
+    """Raised when Exa cannot provide safe, valid webpage content."""
+
+
+class _ExaMcpClient:
+    """Bounded synchronous client for Exa's fixed MCP endpoint."""
+
+    action = "Exa request"
+    error_type: type[RuntimeError] = RuntimeError
 
     def __init__(
         self,
@@ -34,20 +43,19 @@ def __init__(
         self.max_response_bytes = max_response_bytes
         self.transport = transport
 
-    def search(self, query: str) -> str:
+    def _call_tool(
+        self,
+        tool: str,
+        arguments: dict[str, object],
+        max_content_chars: int,
+    ) -> str:
         payload = {
             "jsonrpc": "2.0",
             "id": 1,
             "method": "tools/call",
             "params": {
-                "name": EXA_SEARCH_TOOL,
-                "arguments": {
-                    "query": query,
-                    "type": "auto",
-                    "numResults": MAX_RESULTS,
-                    "livecrawl": "fallback",
-                    "contextMaxCharacters": MAX_CONTENT_CHARS,
-                },
+                "name": tool,
+                "arguments": arguments,
             },
         }
 
@@ -66,15 +74,15 @@ def search(self, query: str) -> str:
                     response.raise_for_status()
                     body = self._read_response(response)
         except httpx.TimeoutException as exc:
-            raise ExaSearchError("web search timed out") from exc
+            raise self.error_type(f"{self.action} timed out") from exc
         except httpx.HTTPStatusError as exc:
-            raise ExaSearchError(
-                f"web search failed with HTTP status {exc.response.status_code}"
+            raise self.error_type(
+                f"{self.action} failed with HTTP status {exc.response.status_code}"
             ) from exc
         except httpx.HTTPError as exc:
-            raise ExaSearchError("web search request failed") from exc
+            raise self.error_type(f"{self.action} request failed") from exc
 
-        return self._parse_response(body)
+        return self._parse_response(body, max_content_chars)
 
     def _read_response(self, response: httpx.Response) -> bytes:
         content_length = response.headers.get("content-length")
@@ -87,20 +95,24 @@ def _read_response(self, response: httpx.Response) -> bytes:
                 declared_length is not None
                 and declared_length > self.max_response_bytes
             ):
-                raise ExaSearchError("web search response exceeded the size limit")
+                raise self.error_type(
+                    f"{self.action} response exceeded the size limit"
+                )
 
         body = bytearray()
         for chunk in response.iter_bytes():
             if len(body) + len(chunk) > self.max_response_bytes:
-                raise ExaSearchError("web search response exceeded the size limit")
+                raise self.error_type(
+                    f"{self.action} response exceeded the size limit"
+                )
             body.extend(chunk)
         return bytes(body)
 
-    def _parse_response(self, body: bytes) -> str:
+    def _parse_response(self, body: bytes, max_content_chars: int) -> str:
         try:
             text = body.decode("utf-8")
         except UnicodeDecodeError as exc:
-            raise ExaSearchError("web search returned invalid JSON") from exc
+            raise self.error_type(f"{self.action} returned invalid JSON") from exc
 
         candidates = [text.strip()]
         candidates.extend(
@@ -117,11 +129,48 @@ def _parse_response(self, body: bytes) -> str:
             parsed_payload = True
             content = _text_content(payload)
             if content is not None:
-                return content[:MAX_CONTENT_CHARS]
+                return content[:max_content_chars]
 
         if not parsed_payload:
-            raise ExaSearchError("web search returned invalid JSON")
-        raise ExaSearchError("web search returned an invalid response")
+            raise self.error_type(f"{self.action} returned invalid JSON")
+        raise self.error_type(f"{self.action} returned an invalid response")
+
+
+class ExaSearchClient(_ExaMcpClient):
+    """Client for Exa's web-search MCP tool."""
+
+    action = "web search"
+    error_type = ExaSearchError
+
+    def search(self, query: str) -> str:
+        return self._call_tool(
+            EXA_SEARCH_TOOL,
+            {
+                "query": query,
+                "type": "auto",
+                "numResults": MAX_RESULTS,
+                "livecrawl": "fallback",
+                "contextMaxCharacters": MAX_CONTENT_CHARS,
+            },
+            MAX_CONTENT_CHARS,
+        )
+
+
+class ExaFetchClient(_ExaMcpClient):
+    """Client for Exa's webpage-fetch MCP tool."""
+
+    action = "website fetch"
+    error_type = ExaFetchError
+
+    def fetch(self, url: str) -> str:
+        return self._call_tool(
+            EXA_FETCH_TOOL,
+            {
+                "urls": [url],
+                "maxCharacters": MAX_FETCH_CONTENT_CHARS,
+            },
+            MAX_FETCH_CONTENT_CHARS,
+        )
 
 
 def _text_content(payload: object) -> str | None:

plain_agent/tools/web_search/tool.py → plain_agent/tools/web/search.py

@@ -1,4 +1,4 @@
-"""Model-facing web-search tool."""
+"""Model-facing web search tool."""
 
 from pathlib import Path
 
@@ -9,7 +9,7 @@
 )
 from plain_agent.tools.permissions.network_permission import NetworkPermissionRequest
 from plain_agent.tools.utils import error, ok
-from plain_agent.tools.web_search.exa_mcp import (
+from plain_agent.tools.web.providers.exa import (
     EXA_DESTINATION,
     ExaSearchClient,
     ExaSearchError,