feat: merge http/web_fetch tools, add tool output stash for large responses (#578)

* feat: merge http/web_fetch tools, add tool output stash for large responses

Merge `web_fetch` into `http` tool with smart approval: plain GETs (no
headers, no body) run without approval and follow redirects with SSRF
re-validation per hop; all other requests require approval as before.

Add `tool_output_stash` on JobContext so full tool outputs are preserved
before safety-layer truncation. The `json` tool gains a
`source_tool_call_id` parameter to reference stashed outputs, enabling
reliable parsing of large API responses that exceed the 100KB context
limit.

Other improvements:
- Descriptive User-Agent header using CARGO_PKG_VERSION
- Truncation now keeps partial data + hint about source_tool_call_id
- System prompt reinforces tool_calls over narration
- json tool query/stringify handle pre-parsed (non-string) data

[skip-regression-check]

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* chore: delete dead web_fetch.rs (merged into http tool)

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* style: fix rustfmt formatting

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* style: rename shadowed data binding for clarity in json tool

Address PR review: rename owned `data` to `data_value` before
re-binding as `let data = &data_value` to make ownership explicit.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix(ci): mark network-dependent trace tests as #[ignore]

The weather_sf and baseball_stats tests hit live external APIs (wttr.in,
ESPN) which are unreliable in CI. Mark them #[ignore] so they don't
block the pipeline. Run locally with `--ignored` to include them.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: replay recorded HTTP exchanges in trace tests instead of hitting live APIs

Wire ReplayingHttpInterceptor into TestRig when the trace fixture
contains http_exchanges. This replays recorded responses instead of
making live network calls, making tests deterministic and CI-stable.

Add captured HTTP responses to weather_sf.json (wttr.in) and
baseball_stats.json (ESPN API) fixtures.

Revert #[ignore] on both tests — they now run offline.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: recover inline bracket-format tool calls from LLM text responses

When flatten_tool_messages converts tool calls to text like
`[Called tool `http` with arguments: {...}]` for NEAR AI compatibility,
the LLM sometimes echoes this format back in its text responses instead
of using proper tool_calls. Add recovery for this bracket format in
recover_tool_calls_from_content and strip it in clean_response so
users don't see raw tool call syntax.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>

src/tools/builtin/http.rs

@@ -1,4 +1,12 @@
 //! HTTP request tool.
+//!
+//! Unified HTTP tool that handles both simple page/API fetches (GET, no auth)
+//! and full API calls (any method, custom headers, credential injection).
+//!
+//! - Plain GET without auth headers/body → no approval needed, follows redirects
+//! - Everything else → requires approval
+//!
+//! Replaces the former `web_fetch` tool which was a separate GET-only tool.
 
 use std::collections::HashMap;
 use std::net::{IpAddr, ToSocketAddrs};
@@ -25,6 +33,16 @@ use crate::tools::builtin::convert_html_to_markdown;
 /// HTTP wrapper uses the same limit for consistency.
 const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
 
+/// Maximum number of redirects to follow for simple GET requests.
+const MAX_REDIRECTS: usize = 3;
+
+/// Descriptive User-Agent so public APIs don't reject bare requests.
+const USER_AGENT: &str = concat!(
+    "IronClaw-Agent/",
+    env!("CARGO_PKG_VERSION"),
+    " (https://github.com/nearai/ironclaw)"
+);
+
 /// Tool for making HTTP requests.
 pub struct HttpTool {
     client: Client,
@@ -38,6 +56,7 @@ impl HttpTool {
         let client = Client::builder()
             .timeout(Duration::from_secs(30))
             .redirect(reqwest::redirect::Policy::none())
+            .user_agent(USER_AGENT)
             .build()
             .expect("Failed to create HTTP client");
 
@@ -201,7 +220,10 @@ impl Tool for HttpTool {
     }
 
     fn description(&self) -> &str {
-        "Make HTTP requests to external APIs. Supports GET, POST, PUT, DELETE methods."
+        "Make HTTP requests. Simple GET requests (no auth, no custom headers) run without \
+         approval and follow redirects — use for fetching weather, public JSON APIs, web pages, \
+         and documentation. Requests with authentication, custom headers, or non-GET methods \
+         (POST, PUT, DELETE, PATCH) require user approval."
     }
 
     fn parameters_schema(&self) -> serde_json::Value {
@@ -368,25 +390,108 @@ impl Tool for HttpTool {
             return Ok(ToolOutput::success(result, start.elapsed()).with_raw(recorded.body));
         }
 
-        // Execute request
-        let response = request.send().await.map_err(|e| {
-            if e.is_timeout() {
-                ToolError::Timeout(Duration::from_secs(30))
-            } else {
-                ToolError::ExternalService(e.to_string())
+        // Determine if this is a simple GET (eligible for redirect following).
+        let is_simple_get =
+            method.eq_ignore_ascii_case("GET") && headers_vec.is_empty() && body_bytes.is_none();
+
+        // Execute request, optionally following redirects for simple GETs.
+        let response = if is_simple_get {
+            let mut redirects_remaining = MAX_REDIRECTS;
+            loop {
+                let resp = self
+                    .client
+                    .get(parsed_url.clone())
+                    .header(
+                        reqwest::header::ACCEPT,
+                        "text/markdown, text/html;q=0.9, application/json;q=0.9, */*;q=0.8",
+                    )
+                    .send()
+                    .await
+                    .map_err(|e| {
+                        if e.is_timeout() {
+                            ToolError::Timeout(Duration::from_secs(30))
+                        } else {
+                            ToolError::ExternalService(e.to_string())
+                        }
+                    })?;
+
+                let status = resp.status().as_u16();
+                if (300..400).contains(&status) {
+                    if redirects_remaining == 0 {
+                        return Err(ToolError::ExecutionFailed(format!(
+                            "too many redirects (max {})",
+                            MAX_REDIRECTS
+                        )));
+                    }
+
+                    let location = resp
+                        .headers()
+                        .get(reqwest::header::LOCATION)
+                        .and_then(|v| v.to_str().ok())
+                        .ok_or_else(|| {
+                            ToolError::ExecutionFailed(format!(
+                                "redirect (HTTP {}) has no Location header",
+                                status
+                            ))
+                        })?;
+
+                    let next_url_str =
+                        if location.starts_with("http://") || location.starts_with("https://") {
+                            location.to_string()
+                        } else {
+                            parsed_url
+                                .join(location)
+                                .map(|u| u.to_string())
+                                .map_err(|e| {
+                                    ToolError::ExecutionFailed(format!(
+                                        "could not resolve relative redirect '{}': {}",
+                                        location, e
+                                    ))
+                                })?
+                        };
+
+                    // SSRF re-validation on every hop.
+                    parsed_url = validate_url(&next_url_str)?;
+                    let detector = LeakDetector::new();
+                    detector
+                        .scan_http_request(parsed_url.as_str(), &[], None)
+                        .map_err(|e| ToolError::NotAuthorized(e.to_string()))?;
+
+                    redirects_remaining -= 1;
+                    tracing::debug!(
+                        to = %parsed_url,
+                        hops_left = redirects_remaining,
+                        "http tool following redirect"
+                    );
+                    continue;
+                }
+
+                break resp;
             }
-        })?;
+        } else {
+            let resp = request.send().await.map_err(|e| {
+                if e.is_timeout() {
+                    ToolError::Timeout(Duration::from_secs(30))
+                } else {
+                    ToolError::ExternalService(e.to_string())
+                }
+            })?;
+
+            let status = resp.status().as_u16();
+
+            // Block redirects for non-simple requests (potential SSRF)
+            if (300..400).contains(&status) {
+                return Err(ToolError::NotAuthorized(format!(
+                    "request returned redirect (HTTP {}), which is blocked to prevent SSRF",
+                    status
+                )));
+            }
+
+            resp
+        };
 
         let status = response.status().as_u16();
 
-        // Block redirects: the server tried to send us elsewhere (potential SSRF)
-        if (300..400).contains(&status) {
-            return Err(ToolError::NotAuthorized(format!(
-                "request returned redirect (HTTP {}), which is blocked to prevent SSRF",
-                status
-            )));
-        }
-
         let headers: HashMap<String, String> = response
             .headers()
             .iter()
@@ -496,6 +601,25 @@ impl Tool for HttpTool {
         {
             return ApprovalRequirement::Always;
         }
+        // 3. Plain GET without headers or body → no approval needed
+        let method = params
+            .get("method")
+            .and_then(|v| v.as_str())
+            .unwrap_or("GET");
+        let has_headers = params
+            .get("headers")
+            .map(|h| match h {
+                serde_json::Value::Array(a) => !a.is_empty(),
+                serde_json::Value::Object(o) => !o.is_empty(),
+                _ => false,
+            })
+            .unwrap_or(false);
+        let has_body = params.get("body").is_some();
+
+        if method.eq_ignore_ascii_case("GET") && !has_headers && !has_body {
+            return ApprovalRequirement::Never;
+        }
+
         // Default: outbound HTTP still needs approval unless auto-approved
         ApprovalRequirement::UnlessAutoApproved
     }
@@ -622,12 +746,37 @@ mod tests {
     // ── Approval requirement tests ──────────────────────────────────────
 
     #[test]
-    fn test_no_auth_headers_returns_unless_auto_approved() {
+    fn test_plain_get_returns_never() {
         let tool = HttpTool::new();
         let params = serde_json::json!({
             "method": "GET",
             "url": "https://api.example.com/data"
         });
+        assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never);
+    }
+
+    #[test]
+    fn test_post_returns_unless_auto_approved() {
+        let tool = HttpTool::new();
+        let params = serde_json::json!({
+            "method": "POST",
+            "url": "https://api.example.com/data",
+            "body": {"key": "value"}
+        });
+        assert_eq!(
+            tool.requires_approval(&params),
+            ApprovalRequirement::UnlessAutoApproved
+        );
+    }
+
+    #[test]
+    fn test_get_with_headers_returns_unless_auto_approved() {
+        let tool = HttpTool::new();
+        let params = serde_json::json!({
+            "method": "GET",
+            "url": "https://api.example.com/data",
+            "headers": [{"name": "X-Custom", "value": "test"}]
+        });
         assert_eq!(
             tool.requires_approval(&params),
             ApprovalRequirement::UnlessAutoApproved
@@ -725,30 +874,24 @@ mod tests {
     }
 
     #[test]
-    fn test_empty_headers_return_unless_auto_approved() {
+    fn test_empty_headers_get_returns_never() {
         let tool = HttpTool::new();
 
-        // Empty object
+        // Empty object — still a plain GET
         let params = serde_json::json!({
             "method": "GET",
             "url": "https://example.com",
             "headers": {}
         });
-        assert_eq!(
-            tool.requires_approval(&params),
-            ApprovalRequirement::UnlessAutoApproved
-        );
+        assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never);
 
-        // Empty array
+        // Empty array — still a plain GET
         let params = serde_json::json!({
             "method": "GET",
             "url": "https://example.com",
             "headers": []
         });
-        assert_eq!(
-            tool.requires_approval(&params),
-            ApprovalRequirement::UnlessAutoApproved
-        );
+        assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never);
     }
 
     // ── Credential registry approval tests ─────────────────────────────
@@ -783,7 +926,7 @@ mod tests {
     }
 
     #[test]
-    fn test_host_without_credential_mapping_returns_unless_auto_approved() {
+    fn test_host_without_credential_mapping_get_returns_never() {
         use crate::tools::wasm::SharedCredentialRegistry;
 
         let registry = Arc::new(SharedCredentialRegistry::new());
@@ -799,10 +942,19 @@ mod tests {
             ))),
         );
 
+        // Plain GET with no credentials → Never
         let params = serde_json::json!({
             "method": "GET",
             "url": "https://api.example.com/data"
         });
+        assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never);
+
+        // POST with no credentials → UnlessAutoApproved
+        let params = serde_json::json!({
+            "method": "POST",
+            "url": "https://api.example.com/data",
+            "body": {"key": "value"}
+        });
         assert_eq!(
             tool.requires_approval(&params),
             ApprovalRequirement::UnlessAutoApproved

src/tools/builtin/json.rs

@@ -15,7 +15,9 @@ impl Tool for JsonTool {
     }
 
     fn description(&self) -> &str {
-        "Parse, query, and transform JSON data. Supports JSONPath-like queries."
+        "Parse, query, and transform JSON data. Supports JSONPath-like queries. \
+         Use `source_tool_call_id` to reference the full output of a previous tool call \
+         (avoids truncation issues with large responses)."
     }
 
     fn parameters_schema(&self) -> serde_json::Value {
@@ -28,27 +30,48 @@ impl Tool for JsonTool {
                     "description": "The JSON operation to perform"
                 },
                 "data": {
-                    "description": "JSON input data. Pass a string for parse, or any JSON value (object, array, string, number, boolean, null) otherwise."
+                    "description": "JSON input data. Pass a string for parse, or any JSON value otherwise. Not required when source_tool_call_id is provided."
+                },
+                "source_tool_call_id": {
+                    "type": "string",
+                    "description": "Reference a previous tool call's full output by its ID (e.g., 'call_abc123'). Use this instead of data when the previous tool output was large and may have been truncated."
                 },
                 "path": {
                     "type": "string",
                     "description": "JSONPath-like path for query operation (e.g., 'foo.bar[0].baz')"
                 }
             },
-            "required": ["operation", "data"]
+            "required": ["operation"]
         })
     }
 
     async fn execute(
         &self,
         params: serde_json::Value,
-        _ctx: &JobContext,
+        ctx: &JobContext,
     ) -> Result<ToolOutput, ToolError> {
         let start = std::time::Instant::now();
 
         let operation = require_str(&params, "operation")?;
 
-        let data = require_param(&params, "data")?;
+        // Resolve data: from stash (via source_tool_call_id) or from params
+        let data_value =
+            if let Some(ref_id) = params.get("source_tool_call_id").and_then(|v| v.as_str()) {
+                let stash = ctx.tool_output_stash.read().await;
+                let full_output = stash.get(ref_id).ok_or_else(|| {
+                    ToolError::InvalidParameters(format!(
+                        "no tool output found for call ID '{}'. Available IDs: {:?}",
+                        ref_id,
+                        stash.keys().collect::<Vec<_>>()
+                    ))
+                })?;
+                // Parse the stashed output as JSON, or wrap as string
+                serde_json::from_str::<serde_json::Value>(full_output)
+                    .unwrap_or_else(|_| serde_json::Value::String(full_output.clone()))
+            } else {
+                require_param(&params, "data")?.clone()
+            };
+        let data = &data_value;
 
         let result = match operation {
             "parse" => {
@@ -64,7 +87,11 @@ impl Tool for JsonTool {
                 parsed
             }
             "stringify" => {
-                let value = parse_json_input(data)?;
+                let value = if data.is_string() {
+                    parse_json_input(data)?
+                } else {
+                    data.clone()
+                };
                 let json_str = serde_json::to_string_pretty(&value).map_err(|e| {
                     ToolError::ExecutionFailed(format!("failed to stringify: {}", e))
                 })?;
@@ -76,7 +103,11 @@ impl Tool for JsonTool {
                     ToolError::InvalidParameters("missing 'path' parameter for query".to_string())
                 })?;
 
-                let value = parse_json_input(data)?;
+                let value = if data.is_string() {
+                    parse_json_input(data)?
+                } else {
+                    data.clone()
+                };
                 query_json(&value, path)?
             }
             "validate" => {
@@ -190,6 +221,54 @@ mod tests {
         assert!(err.to_string().contains("invalid JSON input"));
     }
 
+    #[tokio::test]
+    async fn test_query_with_object_data_from_stash() {
+        use crate::context::JobContext;
+
+        let ctx = JobContext::with_user("test", "chat", "test-session");
+
+        // Simulate stashed output: the http tool stores serialized JSON
+        // containing {"status": 200, "body": {"leagues": [{"name": "MLB"}]}}
+        let stashed = r#"{"status": 200, "body": {"leagues": [{"name": "MLB"}]}}"#;
+        ctx.tool_output_stash
+            .write()
+            .await
+            .insert("call_http_01".to_string(), stashed.to_string());
+
+        let tool = JsonTool;
+        let params = serde_json::json!({
+            "operation": "query",
+            "source_tool_call_id": "call_http_01",
+            "path": "body.leagues[0].name"
+        });
+
+        let result = tool.execute(params, &ctx).await.unwrap();
+        assert_eq!(result.result, serde_json::json!("MLB"));
+    }
+
+    #[tokio::test]
+    async fn test_stringify_with_object_data_from_stash() {
+        use crate::context::JobContext;
+
+        let ctx = JobContext::with_user("test", "chat", "test-session");
+
+        let stashed = r#"{"key": "value"}"#;
+        ctx.tool_output_stash
+            .write()
+            .await
+            .insert("call_01".to_string(), stashed.to_string());
+
+        let tool = JsonTool;
+        let params = serde_json::json!({
+            "operation": "stringify",
+            "source_tool_call_id": "call_01"
+        });
+
+        let result = tool.execute(params, &ctx).await.unwrap();
+        let stringified = result.result.as_str().unwrap();
+        assert!(stringified.contains("\"key\": \"value\""));
+    }
+
     #[test]
     fn test_json_tool_schema_data_is_freeform() {
         let schema = JsonTool.parameters_schema();

src/tools/builtin/mod.rs

@@ -14,7 +14,6 @@ pub mod secrets_tools;
 pub(crate) mod shell;
 pub mod skill_tools;
 mod time;
-mod web_fetch;
 
 pub use echo::EchoTool;
 pub use extension_tools::{
@@ -36,8 +35,6 @@ pub use secrets_tools::{SecretDeleteTool, SecretListTool};
 pub use shell::ShellTool;
 pub use skill_tools::{SkillInstallTool, SkillListTool, SkillRemoveTool, SkillSearchTool};
 pub use time::TimeTool;
-pub use web_fetch::WebFetchTool;
-
 mod html_converter;
 
 pub use html_converter::convert_html_to_markdown;

src/tools/builtin/web_fetch.rs

@@ -1,378 +0,0 @@
-//! Web fetch tool — GET a URL and return its content as clean Markdown.
-//!
-//! Distinct from the generic `http` tool (which handles API calls with full
-//! method/header/body control). `web_fetch` is purpose-built for reading web
-//! pages, articles, and documentation:
-//!
-//! - GET-only, no custom headers or body
-//! - Always attempts HTML → Markdown conversion via Readability
-//! - Returns structured output: `{url, final_url, status, title, content, word_count}`
-//! - Auto-approved (no confirmation prompt)
-//! - Follows up to 3 redirects, SSRF-validating each hop
-//!
-//! All the same security infrastructure as `http`:
-//! HTTPS-only, SSRF protection, DNS rebinding defence, outbound/inbound leak
-//! scanning, 5 MB response cap.
-
-use std::time::{Duration, Instant};
-
-use async_trait::async_trait;
-use futures::StreamExt;
-use reqwest::Client;
-
-use crate::context::JobContext;
-use crate::safety::LeakDetector;
-use crate::tools::builtin::http::validate_url;
-use crate::tools::tool::{ApprovalRequirement, Tool, ToolError, ToolOutput, ToolRateLimitConfig};
-
-#[cfg(feature = "html-to-markdown")]
-use crate::tools::builtin::convert_html_to_markdown;
-
-/// Maximum response body size — matches the `http` tool limit.
-const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
-
-/// Maximum number of redirects to follow before giving up.
-const MAX_REDIRECTS: usize = 3;
-
-/// Chrome-like User-Agent — many sites block default `reqwest` strings.
-const USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) \
-    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-
-/// Extract the `<title>` text from raw HTML without a full DOM parser.
-///
-/// Uses `to_ascii_lowercase()` (not `to_lowercase()`) so that byte offsets
-/// remain valid across both strings.  HTML tag names are ASCII-only, so
-/// ASCII-only case folding is sufficient.  Unicode `to_lowercase()` can
-/// change byte lengths (e.g. `İ` → `i\u{307}`), making offsets derived
-/// from the lowercased string invalid when used to index into the original.
-fn extract_title(html: &str) -> Option<String> {
-    let lower = html.to_ascii_lowercase();
-    let tag_start = lower.find("<title")?;
-    let tag_end = html[tag_start..].find('>')? + tag_start + 1;
-    let close = lower[tag_end..].find("</title>")? + tag_end;
-    let title = html[tag_end..close].trim().to_string();
-    if title.is_empty() { None } else { Some(title) }
-}
-
-/// Web fetch tool — retrieve a URL and return clean Markdown content.
-pub struct WebFetchTool {
-    client: Client,
-    leak_detector: LeakDetector,
-}
-
-impl WebFetchTool {
-    /// Create a new `WebFetchTool` with a Chrome-like UA and no auto-redirects.
-    ///
-    /// Redirects are followed manually (up to [`MAX_REDIRECTS`] hops) so that
-    /// each `Location` URL is SSRF-validated before the next request is sent.
-    pub fn new() -> Self {
-        let client = Client::builder()
-            .timeout(Duration::from_secs(30))
-            .redirect(reqwest::redirect::Policy::none())
-            .user_agent(USER_AGENT)
-            .build()
-            .expect("Failed to create HTTP client for web_fetch");
-
-        Self {
-            client,
-            leak_detector: LeakDetector::new(),
-        }
-    }
-}
-
-impl Default for WebFetchTool {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-#[async_trait]
-impl Tool for WebFetchTool {
-    fn name(&self) -> &str {
-        "web_fetch"
-    }
-
-    fn description(&self) -> &str {
-        "Fetch a URL and extract its content as clean Markdown. \
-         Use for reading articles, documentation, and web pages. \
-         For API calls (POST, custom headers, authentication), use the `http` tool instead."
-    }
-
-    fn parameters_schema(&self) -> serde_json::Value {
-        serde_json::json!({
-            "type": "object",
-            "properties": {
-                "url": {
-                    "type": "string",
-                    "description": "HTTPS URL to fetch. Must be a public URL (no localhost or private IPs)."
-                }
-            },
-            "required": ["url"],
-            "additionalProperties": false
-        })
-    }
-
-    async fn execute(
-        &self,
-        params: serde_json::Value,
-        _ctx: &JobContext,
-    ) -> Result<ToolOutput, ToolError> {
-        let start = Instant::now();
-
-        let url_str = params
-            .get("url")
-            .and_then(|v| v.as_str())
-            .ok_or_else(|| ToolError::InvalidParameters("'url' is required".to_string()))?;
-
-        // SSRF defence: HTTPS-only, no localhost, no private IPs, DNS rebinding check.
-        let mut current_url = validate_url(url_str)?;
-
-        // Outbound leak scan — reject if URL contains secrets.
-        self.leak_detector
-            .scan_http_request(current_url.as_str(), &[], None)
-            .map_err(|e| ToolError::NotAuthorized(e.to_string()))?;
-
-        // Follow redirects manually so every hop is SSRF-validated.
-        let response = {
-            let mut redirects_remaining = MAX_REDIRECTS;
-            loop {
-                let resp = self
-                    .client
-                    .get(current_url.clone())
-                    .header(
-                        reqwest::header::ACCEPT,
-                        "text/markdown, text/html;q=0.9, */*;q=0.8",
-                    )
-                    .send()
-                    .await
-                    .map_err(|e| {
-                        if e.is_timeout() {
-                            ToolError::Timeout(Duration::from_secs(30))
-                        } else {
-                            ToolError::ExternalService(e.to_string())
-                        }
-                    })?;
-
-                let status = resp.status().as_u16();
-
-                if (300..400).contains(&status) {
-                    if redirects_remaining == 0 {
-                        return Err(ToolError::ExecutionFailed(format!(
-                            "too many redirects (max {})",
-                            MAX_REDIRECTS
-                        )));
-                    }
-
-                    let location = resp
-                        .headers()
-                        .get(reqwest::header::LOCATION)
-                        .and_then(|v| v.to_str().ok())
-                        .ok_or_else(|| {
-                            ToolError::ExecutionFailed(format!(
-                                "redirect (HTTP {}) has no Location header",
-                                status
-                            ))
-                        })?;
-
-                    // Resolve relative redirects against the current URL.
-                    let next_url_str =
-                        if location.starts_with("http://") || location.starts_with("https://") {
-                            location.to_string()
-                        } else {
-                            // Relative redirect — join with current URL.
-                            current_url
-                                .join(location)
-                                .map(|u| u.to_string())
-                                .map_err(|e| {
-                                    ToolError::ExecutionFailed(format!(
-                                        "could not resolve relative redirect '{}': {}",
-                                        location, e
-                                    ))
-                                })?
-                        };
-
-                    // SSRF re-validation on every hop.
-                    current_url = validate_url(&next_url_str)?;
-                    self.leak_detector
-                        .scan_http_request(current_url.as_str(), &[], None)
-                        .map_err(|e| ToolError::NotAuthorized(e.to_string()))?;
-
-                    redirects_remaining -= 1;
-                    tracing::debug!(
-                        to = %current_url,
-                        hops_left = redirects_remaining,
-                        "web_fetch following redirect"
-                    );
-                    continue;
-                }
-
-                break resp;
-            }
-        };
-
-        let status = response.status().as_u16();
-
-        // Detect content type before consuming the response.
-        let content_type = response
-            .headers()
-            .get(reqwest::header::CONTENT_TYPE)
-            .and_then(|v| v.to_str().ok())
-            .unwrap_or("")
-            .to_lowercase();
-
-        // Pre-check Content-Length to reject obviously oversized responses.
-        if let Some(content_length) = response.headers().get(reqwest::header::CONTENT_LENGTH)
-            && let Ok(s) = content_length.to_str()
-            && let Ok(len) = s.parse::<usize>()
-            && len > MAX_RESPONSE_SIZE
-        {
-            return Err(ToolError::ExecutionFailed(format!(
-                "Response Content-Length ({} bytes) exceeds maximum allowed size ({} bytes)",
-                len, MAX_RESPONSE_SIZE
-            )));
-        }
-
-        // Stream body with a hard 5 MB cap.
-        let mut body: Vec<u8> = Vec::new();
-        let mut stream = response.bytes_stream();
-        while let Some(chunk) = StreamExt::next(&mut stream).await {
-            let chunk = chunk.map_err(|e| {
-                ToolError::ExternalService(format!("failed to read response body: {}", e))
-            })?;
-            if body.len() + chunk.len() > MAX_RESPONSE_SIZE {
-                return Err(ToolError::ExecutionFailed(format!(
-                    "Response body exceeds maximum allowed size ({} bytes)",
-                    MAX_RESPONSE_SIZE
-                )));
-            }
-            body.extend_from_slice(&chunk);
-        }
-
-        let raw_text = String::from_utf8_lossy(&body).into_owned();
-
-        // HTML → Markdown conversion (always attempted for HTML responses).
-        let is_html = content_type.contains("text/html");
-
-        let (content, title) = if is_html {
-            let title = extract_title(&raw_text);
-
-            #[cfg(feature = "html-to-markdown")]
-            let content = match convert_html_to_markdown(&raw_text, current_url.as_str()) {
-                Ok(md) => md,
-                Err(e) => {
-                    tracing::warn!(
-                        url = %current_url,
-                        error = %e,
-                        "HTML-to-markdown conversion failed, returning raw text"
-                    );
-                    raw_text.clone()
-                }
-            };
-
-            #[cfg(not(feature = "html-to-markdown"))]
-            let content = raw_text.clone();
-
-            (content, title)
-        } else {
-            (raw_text.clone(), None)
-        };
-
-        let word_count = content.split_whitespace().count();
-
-        let result = serde_json::json!({
-            "url": url_str,
-            "final_url": current_url.as_str(),
-            "status": status,
-            "title": title,
-            "content": content,
-            "word_count": word_count,
-        });
-
-        Ok(ToolOutput::success(result, start.elapsed()).with_raw(raw_text))
-    }
-
-    fn estimated_duration(&self, _params: &serde_json::Value) -> Option<Duration> {
-        Some(Duration::from_secs(5))
-    }
-
-    fn requires_sanitization(&self) -> bool {
-        true // External data always needs sanitization
-    }
-
-    fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement {
-        // Web fetch is always auto-approved — the SSRF/leak protections are
-        // unconditional, and reading public web pages doesn't require confirmation.
-        ApprovalRequirement::Never
-    }
-
-    fn rate_limit_config(&self) -> Option<ToolRateLimitConfig> {
-        Some(ToolRateLimitConfig::new(30, 500)) // same as http tool
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn extract_title_finds_basic_title() {
-        let html = "<html><head><title>Hello World</title></head><body></body></html>";
-        assert_eq!(extract_title(html), Some("Hello World".to_string()));
-    }
-
-    #[test]
-    fn extract_title_trims_whitespace() {
-        let html = "<html><head><title>  Spaced Title  </title></head></html>";
-        assert_eq!(extract_title(html), Some("Spaced Title".to_string()));
-    }
-
-    #[test]
-    fn extract_title_returns_none_when_absent() {
-        let html = "<html><head></head><body>No title</body></html>";
-        assert_eq!(extract_title(html), None);
-    }
-
-    #[test]
-    fn extract_title_handles_case_insensitive_tag() {
-        let html = "<html><head><TITLE>Case Test</TITLE></head></html>";
-        assert_eq!(extract_title(html), Some("Case Test".to_string()));
-    }
-
-    #[test]
-    fn extract_title_with_non_ascii_before_tag() {
-        // Turkish dotless-ı (U+0131) is 2 bytes in UTF-8 and lowercases to
-        // ASCII 'i' (1 byte).  Using to_lowercase() would shift the byte offset
-        // of '<title>' so that html[tag_start..] panics at a non-char boundary.
-        // to_ascii_lowercase() preserves byte lengths and must not panic.
-        let html = "<html><head><meta charset=\"utf-8\"/><title>ıTitle</title></head></html>";
-        let result = extract_title(html);
-        assert!(
-            result.is_some(),
-            "should extract title with non-ASCII content"
-        );
-        assert!(result.unwrap().contains("Title"));
-    }
-
-    #[test]
-    fn extract_title_with_tag_attributes() {
-        // <title lang="en"> has attributes — ensure the '>' scan still lands correctly.
-        let html = "<html><head><title lang=\"en\">Attributed</title></head></html>";
-        assert_eq!(extract_title(html), Some("Attributed".to_string()));
-    }
-
-    #[test]
-    fn web_fetch_tool_name_and_schema() {
-        let tool = WebFetchTool::new();
-        assert_eq!(tool.name(), "web_fetch");
-        let schema = tool.parameters_schema();
-        assert_eq!(schema["required"][0], "url");
-        assert_eq!(schema["properties"]["url"]["type"], "string");
-    }
-
-    #[test]
-    fn web_fetch_never_requires_approval() {
-        let tool = WebFetchTool::new();
-        let params = serde_json::json!({"url": "https://example.com"});
-        assert_eq!(tool.requires_approval(&params), ApprovalRequirement::Never);
-    }
-}

src/tools/registry.rs

@@ -20,7 +20,7 @@ use crate::tools::builtin::{
     JobStatusTool, JsonTool, ListDirTool, ListJobsTool, MemoryReadTool, MemorySearchTool,
     MemoryTreeTool, MemoryWriteTool, PromptQueue, ReadFileTool, ShellTool, SkillInstallTool,
     SkillListTool, SkillRemoveTool, SkillSearchTool, TimeTool, ToolActivateTool, ToolAuthTool,
-    ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool, WebFetchTool, WriteFileTool,
+    ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool, WriteFileTool,
 };
 use crate::tools::rate_limiter::RateLimiter;
 use crate::tools::tool::{Tool, ToolDomain};
@@ -68,7 +68,6 @@ const PROTECTED_TOOL_NAMES: &[&str] = &[
     "skill_install",
     "skill_remove",
     "message",
-    "web_fetch",
 ];
 
 /// Registry of available tools.
@@ -230,7 +229,6 @@ impl ToolRegistry {
             http = http.with_credentials(Arc::clone(cr), Arc::clone(ss));
         }
         self.register_sync(Arc::new(http));
-        self.register_sync(Arc::new(WebFetchTool::new()));
 
         tracing::info!("Registered {} built-in tools", self.count());
     }