feat: merge http/web_fetch tools, add tool output stash for large responses (#578)

* feat: merge http/web_fetch tools, add tool output stash for large responses

Merge `web_fetch` into `http` tool with smart approval: plain GETs (no
headers, no body) run without approval and follow redirects with SSRF
re-validation per hop; all other requests require approval as before.

Add `tool_output_stash` on JobContext so full tool outputs are preserved
before safety-layer truncation. The `json` tool gains a
`source_tool_call_id` parameter to reference stashed outputs, enabling
reliable parsing of large API responses that exceed the 100KB context
limit.

Other improvements:
- Descriptive User-Agent header using CARGO_PKG_VERSION
- Truncation now keeps partial data + hint about source_tool_call_id
- System prompt reinforces tool_calls over narration
- json tool query/stringify handle pre-parsed (non-string) data

[skip-regression-check]

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* chore: delete dead web_fetch.rs (merged into http tool)

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* style: fix rustfmt formatting

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* style: rename shadowed data binding for clarity in json tool

Address PR review: rename owned `data` to `data_value` before
re-binding as `let data = &data_value` to make ownership explicit.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix(ci): mark network-dependent trace tests as #[ignore]

The weather_sf and baseball_stats tests hit live external APIs (wttr.in,
ESPN) which are unreliable in CI. Mark them #[ignore] so they don't
block the pipeline. Run locally with `--ignored` to include them.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: replay recorded HTTP exchanges in trace tests instead of hitting live APIs

Wire ReplayingHttpInterceptor into TestRig when the trace fixture
contains http_exchanges. This replays recorded responses instead of
making live network calls, making tests deterministic and CI-stable.

Add captured HTTP responses to weather_sf.json (wttr.in) and
baseball_stats.json (ESPN API) fixtures.

Revert #[ignore] on both tests — they now run offline.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: recover inline bracket-format tool calls from LLM text responses

When flatten_tool_messages converts tool calls to text like
`[Called tool `http` with arguments: {...}]` for NEAR AI compatibility,
the LLM sometimes echoes this format back in its text responses instead
of using proper tool_calls. Add recovery for this bracket format in
recover_tool_calls_from_content and strip it in clean_response so
users don't see raw tool call syntax.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>

src/llm/reasoning.rs

@@ -689,6 +689,8 @@ Example:
 - If tools return empty or irrelevant results, answer with what you already know rather than retrying
 
 ## Tool Call Style
+- ALWAYS call tools via tool_calls — never just describe what you would do
+- If you say "let me fetch/check/look up X", you MUST include the actual tool call in the same response
 - Do not narrate routine, low-risk tool calls; just call the tool
 - Narrate only when it helps: multi-step work, sensitive actions, or when the user asks
 - For multi-step tasks, call independent tools in parallel when possible
@@ -1131,6 +1133,51 @@ fn recover_tool_calls_from_content(
         }
     }
 
+    // Bracket format from flatten_tool_messages:
+    // [Called tool `name` with arguments: {...}]
+    {
+        let mut remaining = content;
+        while let Some(start) = remaining.find("[Called tool `") {
+            let after_prefix = &remaining[start + "[Called tool `".len()..];
+            let Some(backtick_end) = after_prefix.find('`') else {
+                break;
+            };
+            let name = &after_prefix[..backtick_end];
+            let after_name = &after_prefix[backtick_end + 1..];
+
+            if !tool_names.contains(name) {
+                remaining = after_name;
+                continue;
+            }
+
+            // Look for " with arguments: " followed by JSON until "]"
+            if let Some(args_start) = after_name.strip_prefix(" with arguments: ") {
+                // Find the closing "]" — but the JSON itself may contain "]",
+                // so find the last "]" on this logical line.
+                if let Some(bracket_end) = args_start.rfind(']') {
+                    let args_str = &args_start[..bracket_end];
+                    let arguments = serde_json::from_str::<serde_json::Value>(args_str)
+                        .unwrap_or(serde_json::Value::Object(Default::default()));
+                    calls.push(ToolCall {
+                        id: format!("recovered_{}", calls.len()),
+                        name: name.to_string(),
+                        arguments,
+                    });
+                    remaining = &args_start[bracket_end + 1..];
+                    continue;
+                }
+            }
+
+            // No arguments or malformed — call with empty args
+            calls.push(ToolCall {
+                id: format!("recovered_{}", calls.len()),
+                name: name.to_string(),
+                arguments: serde_json::Value::Object(Default::default()),
+            });
+            remaining = after_name;
+        }
+    }
+
     calls
 }
 
@@ -1174,10 +1221,39 @@ fn clean_response(text: &str) -> String {
         result = strip_pipe_tag(&result, tag);
     }
 
+    // 6b. Strip bracket-format inline tool calls: [Called tool `name` with arguments: {...}]
+    result = strip_bracket_tool_calls(&result);
+
     // 7. Collapse triple+ newlines, trim
     collapse_newlines(&result)
 }
 
+/// Strip bracket-format inline tool calls produced by `flatten_tool_messages`.
+///
+/// Removes patterns like `[Called tool `name` with arguments: {...}]` from text
+/// so the user doesn't see raw tool call syntax when the model echoes it back.
+fn strip_bracket_tool_calls(text: &str) -> String {
+    let mut result = String::with_capacity(text.len());
+    let mut remaining = text;
+    while let Some(start) = remaining.find("[Called tool `") {
+        result.push_str(&remaining[..start]);
+        let after = &remaining[start..];
+        // Find the closing "]" for this bracket expression
+        if let Some(end) = after.find("]\n").map(|i| i + 2).or_else(|| {
+            // If it's at the end of the string, just find "]"
+            after.rfind(']').map(|i| i + 1)
+        }) {
+            remaining = &after[end..];
+        } else {
+            // Malformed — keep the rest
+            result.push_str(after);
+            return result;
+        }
+    }
+    result.push_str(remaining);
+    result
+}
+
 /// Tool-related tags stripped with simple string matching (no code-awareness needed).
 const TOOL_TAGS: &[&str] = &["tool_call", "function_call", "tool_calls"];
 
@@ -1841,4 +1917,32 @@ That's my plan."#;
         assert_eq!(calls.len(), 1);
         assert_eq!(calls[0].name, "tool_list");
     }
+
+    #[test]
+    fn test_recover_bracket_format_tool_call() {
+        let tools = make_tools(&["http"]);
+        let content = "Let me try that. [Called tool `http` with arguments: {\"method\":\"GET\",\"url\":\"https://example.com\"}]";
+        let calls = recover_tool_calls_from_content(content, &tools);
+        assert_eq!(calls.len(), 1);
+        assert_eq!(calls[0].name, "http");
+        assert_eq!(calls[0].arguments["method"], "GET");
+        assert_eq!(calls[0].arguments["url"], "https://example.com");
+    }
+
+    #[test]
+    fn test_recover_bracket_format_unknown_tool_ignored() {
+        let tools = make_tools(&["http"]);
+        let content = "[Called tool `unknown_tool` with arguments: {}]";
+        let calls = recover_tool_calls_from_content(content, &tools);
+        assert!(calls.is_empty());
+    }
+
+    #[test]
+    fn test_clean_response_strips_bracket_tool_calls() {
+        let input = "Let me fetch that.\n[Called tool `http` with arguments: {\"method\":\"GET\",\"url\":\"https://example.com\"}]\nHere are the results.";
+        let cleaned = clean_response(input);
+        assert!(!cleaned.contains("[Called tool"));
+        assert!(cleaned.contains("Let me fetch that."));
+        assert!(cleaned.contains("Here are the results."));
+    }
 }