//! HTTP request tool. use std::collections::HashMap; use std::net::{IpAddr, ToSocketAddrs}; use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; use futures::StreamExt; use reqwest::Client; use crate::context::JobContext; use crate::safety::LeakDetector; use crate::secrets::SecretsStore; use crate::tools::tool::{ApprovalRequirement, Tool, ToolError, ToolOutput, require_str}; use crate::tools::wasm::{InjectedCredentials, SharedCredentialRegistry, inject_credential}; #[cfg(feature = "html-to-markdown")] use crate::tools::builtin::convert_html_to_markdown; /// Maximum response body size (5 MB). /// /// 5 MB is large enough for typical JSON API responses and moderate HTML pages, /// but small enough to prevent OOM from malicious or runaway servers. The WASM /// HTTP wrapper uses the same limit for consistency. const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024; /// Tool for making HTTP requests. pub struct HttpTool { client: Client, credential_registry: Option>, secrets_store: Option>, } impl HttpTool { /// Create a new HTTP tool. pub fn new() -> Self { let client = Client::builder() .timeout(Duration::from_secs(30)) .redirect(reqwest::redirect::Policy::none()) .build() .expect("Failed to create HTTP client"); Self { client, credential_registry: None, secrets_store: None, } } /// Attach a credential registry and secrets store for auto-injection. pub fn with_credentials( mut self, registry: Arc, secrets_store: Arc, ) -> Self { self.credential_registry = Some(registry); self.secrets_store = Some(secrets_store); self } } fn validate_url(url: &str) -> Result { let parsed = reqwest::Url::parse(url) .map_err(|e| ToolError::InvalidParameters(format!("invalid URL: {}", e)))?; if parsed.scheme() != "https" { return Err(ToolError::NotAuthorized( "only https URLs are allowed".to_string(), )); } let host = parsed .host_str() .ok_or_else(|| ToolError::InvalidParameters("URL missing host".to_string()))?; let host_lower = host.to_lowercase(); if host_lower == "localhost" || host_lower.ends_with(".localhost") { return Err(ToolError::NotAuthorized( "localhost is not allowed".to_string(), )); } // Check literal IP addresses if let Ok(ip) = host.parse::() && is_disallowed_ip(&ip) { return Err(ToolError::NotAuthorized( "private or local IPs are not allowed".to_string(), )); } // Resolve hostname and check all resolved IPs against the blocklist. // This prevents DNS rebinding where a hostname resolves to a private IP. let port = parsed.port_or_known_default().unwrap_or(443); let socket_addr = format!("{}:{}", host, port); if let Ok(addrs) = socket_addr.to_socket_addrs() { for addr in addrs { if is_disallowed_ip(&addr.ip()) { return Err(ToolError::NotAuthorized(format!( "hostname '{}' resolves to disallowed IP {}", host, addr.ip() ))); } } } Ok(parsed) } fn is_disallowed_ip(ip: &IpAddr) -> bool { match ip { IpAddr::V4(v4) => { v4.is_private() || v4.is_loopback() || v4.is_link_local() || v4.is_multicast() || v4.is_unspecified() || *v4 == std::net::Ipv4Addr::new(169, 254, 169, 254) } IpAddr::V6(v6) => { v6.is_loopback() || v6.is_unique_local() || v6.is_unicast_link_local() || v6.is_multicast() || v6.is_unspecified() } } } #[cfg(feature = "html-to-markdown")] /// Heuristic: treat as HTML if the `Content-Type` header contains `text/html`. fn is_html_response(headers: &HashMap) -> bool { headers .iter() .find(|(k, _)| k.eq_ignore_ascii_case("content-type")) .map(|(_, v)| v.to_lowercase().contains("text/html")) .unwrap_or(false) } fn parse_headers_param( headers: Option<&serde_json::Value>, ) -> Result, ToolError> { match headers { None => Ok(Vec::new()), Some(serde_json::Value::Object(map)) => { let mut out = Vec::with_capacity(map.len()); for (k, v) in map { let value = v.as_str().ok_or_else(|| { ToolError::InvalidParameters(format!("header '{}' must have a string value", k)) })?; out.push((k.clone(), value.to_string())); } Ok(out) } Some(serde_json::Value::Array(items)) => { let mut out = Vec::with_capacity(items.len()); for (idx, item) in items.iter().enumerate() { let obj = item.as_object().ok_or_else(|| { ToolError::InvalidParameters(format!( "headers[{}] must be an object with 'name' and 'value'", idx )) })?; let name = obj.get("name").and_then(|v| v.as_str()).ok_or_else(|| { ToolError::InvalidParameters(format!("headers[{}].name must be a string", idx)) })?; let value = obj.get("value").and_then(|v| v.as_str()).ok_or_else(|| { ToolError::InvalidParameters(format!("headers[{}].value must be a string", idx)) })?; out.push((name.to_string(), value.to_string())); } Ok(out) } Some(_) => Err(ToolError::InvalidParameters( "'headers' must be an object or an array of {name, value}".to_string(), )), } } /// Extract host from URL in params (for approval checks). fn extract_host_from_params(params: &serde_json::Value) -> Option { params .get("url") .and_then(|u| u.as_str()) .and_then(|u| reqwest::Url::parse(u).ok()) .and_then(|u| u.host_str().map(|h| h.to_string())) } impl Default for HttpTool { fn default() -> Self { Self::new() } } #[async_trait] impl Tool for HttpTool { fn name(&self) -> &str { "http" } fn description(&self) -> &str { "Make HTTP requests to external APIs. Supports GET, POST, PUT, DELETE methods." } fn parameters_schema(&self) -> serde_json::Value { serde_json::json!({ "type": "object", "properties": { "method": { "type": "string", "enum": ["GET", "POST", "PUT", "DELETE", "PATCH"], "description": "HTTP method" }, "url": { "type": "string", "description": "The URL to request" }, "headers": { "type": "array", "description": "Optional headers as a list of {name, value} objects", "items": { "type": "object", "properties": { "name": { "type": "string" }, "value": { "type": "string" } }, "required": ["name", "value"], "additionalProperties": false } }, "body": { "type": ["object", "array", "string", "number", "boolean", "null"], "description": "Request body (for POST/PUT/PATCH)" }, "timeout_secs": { "type": "integer", "description": "Request timeout in seconds (default: 30)" } }, "required": ["method", "url"] }) } async fn execute( &self, params: serde_json::Value, _ctx: &JobContext, ) -> Result { let start = std::time::Instant::now(); let method = require_str(¶ms, "method")?; let url = require_str(¶ms, "url")?; let mut parsed_url = validate_url(url)?; // Parse headers let mut headers_vec = parse_headers_param(params.get("headers"))?; // Build request let mut request = match method.to_uppercase().as_str() { "GET" => self.client.get(parsed_url.clone()), "POST" => self.client.post(parsed_url.clone()), "PUT" => self.client.put(parsed_url.clone()), "DELETE" => self.client.delete(parsed_url.clone()), "PATCH" => self.client.patch(parsed_url.clone()), _ => { return Err(ToolError::InvalidParameters(format!( "unsupported method: {}", method ))); } }; // Add headers for (key, value) in &headers_vec { request = request.header(key.as_str(), value.as_str()); } // Add body if present let body_bytes = if let Some(body) = params.get("body") { if let Some(body_str) = body.as_str() { if let Ok(json_body) = serde_json::from_str::(body_str) { let bytes = serde_json::to_vec(&json_body).map_err(|e| { ToolError::InvalidParameters(format!("invalid body JSON: {}", e)) })?; request = request.json(&json_body); Some(bytes) } else { let bytes = body_str.as_bytes().to_vec(); request = request.body(body_str.to_string()); Some(bytes) } } else { let bytes = serde_json::to_vec(body).map_err(|e| { ToolError::InvalidParameters(format!("invalid body JSON: {}", e)) })?; request = request.json(body); Some(bytes) } } else { None }; // Credential injection from shared registry if let (Some(registry), Some(store)) = ( self.credential_registry.as_ref(), self.secrets_store.as_ref(), ) { let host = parsed_url.host_str().unwrap_or(""); let matched: Vec = registry.find_for_host(host); for mapping in &matched { match store .get_decrypted(&_ctx.user_id, &mapping.secret_name) .await { Ok(secret) => { let mut injected = InjectedCredentials::empty(); inject_credential(&mut injected, &mapping.location, &secret); for (name, value) in &injected.headers { request = request.header(name.as_str(), value.as_str()); headers_vec.push((name.clone(), value.clone())); } for (name, value) in &injected.query_params { parsed_url.query_pairs_mut().append_pair(name, value); request = request.query(&[(name.as_str(), value.as_str())]); } } Err(e) => { tracing::warn!( secret = %mapping.secret_name, error = %e, "Failed to inject credential for HTTP tool" ); } } } } // Leak detection on outbound request (url/headers/body) let detector = LeakDetector::new(); detector .scan_http_request(parsed_url.as_str(), &headers_vec, body_bytes.as_deref()) .map_err(|e| ToolError::NotAuthorized(format!("{}", e)))?; // Execute request let response = request.send().await.map_err(|e| { if e.is_timeout() { ToolError::Timeout(Duration::from_secs(30)) } else { ToolError::ExternalService(e.to_string()) } })?; let status = response.status().as_u16(); // Block redirects: the server tried to send us elsewhere (potential SSRF) if (300..400).contains(&status) { return Err(ToolError::NotAuthorized(format!( "request returned redirect (HTTP {}), which is blocked to prevent SSRF", status ))); } let headers: HashMap = response .headers() .iter() .filter_map(|(k, v)| v.to_str().ok().map(|v| (k.to_string(), v.to_string()))) .collect(); // Pre-check Content-Length header to reject obviously oversized responses // before downloading anything, preventing OOM from malicious servers. if let Some(content_length) = response.headers().get(reqwest::header::CONTENT_LENGTH) && let Ok(s) = content_length.to_str() && let Ok(len) = s.parse::() && len > MAX_RESPONSE_SIZE { tracing::warn!( url = %parsed_url, content_length = len, max = MAX_RESPONSE_SIZE, "Rejected HTTP response: Content-Length exceeds limit" ); return Err(ToolError::ExecutionFailed(format!( "Response Content-Length ({} bytes) exceeds maximum allowed size ({} bytes)", len, MAX_RESPONSE_SIZE ))); } // Stream the response body with a hard size cap. Even if Content-Length was // absent or lied about the size, we stop reading once we exceed the limit. let mut body = Vec::new(); let mut stream = response.bytes_stream(); while let Some(chunk) = StreamExt::next(&mut stream).await { let chunk = chunk.map_err(|e| { ToolError::ExternalService(format!("failed to read response body: {}", e)) })?; if body.len() + chunk.len() > MAX_RESPONSE_SIZE { return Err(ToolError::ExecutionFailed(format!( "Response body exceeds maximum allowed size ({} bytes)", MAX_RESPONSE_SIZE ))); } body.extend_from_slice(&chunk); } let body_bytes = bytes::Bytes::from(body); let body_text = String::from_utf8_lossy(&body_bytes).into_owned(); #[cfg(feature = "html-to-markdown")] let body_text = if is_html_response(&headers) { match convert_html_to_markdown(&body_text, parsed_url.as_str()) { Ok(md) => md, Err(e) => { tracing::warn!(url = %parsed_url, error = %e, "HTML-to-markdown conversion failed, returning raw HTML"); body_text } } } else { body_text }; // Try to parse as JSON, fall back to string let body: serde_json::Value = serde_json::from_str(&body_text) .unwrap_or_else(|_| serde_json::Value::String(body_text.clone())); let result = serde_json::json!({ "status": status, "headers": headers, "body": body }); Ok(ToolOutput::success(result, start.elapsed()).with_raw(body_text)) } fn estimated_duration(&self, _params: &serde_json::Value) -> Option { Some(Duration::from_secs(5)) // Average HTTP request time } fn requires_sanitization(&self) -> bool { true // External data always needs sanitization } fn requires_approval(&self, params: &serde_json::Value) -> ApprovalRequirement { // 1. Manual auth headers/query params in LLM params if crate::safety::params_contain_manual_credentials(params) { return ApprovalRequirement::Always; } // 2. Target host has credential mappings (will be auto-injected) if let Some(ref registry) = self.credential_registry && let Some(host) = extract_host_from_params(params) && registry.has_credentials_for_host(&host) { return ApprovalRequirement::Always; } // Default: outbound HTTP still needs approval unless auto-approved ApprovalRequirement::UnlessAutoApproved } fn rate_limit_config(&self) -> Option { Some(crate::tools::tool::ToolRateLimitConfig::new(30, 500)) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_http_tool_schema_headers_is_array() { let tool = HttpTool::new(); let schema = tool.parameters_schema(); assert_eq!(schema["properties"]["headers"]["type"], "array"); } #[test] fn test_validate_url_rejects_http() { let err = validate_url("http://example.com").unwrap_err(); assert!(err.to_string().contains("https")); } #[test] fn test_validate_url_rejects_localhost() { let err = validate_url("https://localhost:8080").unwrap_err(); assert!(err.to_string().contains("localhost")); } #[test] fn test_validate_url_accepts_https_public() { let url = validate_url("https://example.com").unwrap(); assert_eq!(url.host_str(), Some("example.com")); } #[test] fn test_validate_url_rejects_private_ip_literal() { let err = validate_url("https://192.168.1.1/api").unwrap_err(); assert!(err.to_string().contains("private")); } #[test] fn test_validate_url_rejects_loopback_ip() { let err = validate_url("https://127.0.0.1/api").unwrap_err(); assert!(err.to_string().contains("private")); } #[test] fn test_validate_url_rejects_link_local() { let err = validate_url("https://169.254.169.254/latest/meta-data/").unwrap_err(); assert!(err.to_string().contains("private")); } #[test] fn test_is_disallowed_ip_covers_ranges() { use std::net::Ipv4Addr; // Private ranges assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)))); assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(172, 16, 0, 1)))); assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(192, 168, 0, 1)))); // Loopback assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::LOCALHOST))); // Cloud metadata assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new( 169, 254, 169, 254 )))); // Public assert!(!is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)))); } #[test] fn test_max_response_size_is_reasonable() { // MAX_RESPONSE_SIZE should be 5 MB to prevent OOM while allowing typical API responses. assert_eq!(MAX_RESPONSE_SIZE, 5 * 1024 * 1024); } #[test] fn test_parse_headers_param_accepts_object_legacy_shape() { let headers = serde_json::json!({"Authorization": "Bearer token"}); let parsed = parse_headers_param(Some(&headers)).unwrap(); assert_eq!( parsed, vec![("Authorization".to_string(), "Bearer token".to_string())] ); } #[test] fn test_parse_headers_param_accepts_array_shape() { let headers = serde_json::json!([ {"name": "Authorization", "value": "Bearer token"}, {"name": "X-Test", "value": "1"} ]); let parsed = parse_headers_param(Some(&headers)).unwrap(); assert_eq!( parsed, vec![ ("Authorization".to_string(), "Bearer token".to_string()), ("X-Test".to_string(), "1".to_string()) ] ); } #[test] fn test_http_tool_schema_body_has_type() { let schema = HttpTool::new().parameters_schema(); let body = schema .get("properties") .and_then(|p| p.get("body")) .expect("body schema missing"); assert!( body.get("type").is_some(), "body schema must include a type for OpenAI-compatible tool validation" ); } // ── Approval requirement tests ────────────────────────────────────── #[test] fn test_no_auth_headers_returns_unless_auto_approved() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data" }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved ); } #[test] fn test_auth_header_object_format_returns_always() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data", "headers": {"Authorization": "Bearer token123"} }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_auth_header_array_format_returns_always() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data", "headers": [{"name": "Authorization", "value": "Bearer token123"}] }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_auth_header_case_insensitive() { let tool = HttpTool::new(); // Object format with mixed case let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": {"AUTHORIZATION": "Bearer x"} }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); // Array format with mixed case let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": [{"name": "X-Api-Key", "value": "key123"}] }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_all_auth_header_names_detected() { let tool = HttpTool::new(); for header_name in [ "authorization", "x-api-key", "cookie", "proxy-authorization", "x-auth-token", "api-key", "x-token", "x-access-token", "x-session-token", "x-csrf-token", "x-secret", "x-api-secret", ] { let mut headers = serde_json::Map::new(); headers.insert(header_name.to_string(), serde_json::json!("value")); let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": headers }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::Always, "Header '{}' should trigger Always approval", header_name ); } } #[test] fn test_non_auth_headers_return_unless_auto_approved() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": {"Content-Type": "application/json", "Accept": "text/html"} }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved ); } #[test] fn test_empty_headers_return_unless_auto_approved() { let tool = HttpTool::new(); // Empty object let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": {} }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved ); // Empty array let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": [] }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved ); } // ── Credential registry approval tests ───────────────────────────── #[test] fn test_host_with_credential_mapping_returns_always() { use crate::secrets::CredentialMapping; use crate::tools::wasm::SharedCredentialRegistry; let registry = Arc::new(SharedCredentialRegistry::new()); registry.add_mappings(vec![CredentialMapping::bearer( "openai_key", "api.openai.com", )]); let tool = HttpTool::new().with_credentials( registry, // secrets_store is not used in requires_approval, just needs to be present Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( "0123456789abcdef0123456789abcdef".to_string(), )) .unwrap(), ))), ); let params = serde_json::json!({ "method": "GET", "url": "https://api.openai.com/v1/models" }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_host_without_credential_mapping_returns_unless_auto_approved() { use crate::tools::wasm::SharedCredentialRegistry; let registry = Arc::new(SharedCredentialRegistry::new()); // Empty registry - no credential mappings let tool = HttpTool::new().with_credentials( registry, Arc::new(crate::secrets::InMemorySecretsStore::new(Arc::new( crate::secrets::SecretsCrypto::new(secrecy::SecretString::from( "0123456789abcdef0123456789abcdef".to_string(), )) .unwrap(), ))), ); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data" }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved ); } #[test] fn test_url_query_param_credential_returns_always() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data?api_key=secret123" }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_bearer_value_in_custom_header_returns_always() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": {"X-Custom": "Bearer sk-test123"} }); assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Always); } #[test] fn test_extract_host_from_params_valid() { let params = serde_json::json!({ "url": "https://api.example.com/path" }); assert_eq!( extract_host_from_params(¶ms), Some("api.example.com".to_string()) ); } #[test] fn test_extract_host_from_params_missing_url() { let params = serde_json::json!({"method": "GET"}); assert_eq!(extract_host_from_params(¶ms), None); } }