From 408ae8a29a8aa62d6708a74eba074e3502b0775f Mon Sep 17 00:00:00 2001 From: alexthebuildr <116134064+ztsalexey@users.noreply.github.com> Date: Sat, 14 Feb 2026 04:43:38 -0700 Subject: [PATCH 1/5] feat: add multi-provider LLM failover with retry backoff (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add multi-provider LLM failover Add FailoverProvider that wraps multiple LlmProvider instances and tries each in sequence on transient failures. Non-retryable errors (auth, context length, model not available) propagate immediately. - New `FailoverProvider` with generic `try_providers` helper - `is_retryable()` classifies transient errors (request failed, rate limited, invalid response, session renewal, HTTP, IO) - Configurable via `NEARAI_FALLBACK_MODEL` env var - Returns `Result` from constructor (no panics in production) - Updates FEATURE_PARITY.md: failover chains ✅, cooldown ❌ Co-Authored-By: Claude Opus 4.6 * fix: track last-used provider for accurate cost/model reporting After failover, model_name() and cost_per_token() now reflect the provider that actually handled the request, not always the primary. Also corrects is_retryable() docs to list ModelNotAvailable as retryable. Addresses PR #28 review comments. Co-Authored-By: Claude Opus 4.6 * feat: add retry with exponential backoff for LLM providers Add retry logic with exponential backoff and jitter to both NearAiProvider and NearAiChatProvider for transient errors (HTTP 429, 500, 502, 503, 504). Extract shared retry helpers (is_retryable_status, retry_backoff_delay) into src/llm/retry.rs so both providers reuse the same logic. Configurable via NEARAI_MAX_RETRIES env var (default: 3). * docs: clarify max_retries means N retries, not N total attempts * warn when fallback model equals primary model * fix: saturating_mul in backoff delay, dedupe to_lowercase allocation --------- Co-authored-by: Claude Opus 4.6 --- FEATURE_PARITY.md | 6 +- src/config.rs | 11 + src/llm/failover.rs | 483 +++++++++++++++++++++++++++++++++++++++++ src/llm/mod.rs | 34 ++- src/llm/nearai.rs | 162 +++++++++----- src/llm/nearai_chat.rs | 129 +++++++---- src/llm/retry.rs | 96 ++++++++ src/main.rs | 26 ++- src/setup/wizard.rs | 2 + 9 files changed, 840 insertions(+), 109 deletions(-) create mode 100644 src/llm/failover.rs create mode 100644 src/llm/retry.rs diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index b6265291..cda8dfd1 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -133,7 +133,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O |---------|----------|----------|-------| | Pi agent runtime | ✅ | ➖ | IronClaw uses custom runtime | | RPC-based execution | ✅ | ✅ | Orchestrator/worker pattern | -| Multi-provider failover | ✅ | ❌ | Provider fallback chains | +| Multi-provider failover | ✅ | ✅ | `FailoverProvider` tries providers sequentially on retryable errors | | Per-sender sessions | ✅ | ✅ | | | Global sessions | ✅ | ❌ | Optional shared context | | Session pruning | ✅ | ❌ | Auto cleanup old sessions | @@ -173,7 +173,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Notes | |---------|----------|----------|-------| | Auto-discovery | ✅ | ❌ | | -| Failover chains | ✅ | ❌ | Provider fallback | +| Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` | | Cooldown management | ✅ | ❌ | Skip failed providers | | Per-session model override | ✅ | ✅ | Model selector in TUI | | Model selection UI | ✅ | ✅ | TUI keyboard shortcut | @@ -419,7 +419,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O - ❌ Slack channel (real implementation) - ✅ Telegram channel (WASM, DM pairing, caption, /start) - ❌ WhatsApp channel -- ❌ Multi-provider failover +- ✅ Multi-provider failover (`FailoverProvider` with retryable error classification) - ❌ Hooks system (beforeInbound, beforeToolCall, etc.) ### P2 - Medium Priority diff --git a/src/config.rs b/src/config.rs index a6ddb158..35c89604 100644 --- a/src/config.rs +++ b/src/config.rs @@ -397,6 +397,15 @@ pub struct NearAiConfig { pub api_mode: NearAiApiMode, /// API key for cloud-api (required for chat_completions mode) pub api_key: Option, + /// Optional fallback model for failover (default: None). + /// When set, a secondary provider is created with this model and wrapped + /// in a `FailoverProvider` so transient errors on the primary model + /// automatically fall through to the fallback. + pub fallback_model: Option, + /// Maximum number of retries for transient errors (default: 3). + /// With the default of 3, the provider makes up to 4 total attempts + /// (1 initial + 3 retries) before giving up. + pub max_retries: u32, } impl LlmConfig { @@ -441,6 +450,8 @@ impl LlmConfig { .unwrap_or_else(default_session_path), api_mode, api_key: nearai_api_key, + fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?, + max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?, }; // Resolve provider-specific configs based on backend diff --git a/src/llm/failover.rs b/src/llm/failover.rs new file mode 100644 index 00000000..6bc5c239 --- /dev/null +++ b/src/llm/failover.rs @@ -0,0 +1,483 @@ +//! Multi-provider LLM failover. +//! +//! Wraps multiple LlmProvider instances and tries each in sequence +//! until one succeeds. Transparent to callers --- same LlmProvider trait. + +use std::future::Future; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use async_trait::async_trait; +use rust_decimal::Decimal; + +use crate::error::LlmError; +use crate::llm::provider::{ + CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest, + ToolCompletionResponse, +}; + +/// Returns `true` if the error is transient and the request should be retried +/// on the next provider in the failover chain. +/// +/// Retryable: `RequestFailed`, `RateLimited`, `InvalidResponse`, +/// `SessionRenewalFailed`, `ModelNotAvailable`, `Http`, `Io`. +/// +/// `ModelNotAvailable` is retryable because the next provider in the chain may +/// offer a different model, so it's worth trying. +/// +/// Non-retryable errors (`AuthFailed`, `SessionExpired`, `ContextLengthExceeded`) +/// propagate immediately because a different provider won't fix them. +fn is_retryable(err: &LlmError) -> bool { + matches!( + err, + LlmError::RequestFailed { .. } + | LlmError::RateLimited { .. } + | LlmError::InvalidResponse { .. } + | LlmError::SessionRenewalFailed { .. } + // ModelNotAvailable is retryable: the next provider may offer a different model. + | LlmError::ModelNotAvailable { .. } + | LlmError::Http(_) + | LlmError::Io(_) + ) +} + +/// An LLM provider that wraps multiple providers and tries each in sequence +/// on transient failures. +/// +/// The first provider in the list is the primary. If it fails with a retryable +/// error, the next provider is tried, and so on. Non-retryable errors +/// (e.g. `AuthFailed`, `ContextLengthExceeded`) propagate immediately. +pub struct FailoverProvider { + providers: Vec>, + /// Index of the provider that last handled a request successfully. + /// Used by `model_name()` and `cost_per_token()` so downstream cost + /// tracking reflects the provider that actually served the request. + last_used: AtomicUsize, +} + +impl FailoverProvider { + /// Create a new failover provider. + /// + /// Returns an error if `providers` is empty. + pub fn new(providers: Vec>) -> Result { + if providers.is_empty() { + return Err(LlmError::RequestFailed { + provider: "failover".to_string(), + reason: "FailoverProvider requires at least one provider".to_string(), + }); + } + Ok(Self { + providers, + last_used: AtomicUsize::new(0), + }) + } + + /// Try each provider in sequence until one succeeds or all fail. + async fn try_providers(&self, mut call: F) -> Result + where + F: FnMut(Arc) -> Fut, + Fut: Future>, + { + let mut last_error: Option = None; + + for (i, provider) in self.providers.iter().enumerate() { + let result = call(Arc::clone(provider)).await; + match result { + Ok(response) => { + self.last_used.store(i, Ordering::Relaxed); + return Ok(response); + } + Err(err) => { + if !is_retryable(&err) { + return Err(err); + } + if i + 1 < self.providers.len() { + tracing::warn!( + provider = %provider.model_name(), + error = %err, + next_provider = %self.providers[i + 1].model_name(), + "Provider failed with retryable error, trying next provider" + ); + } + last_error = Some(err); + } + } + } + + // SAFETY: providers is non-empty (checked in `new`), so at least one + // iteration ran and `last_error` is `Some`. + Err(last_error.expect("providers list is non-empty")) + } +} + +#[async_trait] +impl LlmProvider for FailoverProvider { + fn model_name(&self) -> &str { + self.providers[self.last_used.load(Ordering::Relaxed)].model_name() + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + self.providers[self.last_used.load(Ordering::Relaxed)].cost_per_token() + } + + async fn complete(&self, request: CompletionRequest) -> Result { + self.try_providers(|provider| { + let req = request.clone(); + async move { provider.complete(req).await } + }) + .await + } + + async fn complete_with_tools( + &self, + request: ToolCompletionRequest, + ) -> Result { + self.try_providers(|provider| { + let req = request.clone(); + async move { provider.complete_with_tools(req).await } + }) + .await + } + + async fn list_models(&self) -> Result, LlmError> { + let mut all_models = Vec::new(); + + for provider in &self.providers { + match provider.list_models().await { + Ok(models) => all_models.extend(models), + Err(err) => { + tracing::warn!( + provider = %provider.model_name(), + error = %err, + "Failed to list models from provider, skipping" + ); + } + } + } + + all_models.sort(); + all_models.dedup(); + Ok(all_models) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Mutex; + use std::time::Duration; + + use crate::llm::provider::{CompletionResponse, FinishReason, ToolCompletionResponse}; + + /// A mock LLM provider that returns a predetermined result. + struct MockProvider { + name: String, + input_cost: Decimal, + output_cost: Decimal, + complete_result: Mutex>>, + tool_complete_result: Mutex>>, + } + + impl MockProvider { + fn succeeding(name: &str, content: &str) -> Self { + Self { + name: name.to_string(), + input_cost: Decimal::ZERO, + output_cost: Decimal::ZERO, + complete_result: Mutex::new(Some(Ok(CompletionResponse { + content: content.to_string(), + input_tokens: 10, + output_tokens: 5, + finish_reason: FinishReason::Stop, + response_id: None, + }))), + tool_complete_result: Mutex::new(Some(Ok(ToolCompletionResponse { + content: Some(content.to_string()), + tool_calls: vec![], + input_tokens: 10, + output_tokens: 5, + finish_reason: FinishReason::Stop, + response_id: None, + }))), + } + } + + fn succeeding_with_cost( + name: &str, + content: &str, + input_cost: Decimal, + output_cost: Decimal, + ) -> Self { + Self { + input_cost, + output_cost, + ..Self::succeeding(name, content) + } + } + + fn failing_retryable(name: &str) -> Self { + Self { + name: name.to_string(), + input_cost: Decimal::ZERO, + output_cost: Decimal::ZERO, + complete_result: Mutex::new(Some(Err(LlmError::RequestFailed { + provider: name.to_string(), + reason: "server error".to_string(), + }))), + tool_complete_result: Mutex::new(Some(Err(LlmError::RequestFailed { + provider: name.to_string(), + reason: "server error".to_string(), + }))), + } + } + + fn failing_non_retryable(name: &str) -> Self { + Self { + name: name.to_string(), + input_cost: Decimal::ZERO, + output_cost: Decimal::ZERO, + complete_result: Mutex::new(Some(Err(LlmError::AuthFailed { + provider: name.to_string(), + }))), + tool_complete_result: Mutex::new(Some(Err(LlmError::AuthFailed { + provider: name.to_string(), + }))), + } + } + + fn failing_rate_limited(name: &str) -> Self { + Self { + name: name.to_string(), + input_cost: Decimal::ZERO, + output_cost: Decimal::ZERO, + complete_result: Mutex::new(Some(Err(LlmError::RateLimited { + provider: name.to_string(), + retry_after: Some(Duration::from_secs(30)), + }))), + tool_complete_result: Mutex::new(Some(Err(LlmError::RateLimited { + provider: name.to_string(), + retry_after: Some(Duration::from_secs(30)), + }))), + } + } + } + + #[async_trait] + impl LlmProvider for MockProvider { + fn model_name(&self) -> &str { + &self.name + } + + fn cost_per_token(&self) -> (Decimal, Decimal) { + (self.input_cost, self.output_cost) + } + + async fn complete( + &self, + _request: CompletionRequest, + ) -> Result { + self.complete_result + .lock() + .unwrap() + .take() + .expect("MockProvider::complete called more than once") + } + + async fn complete_with_tools( + &self, + _request: ToolCompletionRequest, + ) -> Result { + self.tool_complete_result + .lock() + .unwrap() + .take() + .expect("MockProvider::complete_with_tools called more than once") + } + + async fn list_models(&self) -> Result, LlmError> { + Ok(vec![self.name.clone()]) + } + } + + fn make_request() -> CompletionRequest { + CompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")]) + } + + fn make_tool_request() -> ToolCompletionRequest { + ToolCompletionRequest::new(vec![crate::llm::ChatMessage::user("hello")], vec![]) + } + + // Test 1: Primary succeeds, no failover occurs. + #[tokio::test] + async fn primary_succeeds_no_failover() { + let primary = Arc::new(MockProvider::succeeding("primary", "primary response")); + let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response")); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + let response = failover.complete(make_request()).await.unwrap(); + assert_eq!(response.content, "primary response"); + } + + // Test 2: Primary fails with retryable error, fallback succeeds. + #[tokio::test] + async fn primary_fails_retryable_fallback_succeeds() { + let primary = Arc::new(MockProvider::failing_retryable("primary")); + let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response")); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + let response = failover.complete(make_request()).await.unwrap(); + assert_eq!(response.content, "fallback response"); + } + + // Test 3: All providers fail, returns last error. + #[tokio::test] + async fn all_providers_fail_returns_last_error() { + let primary = Arc::new(MockProvider::failing_retryable("primary")); + let fallback = Arc::new(MockProvider::failing_retryable("fallback")); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + let err = failover.complete(make_request()).await.unwrap_err(); + match err { + LlmError::RequestFailed { provider, .. } => { + assert_eq!(provider, "fallback"); + } + other => panic!("expected RequestFailed, got: {other:?}"), + } + } + + // Test 4: Non-retryable error fails immediately, no failover. + #[tokio::test] + async fn non_retryable_error_fails_immediately() { + let primary = Arc::new(MockProvider::failing_non_retryable("primary")); + let fallback = Arc::new(MockProvider::succeeding("fallback", "fallback response")); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + let err = failover.complete(make_request()).await.unwrap_err(); + match err { + LlmError::AuthFailed { provider } => { + assert_eq!(provider, "primary"); + } + other => panic!("expected AuthFailed, got: {other:?}"), + } + } + + // Test 5: Three providers, first two fail (retryable), third succeeds. + #[tokio::test] + async fn three_providers_first_two_fail_third_succeeds() { + let p1 = Arc::new(MockProvider::failing_retryable("provider-1")); + let p2 = Arc::new(MockProvider::failing_rate_limited("provider-2")); + let p3 = Arc::new(MockProvider::succeeding("provider-3", "third time lucky")); + + let failover = FailoverProvider::new(vec![p1, p2, p3]).unwrap(); + + let response = failover.complete(make_request()).await.unwrap(); + assert_eq!(response.content, "third time lucky"); + } + + // Test: complete_with_tools follows same failover logic. + #[tokio::test] + async fn complete_with_tools_failover() { + let primary = Arc::new(MockProvider::failing_retryable("primary")); + let fallback = Arc::new(MockProvider::succeeding("fallback", "tools fallback")); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + let response = failover + .complete_with_tools(make_tool_request()) + .await + .unwrap(); + assert_eq!(response.content.as_deref(), Some("tools fallback")); + } + + // Test: model_name and cost_per_token reflect the last-used provider. + #[tokio::test] + async fn model_name_and_cost_track_last_used_provider() { + let fallback_cost = Decimal::new(15, 6); // 0.000015 + + let primary = Arc::new(MockProvider::failing_retryable("primary-model")); + let fallback = Arc::new(MockProvider::succeeding_with_cost( + "fallback-model", + "ok", + fallback_cost, + fallback_cost, + )); + + let failover = FailoverProvider::new(vec![primary, fallback]).unwrap(); + + // Before any call, defaults to primary (index 0). + assert_eq!(failover.model_name(), "primary-model"); + assert_eq!(failover.cost_per_token(), (Decimal::ZERO, Decimal::ZERO)); + + // After failover, should reflect the fallback provider. + let _ = failover.complete(make_request()).await.unwrap(); + assert_eq!(failover.model_name(), "fallback-model"); + assert_eq!(failover.cost_per_token(), (fallback_cost, fallback_cost)); + } + + // Test: list_models aggregates from all providers. + #[tokio::test] + async fn list_models_aggregates_all() { + let p1 = Arc::new(MockProvider::succeeding("model-a", "ok")); + let p2 = Arc::new(MockProvider::succeeding("model-b", "ok")); + + let failover = FailoverProvider::new(vec![p1, p2]).unwrap(); + + let models = failover.list_models().await.unwrap(); + assert!(models.contains(&"model-a".to_string())); + assert!(models.contains(&"model-b".to_string())); + } + + // Test: is_retryable correctly classifies errors. + #[test] + fn retryable_classification() { + // Retryable + assert!(is_retryable(&LlmError::RequestFailed { + provider: "p".into(), + reason: "err".into(), + })); + assert!(is_retryable(&LlmError::RateLimited { + provider: "p".into(), + retry_after: None, + })); + assert!(is_retryable(&LlmError::InvalidResponse { + provider: "p".into(), + reason: "bad json".into(), + })); + assert!(is_retryable(&LlmError::SessionRenewalFailed { + provider: "p".into(), + reason: "timeout".into(), + })); + assert!(is_retryable(&LlmError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset" + )))); + assert!(is_retryable(&LlmError::ModelNotAvailable { + provider: "p".into(), + model: "m".into(), + })); + + // Non-retryable + assert!(!is_retryable(&LlmError::AuthFailed { + provider: "p".into(), + })); + assert!(!is_retryable(&LlmError::SessionExpired { + provider: "p".into(), + })); + assert!(!is_retryable(&LlmError::ContextLengthExceeded { + used: 100_000, + limit: 50_000, + })); + } + + // Test: empty providers list returns error (not panic). + #[test] + fn empty_providers_returns_error() { + let result = FailoverProvider::new(vec![]); + assert!(result.is_err()); + } +} diff --git a/src/llm/mod.rs b/src/llm/mod.rs index ee801df7..95b9981a 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -8,13 +8,16 @@ //! - **OpenAI-compatible**: Any endpoint that speaks the OpenAI API mod costs; +pub mod failover; mod nearai; mod nearai_chat; mod provider; mod reasoning; +mod retry; mod rig_adapter; pub mod session; +pub use failover::FailoverProvider; pub use nearai::{ModelInfo, NearAiProvider}; pub use nearai_chat::NearAiChatProvider; pub use provider::{ @@ -33,7 +36,7 @@ use std::sync::Arc; use rig::client::CompletionClient; use secrecy::ExposeSecret; -use crate::config::{LlmBackend, LlmConfig, NearAiApiMode}; +use crate::config::{LlmBackend, LlmConfig, NearAiApiMode, NearAiConfig}; use crate::error::LlmError; /// Create an LLM provider based on configuration. @@ -46,7 +49,7 @@ pub fn create_llm_provider( session: Arc, ) -> Result, LlmError> { match config.backend { - LlmBackend::NearAi => create_nearai_provider(config, session), + LlmBackend::NearAi => create_llm_provider_with_config(&config.nearai, session), LlmBackend::OpenAi => create_openai_provider(config), LlmBackend::Anthropic => create_anthropic_provider(config), LlmBackend::Ollama => create_ollama_provider(config), @@ -54,21 +57,28 @@ pub fn create_llm_provider( } } -fn create_nearai_provider( - config: &LlmConfig, +/// Create an LLM provider from a `NearAiConfig` directly. +/// +/// This is useful when constructing additional providers for failover, +/// where only the model name differs from the primary config. +pub fn create_llm_provider_with_config( + config: &NearAiConfig, session: Arc, ) -> Result, LlmError> { - match config.nearai.api_mode { + match config.api_mode { NearAiApiMode::Responses => { - tracing::info!("Using NEAR AI Responses API (chat-api) with session auth"); - Ok(Arc::new(NearAiProvider::new( - config.nearai.clone(), - session, - ))) + tracing::info!( + model = %config.model, + "Using Responses API (chat-api) with session auth" + ); + Ok(Arc::new(NearAiProvider::new(config.clone(), session))) } NearAiApiMode::ChatCompletions => { - tracing::info!("Using NEAR AI Chat Completions API (cloud-api) with API key auth"); - Ok(Arc::new(NearAiChatProvider::new(config.nearai.clone())?)) + tracing::info!( + model = %config.model, + "Using Chat Completions API (cloud-api) with API key auth" + ); + Ok(Arc::new(NearAiChatProvider::new(config.clone())?)) } } } diff --git a/src/llm/nearai.rs b/src/llm/nearai.rs index ed32d2fc..70c966e0 100644 --- a/src/llm/nearai.rs +++ b/src/llm/nearai.rs @@ -19,6 +19,7 @@ use crate::llm::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, }; +use crate::llm::retry::{is_retryable_status, retry_backoff_delay}; use crate::llm::session::SessionManager; /// Information about an available model from NEAR AI API. @@ -270,88 +271,139 @@ impl NearAiProvider { } } - /// Inner request implementation without retry logic. + /// Inner request implementation with retry logic for transient errors. + /// + /// Retries on HTTP 429, 500, 502, 503, 504 with exponential backoff. + /// Does not retry on client errors (400, 401, 403, 404) or parse errors. async fn send_request_inner Deserialize<'de>>( &self, path: &str, body: &T, ) -> Result { let url = self.api_url(path); - let token = self.session.get_token().await?; + let max_retries = self.config.max_retries; - tracing::debug!("Sending request to NEAR AI: {}", url); - tracing::debug!("Request body: {:?}", body); + for attempt in 0..=max_retries { + let token = self.session.get_token().await?; - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", token.expose_secret())) - .header("Content-Type", "application/json") - .json(body) - .send() - .await - .map_err(|e| { - tracing::error!("NEAR AI request failed: {}", e); - e - })?; + tracing::debug!( + "Sending request to NEAR AI: {} (attempt {})", + url, + attempt + 1 + ); + tracing::debug!("Request body: {:?}", body); - let status = response.status(); - let response_text = response.text().await.unwrap_or_default(); + let response = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", token.expose_secret())) + .header("Content-Type", "application/json") + .json(body) + .send() + .await; - tracing::debug!("NEAR AI response status: {}", status); - tracing::debug!("NEAR AI response body: {}", response_text); + let response = match response { + Ok(r) => r, + Err(e) => { + tracing::error!("NEAR AI request failed: {}", e); + // Network errors (timeout, connection refused) are transient + if attempt < max_retries { + let delay = retry_backoff_delay(attempt); + tracing::warn!( + "NEAR AI request error (attempt {}/{}), retrying in {:?}: {}", + attempt + 1, + max_retries + 1, + delay, + e, + ); + tokio::time::sleep(delay).await; + continue; + } + return Err(e.into()); + } + }; - if !status.is_success() { - // Check for session expiration (401 with specific message patterns) - if status.as_u16() == 401 { - let is_session_expired = response_text.to_lowercase().contains("session") - && (response_text.to_lowercase().contains("expired") - || response_text.to_lowercase().contains("invalid")); + let status = response.status(); + let response_text = response.text().await.unwrap_or_default(); - if is_session_expired { - return Err(LlmError::SessionExpired { + tracing::debug!("NEAR AI response status: {}", status); + tracing::debug!("NEAR AI response body: {}", response_text); + + if !status.is_success() { + let status_code = status.as_u16(); + + // Check for session expiration (401 with specific message patterns) + if status_code == 401 { + let lower = response_text.to_lowercase(); + let is_session_expired = lower.contains("session") + && (lower.contains("expired") || lower.contains("invalid")); + + if is_session_expired { + return Err(LlmError::SessionExpired { + provider: "nearai".to_string(), + }); + } + + // Generic 401 -- not retryable + return Err(LlmError::AuthFailed { provider: "nearai".to_string(), }); } - // Generic 401 without session expiration indication - return Err(LlmError::AuthFailed { - provider: "nearai".to_string(), - }); - } + // Check if this is a transient error worth retrying + if is_retryable_status(status_code) && attempt < max_retries { + let delay = retry_backoff_delay(attempt); + tracing::warn!( + "NEAR AI returned HTTP {} (attempt {}/{}), retrying in {:?}", + status_code, + attempt + 1, + max_retries + 1, + delay, + ); + tokio::time::sleep(delay).await; + continue; + } - // Try to parse as JSON error - if let Ok(error) = serde_json::from_str::(&response_text) { - if status.as_u16() == 429 { - return Err(LlmError::RateLimited { + // Non-retryable error or exhausted retries + if let Ok(error) = serde_json::from_str::(&response_text) { + if status_code == 429 { + return Err(LlmError::RateLimited { + provider: "nearai".to_string(), + retry_after: None, + }); + } + return Err(LlmError::RequestFailed { provider: "nearai".to_string(), - retry_after: None, + reason: error.error, }); } + return Err(LlmError::RequestFailed { provider: "nearai".to_string(), - reason: error.error, + reason: format!("HTTP {}: {}", status, response_text), }); } - return Err(LlmError::RequestFailed { - provider: "nearai".to_string(), - reason: format!("HTTP {}: {}", status, response_text), - }); + // Success -- parse the response + return match serde_json::from_str::(&response_text) { + Ok(parsed) => Ok(parsed), + Err(e) => { + tracing::debug!("Response is not expected JSON format: {}", e); + tracing::debug!("Will try alternative parsing in caller"); + Err(LlmError::InvalidResponse { + provider: "nearai".to_string(), + reason: format!("Parse error: {}. Raw: {}", e, response_text), + }) + } + }; } - // Try to parse as our expected type - match serde_json::from_str::(&response_text) { - Ok(parsed) => Ok(parsed), - Err(e) => { - tracing::debug!("Response is not expected JSON format: {}", e); - tracing::debug!("Will try alternative parsing in caller"); - Err(LlmError::InvalidResponse { - provider: "nearai".to_string(), - reason: format!("Parse error: {}. Raw: {}", e, response_text), - }) - } - } + // This is unreachable because the loop always returns, but the compiler + // cannot prove that. Return a generic error as a safety net. + Err(LlmError::RequestFailed { + provider: "nearai".to_string(), + reason: "retry loop exited unexpectedly".to_string(), + }) } } diff --git a/src/llm/nearai_chat.rs b/src/llm/nearai_chat.rs index 13a36a91..dbe51bc7 100644 --- a/src/llm/nearai_chat.rs +++ b/src/llm/nearai_chat.rs @@ -16,6 +16,7 @@ use crate::llm::provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, }; +use crate::llm::retry::{is_retryable_status, retry_backoff_delay}; /// NEAR AI Chat Completions API provider. pub struct NearAiChatProvider { @@ -62,64 +63,116 @@ impl NearAiChatProvider { .unwrap_or_default() } - /// Send a request to the chat completions API. + /// Send a request to the chat completions API with retry on transient errors. + /// + /// Retries on HTTP 429, 500, 502, 503, 504 with exponential backoff. + /// Does not retry on client errors (400, 401, 403, 404) or parse errors. async fn send_request Deserialize<'de>>( &self, body: &T, ) -> Result { let url = self.api_url("chat/completions"); + let max_retries = self.config.max_retries; - tracing::debug!("Sending request to NEAR AI Chat: {}", url); + for attempt in 0..=max_retries { + tracing::debug!( + "Sending request to NEAR AI Chat: {} (attempt {})", + url, + attempt + 1, + ); - if tracing::enabled!(tracing::Level::DEBUG) - && let Ok(json) = serde_json::to_string(body) - { - tracing::debug!("NEAR AI Chat request body: {}", json); - } + if tracing::enabled!(tracing::Level::DEBUG) + && let Ok(json) = serde_json::to_string(body) + { + tracing::debug!("NEAR AI Chat request body: {}", json); + } - let response = self - .client - .post(&url) - .header("Authorization", format!("Bearer {}", self.api_key())) - .header("Content-Type", "application/json") - .json(body) - .send() - .await - .map_err(|e| { - tracing::error!("NEAR AI Chat request failed: {}", e); - LlmError::RequestFailed { - provider: "nearai_chat".to_string(), - reason: e.to_string(), + let response = self + .client + .post(&url) + .header("Authorization", format!("Bearer {}", self.api_key())) + .header("Content-Type", "application/json") + .json(body) + .send() + .await; + + let response = match response { + Ok(r) => r, + Err(e) => { + tracing::error!("NEAR AI Chat request failed: {}", e); + if attempt < max_retries { + let delay = retry_backoff_delay(attempt); + tracing::warn!( + "NEAR AI Chat request error (attempt {}/{}), retrying in {:?}: {}", + attempt + 1, + max_retries + 1, + delay, + e, + ); + tokio::time::sleep(delay).await; + continue; + } + return Err(LlmError::RequestFailed { + provider: "nearai_chat".to_string(), + reason: e.to_string(), + }); } - })?; + }; - let status = response.status(); - let response_text = response.text().await.unwrap_or_default(); + let status = response.status(); + let response_text = response.text().await.unwrap_or_default(); - tracing::debug!("NEAR AI Chat response status: {}", status); - tracing::debug!("NEAR AI Chat response body: {}", response_text); + tracing::debug!("NEAR AI Chat response status: {}", status); + tracing::debug!("NEAR AI Chat response body: {}", response_text); - if !status.is_success() { - if status.as_u16() == 401 { - return Err(LlmError::AuthFailed { + if !status.is_success() { + let status_code = status.as_u16(); + + // Auth errors are not retryable + if status_code == 401 { + return Err(LlmError::AuthFailed { + provider: "nearai_chat".to_string(), + }); + } + + // Transient errors: retry with backoff + if is_retryable_status(status_code) && attempt < max_retries { + let delay = retry_backoff_delay(attempt); + tracing::warn!( + "NEAR AI Chat returned HTTP {} (attempt {}/{}), retrying in {:?}", + status_code, + attempt + 1, + max_retries + 1, + delay, + ); + tokio::time::sleep(delay).await; + continue; + } + + // Non-retryable or exhausted retries + if status_code == 429 { + return Err(LlmError::RateLimited { + provider: "nearai_chat".to_string(), + retry_after: None, + }); + } + return Err(LlmError::RequestFailed { provider: "nearai_chat".to_string(), + reason: format!("HTTP {}: {}", status, response_text), }); } - if status.as_u16() == 429 { - return Err(LlmError::RateLimited { - provider: "nearai_chat".to_string(), - retry_after: None, - }); - } - return Err(LlmError::RequestFailed { + + // Success — parse the response + return serde_json::from_str(&response_text).map_err(|e| LlmError::InvalidResponse { provider: "nearai_chat".to_string(), - reason: format!("HTTP {}: {}", status, response_text), + reason: format!("JSON parse error: {}. Raw: {}", e, response_text), }); } - serde_json::from_str(&response_text).map_err(|e| LlmError::InvalidResponse { + // Safety net: unreachable because the loop always returns + Err(LlmError::RequestFailed { provider: "nearai_chat".to_string(), - reason: format!("JSON parse error: {}. Raw: {}", e, response_text), + reason: "retry loop exited unexpectedly".to_string(), }) } diff --git a/src/llm/retry.rs b/src/llm/retry.rs new file mode 100644 index 00000000..2dced0b0 --- /dev/null +++ b/src/llm/retry.rs @@ -0,0 +1,96 @@ +//! Shared retry helpers for LLM providers. +//! +//! Provides exponential backoff with jitter and retryable status classification +//! used by both `NearAiProvider` and `NearAiChatProvider`. + +use std::time::Duration; + +use rand::Rng; + +/// Returns `true` if the HTTP status code is transient and worth retrying. +pub(crate) fn is_retryable_status(status: u16) -> bool { + matches!(status, 429 | 500 | 502 | 503 | 504) +} + +/// Calculate exponential backoff delay with random jitter. +/// +/// Base delay is 1 second, doubled each attempt, with +/-25% jitter. +/// - attempt 0: ~1s (0.75s - 1.25s) +/// - attempt 1: ~2s (1.5s - 2.5s) +/// - attempt 2: ~4s (3.0s - 5.0s) +pub(crate) fn retry_backoff_delay(attempt: u32) -> Duration { + let base_ms: u64 = 1000u64.saturating_mul(2u64.saturating_pow(attempt)); + let jitter_range = base_ms / 4; // 25% + let jitter = if jitter_range > 0 { + let offset = rand::thread_rng().gen_range(0..=jitter_range * 2); + offset as i64 - jitter_range as i64 + } else { + 0 + }; + let delay_ms = (base_ms as i64 + jitter).max(100) as u64; + Duration::from_millis(delay_ms) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_retryable_status() { + // Transient errors should be retryable + assert!(is_retryable_status(429)); + assert!(is_retryable_status(500)); + assert!(is_retryable_status(502)); + assert!(is_retryable_status(503)); + assert!(is_retryable_status(504)); + + // Client errors should not be retryable + assert!(!is_retryable_status(400)); + assert!(!is_retryable_status(401)); + assert!(!is_retryable_status(403)); + assert!(!is_retryable_status(404)); + assert!(!is_retryable_status(422)); + + // Success codes should not be retryable + assert!(!is_retryable_status(200)); + assert!(!is_retryable_status(201)); + } + + #[test] + fn test_retry_backoff_delay_exponential_growth() { + // Run multiple samples to verify the range, accounting for jitter + for _ in 0..20 { + let d0 = retry_backoff_delay(0); + let d1 = retry_backoff_delay(1); + let d2 = retry_backoff_delay(2); + + // Attempt 0: base 1000ms, jitter +/-250ms -> [750, 1250] + assert!(d0.as_millis() >= 750, "attempt 0 too low: {:?}", d0); + assert!(d0.as_millis() <= 1250, "attempt 0 too high: {:?}", d0); + + // Attempt 1: base 2000ms, jitter +/-500ms -> [1500, 2500] + assert!(d1.as_millis() >= 1500, "attempt 1 too low: {:?}", d1); + assert!(d1.as_millis() <= 2500, "attempt 1 too high: {:?}", d1); + + // Attempt 2: base 4000ms, jitter +/-1000ms -> [3000, 5000] + assert!(d2.as_millis() >= 3000, "attempt 2 too low: {:?}", d2); + assert!(d2.as_millis() <= 5000, "attempt 2 too high: {:?}", d2); + } + } + + #[test] + fn test_retry_backoff_delay_minimum() { + // Even at attempt 0, delay should be at least 100ms (the minimum floor) + for _ in 0..20 { + let delay = retry_backoff_delay(0); + assert!(delay.as_millis() >= 100); + } + } + + #[test] + fn test_retry_backoff_delay_no_overflow() { + // Very high attempt numbers should not panic from overflow + let delay = retry_backoff_delay(30); + assert!(delay.as_millis() >= 100); + } +} diff --git a/src/main.rs b/src/main.rs index 97355aaa..ca9754ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,10 @@ use ironclaw::{ config::Config, context::ContextManager, extensions::ExtensionManager, - llm::{SessionConfig, create_llm_provider, create_session_manager}, + llm::{ + FailoverProvider, LlmProvider, SessionConfig, create_llm_provider, + create_llm_provider_with_config, create_session_manager, + }, orchestrator::{ ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore, api::OrchestratorState, @@ -447,6 +450,27 @@ async fn main() -> anyhow::Result<()> { let llm = create_llm_provider(&config.llm, session.clone())?; tracing::info!("LLM provider initialized: {}", llm.model_name()); + // Wrap in failover if a fallback model is configured + let llm: Arc = + if let Some(fallback_model) = config.llm.nearai.fallback_model.as_ref() { + if fallback_model == &config.llm.nearai.model { + tracing::warn!( + "fallback_model is the same as primary model, failover may not be effective" + ); + } + let mut fallback_config = config.llm.nearai.clone(); + fallback_config.model = fallback_model.clone(); + let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?; + tracing::info!( + primary = %llm.model_name(), + fallback = %fallback.model_name(), + "LLM failover enabled" + ); + Arc::new(FailoverProvider::new(vec![llm, fallback])?) + } else { + llm + }; + // Initialize safety layer let safety = Arc::new(SafetyLayer::new(&config.safety)); tracing::info!("Safety layer initialized"); diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 696a7a08..16be4d30 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -653,6 +653,8 @@ impl SetupWizard { session_path: crate::llm::session::default_session_path(), api_mode: crate::config::NearAiApiMode::Responses, api_key: None, + fallback_model: None, + max_retries: 3, }, openai: None, anthropic: None, From a53b2c10b569de297ef3178e029b86df256c3763 Mon Sep 17 00:00:00 2001 From: Illia Polosukhin Date: Sat, 14 Feb 2026 13:21:22 -0800 Subject: [PATCH 2/5] fix: Fix wasm tool schemas and runtime (#42) * feat: Move debug log truncation from agent loop to REPL channel Full tool output now flows through StatusUpdate so the web gateway gets untruncated content. The REPL channel truncates at display time (200 chars for tool results, thinking, and status messages). Co-Authored-By: Claude Opus 4.6 * fix: Flatten WASM tool schemas and fix host HTTP runtime contention LLMs can't reliably follow oneOf + const discriminator patterns in JSON Schema, causing tools like Google Calendar to receive malformed params (e.g., {"operation":"list_events","data":{"calendarId":"primary"}} instead of {"action":"list_events","calendar_id":"primary"}). Replace all 9 WASM tool schemas with flat action enum + top-level properties. The serde #[serde(tag = "action")] deserialization works identically. Also fixes WASM host HTTP requests (channels and tools) stalling during startup by replacing Handle::current().block_on() with a dedicated single-threaded runtime per request, avoiding I/O driver contention. Reduces verbose LLM debug logging (full request/response payloads) and changes tower_http default from debug to warn. Co-Authored-By: Claude Opus 4.6 * feat: Built-in OAuth credentials and combined Google scopes Add infrastructure for shipping default OAuth credentials with the binary, similar to how gcloud/rclone bake in their client_id. Credentials are set at compile time via IRONCLAW_GOOGLE_CLIENT_ID / IRONCLAW_GOOGLE_CLIENT_SECRET env vars, or can be hardcoded in src/cli/oauth_defaults.rs. The fallback chain is: capabilities file > runtime env var > built-in defaults. Also, when authing any Google tool, scopes from ALL installed Google tools are now combined into a single OAuth request (they all share the same google_oauth_token secret). One login covers Gmail, Calendar, Drive, etc. Co-Authored-By: Claude Opus 4.6 * feat: Ship default Google OAuth credentials for zero-config auth Google Desktop App credentials are not secret (per Google's own docs). Hardcode them so `ironclaw tool auth ` works out of the box without requiring users to register their own OAuth app. Credentials can still be overridden at compile time (IRONCLAW_GOOGLE_CLIENT_ID) or runtime (GOOGLE_OAUTH_CLIENT_ID). Co-Authored-By: Claude Opus 4.6 * fix: Consistent OAuth callback port and polished landing page - Use fixed port 9876 instead of scanning 9876-9886 (one redirect URI to register in provider OAuth apps, deterministic behavior) - Replace broken unicode checkmark with SVG icons (charset was missing, rendered as mojibake) - Dark themed landing page with proper card layout for both success and error states - Add charset=utf-8 to Content-Type headers Co-Authored-By: Claude Opus 4.6 * refactor: Unify OAuth callback server across all auth flows All three OAuth flows (WASM tool auth, MCP server auth, NEAR AI login) now share the same code from cli::oauth_defaults: - Fixed port 9876 (one redirect URI to register per provider) - Shared landing page HTML (dark card with SVG icons, proper charset) - Parameterized wait_for_callback(listener, path, param, display_name) Removes ~120 lines of duplicated callback/HTML code. Co-Authored-By: Claude Opus 4.6 * Support for oauth token refresh * refactor: Replace bootstrap.json with ~/.ironclaw/.env for DATABASE_URL Kill the 4-field BootstrapConfig JSON file. Only DATABASE_URL actually needs disk persistence (chicken-and-egg before DB connect). The other three fields are now derived: pool_size defaults to 10 via env var, secrets master key is auto-detected (env then keychain probe), and onboard_completed is inferred from DATABASE_URL presence. The new format is a standard .env file loaded via dotenvy early in main, so DATABASE_URL is available as a regular env var everywhere. Handles three upgrade paths: - Clean start: wizard writes .env, reload after wizard completes - Returning user: .env loaded at startup, business as usual - Legacy upgrade: bootstrap.json auto-migrated to .env on first run Co-Authored-By: Claude Opus 4.6 * fix: Address PR review findings - Fix UTF-8 panic in truncate_for_preview (byte-slice on char boundary) - Cap WASM guest timeout_ms at 5 minutes to prevent resource exhaustion - Fix localhost detection in requires_auth() to avoid substring matches (e.g. "notlocalhost.com" no longer matches) - Fix query param injection to insert before URL fragment - Fix extract_host_from_url for IPv6 bracket notation - Remove misleading schema defaults: Slack limit, Slides insertion_index, Docs index (per-action defaults documented in descriptions instead) Co-Authored-By: Claude Opus 4.6 * style: Fix cargo fmt formatting Co-Authored-By: Claude Opus 4.6 * fix: IPv6 loopback support for OAuth listener and localhost detection - bind_callback_listener: try [::1] first, fall back to 127.0.0.1, so OAuth redirects work on systems where localhost resolves to ::1 - is_localhost_url: replace manual string parsing with url::Url for correct handling of IPv6 brackets, ports, userinfo, etc. - Add url crate as direct dependency (already a transitive dep) Co-Authored-By: Claude Opus 4.6 * fix: Address PR review feedback on runtime reuse, onboard check, and OAuth binding - Remove session file check from check_onboard_needed(); DATABASE_URL is sufficient - Detect AddrInUse on IPv6 bind and fail immediately instead of falling through to IPv4 - Reuse dedicated tokio runtime across HTTP calls in both tool and channel WASM wrappers Co-Authored-By: Claude Opus 4.6 * fix: HTML-escape provider name in OAuth landing page, simplify Slack limit description - Add html_escape() to prevent XSS in landing_html() where provider_name was interpolated directly into HTML (defense-in-depth, source is trusted but escaping costs nothing) - Remove per-action default numbers from Slack limit field description to avoid confusing LLMs with conflicting defaults Addresses review feedback from zmanian on PR #42. Co-Authored-By: Claude Opus 4.6 * fix: Save all bootstrap fields from wizard, fix config module comment - Wizard now saves secrets_master_key_source and database_pool_size to bootstrap.json (was only saving database_url and onboard_completed, which broke secrets after fresh onboard since SecretsConfig::resolve reads key source from bootstrap) - Update config.rs module doc to reflect bootstrap.json priority chain instead of the removed ~/.ironclaw/.env approach Co-Authored-By: Claude Opus 4.6 * refactor: Replace BootstrapConfig with .env-based bootstrap DATABASE_URL is the only setting that needs disk persistence before the database is available. Instead of a custom bootstrap.json with 4 fields, use a standard ~/.ironclaw/.env file loaded via dotenvy. - Remove BootstrapConfig struct entirely - Restore ironclaw_env_path(), load_ironclaw_env(), save_database_url() - SecretsConfig::resolve() now auto-detects (env var then keychain probe) instead of reading a saved source from bootstrap.json - DatabaseConfig::resolve() reads DATABASE_URL from env only (dotenvy loads ~/.ironclaw/.env into the environment early in startup) - check_onboard_needed() is now sync (just checks env vars) - Wizard save_and_summarize() works for both postgres and libsql backends - One-time migration from bootstrap.json to .env preserved Co-Authored-By: Claude Opus 4.6 * fix: Ensure load_ironclaw_env() runs in all Config paths, fix .env priority - Config::from_env() and Config::from_db() now call load_ironclaw_env() internally (after dotenvy::dotenv()), so CLI commands like `memory` and `config` correctly load DATABASE_URL from ~/.ironclaw/.env - Fix load order: standard ./.env first (higher priority), then ~/.ironclaw/.env, matching the documented priority chain - Collapse nested if/if-let into let-chains (clippy::collapsible_if) in oauth_defaults.rs, tool.rs, and secrets/store.rs - Fix rename_to_migrated to take &Path instead of &PathBuf Co-Authored-By: Claude Opus 4.6 * fix: Address PR review comments (quoting, SSRF, error mapping) - Quote DATABASE_URL in .env writes so `#` in passwords isn't treated as a dotenv comment (e.g., `DATABASE_URL="postgres://..."`) - Add SSRF defenses to refresh_oauth_token(): require HTTPS, reject private/loopback IPs (with DNS resolution), disable redirects. token_url comes from tool capabilities JSON, so a malicious tool could otherwise exfiltrate refresh tokens. - Fix IPv4 bind error mapping: only map AddrInUse to PortInUse, use generic Io variant for other bind failures Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- Cargo.lock | 14 +- Cargo.toml | 5 +- examples/test_heartbeat.rs | 1 - src/agent/agent_loop.rs | 4 +- src/bootstrap.rs | 393 +++++++----- src/channels/repl.rs | 14 +- src/channels/wasm/wrapper.rs | 94 ++- src/cli/config.rs | 80 +-- src/cli/mod.rs | 1 + src/cli/oauth_defaults.rs | 343 ++++++++++ src/cli/status.rs | 32 +- src/cli/tool.rs | 207 +++--- src/config.rs | 80 +-- src/history/store.rs | 5 + src/llm/nearai.rs | 12 +- src/llm/session.rs | 177 +---- src/main.rs | 52 +- src/secrets/store.rs | 41 +- src/settings.rs | 90 +-- src/setup/channels.rs | 44 +- src/setup/wizard.rs | 86 ++- src/tools/mcp/auth.rs | 87 +-- src/tools/mcp/client.rs | 13 +- src/tools/mcp/config.rs | 83 ++- src/tools/registry.rs | 17 +- src/tools/wasm/credential_injector.rs | 4 +- src/tools/wasm/loader.rs | 186 +++++- src/tools/wasm/mod.rs | 2 +- src/tools/wasm/wrapper.rs | 891 +++++++++++++++++++++++++- tools-src/gmail/src/lib.rs | 151 ++--- tools-src/google-calendar/src/lib.rs | 220 ++----- tools-src/google-docs/src/lib.rs | 337 +++------- tools-src/google-drive/src/lib.rs | 274 +++----- tools-src/google-sheets/src/lib.rs | 302 +++------ tools-src/google-slides/src/lib.rs | 410 +++--------- tools-src/okta/src/lib.rs | 61 +- tools-src/slack/src/lib.rs | 100 +-- tools-src/telegram/src/lib.rs | 182 ++---- 38 files changed, 2794 insertions(+), 2301 deletions(-) create mode 100644 src/cli/oauth_defaults.rs diff --git a/Cargo.lock b/Cargo.lock index 6d64e505..a7b7f585 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2210,11 +2210,11 @@ dependencies = [ "hyper 1.8.1", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", - "webpki-roots", ] [[package]] @@ -2548,6 +2548,7 @@ dependencies = [ "tower-http 0.6.8", "tracing", "tracing-subscriber", + "url", "urlencoding", "uuid", "wasmparser 0.220.1", @@ -4033,6 +4034,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -4050,7 +4052,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", ] [[package]] @@ -6237,15 +6238,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "which" version = "4.4.2" diff --git a/Cargo.toml b/Cargo.toml index 6a11219a..e00dd628 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } futures = "0.3" # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } # Serialization serde = { version = "1", features = ["derive"] } @@ -84,7 +84,8 @@ fs4 = "0.6" # Secrecy for sensitive values secrecy = { version = "0.10", features = ["serde"] } -# URL encoding for OAuth flow +# URL parsing and encoding +url = "2" urlencoding = "2" # Open URLs in browser diff --git a/examples/test_heartbeat.rs b/examples/test_heartbeat.rs index 188009bb..fcb9333d 100644 --- a/examples/test_heartbeat.rs +++ b/examples/test_heartbeat.rs @@ -81,7 +81,6 @@ async fn main() -> anyhow::Result<()> { let session = create_session_manager(SessionConfig { auth_base_url: config.llm.nearai.auth_base_url.clone(), session_path: config.llm.nearai.session_path.clone(), - ..Default::default() }) .await; let llm = create_llm_provider(&config.llm, session)?; diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index fa9cfb9a..02912a15 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -1236,7 +1236,7 @@ impl Agent { &message.channel, StatusUpdate::ToolResult { name: tc.name.clone(), - preview: truncate_for_preview(output, 200), + preview: output.clone(), }, &message.metadata, ) @@ -1710,7 +1710,7 @@ impl Agent { &message.channel, StatusUpdate::ToolResult { name: pending.tool_name.clone(), - preview: truncate_for_preview(output, 200), + preview: output.clone(), }, &message.metadata, ) diff --git a/src/bootstrap.rs b/src/bootstrap.rs index e24b996e..6c14efdf 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -1,147 +1,128 @@ -//! Bootstrap configuration for IronClaw. +//! Bootstrap helpers for IronClaw. //! -//! These are the only settings that MUST live on disk because they're needed -//! before the database connection is established. Everything else lives in the -//! `settings` table in PostgreSQL. +//! The only setting that truly needs disk persistence before the database is +//! available is `DATABASE_URL` (chicken-and-egg: can't connect to DB without +//! it). Everything else is auto-detected or read from env vars. //! -//! File: `~/.ironclaw/bootstrap.json` +//! File: `~/.ironclaw/.env` (standard dotenvy format) use std::path::PathBuf; -use serde::{Deserialize, Serialize}; - -use crate::settings::KeySource; - -/// Minimal config needed to connect to the database and decrypt secrets. -/// -/// This is the only JSON file IronClaw reads from disk at startup. -/// All other configuration lives in the `settings` table in PostgreSQL. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BootstrapConfig { - /// Database connection URL (postgres://...). - #[serde(default)] - pub database_url: Option, - - /// Database connection pool size. - #[serde(default)] - pub database_pool_size: Option, - - /// Source for the secrets master key. - #[serde(default)] - pub secrets_master_key_source: KeySource, - - /// Whether onboarding wizard has been completed. - #[serde(default)] - pub onboard_completed: bool, +/// Path to the IronClaw-specific `.env` file: `~/.ironclaw/.env`. +pub fn ironclaw_env_path() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw") + .join(".env") } -impl Default for BootstrapConfig { - fn default() -> Self { - Self { - database_url: None, - database_pool_size: None, - secrets_master_key_source: KeySource::None, - onboard_completed: false, - } +/// Load env vars from `~/.ironclaw/.env` (in addition to the standard `.env`). +/// +/// Call this **after** `dotenvy::dotenv()` so that the standard `./.env` +/// takes priority over `~/.ironclaw/.env`. dotenvy never overwrites +/// existing env vars, so the effective priority is: +/// +/// explicit env vars > `./.env` > `~/.ironclaw/.env` +/// +/// If `~/.ironclaw/.env` doesn't exist but the legacy `bootstrap.json` does, +/// extracts `DATABASE_URL` from it and writes the `.env` file (one-time +/// upgrade from the old config format). +pub fn load_ironclaw_env() { + let path = ironclaw_env_path(); + + if !path.exists() { + // One-time upgrade: extract DATABASE_URL from legacy bootstrap.json + migrate_bootstrap_json_to_env(&path); + } + + if path.exists() { + let _ = dotenvy::from_path(&path); } } -impl BootstrapConfig { - /// Default bootstrap file path: `~/.ironclaw/bootstrap.json`. - pub fn default_path() -> PathBuf { - dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw") - .join("bootstrap.json") +/// If `bootstrap.json` exists, pull `database_url` out of it and write `.env`. +fn migrate_bootstrap_json_to_env(env_path: &std::path::Path) { + let ironclaw_dir = env_path + .parent() + .unwrap_or_else(|| std::path::Path::new(".")); + let bootstrap_path = ironclaw_dir.join("bootstrap.json"); + + if !bootstrap_path.exists() { + return; } - /// Legacy settings.json path (for migration detection). - pub fn legacy_settings_path() -> PathBuf { - dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw") - .join("settings.json") - } + let content = match std::fs::read_to_string(&bootstrap_path) { + Ok(c) => c, + Err(_) => return, + }; - /// Load from the default path, falling back to legacy settings.json, - /// then to defaults if neither exists. - pub fn load() -> Self { - let bootstrap_path = Self::default_path(); - if bootstrap_path.exists() { - return Self::load_from(&bootstrap_path); + // Minimal parse: just grab database_url from the JSON + let parsed: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(_) => return, + }; + + if let Some(url) = parsed.get("database_url").and_then(|v| v.as_str()) { + if let Some(parent) = env_path.parent() + && let Err(e) = std::fs::create_dir_all(parent) + { + eprintln!("Warning: failed to create {}: {}", parent.display(), e); + return; } - - // Fall back to legacy settings.json (extract just the 4 bootstrap fields) - let legacy_path = Self::legacy_settings_path(); - if legacy_path.exists() { - return Self::load_from_legacy(&legacy_path); + if let Err(e) = std::fs::write(env_path, format!("DATABASE_URL=\"{}\"\n", url)) { + eprintln!("Warning: failed to migrate bootstrap.json to .env: {}", e); + return; } - - Self::default() - } - - /// Load from a specific path. - pub fn load_from(path: &PathBuf) -> Self { - match std::fs::read_to_string(path) { - Ok(data) => serde_json::from_str(&data).unwrap_or_default(), - Err(_) => Self::default(), - } - } - - /// Extract bootstrap fields from a legacy settings.json. - fn load_from_legacy(path: &PathBuf) -> Self { - match std::fs::read_to_string(path) { - Ok(data) => { - // The legacy Settings struct is a superset; serde will ignore extra fields. - serde_json::from_str(&data).unwrap_or_default() - } - Err(_) => Self::default(), - } - } - - /// Save to the default path. - pub fn save(&self) -> std::io::Result<()> { - self.save_to(&Self::default_path()) - } - - /// Save to a specific path. - pub fn save_to(&self, path: &PathBuf) -> std::io::Result<()> { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent)?; - } - let json = serde_json::to_string_pretty(self) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; - std::fs::write(path, json) + rename_to_migrated(&bootstrap_path); + eprintln!( + "Migrated DATABASE_URL from bootstrap.json to {}", + env_path.display() + ); } } -/// One-time migration from disk config files to the database settings table. +/// Write `DATABASE_URL` to `~/.ironclaw/.env`. /// -/// On first boot after upgrade, checks if: -/// 1. `~/.ironclaw/settings.json` exists -/// 2. The DB settings table is empty for this user +/// Creates the parent directory if it doesn't exist. +/// The value is double-quoted so that `#` (common in URL-encoded passwords) +/// and other shell-special characters are preserved by dotenvy. +pub fn save_database_url(url: &str) -> std::io::Result<()> { + let path = ironclaw_env_path(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&path, format!("DATABASE_URL=\"{}\"\n", url)) +} + +/// One-time migration of legacy `~/.ironclaw/settings.json` into the database. /// -/// If both conditions hold, migrates settings, MCP servers, and session data -/// to the database, writes `bootstrap.json`, and renames old files to `.migrated`. +/// Only runs when a `settings.json` exists on disk AND the DB has no settings +/// yet. After the wizard writes directly to the DB, this path is only hit by +/// users upgrading from the old disk-only configuration. +/// +/// After syncing, renames `settings.json` to `.migrated` so it won't trigger again. pub async fn migrate_disk_to_db( store: &dyn crate::db::Database, user_id: &str, ) -> Result<(), MigrationError> { - let legacy_settings_path = BootstrapConfig::legacy_settings_path(); + let ironclaw_dir = dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw"); + let legacy_settings_path = ironclaw_dir.join("settings.json"); + if !legacy_settings_path.exists() { tracing::debug!("No legacy settings.json found, skipping disk-to-DB migration"); return Ok(()); } - // Only migrate if DB is empty for this user + // If DB already has settings, this is not a first boot, the wizard already + // wrote directly to the DB. Just clean up the stale file. let has_settings = store.has_settings(user_id).await.map_err(|e| { MigrationError::Database(format!("Failed to check existing settings: {}", e)) })?; if has_settings { - tracing::debug!( - "DB already has settings for user '{}', skipping migration", - user_id - ); + tracing::info!("DB already has settings, renaming stale settings.json"); + rename_to_migrated(&legacy_settings_path); return Ok(()); } @@ -160,22 +141,14 @@ pub async fn migrate_disk_to_db( tracing::info!("Migrated {} settings to database", db_map.len()); } - // 2. Write bootstrap.json with the 4 essential fields - let bootstrap = BootstrapConfig { - database_url: settings.database_url.clone(), - database_pool_size: settings.database_pool_size, - secrets_master_key_source: settings.secrets_master_key_source, - onboard_completed: settings.onboard_completed, - }; - bootstrap - .save() - .map_err(|e| MigrationError::Io(format!("Failed to write bootstrap.json: {}", e)))?; - tracing::info!("Wrote bootstrap.json"); + // 2. Write DATABASE_URL to ~/.ironclaw/.env + if let Some(ref url) = settings.database_url { + save_database_url(url) + .map_err(|e| MigrationError::Io(format!("Failed to write .env: {}", e)))?; + tracing::info!("Wrote DATABASE_URL to {}", ironclaw_env_path().display()); + } // 3. Migrate mcp-servers.json if it exists - let ironclaw_dir = dirs::home_dir() - .unwrap_or_else(|| PathBuf::from(".")) - .join(".ironclaw"); let mcp_path = ironclaw_dir.join("mcp-servers.json"); if mcp_path.exists() { match std::fs::read_to_string(&mcp_path) { @@ -236,12 +209,19 @@ pub async fn migrate_disk_to_db( // 5. Rename settings.json to .migrated (don't delete, safety net) rename_to_migrated(&legacy_settings_path); + // 6. Clean up old bootstrap.json if it exists (superseded by .env) + let old_bootstrap = ironclaw_dir.join("bootstrap.json"); + if old_bootstrap.exists() { + rename_to_migrated(&old_bootstrap); + tracing::info!("Renamed old bootstrap.json to .migrated"); + } + tracing::info!("Disk-to-DB migration complete"); Ok(()) } /// Rename a file to `.migrated` as a safety net. -fn rename_to_migrated(path: &PathBuf) { +fn rename_to_migrated(path: &std::path::Path) { let mut migrated = path.as_os_str().to_owned(); migrated.push(".migrated"); if let Err(e) = std::fs::rename(path, &migrated) { @@ -264,62 +244,145 @@ mod tests { use tempfile::tempdir; #[test] - fn test_bootstrap_save_load() { + fn test_save_and_load_database_url() { let dir = tempdir().unwrap(); - let path = dir.path().join("bootstrap.json"); + let env_path = dir.path().join(".env"); - let config = BootstrapConfig { - database_url: Some("postgres://localhost/test".to_string()), - database_pool_size: Some(5), - secrets_master_key_source: KeySource::Keychain, - onboard_completed: true, - }; + // Write in the quoted format that save_database_url uses + let url = "postgres://localhost:5432/ironclaw_test"; + std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap(); - config.save_to(&path).unwrap(); - - let loaded = BootstrapConfig::load_from(&path); + // Verify the content is a valid dotenv line (quoted) + let content = std::fs::read_to_string(&env_path).unwrap(); assert_eq!( - loaded.database_url, - Some("postgres://localhost/test".to_string()) + content, + "DATABASE_URL=\"postgres://localhost:5432/ironclaw_test\"\n" ); - assert_eq!(loaded.database_pool_size, Some(5)); - assert_eq!(loaded.secrets_master_key_source, KeySource::Keychain); - assert!(loaded.onboard_completed); + + // Verify dotenvy can parse it (strips quotes automatically) + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].0, "DATABASE_URL"); + assert_eq!(parsed[0].1, url); } #[test] - fn test_bootstrap_from_legacy_settings() { + fn test_save_database_url_with_hash_in_password() { let dir = tempdir().unwrap(); - let path = dir.path().join("settings.json"); + let env_path = dir.path().join(".env"); - // Write a legacy settings.json with many extra fields - let legacy = serde_json::json!({ - "database_url": "postgres://localhost/ironclaw", - "database_pool_size": 10, + // URLs with # in the password are common (URL-encoded special chars). + // Without quoting, dotenvy treats # as a comment delimiter. + let url = "postgres://user:p%23ss@localhost:5432/ironclaw"; + std::fs::write(&env_path, format!("DATABASE_URL=\"{}\"\n", url)).unwrap(); + + let parsed: Vec<(String, String)> = dotenvy::from_path_iter(&env_path) + .unwrap() + .filter_map(|r| r.ok()) + .collect(); + assert_eq!(parsed.len(), 1); + assert_eq!(parsed[0].0, "DATABASE_URL"); + assert_eq!(parsed[0].1, url); + } + + #[test] + fn test_save_database_url_creates_parent_dirs() { + let dir = tempdir().unwrap(); + let nested = dir.path().join("deep").join("nested"); + let env_path = nested.join(".env"); + + // Parent doesn't exist yet + assert!(!nested.exists()); + + // The global function uses a fixed path, so we test the logic directly + std::fs::create_dir_all(&nested).unwrap(); + std::fs::write(&env_path, "DATABASE_URL=postgres://test\n").unwrap(); + + assert!(env_path.exists()); + let content = std::fs::read_to_string(&env_path).unwrap(); + assert!(content.contains("DATABASE_URL=postgres://test")); + } + + #[test] + fn test_ironclaw_env_path() { + let path = ironclaw_env_path(); + assert!(path.ends_with(".ironclaw/.env")); + } + + #[test] + fn test_migrate_bootstrap_json_to_env() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + let bootstrap_path = dir.path().join("bootstrap.json"); + + // Write a legacy bootstrap.json + let bootstrap_json = serde_json::json!({ + "database_url": "postgres://localhost/ironclaw_upgrade", + "database_pool_size": 5, "secrets_master_key_source": "keychain", - "onboard_completed": true, - "selected_model": "claude-3-5-sonnet", - "agent": { "name": "testbot", "max_parallel_jobs": 3 }, - "heartbeat": { "enabled": true } + "onboard_completed": true }); - std::fs::write(&path, serde_json::to_string_pretty(&legacy).unwrap()).unwrap(); + std::fs::write( + &bootstrap_path, + serde_json::to_string_pretty(&bootstrap_json).unwrap(), + ) + .unwrap(); - let config = BootstrapConfig::load_from_legacy(&path); + assert!(!env_path.exists()); + assert!(bootstrap_path.exists()); + + // Run the migration + migrate_bootstrap_json_to_env(&env_path); + + // .env should now exist with DATABASE_URL + assert!(env_path.exists()); + let content = std::fs::read_to_string(&env_path).unwrap(); assert_eq!( - config.database_url, - Some("postgres://localhost/ironclaw".to_string()) + content, + "DATABASE_URL=\"postgres://localhost/ironclaw_upgrade\"\n" ); - assert_eq!(config.database_pool_size, Some(10)); - assert_eq!(config.secrets_master_key_source, KeySource::Keychain); - assert!(config.onboard_completed); + + // bootstrap.json should be renamed to .migrated + assert!(!bootstrap_path.exists()); + assert!(dir.path().join("bootstrap.json.migrated").exists()); } #[test] - fn test_bootstrap_defaults() { - let config = BootstrapConfig::default(); - assert!(config.database_url.is_none()); - assert!(config.database_pool_size.is_none()); - assert_eq!(config.secrets_master_key_source, KeySource::None); - assert!(!config.onboard_completed); + fn test_migrate_bootstrap_json_no_database_url() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + let bootstrap_path = dir.path().join("bootstrap.json"); + + // bootstrap.json with no database_url + let bootstrap_json = serde_json::json!({ + "onboard_completed": false + }); + std::fs::write( + &bootstrap_path, + serde_json::to_string_pretty(&bootstrap_json).unwrap(), + ) + .unwrap(); + + migrate_bootstrap_json_to_env(&env_path); + + // .env should NOT be created + assert!(!env_path.exists()); + // bootstrap.json should remain (no migration happened) + assert!(bootstrap_path.exists()); + } + + #[test] + fn test_migrate_bootstrap_json_missing() { + let dir = tempdir().unwrap(); + let env_path = dir.path().join(".env"); + + // No bootstrap.json at all + migrate_bootstrap_json_to_env(&env_path); + + // Nothing should happen + assert!(!env_path.exists()); } } diff --git a/src/channels/repl.rs b/src/channels/repl.rs index ea91082b..1dde2f47 100644 --- a/src/channels/repl.rs +++ b/src/channels/repl.rs @@ -37,6 +37,9 @@ use crate::agent::truncate_for_preview; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::error::ChannelError; +/// Max characters for tool result previews in the terminal. +const CLI_TOOL_RESULT_MAX: usize = 200; + /// Max characters for thinking/status messages in the terminal. const CLI_STATUS_MAX: usize = 200; @@ -265,7 +268,7 @@ impl Channel for ReplChannel { std::thread::spawn(move || { // Single message mode: send it and return if let Some(msg) = single_message { - let incoming = IncomingMessage::new("repl", "user", &msg); + let incoming = IncomingMessage::new("repl", "default", &msg); let _ = tx.blocking_send(incoming); return; } @@ -333,21 +336,21 @@ impl Channel for ReplChannel { _ => {} } - let msg = IncomingMessage::new("repl", "user", line); + let msg = IncomingMessage::new("repl", "default", line); if tx.blocking_send(msg).is_err() { break; } } Err(ReadlineError::Interrupted) => { // Ctrl+C: send /interrupt - let msg = IncomingMessage::new("repl", "user", "/interrupt"); + let msg = IncomingMessage::new("repl", "default", "/interrupt"); if tx.blocking_send(msg).is_err() { break; } } Err(ReadlineError::Eof) => { // Ctrl+D: send /quit so the agent loop runs graceful shutdown - let msg = IncomingMessage::new("repl", "user", "/quit"); + let msg = IncomingMessage::new("repl", "default", "/quit"); let _ = tx.blocking_send(msg); break; } @@ -418,7 +421,8 @@ impl Channel for ReplChannel { } } StatusUpdate::ToolResult { name: _, preview } => { - eprintln!(" \x1b[90m{preview}\x1b[0m"); + let display = truncate_for_preview(&preview, CLI_TOOL_RESULT_MAX); + eprintln!(" \x1b[90m{display}\x1b[0m"); } StatusUpdate::StreamChunk(chunk) => { // Print separator on the false-to-true transition diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 5d34e40c..212334a6 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -76,6 +76,9 @@ struct ChannelStoreData { credentials: HashMap, /// Pairing store for DM pairing (guest access control). pairing_store: Arc, + /// Dedicated tokio runtime for HTTP requests, lazily initialized. + /// Reused across multiple `http_request` calls within one execution. + http_runtime: Option, } impl ChannelStoreData { @@ -96,6 +99,7 @@ impl ChannelStoreData { table: ResourceTable::new(), credentials, pairing_store, + http_runtime: None, } } @@ -283,10 +287,25 @@ impl near::agent::channel_host::Host for ChannelStoreData { .map(|h| h.max_response_bytes) .unwrap_or(10 * 1024 * 1024); - // Make the HTTP request using blocking I/O - // We're already in a spawn_blocking context, so we can use block_on - let result = tokio::runtime::Handle::current().block_on(async { - let client = reqwest::Client::new(); + // Make the HTTP request using a dedicated single-threaded runtime. + // We're inside spawn_blocking, so we can't rely on the main runtime's + // I/O driver (it may be busy with WASM compilation or other startup work). + // A dedicated runtime gives us our own I/O driver and avoids contention. + // The runtime is lazily created and reused across calls within one execution. + if self.http_runtime.is_none() { + self.http_runtime = Some( + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("Failed to create HTTP runtime: {e}"))?, + ); + } + let rt = self.http_runtime.as_ref().expect("just initialized"); + let result = rt.block_on(async { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .map_err(|e| format!("Failed to build HTTP client: {e}"))?; let mut request = match method.to_uppercase().as_str() { "GET" => client.get(&url), @@ -308,9 +327,9 @@ impl near::agent::channel_host::Host for ChannelStoreData { request = request.body(body_bytes); } - // Send request with caller-specified timeout (default 30s). - // Cap at callback_timeout to prevent outliving the host wrapper. - let timeout = std::time::Duration::from_millis(timeout_ms.unwrap_or(30_000) as u64); + // Send request with caller-specified timeout (default 30s, max 5min). + let timeout_ms = timeout_ms.unwrap_or(30_000).min(300_000) as u64; + let timeout = std::time::Duration::from_millis(timeout_ms); let response = request.timeout(timeout).send().await.map_err(|e| { // Walk the full error chain so we get the actual root cause // (DNS, TLS, connection refused, etc.) instead of just @@ -795,7 +814,21 @@ impl WasmChannel { .await; match result { - Ok(Ok((config, _host_state))) => { + Ok(Ok((config, mut host_state))) => { + // Surface WASM guest logs (errors/warnings from webhook setup, etc.) + for entry in host_state.take_logs() { + match entry.level { + crate::tools::wasm::LogLevel::Error => { + tracing::error!(channel = %self.name, "{}", entry.message); + } + crate::tools::wasm::LogLevel::Warn => { + tracing::warn!(channel = %self.name, "{}", entry.message); + } + _ => { + tracing::debug!(channel = %self.name, "{}", entry.message); + } + } + } tracing::info!( channel = %self.name, display_name = %config.display_name, @@ -2615,15 +2648,52 @@ mod tests { assert_eq!(store.redact_credentials(input), input); } - /// Verify that the block_on-inside-spawn_blocking pattern used by the WASM - /// channel HTTP host function doesn't deadlock or panic. + /// Verify that WASM HTTP host functions work using a dedicated + /// current-thread runtime inside spawn_blocking. #[tokio::test] - async fn test_block_on_inside_spawn_blocking_does_not_deadlock() { + async fn test_dedicated_runtime_inside_spawn_blocking() { let result = tokio::task::spawn_blocking(|| { - tokio::runtime::Handle::current().block_on(async { 42 }) + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime"); + rt.block_on(async { 42 }) }) .await .expect("spawn_blocking panicked"); assert_eq!(result, 42); } + + /// Verify a real HTTP request works using the dedicated-runtime pattern. + /// This catches DNS, TLS, and I/O driver issues that trivial tests miss. + #[tokio::test] + #[ignore] // requires network + async fn test_dedicated_runtime_real_http() { + let result = tokio::task::spawn_blocking(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime"); + rt.block_on(async { + let client = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .expect("failed to build client"); + let resp = client + .get("https://api.telegram.org/bot000/getMe") + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + match resp { + Ok(r) => r.status().as_u16(), + Err(e) if e.is_timeout() => panic!("request timed out: {e}"), + Err(e) => panic!("unexpected error: {e}"), + } + }) + }) + .await + .expect("spawn_blocking panicked"); + // 404 because "000" is not a valid bot token + assert_eq!(result, 404); + } } diff --git a/src/cli/config.rs b/src/cli/config.rs index 8835b2c0..f91e9241 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -48,8 +48,6 @@ pub enum ConfigCommand { /// Connects to the database to read/write settings. Falls back to disk /// if the database is not available. pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> { - let _ = dotenvy::dotenv(); - // Try to connect to the DB for settings access let db: Option> = match connect_db().await { Ok(d) => Some(d), @@ -92,7 +90,7 @@ async fn load_settings(store: Option<&dyn crate::db::Database>) -> Settings { _ => {} } } - Settings::load() + Settings::default() } /// List all settings. @@ -155,19 +153,17 @@ async fn set_setting( .set(path, value) .map_err(|e| anyhow::anyhow!("{}", e))?; - // Save to DB if available, otherwise disk - if let Some(store) = store { - let json_value = match serde_json::from_str::(value) { - Ok(v) => v, - Err(_) => serde_json::Value::String(value.to_string()), - }; - store - .set_setting(DEFAULT_USER_ID, path, &json_value) - .await - .map_err(|e| anyhow::anyhow!("Failed to save to database: {}", e))?; - } else { - settings.save()?; - } + let store = store.ok_or_else(|| { + anyhow::anyhow!("Database connection required to save settings. Check DATABASE_URL.") + })?; + let json_value = match serde_json::from_str::(value) { + Ok(v) => v, + Err(_) => serde_json::Value::String(value.to_string()), + }; + store + .set_setting(DEFAULT_USER_ID, path, &json_value) + .await + .map_err(|e| anyhow::anyhow!("Failed to save to database: {}", e))?; println!("Set {} = {}", path, value); Ok(()) @@ -180,17 +176,13 @@ async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> a .get(path) .ok_or_else(|| anyhow::anyhow!("Unknown setting: {}", path))?; - // Delete from DB (falling back to default) or reset on disk - if let Some(store) = store { - store - .delete_setting(DEFAULT_USER_ID, path) - .await - .map_err(|e| anyhow::anyhow!("Failed to delete setting from database: {}", e))?; - } else { - let mut settings = Settings::load(); - settings.reset(path).map_err(|e| anyhow::anyhow!("{}", e))?; - settings.save()?; - } + let store = store.ok_or_else(|| { + anyhow::anyhow!("Database connection required to reset settings. Check DATABASE_URL.") + })?; + store + .delete_setting(DEFAULT_USER_ID, path) + .await + .map_err(|e| anyhow::anyhow!("Failed to delete setting from database: {}", e))?; println!("Reset {} to default: {}", path, default_value); Ok(()) @@ -200,37 +192,13 @@ async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> a fn show_path(has_db: bool) -> anyhow::Result<()> { if has_db { println!("Settings stored in: database (settings table)"); - println!( - "Bootstrap config: {}", - crate::bootstrap::BootstrapConfig::default_path().display() - ); } else { - let path = Settings::default_path(); - println!("Settings stored in: {} (disk fallback)", path.display()); - - if path.exists() { - let metadata = std::fs::metadata(&path)?; - println!(" Size: {} bytes", metadata.len()); - if let Ok(modified) = metadata.modified() { - use std::time::SystemTime; - let duration = SystemTime::now() - .duration_since(modified) - .unwrap_or_default(); - let secs = duration.as_secs(); - if secs < 60 { - println!(" Modified: {} seconds ago", secs); - } else if secs < 3600 { - println!(" Modified: {} minutes ago", secs / 60); - } else if secs < 86400 { - println!(" Modified: {} hours ago", secs / 3600); - } else { - println!(" Modified: {} days ago", secs / 86400); - } - } - } else { - println!(" (does not exist, using defaults)"); - } + println!("Settings stored in: PostgreSQL (not connected, using defaults)"); } + println!( + "Env config: {}", + crate::bootstrap::ironclaw_env_path().display() + ); Ok(()) } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 77ed1f3d..06715d8c 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -12,6 +12,7 @@ mod config; mod mcp; pub mod memory; +pub mod oauth_defaults; mod pairing; pub mod status; mod tool; diff --git a/src/cli/oauth_defaults.rs b/src/cli/oauth_defaults.rs new file mode 100644 index 00000000..eea91a71 --- /dev/null +++ b/src/cli/oauth_defaults.rs @@ -0,0 +1,343 @@ +//! Shared OAuth infrastructure: built-in credentials, callback server, landing pages. +//! +//! Every OAuth flow in the codebase (WASM tool auth, MCP server auth, NEAR AI login) +//! uses the same callback port, landing page, and listener logic from this module. +//! +//! # Built-in Credentials +//! +//! Many CLI tools (gcloud, rclone, gdrive) ship with default OAuth credentials +//! so users don't need to register their own OAuth app. Google explicitly +//! documents that client_secret for "Desktop App" / "Installed App" types +//! is NOT actually secret. +//! +//! Default credentials are hardcoded below. They can be overridden at: +//! +//! - **Compile time**: Set IRONCLAW_GOOGLE_CLIENT_ID / IRONCLAW_GOOGLE_CLIENT_SECRET +//! env vars before building to replace the hardcoded defaults. +//! - **Runtime**: Users can set GOOGLE_OAUTH_CLIENT_ID / GOOGLE_OAUTH_CLIENT_SECRET +//! env vars, which take priority over built-in defaults. + +use std::time::Duration; + +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpListener; + +// ── Built-in credentials ──────────────────────────────────────────────── + +pub struct OAuthCredentials { + pub client_id: &'static str, + pub client_secret: &'static str, +} + +/// Google OAuth "Desktop App" credentials, shared across all Google tools. +/// Compile-time env vars override the hardcoded defaults below. +const GOOGLE_CLIENT_ID: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_ID") { + Some(v) => v, + None => "564604149681-efo25d43rs85v0tibdepsmdv5dsrhhr0.apps.googleusercontent.com", +}; +const GOOGLE_CLIENT_SECRET: &str = match option_env!("IRONCLAW_GOOGLE_CLIENT_SECRET") { + Some(v) => v, + None => "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", +}; + +/// Returns built-in OAuth credentials for a provider, keyed by secret_name. +/// +/// The secret_name comes from the tool's capabilities.json `auth.secret_name` field. +/// Returns `None` if no built-in credentials are configured for that provider. +pub fn builtin_credentials(secret_name: &str) -> Option { + match secret_name { + "google_oauth_token" => Some(OAuthCredentials { + client_id: GOOGLE_CLIENT_ID, + client_secret: GOOGLE_CLIENT_SECRET, + }), + _ => None, + } +} + +// ── Shared callback server ────────────────────────────────────────────── + +/// Fixed port for all OAuth callbacks. +/// +/// Every redirect URI registered with providers must use this port: +/// `http://localhost:9876/callback` (or `/auth/callback` for NEAR AI). +pub const OAUTH_CALLBACK_PORT: u16 = 9876; + +/// Error from the OAuth callback listener. +#[derive(Debug, thiserror::Error)] +pub enum OAuthCallbackError { + #[error("Port {0} is in use (another auth flow running?): {1}")] + PortInUse(u16, String), + + #[error("Authorization denied by user")] + Denied, + + #[error("Timed out waiting for authorization")] + Timeout, + + #[error("IO error: {0}")] + Io(String), +} + +/// Bind the OAuth callback listener on the fixed port. +/// +/// Tries IPv6 loopback (`[::1]`) first so that `http://localhost:…` redirects +/// work on systems where `localhost` resolves to `::1`. Falls back to IPv4 +/// (`127.0.0.1`) only if IPv6 fails for a reason other than `AddrInUse` +/// (e.g., IPv6 not supported on the host). If the port is already occupied +/// on IPv6, the port is occupied period, so we fail immediately. +pub async fn bind_callback_listener() -> Result { + let ipv6_addr = format!("[::1]:{}", OAUTH_CALLBACK_PORT); + match TcpListener::bind(&ipv6_addr).await { + Ok(listener) => return Ok(listener), + Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => { + return Err(OAuthCallbackError::PortInUse( + OAUTH_CALLBACK_PORT, + e.to_string(), + )); + } + Err(_) => { + // IPv6 not available on this host, fall back to IPv4 + } + } + TcpListener::bind(format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT)) + .await + .map_err(|e| { + if e.kind() == std::io::ErrorKind::AddrInUse { + OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string()) + } else { + OAuthCallbackError::Io(e.to_string()) + } + }) +} + +/// Wait for an OAuth callback and extract a query parameter value. +/// +/// Listens for a GET request matching `path_prefix` (e.g., "/callback" or "/auth/callback"), +/// extracts the value of `param_name` (e.g., "code" or "token"), and shows a branded +/// landing page using `display_name` (e.g., "Google", "Notion", "NEAR AI"). +/// +/// Times out after 5 minutes. +pub async fn wait_for_callback( + listener: TcpListener, + path_prefix: &str, + param_name: &str, + display_name: &str, +) -> Result { + let path_prefix = path_prefix.to_string(); + let param_name = param_name.to_string(); + let display_name = display_name.to_string(); + + tokio::time::timeout(Duration::from_secs(300), async move { + loop { + let (mut socket, _) = listener + .accept() + .await + .map_err(|e| OAuthCallbackError::Io(e.to_string()))?; + + let mut reader = BufReader::new(&mut socket); + let mut request_line = String::new(); + reader + .read_line(&mut request_line) + .await + .map_err(|e| OAuthCallbackError::Io(e.to_string()))?; + + if let Some(path) = request_line.split_whitespace().nth(1) + && path.starts_with(&path_prefix) + && let Some(query) = path.split('?').nth(1) + { + // Check for error first + if query.contains("error=") { + let html = landing_html(&display_name, false); + let response = format!( + "HTTP/1.1 400 Bad Request\r\n\ + Content-Type: text/html; charset=utf-8\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + html + ); + let _ = socket.write_all(response.as_bytes()).await; + return Err(OAuthCallbackError::Denied); + } + + // Look for the target parameter + for param in query.split('&') { + let parts: Vec<&str> = param.splitn(2, '=').collect(); + if parts.len() == 2 && parts[0] == param_name { + let value = urlencoding::decode(parts[1]) + .unwrap_or_else(|_| parts[1].into()) + .into_owned(); + + let html = landing_html(&display_name, true); + let response = format!( + "HTTP/1.1 200 OK\r\n\ + Content-Type: text/html; charset=utf-8\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + html + ); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + + return Ok(value); + } + } + } + + // Not the callback we're looking for + let response = "HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(response.as_bytes()).await; + } + }) + .await + .map_err(|_| OAuthCallbackError::Timeout)? +} + +/// Escape a string for safe interpolation into HTML content. +fn html_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(c), + } + } + out +} + +/// HTML landing page shown in the browser after an OAuth redirect. +pub fn landing_html(provider_name: &str, success: bool) -> String { + let safe_name = html_escape(provider_name); + let (icon, heading, subtitle, accent) = if success { + ( + r##"
+ +
"##, + format!("{} Connected", safe_name), + "You can close this window and return to your terminal.", + "#22c55e", + ) + } else { + ( + r##"
+ +
"##, + "Authorization Failed".to_string(), + "The request was denied. You can close this window and try again.", + "#ef4444", + ) + }; + + format!( + r#" + + + + +IronClaw - {heading} + + + +
+ {icon} +

{heading}

+

{subtitle}

+
IronClaw
+
+ +"#, + heading = heading, + icon = icon, + subtitle = subtitle, + accent = accent, + ) +} + +#[cfg(test)] +mod tests { + use crate::cli::oauth_defaults::{builtin_credentials, landing_html}; + + #[test] + fn test_unknown_provider_returns_none() { + assert!(builtin_credentials("unknown_token").is_none()); + } + + #[test] + fn test_google_returns_based_on_compile_env() { + let creds = builtin_credentials("google_oauth_token"); + assert!(creds.is_some()); + let creds = creds.unwrap(); + assert!(!creds.client_id.is_empty()); + assert!(!creds.client_secret.is_empty()); + } + + #[test] + fn test_landing_html_success_contains_key_elements() { + let html = landing_html("Google", true); + assert!(html.contains("Google Connected")); + assert!(html.contains("charset")); + assert!(html.contains("IronClaw")); + assert!(html.contains("#22c55e")); // green accent + assert!(!html.contains("Failed")); + } + + #[test] + fn test_landing_html_escapes_provider_name() { + let html = landing_html("", true); + assert!(!html.contains("