//! LLM integration for the agent. //! //! Supports multiple backends: //! - **NEAR AI** (default): Session token or API key auth via Chat Completions API //! - **OpenAI**: Direct API access with your own key //! - **Anthropic**: Direct API access with your own key //! - **Ollama**: Local model inference //! - **OpenAI-compatible**: Any endpoint that speaks the OpenAI API pub mod circuit_breaker; pub mod costs; pub mod failover; mod nearai_chat; mod provider; mod reasoning; pub mod response_cache; pub mod retry; mod rig_adapter; pub mod session; pub mod smart_routing; pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider}; pub use failover::{CooldownConfig, FailoverProvider}; pub use nearai_chat::{ModelInfo, NearAiChatProvider}; pub use provider::{ ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult, }; pub use reasoning::{ ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, TokenUsage, ToolSelection, }; pub use response_cache::{CachedProvider, ResponseCacheConfig}; pub use retry::{RetryConfig, RetryProvider}; pub use rig_adapter::RigAdapter; pub use session::{SessionConfig, SessionManager, create_session_manager}; pub use smart_routing::{SmartRoutingConfig, SmartRoutingProvider, TaskComplexity}; use std::sync::Arc; use rig::client::CompletionClient; use secrecy::ExposeSecret; use crate::config::{LlmBackend, LlmConfig, NearAiConfig}; use crate::error::LlmError; /// Create an LLM provider based on configuration. /// /// - `NearAi` backend: Uses session manager for authentication (Responses API) /// or API key (Chat Completions API) /// - Other backends: Use rig-core adapter with provider-specific clients pub fn create_llm_provider( config: &LlmConfig, session: Arc, ) -> Result, LlmError> { match config.backend { LlmBackend::NearAi => create_llm_provider_with_config(&config.nearai, session), LlmBackend::OpenAi => create_openai_provider(config), LlmBackend::Anthropic => create_anthropic_provider(config), LlmBackend::Ollama => create_ollama_provider(config), LlmBackend::OpenAiCompatible => create_openai_compatible_provider(config), LlmBackend::Tinfoil => create_tinfoil_provider(config), } } /// Create an LLM provider from a `NearAiConfig` directly. /// /// This is useful when constructing additional providers for failover, /// where only the model name differs from the primary config. pub fn create_llm_provider_with_config( config: &NearAiConfig, session: Arc, ) -> Result, LlmError> { let auth_mode = if config.api_key.is_some() { "API key" } else { "session token" }; tracing::info!( model = %config.model, base_url = %config.base_url, auth = auth_mode, "Using NEAR AI (Chat Completions API)" ); Ok(Arc::new(NearAiChatProvider::new(config.clone(), session)?)) } fn create_openai_provider(config: &LlmConfig) -> Result, LlmError> { let oai = config.openai.as_ref().ok_or_else(|| LlmError::AuthFailed { provider: "openai".to_string(), })?; use rig::providers::openai; // Use CompletionsClient (Chat Completions API) instead of the default Client // (Responses API). The Responses API path in rig-core panics when tool results // are sent back because ironclaw doesn't thread `call_id` through its ToolCall // type. The Chat Completions API works correctly with the existing code. let client: openai::CompletionsClient = if let Some(ref base_url) = oai.base_url { tracing::info!( "Using OpenAI direct API (chat completions, model: {}, base_url: {})", oai.model, base_url, ); openai::Client::builder() .base_url(base_url) .api_key(oai.api_key.expose_secret()) .build() } else { tracing::info!( "Using OpenAI direct API (chat completions, model: {}, base_url: default)", oai.model, ); openai::Client::new(oai.api_key.expose_secret()) } .map_err(|e| LlmError::RequestFailed { provider: "openai".to_string(), reason: format!("Failed to create OpenAI client: {}", e), })? .completions_api(); let model = client.completion_model(&oai.model); Ok(Arc::new(RigAdapter::new(model, &oai.model))) } fn create_anthropic_provider(config: &LlmConfig) -> Result, LlmError> { let anth = config .anthropic .as_ref() .ok_or_else(|| LlmError::AuthFailed { provider: "anthropic".to_string(), })?; use rig::providers::anthropic; let client: anthropic::Client = if let Some(ref base_url) = anth.base_url { anthropic::Client::builder() .api_key(anth.api_key.expose_secret()) .base_url(base_url) .build() } else { anthropic::Client::new(anth.api_key.expose_secret()) } .map_err(|e| LlmError::RequestFailed { provider: "anthropic".to_string(), reason: format!("Failed to create Anthropic client: {}", e), })?; let model = client.completion_model(&anth.model); tracing::info!( "Using Anthropic direct API (model: {}, base_url: {})", anth.model, anth.base_url.as_deref().unwrap_or("default"), ); Ok(Arc::new(RigAdapter::new(model, &anth.model))) } fn create_ollama_provider(config: &LlmConfig) -> Result, LlmError> { let oll = config.ollama.as_ref().ok_or_else(|| LlmError::AuthFailed { provider: "ollama".to_string(), })?; use rig::client::Nothing; use rig::providers::ollama; let client: ollama::Client = ollama::Client::builder() .base_url(&oll.base_url) .api_key(Nothing) .build() .map_err(|e| LlmError::RequestFailed { provider: "ollama".to_string(), reason: format!("Failed to create Ollama client: {}", e), })?; let model = client.completion_model(&oll.model); tracing::info!( "Using Ollama (base_url: {}, model: {})", oll.base_url, oll.model ); Ok(Arc::new(RigAdapter::new(model, &oll.model))) } const TINFOIL_BASE_URL: &str = "https://inference.tinfoil.sh/v1"; fn create_tinfoil_provider(config: &LlmConfig) -> Result, LlmError> { let tf = config .tinfoil .as_ref() .ok_or_else(|| LlmError::AuthFailed { provider: "tinfoil".to_string(), })?; use rig::providers::openai; let client: openai::Client = openai::Client::builder() .base_url(TINFOIL_BASE_URL) .api_key(tf.api_key.expose_secret()) .build() .map_err(|e| LlmError::RequestFailed { provider: "tinfoil".to_string(), reason: format!("Failed to create Tinfoil client: {}", e), })?; // Tinfoil currently only supports the Chat Completions API and not the newer Responses API, // so we must explicitly select the completions API here (unlike other OpenAI-compatible providers). let client = client.completions_api(); let model = client.completion_model(&tf.model); tracing::info!("Using Tinfoil private inference (model: {})", tf.model); Ok(Arc::new(RigAdapter::new(model, &tf.model))) } fn create_openai_compatible_provider(config: &LlmConfig) -> Result, LlmError> { let compat = config .openai_compatible .as_ref() .ok_or_else(|| LlmError::AuthFailed { provider: "openai_compatible".to_string(), })?; use rig::providers::openai; let client: openai::CompletionsClient = openai::Client::builder() .base_url(&compat.base_url) .api_key( compat .api_key .as_ref() .map(|k| k.expose_secret().to_string()) .unwrap_or_else(|| "no-key".to_string()), ) .build() .map_err(|e| LlmError::RequestFailed { provider: "openai_compatible".to_string(), reason: format!("Failed to create OpenAI-compatible client: {}", e), })? .completions_api(); let model = client.completion_model(&compat.model); tracing::info!( "Using OpenAI-compatible endpoint (chat completions, base_url: {}, model: {})", compat.base_url, compat.model ); Ok(Arc::new(RigAdapter::new(model, &compat.model))) } /// Create a cheap/fast LLM provider for lightweight tasks (heartbeat, routing, evaluation). /// /// Uses `NEARAI_CHEAP_MODEL` if set, otherwise falls back to the main provider. /// Currently only supports NEAR AI backend. pub fn create_cheap_llm_provider( config: &LlmConfig, session: Arc, ) -> Result>, LlmError> { let Some(ref cheap_model) = config.nearai.cheap_model else { return Ok(None); }; if config.backend != LlmBackend::NearAi { tracing::warn!( "NEARAI_CHEAP_MODEL is set but LLM_BACKEND is {:?}, not NearAi. \ Cheap model setting will be ignored.", config.backend ); return Ok(None); } let mut cheap_config = config.nearai.clone(); cheap_config.model = cheap_model.clone(); Ok(Some(Arc::new(NearAiChatProvider::new( cheap_config, session, )?))) } /// Build the full LLM provider chain with all configured wrappers. /// /// Applies decorators in this order: /// 1. Raw provider (from config) /// 2. RetryProvider (per-provider retry with exponential backoff) /// 3. SmartRoutingProvider (cheap/primary split when cheap model is configured) /// 4. FailoverProvider (fallback model when primary fails) /// 5. CircuitBreakerProvider (fast-fail when backend is degraded) /// 6. CachedProvider (in-memory response cache) /// /// Also returns a separate cheap LLM provider for heartbeat/evaluation (not /// part of the chain — it's a standalone provider for explicitly cheap tasks). /// /// This is the single source of truth for provider chain construction, /// called by both `main.rs` and `app.rs`. #[allow(clippy::type_complexity)] pub fn build_provider_chain( config: &LlmConfig, session: Arc, ) -> Result<(Arc, Option>), LlmError> { let llm = create_llm_provider(config, session.clone())?; tracing::info!("LLM provider initialized: {}", llm.model_name()); // 1. Retry let retry_config = RetryConfig { max_retries: config.nearai.max_retries, }; let llm: Arc = if retry_config.max_retries > 0 { tracing::info!( max_retries = retry_config.max_retries, "LLM retry wrapper enabled" ); Arc::new(RetryProvider::new(llm, retry_config.clone())) } else { llm }; // 2. Smart routing (cheap/primary split) let llm: Arc = if let Some(ref cheap_model) = config.nearai.cheap_model { let mut cheap_config = config.nearai.clone(); cheap_config.model = cheap_model.clone(); let cheap = create_llm_provider_with_config(&cheap_config, session.clone())?; let cheap: Arc = if retry_config.max_retries > 0 { Arc::new(RetryProvider::new(cheap, retry_config.clone())) } else { cheap }; tracing::info!( primary = %llm.model_name(), cheap = %cheap.model_name(), "Smart routing enabled" ); Arc::new(SmartRoutingProvider::new( llm, cheap, SmartRoutingConfig { cascade_enabled: config.nearai.smart_routing_cascade, ..SmartRoutingConfig::default() }, )) } else { llm }; // 3. Failover let llm: Arc = if let Some(ref fallback_model) = config.nearai.fallback_model { if fallback_model == &config.nearai.model { tracing::warn!( "fallback_model is the same as primary model, failover may not be effective" ); } let mut fallback_config = config.nearai.clone(); fallback_config.model = fallback_model.clone(); let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?; tracing::info!( primary = %llm.model_name(), fallback = %fallback.model_name(), "LLM failover enabled" ); let fallback: Arc = if retry_config.max_retries > 0 { Arc::new(RetryProvider::new(fallback, retry_config.clone())) } else { fallback }; let cooldown_config = CooldownConfig { cooldown_duration: std::time::Duration::from_secs(config.nearai.failover_cooldown_secs), failure_threshold: config.nearai.failover_cooldown_threshold, }; Arc::new(FailoverProvider::with_cooldown( vec![llm, fallback], cooldown_config, )?) } else { llm }; // 4. Circuit breaker let llm: Arc = if let Some(threshold) = config.nearai.circuit_breaker_threshold { let cb_config = CircuitBreakerConfig { failure_threshold: threshold, recovery_timeout: std::time::Duration::from_secs( config.nearai.circuit_breaker_recovery_secs, ), ..CircuitBreakerConfig::default() }; tracing::info!( threshold, recovery_secs = config.nearai.circuit_breaker_recovery_secs, "LLM circuit breaker enabled" ); Arc::new(CircuitBreakerProvider::new(llm, cb_config)) } else { llm }; // 5. Response cache let llm: Arc = if config.nearai.response_cache_enabled { let rc_config = ResponseCacheConfig { ttl: std::time::Duration::from_secs(config.nearai.response_cache_ttl_secs), max_entries: config.nearai.response_cache_max_entries, }; tracing::info!( ttl_secs = config.nearai.response_cache_ttl_secs, max_entries = config.nearai.response_cache_max_entries, "LLM response cache enabled" ); Arc::new(CachedProvider::new(llm, rc_config)) } else { llm }; // Standalone cheap LLM for heartbeat/evaluation (not part of the chain) let cheap_llm = create_cheap_llm_provider(config, session)?; if let Some(ref cheap) = cheap_llm { tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name()); } Ok((llm, cheap_llm)) } #[cfg(test)] mod tests { use super::*; use crate::config::{LlmBackend, NearAiConfig}; use std::path::PathBuf; fn test_nearai_config() -> NearAiConfig { NearAiConfig { model: "test-model".to_string(), cheap_model: None, base_url: "https://api.near.ai".to_string(), auth_base_url: "https://private.near.ai".to_string(), session_path: PathBuf::from("/tmp/test-session.json"), api_key: None, fallback_model: None, max_retries: 3, circuit_breaker_threshold: None, circuit_breaker_recovery_secs: 30, response_cache_enabled: false, response_cache_ttl_secs: 3600, response_cache_max_entries: 1000, failover_cooldown_secs: 300, failover_cooldown_threshold: 3, smart_routing_cascade: true, } } fn test_llm_config() -> LlmConfig { LlmConfig { backend: LlmBackend::NearAi, nearai: test_nearai_config(), openai: None, anthropic: None, ollama: None, openai_compatible: None, tinfoil: None, } } #[test] fn test_create_cheap_llm_provider_returns_none_when_not_configured() { let config = test_llm_config(); let session = Arc::new(SessionManager::new(SessionConfig::default())); let result = create_cheap_llm_provider(&config, session); assert!(result.is_ok()); assert!(result.unwrap().is_none()); } #[test] fn test_create_cheap_llm_provider_creates_provider_when_configured() { let mut config = test_llm_config(); config.nearai.cheap_model = Some("cheap-test-model".to_string()); let session = Arc::new(SessionManager::new(SessionConfig::default())); let result = create_cheap_llm_provider(&config, session); assert!(result.is_ok()); let provider = result.unwrap(); assert!(provider.is_some()); assert_eq!(provider.unwrap().model_name(), "cheap-test-model"); } #[test] fn test_create_cheap_llm_provider_ignored_for_non_nearai_backend() { let mut config = test_llm_config(); config.backend = LlmBackend::OpenAi; config.nearai.cheap_model = Some("cheap-test-model".to_string()); let session = Arc::new(SessionManager::new(SessionConfig::default())); let result = create_cheap_llm_provider(&config, session); assert!(result.is_ok()); assert!(result.unwrap().is_none()); } }