From 200aed16cdb9f3d43375749d869b87c78a4f81c0 Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Sun, 8 Mar 2026 00:30:52 -0800 Subject: [PATCH] feat: configurable LLM request timeout via LLM_REQUEST_TIMEOUT_SECS (#615) (#630) Add LLM_REQUEST_TIMEOUT_SECS env var (default: 120) to configure the HTTP request timeout for LLM API calls. Primarily useful for local models (Ollama, vLLM, LM Studio) that need more time for prompt evaluation on consumer hardware. The timeout is applied to the NearAI provider's HTTP client. Other providers (Anthropic, OpenAI) use rig-core's default client. - Add request_timeout_secs field to LlmConfig - Thread timeout through create_llm_provider -> NearAiChatProvider - Add NearAiChatProvider::new_with_timeout constructor - Add .env.example documentation - 2 regression tests for default and custom timeout values Co-authored-by: Claude Opus 4.6 --- .env.example | 1 + src/config/llm.rs | 34 ++++++++++++++++++++++++++++++++++ src/llm/mod.rs | 25 +++++++++++++++++++++---- src/llm/nearai_chat.rs | 19 +++++++++++++++---- src/setup/wizard.rs | 1 + 5 files changed, 72 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 258fd79f..1200400d 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,7 @@ DATABASE_POOL_SIZE=10 # LLM Provider # LLM_BACKEND=nearai # default # Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil +# LLM_REQUEST_TIMEOUT_SECS=120 # Increase for local LLMs (Ollama, vLLM, LM Studio) # === Anthropic Direct === # Two auth modes: diff --git a/src/config/llm.rs b/src/config/llm.rs index a2670cc9..9d374428 100644 --- a/src/config/llm.rs +++ b/src/config/llm.rs @@ -103,6 +103,10 @@ pub struct LlmConfig { /// Resolved provider config for registry-based providers. /// `None` when backend is "nearai". pub provider: Option, + /// HTTP request timeout in seconds for LLM API calls. + /// Default: 120. Increase for local LLMs (Ollama, vLLM, LM Studio) that + /// need more time for prompt evaluation on consumer hardware. + pub request_timeout_secs: u64, } /// NEAR AI configuration. @@ -165,6 +169,7 @@ impl LlmConfig { smart_routing_cascade: false, }, provider: None, + request_timeout_secs: 120, } } @@ -254,6 +259,8 @@ impl LlmConfig { )?) }; + let request_timeout_secs = parse_optional_env("LLM_REQUEST_TIMEOUT_SECS", 120)?; + Ok(Self { backend: if is_nearai { "nearai".to_string() @@ -265,6 +272,7 @@ impl LlmConfig { session, nearai, provider, + request_timeout_secs, }) } @@ -1016,4 +1024,30 @@ mod tests { assert_eq!(parsed, variant, "round-trip failed for {s}"); } } + + #[test] + fn test_request_timeout_defaults_to_120() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::remove_var("LLM_REQUEST_TIMEOUT_SECS"); + } + let config = LlmConfig::resolve(&Settings::default()).expect("resolve"); + assert_eq!(config.request_timeout_secs, 120); + } + + #[test] + fn test_request_timeout_configurable() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + // SAFETY: Under ENV_MUTEX. + unsafe { + std::env::set_var("LLM_REQUEST_TIMEOUT_SECS", "300"); + } + let config = LlmConfig::resolve(&Settings::default()).expect("resolve"); + assert_eq!(config.request_timeout_secs, 300); + // SAFETY: Cleanup + unsafe { + std::env::remove_var("LLM_REQUEST_TIMEOUT_SECS"); + } + } } diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 81a33940..8945a887 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -58,8 +58,10 @@ pub fn create_llm_provider( config: &LlmConfig, session: Arc, ) -> Result, LlmError> { + let timeout = config.request_timeout_secs; + if config.backend == "nearai" || config.backend == "near_ai" || config.backend == "near" { - return create_llm_provider_with_config(&config.nearai, session); + return create_llm_provider_with_config(&config.nearai, session, timeout); } let reg_config = config @@ -79,6 +81,7 @@ pub fn create_llm_provider( pub fn create_llm_provider_with_config( config: &NearAiConfig, session: Arc, + request_timeout_secs: u64, ) -> Result, LlmError> { let auth_mode = if config.api_key.is_some() { "API key" @@ -89,9 +92,14 @@ pub fn create_llm_provider_with_config( model = %config.model, base_url = %config.base_url, auth = auth_mode, + timeout_secs = request_timeout_secs, "Using NEAR AI (Chat Completions API)" ); - Ok(Arc::new(NearAiChatProvider::new(config.clone(), session)?)) + Ok(Arc::new(NearAiChatProvider::new_with_timeout( + config.clone(), + session, + request_timeout_secs, + )?)) } /// Create a provider from a registry-resolved config. @@ -365,7 +373,11 @@ pub fn build_provider_chain( let llm: Arc = if let Some(ref cheap_model) = config.nearai.cheap_model { let mut cheap_config = config.nearai.clone(); cheap_config.model = cheap_model.clone(); - let cheap = create_llm_provider_with_config(&cheap_config, session.clone())?; + let cheap = create_llm_provider_with_config( + &cheap_config, + session.clone(), + config.request_timeout_secs, + )?; let cheap: Arc = if retry_config.max_retries > 0 { Arc::new(RetryProvider::new(cheap, retry_config.clone())) } else { @@ -397,7 +409,11 @@ pub fn build_provider_chain( } let mut fallback_config = config.nearai.clone(); fallback_config.model = fallback_model.clone(); - let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?; + let fallback = create_llm_provider_with_config( + &fallback_config, + session.clone(), + config.request_timeout_secs, + )?; tracing::info!( primary = %llm.model_name(), fallback = %fallback.model_name(), @@ -503,6 +519,7 @@ mod tests { session: SessionConfig::default(), nearai: test_nearai_config(), provider: None, + request_timeout_secs: 120, } } diff --git a/src/llm/nearai_chat.rs b/src/llm/nearai_chat.rs index 7637cc08..659dd956 100644 --- a/src/llm/nearai_chat.rs +++ b/src/llm/nearai_chat.rs @@ -58,17 +58,28 @@ impl NearAiChatProvider { /// By default this enables tool-message flattening for compatibility with /// providers that reject `role: "tool"` messages. pub fn new(config: NearAiConfig, session: Arc) -> Result { - Self::new_with_flatten(config, session, true) + Self::new_with_options(config, session, true, 120) } - /// Create a chat completions provider with configurable tool-message flattening. - pub fn new_with_flatten( + /// Create a new provider with a custom request timeout. + pub fn new_with_timeout( + config: NearAiConfig, + session: Arc, + request_timeout_secs: u64, + ) -> Result { + Self::new_with_options(config, session, true, request_timeout_secs) + } + + /// Create a chat completions provider with configurable tool-message flattening + /// and request timeout. + pub fn new_with_options( config: NearAiConfig, session: Arc, flatten_tool_messages: bool, + request_timeout_secs: u64, ) -> Result { let client = Client::builder() - .timeout(std::time::Duration::from_secs(120)) + .timeout(std::time::Duration::from_secs(request_timeout_secs)) .build() .map_err(|e| LlmError::RequestFailed { provider: "nearai_chat".to_string(), diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 2f74ae05..19bf9848 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -1491,6 +1491,7 @@ impl SetupWizard { smart_routing_cascade: true, }, provider: None, + request_timeout_secs: 120, }; match create_llm_provider(&config, session) {