mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c1ca3bb91c | ||
|
|
e499795b8c | ||
|
|
5e1da4827a | ||
|
|
d04af5cd75 | ||
|
|
956037c4d3 | ||
|
|
68a1851c19 |
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.5.0](https://github.com/nearai/ironclaw/compare/v0.4.0...v0.5.0) - 2026-02-17
|
||||
|
||||
### Added
|
||||
|
||||
- add cooldown management to FailoverProvider ([#114](https://github.com/nearai/ironclaw/pull/114))
|
||||
|
||||
## [0.4.0](https://github.com/nearai/ironclaw/compare/v0.3.0...v0.4.0) - 2026-02-17
|
||||
|
||||
### Added
|
||||
|
||||
Generated
+1
-1
@@ -2490,7 +2490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ironclaw"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"aho-corasick",
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ exclude = [
|
||||
|
||||
[package]
|
||||
name = "ironclaw"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.92"
|
||||
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
||||
|
||||
+1
-1
@@ -174,7 +174,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|---------|----------|----------|-------|
|
||||
| Auto-discovery | ✅ | ❌ | |
|
||||
| Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` |
|
||||
| Cooldown management | ✅ | ❌ | Skip failed providers |
|
||||
| Cooldown management | ✅ | ✅ | Lock-free per-provider cooldown in `FailoverProvider` |
|
||||
| Per-session model override | ✅ | ✅ | Model selector in TUI |
|
||||
| Model selection UI | ✅ | ✅ | TUI keyboard shortcut |
|
||||
|
||||
|
||||
+16
-10
@@ -1579,6 +1579,9 @@ impl Agent {
|
||||
session: Arc<Mutex<Session>>,
|
||||
thread_id: Uuid,
|
||||
) -> Result<SubmissionResult, Error> {
|
||||
// Lock session first, then undo manager -- consistent with process_user_input
|
||||
// to avoid potential deadlocks.
|
||||
let mut sess = session.lock().await;
|
||||
let undo_mgr = self.session_manager.get_undo_manager(thread_id).await;
|
||||
let mut mgr = undo_mgr.lock().await;
|
||||
|
||||
@@ -1586,7 +1589,6 @@ impl Agent {
|
||||
return Ok(SubmissionResult::ok_with_message("Nothing to undo."));
|
||||
}
|
||||
|
||||
let mut sess = session.lock().await;
|
||||
let thread = sess
|
||||
.threads
|
||||
.get_mut(&thread_id)
|
||||
@@ -1597,12 +1599,10 @@ impl Agent {
|
||||
let current_turn = thread.turn_number();
|
||||
|
||||
if let Some(checkpoint) = mgr.undo(current_turn, current_messages) {
|
||||
// Extract values before consuming the reference
|
||||
let turn_number = checkpoint.turn_number;
|
||||
let messages = checkpoint.messages.clone();
|
||||
let undo_count = mgr.undo_count();
|
||||
// Restore thread from checkpoint
|
||||
thread.restore_from_messages(messages);
|
||||
thread.restore_from_messages(checkpoint.messages);
|
||||
Ok(SubmissionResult::ok_with_message(format!(
|
||||
"Undone to turn {}. {} undo(s) remaining.",
|
||||
turn_number, undo_count
|
||||
@@ -1617,6 +1617,9 @@ impl Agent {
|
||||
session: Arc<Mutex<Session>>,
|
||||
thread_id: Uuid,
|
||||
) -> Result<SubmissionResult, Error> {
|
||||
// Lock session first, then undo manager -- consistent with process_user_input
|
||||
// to avoid potential deadlocks.
|
||||
let mut sess = session.lock().await;
|
||||
let undo_mgr = self.session_manager.get_undo_manager(thread_id).await;
|
||||
let mut mgr = undo_mgr.lock().await;
|
||||
|
||||
@@ -1624,12 +1627,15 @@ impl Agent {
|
||||
return Ok(SubmissionResult::ok_with_message("Nothing to redo."));
|
||||
}
|
||||
|
||||
if let Some(checkpoint) = mgr.redo() {
|
||||
let mut sess = session.lock().await;
|
||||
let thread = sess
|
||||
.threads
|
||||
.get_mut(&thread_id)
|
||||
.ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?;
|
||||
let thread = sess
|
||||
.threads
|
||||
.get_mut(&thread_id)
|
||||
.ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?;
|
||||
|
||||
let current_messages = thread.messages();
|
||||
let current_turn = thread.turn_number();
|
||||
|
||||
if let Some(checkpoint) = mgr.redo(current_turn, current_messages) {
|
||||
thread.restore_from_messages(checkpoint.messages);
|
||||
Ok(SubmissionResult::ok_with_message(format!(
|
||||
"Redone to turn {}.",
|
||||
|
||||
+136
-16
@@ -43,6 +43,10 @@ impl Checkpoint {
|
||||
}
|
||||
|
||||
/// Manager for undo/redo functionality.
|
||||
///
|
||||
/// Each undo/redo operation pops from one stack and pushes the current state
|
||||
/// onto the other, so `undo_count() + redo_count()` stays constant across
|
||||
/// undo/redo cycles (only `checkpoint()` and `clear()` change the total).
|
||||
pub struct UndoManager {
|
||||
/// Stack of past checkpoints (for undo).
|
||||
undo_stack: VecDeque<Checkpoint>,
|
||||
@@ -68,6 +72,14 @@ impl UndoManager {
|
||||
self
|
||||
}
|
||||
|
||||
/// Push a checkpoint onto the undo stack, trimming oldest entries if over limit.
|
||||
fn push_undo(&mut self, checkpoint: Checkpoint) {
|
||||
self.undo_stack.push_back(checkpoint);
|
||||
while self.undo_stack.len() > self.max_checkpoints {
|
||||
self.undo_stack.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a checkpoint at the current state.
|
||||
///
|
||||
/// This clears the redo stack since we're creating a new history branch.
|
||||
@@ -80,24 +92,23 @@ impl UndoManager {
|
||||
// Clear redo stack (new branch of history)
|
||||
self.redo_stack.clear();
|
||||
|
||||
// Create and push checkpoint
|
||||
let checkpoint = Checkpoint::new(turn_number, messages, description);
|
||||
self.undo_stack.push_back(checkpoint);
|
||||
|
||||
// Trim if over limit
|
||||
while self.undo_stack.len() > self.max_checkpoints {
|
||||
self.undo_stack.pop_front();
|
||||
}
|
||||
self.push_undo(checkpoint);
|
||||
}
|
||||
|
||||
/// Undo: pop the last checkpoint and return it.
|
||||
///
|
||||
/// The current state should be saved to redo stack before calling this.
|
||||
/// Saves the current state to the redo stack and pops the most recent
|
||||
/// checkpoint from the undo stack so that repeated undos walk backwards
|
||||
/// through history.
|
||||
///
|
||||
/// Takes ownership of `current_messages`; callers must clone first if
|
||||
/// they need to retain a copy.
|
||||
pub fn undo(
|
||||
&mut self,
|
||||
current_turn: usize,
|
||||
current_messages: Vec<ChatMessage>,
|
||||
) -> Option<&Checkpoint> {
|
||||
) -> Option<Checkpoint> {
|
||||
if self.undo_stack.is_empty() {
|
||||
return None;
|
||||
}
|
||||
@@ -110,9 +121,8 @@ impl UndoManager {
|
||||
);
|
||||
self.redo_stack.push(current);
|
||||
|
||||
// Return the most recent checkpoint without removing it
|
||||
// (we keep it so multiple undos can work)
|
||||
self.undo_stack.back()
|
||||
// Pop and return the most recent checkpoint
|
||||
self.undo_stack.pop_back()
|
||||
}
|
||||
|
||||
/// Pop the last checkpoint from the undo stack.
|
||||
@@ -121,7 +131,29 @@ impl UndoManager {
|
||||
}
|
||||
|
||||
/// Redo: restore a previously undone state.
|
||||
pub fn redo(&mut self) -> Option<Checkpoint> {
|
||||
///
|
||||
/// Saves the current state to the undo stack and pops the most recent
|
||||
/// checkpoint from the redo stack.
|
||||
///
|
||||
/// Takes ownership of `current_messages`; callers must clone first if
|
||||
/// they need to retain a copy.
|
||||
pub fn redo(
|
||||
&mut self,
|
||||
current_turn: usize,
|
||||
current_messages: Vec<ChatMessage>,
|
||||
) -> Option<Checkpoint> {
|
||||
if self.redo_stack.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Save current state to undo stack
|
||||
let current = Checkpoint::new(
|
||||
current_turn,
|
||||
current_messages,
|
||||
format!("Turn {}", current_turn),
|
||||
);
|
||||
self.push_undo(current);
|
||||
|
||||
self.redo_stack.pop()
|
||||
}
|
||||
|
||||
@@ -214,14 +246,16 @@ mod tests {
|
||||
assert!(manager.can_undo());
|
||||
assert!(!manager.can_redo());
|
||||
|
||||
// Undo
|
||||
// Undo - returns owned Checkpoint now
|
||||
let current = vec![ChatMessage::user("Hello"), ChatMessage::assistant("Hi")];
|
||||
let checkpoint = manager.undo(2, current);
|
||||
assert!(checkpoint.is_some());
|
||||
let checkpoint = checkpoint.unwrap();
|
||||
assert_eq!(checkpoint.turn_number, 1);
|
||||
assert!(manager.can_redo());
|
||||
|
||||
// Redo
|
||||
let restored = manager.redo();
|
||||
// Redo - now requires current state parameters
|
||||
let restored = manager.redo(checkpoint.turn_number, checkpoint.messages);
|
||||
assert!(restored.is_some());
|
||||
}
|
||||
|
||||
@@ -249,4 +283,90 @@ mod tests {
|
||||
assert!(restored.is_some());
|
||||
assert_eq!(manager.undo_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_repeated_undo_advances_through_stack() {
|
||||
let mut manager = UndoManager::new();
|
||||
|
||||
// Create 3 checkpoints at turns 0, 1, 2
|
||||
manager.checkpoint(0, vec![], "Turn 0");
|
||||
manager.checkpoint(1, vec![ChatMessage::user("msg1")], "Turn 1");
|
||||
manager.checkpoint(2, vec![ChatMessage::user("msg2")], "Turn 2");
|
||||
assert_eq!(manager.undo_count(), 3);
|
||||
|
||||
// First undo: should return turn 2 checkpoint, stack shrinks to 2
|
||||
let cp1 = manager
|
||||
.undo(3, vec![ChatMessage::user("msg3")])
|
||||
.expect("first undo should succeed");
|
||||
assert_eq!(cp1.turn_number, 2);
|
||||
assert_eq!(manager.undo_count(), 2);
|
||||
|
||||
// Second undo: should return turn 1 checkpoint (different!), stack shrinks to 1
|
||||
let cp2 = manager
|
||||
.undo(cp1.turn_number, cp1.messages)
|
||||
.expect("second undo should succeed");
|
||||
assert_eq!(cp2.turn_number, 1);
|
||||
assert_eq!(manager.undo_count(), 1);
|
||||
|
||||
// Verify we walked backwards through distinct checkpoints
|
||||
assert_ne!(cp1.turn_number, cp2.turn_number);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_undo_redo_cycle_preserves_state() {
|
||||
let mut manager = UndoManager::new();
|
||||
|
||||
let msgs_t0: Vec<ChatMessage> = vec![];
|
||||
let msgs_t1 = vec![ChatMessage::user("hello")];
|
||||
let msgs_t2 = vec![ChatMessage::user("hello"), ChatMessage::assistant("hi")];
|
||||
|
||||
manager.checkpoint(0, msgs_t0, "Turn 0");
|
||||
manager.checkpoint(1, msgs_t1, "Turn 1");
|
||||
|
||||
// Undo from turn 2 -> get turn 1 checkpoint
|
||||
let cp_undo1 = manager
|
||||
.undo(2, msgs_t2.clone())
|
||||
.expect("undo should succeed");
|
||||
assert_eq!(cp_undo1.turn_number, 1);
|
||||
|
||||
// Redo from turn 1 -> get turn 2 state back
|
||||
let cp_redo = manager
|
||||
.redo(cp_undo1.turn_number, cp_undo1.messages)
|
||||
.expect("redo should succeed");
|
||||
assert_eq!(cp_redo.turn_number, 2);
|
||||
assert_eq!(cp_redo.messages.len(), 2);
|
||||
|
||||
// Undo again from turn 2 -> should go back to turn 1 again
|
||||
let cp_undo2 = manager
|
||||
.undo(cp_redo.turn_number, cp_redo.messages)
|
||||
.expect("second undo should succeed");
|
||||
assert_eq!(cp_undo2.turn_number, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_undo_redo_stack_sizes_consistent() {
|
||||
let mut manager = UndoManager::new();
|
||||
|
||||
manager.checkpoint(0, vec![], "Turn 0");
|
||||
manager.checkpoint(1, vec![ChatMessage::user("a")], "Turn 1");
|
||||
manager.checkpoint(2, vec![ChatMessage::user("b")], "Turn 2");
|
||||
|
||||
// Start: undo=3, redo=0, total=3
|
||||
let total = manager.undo_count() + manager.redo_count();
|
||||
assert_eq!(total, 3);
|
||||
|
||||
// After undo: total should still be 3 (one moved from undo to redo,
|
||||
// plus the current state pushed to redo)
|
||||
// Actually: undo pops one (3->2), pushes current to redo (0->1), total=3
|
||||
let cp = manager.undo(3, vec![]).unwrap();
|
||||
assert_eq!(manager.undo_count() + manager.redo_count(), 3);
|
||||
|
||||
// After redo: redo pops one (1->0), pushes current to undo (2->3), total=3
|
||||
let cp2 = manager.redo(cp.turn_number, cp.messages).unwrap();
|
||||
assert_eq!(manager.undo_count() + manager.redo_count(), 3);
|
||||
|
||||
// After another undo: same invariant
|
||||
let _cp3 = manager.undo(cp2.turn_number, cp2.messages).unwrap();
|
||||
assert_eq!(manager.undo_count() + manager.redo_count(), 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ let loadingOlder = false;
|
||||
let jobEvents = new Map(); // job_id -> Array of events
|
||||
let jobListRefreshTimer = null;
|
||||
const JOB_EVENTS_CAP = 500;
|
||||
const MEMORY_SEARCH_QUERY_MAX_LENGTH = 100;
|
||||
|
||||
// --- Auth ---
|
||||
|
||||
@@ -1001,9 +1002,12 @@ function buildBreadcrumb(path) {
|
||||
}
|
||||
|
||||
function searchMemory(query) {
|
||||
const normalizedQuery = normalizeSearchQuery(query);
|
||||
if (!normalizedQuery) return;
|
||||
|
||||
apiFetch('/api/memory/search', {
|
||||
method: 'POST',
|
||||
body: { query, limit: 20 },
|
||||
body: { query: normalizedQuery, limit: 20 },
|
||||
}).then((data) => {
|
||||
const tree = document.getElementById('memory-tree');
|
||||
tree.innerHTML = '';
|
||||
@@ -1014,18 +1018,23 @@ function searchMemory(query) {
|
||||
for (const result of data.results) {
|
||||
const item = document.createElement('div');
|
||||
item.className = 'search-result';
|
||||
const snippet = snippetAround(result.content, query, 120);
|
||||
const snippet = snippetAround(result.content, normalizedQuery, 120);
|
||||
item.innerHTML = '<div class="path">' + escapeHtml(result.path) + '</div>'
|
||||
+ '<div class="snippet">' + highlightQuery(snippet, query) + '</div>';
|
||||
+ '<div class="snippet">' + highlightQuery(snippet, normalizedQuery) + '</div>';
|
||||
item.addEventListener('click', () => readMemoryFile(result.path));
|
||||
tree.appendChild(item);
|
||||
}
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
function normalizeSearchQuery(query) {
|
||||
return (typeof query === 'string' ? query : '').slice(0, MEMORY_SEARCH_QUERY_MAX_LENGTH);
|
||||
}
|
||||
|
||||
function snippetAround(text, query, len) {
|
||||
const normalizedQuery = normalizeSearchQuery(query);
|
||||
const lower = text.toLowerCase();
|
||||
const idx = lower.indexOf(query.toLowerCase());
|
||||
const idx = lower.indexOf(normalizedQuery.toLowerCase());
|
||||
if (idx < 0) return text.substring(0, len);
|
||||
const start = Math.max(0, idx - Math.floor(len / 2));
|
||||
const end = Math.min(text.length, start + len);
|
||||
@@ -1038,11 +1047,11 @@ function snippetAround(text, query, len) {
|
||||
function highlightQuery(text, query) {
|
||||
if (!query) return escapeHtml(text);
|
||||
const escaped = escapeHtml(text);
|
||||
const queryEscaped = query.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const normalizedQuery = normalizeSearchQuery(query);
|
||||
const queryEscaped = normalizedQuery.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const re = new RegExp('(' + queryEscaped + ')', 'gi');
|
||||
return escaped.replace(re, '<mark>$1</mark>');
|
||||
}
|
||||
|
||||
// --- Logs ---
|
||||
|
||||
const LOG_MAX_ENTRIES = 2000;
|
||||
|
||||
@@ -5,7 +5,11 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>IronClaw</title>
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
|
||||
<script
|
||||
src="https://cdn.jsdelivr.net/npm/[email protected]/lib/marked.umd.min.js"
|
||||
integrity="sha384-pN9zSKOnTZwXRtYZAu0PBPEgR2B7DOC1aeLxQ33oJ0oy5iN1we6gm57xldM2irDG"
|
||||
crossorigin="anonymous"
|
||||
></script>
|
||||
</head>
|
||||
<body>
|
||||
<!-- Auth Screen -->
|
||||
|
||||
@@ -419,6 +419,13 @@ pub struct NearAiConfig {
|
||||
/// With the default of 3, the provider makes up to 4 total attempts
|
||||
/// (1 initial + 3 retries) before giving up.
|
||||
pub max_retries: u32,
|
||||
/// Cooldown duration in seconds for the failover provider (default: 300).
|
||||
/// When a provider accumulates enough consecutive failures it is skipped
|
||||
/// for this many seconds.
|
||||
pub failover_cooldown_secs: u64,
|
||||
/// Number of consecutive retryable failures before a provider enters
|
||||
/// cooldown (default: 3).
|
||||
pub failover_cooldown_threshold: u32,
|
||||
}
|
||||
|
||||
impl LlmConfig {
|
||||
@@ -478,6 +485,8 @@ impl LlmConfig {
|
||||
api_key: nearai_api_key,
|
||||
fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?,
|
||||
max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?,
|
||||
failover_cooldown_secs: parse_optional_env("LLM_FAILOVER_COOLDOWN_SECS", 300)?,
|
||||
failover_cooldown_threshold: parse_optional_env("LLM_FAILOVER_THRESHOLD", 3)?,
|
||||
};
|
||||
|
||||
// Resolve provider-specific configs based on backend
|
||||
|
||||
+542
-8
@@ -2,10 +2,15 @@
|
||||
//!
|
||||
//! Wraps multiple LlmProvider instances and tries each in sequence
|
||||
//! until one succeeds. Transparent to callers --- same LlmProvider trait.
|
||||
//!
|
||||
//! Providers that fail repeatedly are temporarily placed in cooldown
|
||||
//! so subsequent requests skip them, reducing latency when a provider
|
||||
//! is known to be down. Cooldown state is lock-free (atomics only).
|
||||
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
@@ -41,61 +46,217 @@ fn is_retryable(err: &LlmError) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Configuration for per-provider cooldown behavior.
|
||||
///
|
||||
/// When a provider accumulates `failure_threshold` consecutive retryable
|
||||
/// failures, it enters cooldown for `cooldown_duration`. During cooldown
|
||||
/// the provider is skipped (unless *all* providers are in cooldown, in
|
||||
/// which case the oldest-cooled one is tried).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CooldownConfig {
|
||||
/// How long a provider stays in cooldown after exceeding the threshold.
|
||||
pub cooldown_duration: Duration,
|
||||
/// Number of consecutive retryable failures before cooldown activates.
|
||||
pub failure_threshold: u32,
|
||||
}
|
||||
|
||||
impl Default for CooldownConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 3,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-provider cooldown state, entirely lock-free.
|
||||
///
|
||||
/// All atomic operations use `Relaxed` ordering — consistent with the
|
||||
/// existing `last_used` field. Stale reads are harmless: the worst case
|
||||
/// is one extra attempt against a provider that just entered cooldown.
|
||||
struct ProviderCooldown {
|
||||
/// Consecutive retryable failures. Reset to 0 on success.
|
||||
failure_count: AtomicU32,
|
||||
/// Nanoseconds since `epoch` when cooldown was activated.
|
||||
/// 0 means the provider is NOT in cooldown.
|
||||
cooldown_activated_nanos: AtomicU64,
|
||||
}
|
||||
|
||||
impl ProviderCooldown {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
failure_count: AtomicU32::new(0),
|
||||
cooldown_activated_nanos: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the provider is currently in cooldown.
|
||||
fn is_in_cooldown(&self, now_nanos: u64, cooldown_nanos: u64) -> bool {
|
||||
let activated = self.cooldown_activated_nanos.load(Ordering::Relaxed);
|
||||
activated != 0 && now_nanos.saturating_sub(activated) < cooldown_nanos
|
||||
}
|
||||
|
||||
/// Record a retryable failure. Returns `true` if the threshold was
|
||||
/// just reached (caller should activate cooldown).
|
||||
fn record_failure(&self, threshold: u32) -> bool {
|
||||
let prev = self.failure_count.fetch_add(1, Ordering::Relaxed);
|
||||
prev + 1 >= threshold
|
||||
}
|
||||
|
||||
/// Activate cooldown at the given timestamp.
|
||||
fn activate_cooldown(&self, now_nanos: u64) {
|
||||
self.cooldown_activated_nanos
|
||||
.store(now_nanos, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Reset failure count and clear cooldown (called on success).
|
||||
fn reset(&self) {
|
||||
self.failure_count.store(0, Ordering::Relaxed);
|
||||
self.cooldown_activated_nanos.store(0, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// An LLM provider that wraps multiple providers and tries each in sequence
|
||||
/// on transient failures.
|
||||
///
|
||||
/// The first provider in the list is the primary. If it fails with a retryable
|
||||
/// error, the next provider is tried, and so on. Non-retryable errors
|
||||
/// (e.g. `AuthFailed`, `ContextLengthExceeded`) propagate immediately.
|
||||
///
|
||||
/// Providers that repeatedly fail with retryable errors are temporarily
|
||||
/// placed in cooldown and skipped, reducing latency.
|
||||
pub struct FailoverProvider {
|
||||
providers: Vec<Arc<dyn LlmProvider>>,
|
||||
/// Index of the provider that last handled a request successfully.
|
||||
/// Used by `model_name()` and `cost_per_token()` so downstream cost
|
||||
/// tracking reflects the provider that actually served the request.
|
||||
last_used: AtomicUsize,
|
||||
/// Per-provider cooldown tracking (same length as `providers`).
|
||||
cooldowns: Vec<ProviderCooldown>,
|
||||
/// Reference instant for computing elapsed nanos. Shared across all
|
||||
/// cooldown timestamps so they are comparable.
|
||||
epoch: Instant,
|
||||
/// Cooldown configuration.
|
||||
cooldown_config: CooldownConfig,
|
||||
}
|
||||
|
||||
impl FailoverProvider {
|
||||
/// Create a new failover provider.
|
||||
/// Create a new failover provider with default cooldown settings.
|
||||
///
|
||||
/// Returns an error if `providers` is empty.
|
||||
pub fn new(providers: Vec<Arc<dyn LlmProvider>>) -> Result<Self, LlmError> {
|
||||
Self::with_cooldown(providers, CooldownConfig::default())
|
||||
}
|
||||
|
||||
/// Create a new failover provider with explicit cooldown configuration.
|
||||
///
|
||||
/// Returns an error if `providers` is empty.
|
||||
pub fn with_cooldown(
|
||||
providers: Vec<Arc<dyn LlmProvider>>,
|
||||
cooldown_config: CooldownConfig,
|
||||
) -> Result<Self, LlmError> {
|
||||
if providers.is_empty() {
|
||||
return Err(LlmError::RequestFailed {
|
||||
provider: "failover".to_string(),
|
||||
reason: "FailoverProvider requires at least one provider".to_string(),
|
||||
});
|
||||
}
|
||||
let cooldowns = (0..providers.len())
|
||||
.map(|_| ProviderCooldown::new())
|
||||
.collect();
|
||||
Ok(Self {
|
||||
providers,
|
||||
last_used: AtomicUsize::new(0),
|
||||
cooldowns,
|
||||
epoch: Instant::now(),
|
||||
cooldown_config,
|
||||
})
|
||||
}
|
||||
|
||||
/// Nanoseconds elapsed since `self.epoch`.
|
||||
///
|
||||
/// Truncates `u128` → `u64` (wraps after ~584 years of continuous
|
||||
/// uptime). Acceptable because `epoch` is set at construction time.
|
||||
fn now_nanos(&self) -> u64 {
|
||||
self.epoch.elapsed().as_nanos() as u64
|
||||
}
|
||||
|
||||
/// Try each provider in sequence until one succeeds or all fail.
|
||||
///
|
||||
/// Providers in cooldown are skipped unless *all* providers are in
|
||||
/// cooldown, in which case the one with the oldest cooldown timestamp
|
||||
/// (most likely to have recovered) is tried.
|
||||
async fn try_providers<T, F, Fut>(&self, mut call: F) -> Result<T, LlmError>
|
||||
where
|
||||
F: FnMut(Arc<dyn LlmProvider>) -> Fut,
|
||||
Fut: Future<Output = Result<T, LlmError>>,
|
||||
{
|
||||
let now_nanos = self.now_nanos();
|
||||
let cooldown_nanos = self.cooldown_config.cooldown_duration.as_nanos() as u64;
|
||||
|
||||
// Partition providers into available and cooled-down.
|
||||
let (mut available, cooled_down): (Vec<usize>, Vec<usize>) = (0..self.providers.len())
|
||||
.partition(|&i| !self.cooldowns[i].is_in_cooldown(now_nanos, cooldown_nanos));
|
||||
|
||||
// Log skipped providers.
|
||||
for &i in &cooled_down {
|
||||
tracing::info!(
|
||||
provider = %self.providers[i].model_name(),
|
||||
"Skipping provider (in cooldown)"
|
||||
);
|
||||
}
|
||||
|
||||
// Never skip ALL providers: if every provider is in cooldown, pick
|
||||
// the one with the oldest cooldown activation (most likely recovered).
|
||||
if available.is_empty() {
|
||||
let oldest = (0..self.providers.len())
|
||||
.min_by_key(|&i| {
|
||||
self.cooldowns[i]
|
||||
.cooldown_activated_nanos
|
||||
.load(Ordering::Relaxed)
|
||||
})
|
||||
.expect("providers list is non-empty");
|
||||
tracing::info!(
|
||||
provider = %self.providers[oldest].model_name(),
|
||||
"All providers in cooldown, trying oldest-cooled provider"
|
||||
);
|
||||
available.push(oldest);
|
||||
}
|
||||
|
||||
let mut last_error: Option<LlmError> = None;
|
||||
|
||||
for (i, provider) in self.providers.iter().enumerate() {
|
||||
for (pos, &i) in available.iter().enumerate() {
|
||||
let provider = &self.providers[i];
|
||||
let result = call(Arc::clone(provider)).await;
|
||||
match result {
|
||||
Ok(response) => {
|
||||
self.last_used.store(i, Ordering::Relaxed);
|
||||
self.cooldowns[i].reset();
|
||||
return Ok(response);
|
||||
}
|
||||
Err(err) => {
|
||||
if !is_retryable(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
if i + 1 < self.providers.len() {
|
||||
|
||||
// Increment failure count; activate cooldown if threshold reached.
|
||||
if self.cooldowns[i].record_failure(self.cooldown_config.failure_threshold) {
|
||||
let nanos = self.now_nanos();
|
||||
self.cooldowns[i].activate_cooldown(nanos);
|
||||
tracing::warn!(
|
||||
provider = %provider.model_name(),
|
||||
threshold = self.cooldown_config.failure_threshold,
|
||||
cooldown_secs = self.cooldown_config.cooldown_duration.as_secs(),
|
||||
"Provider entered cooldown after repeated failures"
|
||||
);
|
||||
}
|
||||
|
||||
if pos + 1 < available.len() {
|
||||
let next_i = available[pos + 1];
|
||||
tracing::warn!(
|
||||
provider = %provider.model_name(),
|
||||
error = %err,
|
||||
next_provider = %self.providers[i + 1].model_name(),
|
||||
next_provider = %self.providers[next_i].model_name(),
|
||||
"Provider failed with retryable error, trying next provider"
|
||||
);
|
||||
}
|
||||
@@ -104,9 +265,9 @@ impl FailoverProvider {
|
||||
}
|
||||
}
|
||||
|
||||
// SAFETY: providers is non-empty (checked in `new`), so at least one
|
||||
// SAFETY: `available` is non-empty (guaranteed above), so at least one
|
||||
// iteration ran and `last_error` is `Some`.
|
||||
Err(last_error.expect("providers list is non-empty"))
|
||||
Err(last_error.expect("available providers list is non-empty"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,7 +327,6 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
use std::sync::Mutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::llm::provider::{CompletionResponse, FinishReason, ToolCompletionResponse};
|
||||
|
||||
@@ -432,6 +592,380 @@ mod tests {
|
||||
assert!(models.contains(&"model-b".to_string()));
|
||||
}
|
||||
|
||||
// --- MultiCallMockProvider for cooldown tests ---
|
||||
//
|
||||
// Unlike `MockProvider` which uses `.take()` (single-use), this mock
|
||||
// tracks a call counter and returns errors for the first N calls,
|
||||
// then succeeds.
|
||||
|
||||
struct MultiCallMockProvider {
|
||||
name: String,
|
||||
/// How many calls should fail before succeeding. 0 = always succeed.
|
||||
fail_count: u32,
|
||||
/// Atomically tracks how many times `complete` has been called.
|
||||
calls: AtomicU32,
|
||||
/// If true, failures are non-retryable (AuthFailed).
|
||||
non_retryable: bool,
|
||||
}
|
||||
|
||||
impl MultiCallMockProvider {
|
||||
/// Always succeeds.
|
||||
fn always_ok(name: &str) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
fail_count: 0,
|
||||
calls: AtomicU32::new(0),
|
||||
non_retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fails with retryable error for the first `n` calls, then succeeds.
|
||||
fn fail_then_ok(name: &str, n: u32) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
fail_count: n,
|
||||
calls: AtomicU32::new(0),
|
||||
non_retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Always fails with retryable error.
|
||||
fn always_fail(name: &str) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
fail_count: u32::MAX,
|
||||
calls: AtomicU32::new(0),
|
||||
non_retryable: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Always fails with non-retryable error.
|
||||
fn always_fail_non_retryable(name: &str) -> Self {
|
||||
Self {
|
||||
name: name.to_string(),
|
||||
fail_count: u32::MAX,
|
||||
calls: AtomicU32::new(0),
|
||||
non_retryable: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn call_count(&self) -> u32 {
|
||||
self.calls.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for MultiCallMockProvider {
|
||||
fn model_name(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
(Decimal::ZERO, Decimal::ZERO)
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: CompletionRequest,
|
||||
) -> Result<CompletionResponse, LlmError> {
|
||||
let n = self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
if n < self.fail_count {
|
||||
if self.non_retryable {
|
||||
return Err(LlmError::AuthFailed {
|
||||
provider: self.name.clone(),
|
||||
});
|
||||
}
|
||||
return Err(LlmError::RequestFailed {
|
||||
provider: self.name.clone(),
|
||||
reason: format!("call {} failed", n),
|
||||
});
|
||||
}
|
||||
Ok(CompletionResponse {
|
||||
content: format!("{} ok", self.name),
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
_request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
let n = self.calls.fetch_add(1, Ordering::Relaxed);
|
||||
if n < self.fail_count {
|
||||
if self.non_retryable {
|
||||
return Err(LlmError::AuthFailed {
|
||||
provider: self.name.clone(),
|
||||
});
|
||||
}
|
||||
return Err(LlmError::RequestFailed {
|
||||
provider: self.name.clone(),
|
||||
reason: format!("call {} failed", n),
|
||||
});
|
||||
}
|
||||
Ok(ToolCompletionResponse {
|
||||
content: Some(format!("{} ok", self.name)),
|
||||
tool_calls: vec![],
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
Ok(vec![self.name.clone()])
|
||||
}
|
||||
}
|
||||
|
||||
// --- Cooldown tests ---
|
||||
|
||||
// Cooldown test 1: Provider enters cooldown after `threshold` consecutive failures.
|
||||
#[tokio::test]
|
||||
async fn cooldown_activates_after_threshold() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 2,
|
||||
};
|
||||
let p1 = Arc::new(MultiCallMockProvider::always_fail("p1"));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config).unwrap();
|
||||
|
||||
// Request 1: p1 fails (count=1, below threshold), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 1);
|
||||
|
||||
// Request 2: p1 fails again (count=2, reaches threshold → cooldown), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 2);
|
||||
|
||||
// Request 3: p1 should be skipped (in cooldown), only p2 called.
|
||||
let prev_p1_calls = p1.call_count();
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
// p1 was NOT called again.
|
||||
assert_eq!(p1.call_count(), prev_p1_calls);
|
||||
}
|
||||
|
||||
// Cooldown test 2: Cooldown expires after duration, provider is retried.
|
||||
#[tokio::test]
|
||||
async fn cooldown_expires_after_duration() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_millis(1),
|
||||
failure_threshold: 1,
|
||||
};
|
||||
// p1 fails once then succeeds (fail_then_ok with n=1 would work,
|
||||
// but we use always_fail to prove it's skipped, then swap).
|
||||
let p1 = Arc::new(MultiCallMockProvider::fail_then_ok("p1", 2));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config).unwrap();
|
||||
|
||||
// Request 1: p1 fails (threshold=1, enters cooldown immediately), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 1);
|
||||
|
||||
// Request 2: p1 in cooldown, skipped. Only p2 called.
|
||||
// (But cooldown is 1ms, so wait a bit to let it expire.)
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
|
||||
// After sleep, cooldown should have expired. p1 gets tried again.
|
||||
// p1 is set to fail 2 times total, so call #2 (index 1) still fails.
|
||||
// But it proves p1 was attempted again after cooldown expired.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(p1.call_count(), 2); // p1 was retried
|
||||
assert_eq!(r.content, "p2 ok"); // p2 handled it
|
||||
|
||||
// Wait again for cooldown to expire, p1 call #3 (index 2) succeeds.
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p1 ok");
|
||||
assert_eq!(p1.call_count(), 3);
|
||||
}
|
||||
|
||||
// Cooldown test 3: Never skip all providers — oldest-cooled one is tried.
|
||||
#[tokio::test]
|
||||
async fn never_skip_all_providers() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 1,
|
||||
};
|
||||
// Both providers always fail.
|
||||
let p1 = Arc::new(MultiCallMockProvider::always_fail("p1"));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_fail("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config).unwrap();
|
||||
|
||||
// Request 1: both tried, both fail, both enter cooldown.
|
||||
let _ = failover.complete(make_request()).await;
|
||||
assert_eq!(p1.call_count(), 1);
|
||||
assert_eq!(p2.call_count(), 1);
|
||||
|
||||
// Request 2: all in cooldown, but the oldest-cooled one (p1, activated
|
||||
// first) should be tried.
|
||||
let prev_total = p1.call_count() + p2.call_count();
|
||||
let _ = failover.complete(make_request()).await;
|
||||
let new_total = p1.call_count() + p2.call_count();
|
||||
// Exactly one more call was made (to the oldest-cooled provider).
|
||||
assert_eq!(new_total, prev_total + 1);
|
||||
}
|
||||
|
||||
// Cooldown test 4: Success resets failure count so it never reaches threshold.
|
||||
//
|
||||
// With threshold=3, accumulate 2 failures then succeed. Verify the
|
||||
// atomic counter is back to 0 and no cooldown was activated. Then
|
||||
// use a second provider pair to show that without the reset, 3
|
||||
// consecutive failures DO trigger cooldown (control case).
|
||||
#[tokio::test]
|
||||
async fn reset_on_success() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 3,
|
||||
};
|
||||
// p1 fails for calls 0,1 then succeeds on call 2+.
|
||||
let p1 = Arc::new(MultiCallMockProvider::fail_then_ok("p1", 2));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config.clone()).unwrap();
|
||||
|
||||
// Request 1: p1 fails (failure_count=1), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
|
||||
// Request 2: p1 fails (failure_count=2, still below threshold=3), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 2);
|
||||
|
||||
// Request 3: p1 succeeds (call index 2) → counter resets to 0.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p1 ok");
|
||||
assert_eq!(p1.call_count(), 3);
|
||||
|
||||
// Verify counter was reset to 0 and no cooldown activated.
|
||||
let nanos = failover.now_nanos();
|
||||
let cooldown_nanos = failover.cooldown_config.cooldown_duration.as_nanos() as u64;
|
||||
assert!(!failover.cooldowns[0].is_in_cooldown(nanos, cooldown_nanos));
|
||||
assert_eq!(
|
||||
failover.cooldowns[0].failure_count.load(Ordering::Relaxed),
|
||||
0
|
||||
);
|
||||
|
||||
// Control: without a success in the middle, 3 failures DO trigger cooldown.
|
||||
let p3 = Arc::new(MultiCallMockProvider::always_fail("p3"));
|
||||
let p4 = Arc::new(MultiCallMockProvider::always_ok("p4"));
|
||||
let control =
|
||||
FailoverProvider::with_cooldown(vec![p3.clone(), p4.clone()], config).unwrap();
|
||||
for _ in 0..3 {
|
||||
let _ = control.complete(make_request()).await.unwrap();
|
||||
}
|
||||
let nanos = control.now_nanos();
|
||||
assert!(control.cooldowns[0].is_in_cooldown(nanos, cooldown_nanos));
|
||||
}
|
||||
|
||||
// Cooldown test 5: threshold-1 failures don't trigger cooldown, threshold does.
|
||||
#[tokio::test]
|
||||
async fn threshold_boundary() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 3,
|
||||
};
|
||||
let p1 = Arc::new(MultiCallMockProvider::always_fail("p1"));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config).unwrap();
|
||||
|
||||
// 2 requests: p1 fails twice (below threshold of 3), not in cooldown.
|
||||
for _ in 0..2 {
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
}
|
||||
assert_eq!(p1.call_count(), 2);
|
||||
|
||||
// p1 should still be available (not in cooldown).
|
||||
let nanos = failover.now_nanos();
|
||||
let cooldown_nanos = failover.cooldown_config.cooldown_duration.as_nanos() as u64;
|
||||
assert!(!failover.cooldowns[0].is_in_cooldown(nanos, cooldown_nanos));
|
||||
|
||||
// 3rd request: p1 fails → reaches threshold → enters cooldown.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 3);
|
||||
|
||||
let nanos = failover.now_nanos();
|
||||
assert!(failover.cooldowns[0].is_in_cooldown(nanos, cooldown_nanos));
|
||||
|
||||
// 4th request: p1 should be skipped.
|
||||
let prev = p1.call_count();
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), prev); // not called
|
||||
}
|
||||
|
||||
// Cooldown test 6: Non-retryable error returns immediately, no failure bump.
|
||||
#[tokio::test]
|
||||
async fn non_retryable_does_not_increment_cooldown() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 1,
|
||||
};
|
||||
let p1 = Arc::new(MultiCallMockProvider::always_fail_non_retryable("p1"));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone()], config).unwrap();
|
||||
|
||||
// Non-retryable error should return immediately.
|
||||
let err = failover.complete(make_request()).await.unwrap_err();
|
||||
assert!(matches!(err, LlmError::AuthFailed { .. }));
|
||||
assert_eq!(p1.call_count(), 1);
|
||||
// p2 should NOT have been called (non-retryable = no failover).
|
||||
assert_eq!(p2.call_count(), 0);
|
||||
|
||||
// p1 should NOT be in cooldown (non-retryable doesn't bump count).
|
||||
let nanos = failover.now_nanos();
|
||||
let cooldown_nanos = failover.cooldown_config.cooldown_duration.as_nanos() as u64;
|
||||
assert!(!failover.cooldowns[0].is_in_cooldown(nanos, cooldown_nanos));
|
||||
}
|
||||
|
||||
// Cooldown test 7: Three providers, first in cooldown, second/third available.
|
||||
#[tokio::test]
|
||||
async fn three_providers_mixed_cooldown() {
|
||||
let config = CooldownConfig {
|
||||
cooldown_duration: Duration::from_secs(300),
|
||||
failure_threshold: 1,
|
||||
};
|
||||
let p1 = Arc::new(MultiCallMockProvider::always_fail("p1"));
|
||||
let p2 = Arc::new(MultiCallMockProvider::always_ok("p2"));
|
||||
let p3 = Arc::new(MultiCallMockProvider::always_ok("p3"));
|
||||
|
||||
let failover =
|
||||
FailoverProvider::with_cooldown(vec![p1.clone(), p2.clone(), p3.clone()], config)
|
||||
.unwrap();
|
||||
|
||||
// Request 1: p1 fails → enters cooldown (threshold=1), p2 succeeds.
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), 1);
|
||||
|
||||
// Request 2: p1 skipped (cooldown), p2 and p3 available.
|
||||
let prev = p1.call_count();
|
||||
let r = failover.complete(make_request()).await.unwrap();
|
||||
assert_eq!(r.content, "p2 ok");
|
||||
assert_eq!(p1.call_count(), prev); // p1 skipped
|
||||
}
|
||||
|
||||
// Test: is_retryable correctly classifies errors.
|
||||
#[test]
|
||||
fn retryable_classification() {
|
||||
|
||||
+3
-1
@@ -17,7 +17,7 @@ mod retry;
|
||||
mod rig_adapter;
|
||||
pub mod session;
|
||||
|
||||
pub use failover::FailoverProvider;
|
||||
pub use failover::{CooldownConfig, FailoverProvider};
|
||||
pub use nearai::{ModelInfo, NearAiProvider};
|
||||
pub use nearai_chat::NearAiChatProvider;
|
||||
pub use provider::{
|
||||
@@ -235,6 +235,8 @@ mod tests {
|
||||
api_key: None,
|
||||
fallback_model: None,
|
||||
max_retries: 3,
|
||||
failover_cooldown_secs: 300,
|
||||
failover_cooldown_threshold: 3,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+19
-6
@@ -428,17 +428,30 @@ impl SessionManager {
|
||||
})?;
|
||||
|
||||
let user_id = self.user_id.read().await.clone();
|
||||
let value = store
|
||||
let value = if let Some(value) = store
|
||||
.get_setting(&user_id, "nearai.session_token")
|
||||
.await
|
||||
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: format!("DB query failed: {}", e),
|
||||
})?
|
||||
.ok_or_else(|| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: "No session in DB".to_string(),
|
||||
})?;
|
||||
})? {
|
||||
value
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"nearai.session_token missing; falling back to legacy nearai.session for backwards compatibility"
|
||||
);
|
||||
store
|
||||
.get_setting(&user_id, "nearai.session")
|
||||
.await
|
||||
.map_err(|e| LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: format!("DB query failed: {}", e),
|
||||
})?
|
||||
.ok_or(LlmError::SessionRenewalFailed {
|
||||
provider: "nearai".to_string(),
|
||||
reason: "No session in DB".to_string(),
|
||||
})?
|
||||
};
|
||||
|
||||
let session: SessionData =
|
||||
serde_json::from_value(value).map_err(|e| LlmError::SessionRenewalFailed {
|
||||
|
||||
+11
-2
@@ -24,7 +24,7 @@ use ironclaw::{
|
||||
extensions::ExtensionManager,
|
||||
hooks::HookRegistry,
|
||||
llm::{
|
||||
FailoverProvider, LlmProvider, SessionConfig, create_cheap_llm_provider,
|
||||
CooldownConfig, FailoverProvider, LlmProvider, SessionConfig, create_cheap_llm_provider,
|
||||
create_llm_provider, create_llm_provider_with_config, create_session_manager,
|
||||
},
|
||||
orchestrator::{
|
||||
@@ -536,7 +536,16 @@ async fn main() -> anyhow::Result<()> {
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
Arc::new(FailoverProvider::new(vec![llm, fallback])?)
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(
|
||||
config.llm.nearai.failover_cooldown_secs,
|
||||
),
|
||||
failure_threshold: config.llm.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
@@ -1022,6 +1022,8 @@ impl SetupWizard {
|
||||
api_key: None,
|
||||
fallback_model: None,
|
||||
max_retries: 3,
|
||||
failover_cooldown_secs: 300,
|
||||
failover_cooldown_threshold: 3,
|
||||
},
|
||||
openai: None,
|
||||
anthropic: None,
|
||||
|
||||
+45
-10
@@ -5,13 +5,18 @@ use std::net::{IpAddr, ToSocketAddrs};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::StreamExt;
|
||||
use reqwest::Client;
|
||||
|
||||
use crate::context::JobContext;
|
||||
use crate::safety::LeakDetector;
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput, require_str};
|
||||
|
||||
/// Maximum response body size (5 MB). Prevents OOM from unbounded responses.
|
||||
/// Maximum response body size (5 MB).
|
||||
///
|
||||
/// 5 MB is large enough for typical JSON API responses and moderate HTML pages,
|
||||
/// but small enough to prevent OOM from malicious or runaway servers. The WASM
|
||||
/// HTTP wrapper uses the same limit for consistency.
|
||||
const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// Tool for making HTTP requests.
|
||||
@@ -230,19 +235,43 @@ impl Tool for HttpTool {
|
||||
.filter_map(|(k, v)| v.to_str().ok().map(|v| (k.to_string(), v.to_string())))
|
||||
.collect();
|
||||
|
||||
// Get response body with size cap to prevent OOM
|
||||
let body_bytes = response.bytes().await.map_err(|e| {
|
||||
ToolError::ExternalService(format!("failed to read response body: {}", e))
|
||||
})?;
|
||||
|
||||
if body_bytes.len() > MAX_RESPONSE_SIZE {
|
||||
// Pre-check Content-Length header to reject obviously oversized responses
|
||||
// before downloading anything, preventing OOM from malicious servers.
|
||||
if let Some(content_length) = response.headers().get(reqwest::header::CONTENT_LENGTH)
|
||||
&& let Ok(s) = content_length.to_str()
|
||||
&& let Ok(len) = s.parse::<usize>()
|
||||
&& len > MAX_RESPONSE_SIZE
|
||||
{
|
||||
tracing::warn!(
|
||||
url = %parsed_url,
|
||||
content_length = len,
|
||||
max = MAX_RESPONSE_SIZE,
|
||||
"Rejected HTTP response: Content-Length exceeds limit"
|
||||
);
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Response body too large ({} bytes, max {})",
|
||||
body_bytes.len(),
|
||||
MAX_RESPONSE_SIZE
|
||||
"Response Content-Length ({} bytes) exceeds maximum allowed size ({} bytes)",
|
||||
len, MAX_RESPONSE_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Stream the response body with a hard size cap. Even if Content-Length was
|
||||
// absent or lied about the size, we stop reading once we exceed the limit.
|
||||
let mut body = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = StreamExt::next(&mut stream).await {
|
||||
let chunk = chunk.map_err(|e| {
|
||||
ToolError::ExternalService(format!("failed to read response body: {}", e))
|
||||
})?;
|
||||
if body.len() + chunk.len() > MAX_RESPONSE_SIZE {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Response body exceeds maximum allowed size ({} bytes)",
|
||||
MAX_RESPONSE_SIZE
|
||||
)));
|
||||
}
|
||||
body.extend_from_slice(&chunk);
|
||||
}
|
||||
let body_bytes = bytes::Bytes::from(body);
|
||||
|
||||
let body_text = String::from_utf8_lossy(&body_bytes).into_owned();
|
||||
|
||||
// Try to parse as JSON, fall back to string
|
||||
@@ -328,4 +357,10 @@ mod tests {
|
||||
// Public
|
||||
assert!(!is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_response_size_is_reasonable() {
|
||||
// MAX_RESPONSE_SIZE should be 5 MB to prevent OOM while allowing typical API responses.
|
||||
assert_eq!(MAX_RESPONSE_SIZE, 5 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user