feat(testing): add FaultInjector framework for StubLlm (#1233)

* feat(testing): add FaultInjector framework for StubLlm (#1220)

Adds a configurable fault injection framework for testing retry, failover,
and circuit breaker behavior. The FaultInjector attaches to StubLlm and
provides per-call control over failure type, timing, and sequencing.

Components:
- FaultType: maps to LlmError variants (RequestFailed, RateLimited,
  AuthFailed, InvalidResponse, IoError, ContextLengthExceeded, SessionExpired)
- FaultAction: Succeed, Fail(FaultType), Delay(Duration)
- FaultMode: SequenceOnce (play then succeed), SequenceLoop (repeat forever),
  Random (seeded xorshift64 PRNG for reproducibility)
- FaultInjector: thread-safe (AtomicU32 counter + Mutex RNG)

Integration:
- StubLlm gains optional fault_injector field via with_fault_injector()
- When set, takes precedence over should_fail/error_kind
- Backward compatible: existing StubLlm usage unchanged

Closes #1220

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* refactor(testing): address review feedback on FaultInjector

- Remove redundant .abs() in random fault comparison
- Extract check_faults() helper to DRY up StubLlm methods
- Guard xorshift seed=0 (fixed point) by mapping to 1
- Add StubLlm integration test (stub_llm_fault_injector_sequence)
- Remove dead seed field from FaultMode::Random
- Move pub mod fault_injection to top of mod.rs
- Add Debug impl for FaultInjector
- Add empty_sequence_always_succeeds test
- Add random_seed_zero_does_not_always_fail test

* fix(testing): address #1233 review -- seed-0 bug, reset(), Debug derive

- Store seed in FaultMode::Random so reset() can re-init the RNG
- Add reset() method for test reproducibility (re-seeds RNG, zeros counter)
- Strengthen seed=0 regression test to 100 iterations with stricter assertion
- Add reset_restores_random_rng_from_stored_seed test
- Debug impl and empty_sequence test were already present from prior commit

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* ci: re-trigger CI with latest changes

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* ci: trigger new run with skip-regression-check label

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix(testing): address PR #1233 review -- error_rate validation and edge cases

- Validate error_rate is in 0.0..=1.0 and not NaN (panics on invalid input)
- Fix error_rate==1.0 edge case: use <= instead of < so 1.0 always fails
- Add regression tests for error_rate validation (NaN, negative, >1.0)
- Add tests for error_rate boundary values (0.0 never fails, 1.0 always fails)
- Add delay action test using tokio::time::pause() for deterministic timing

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>

src/testing/mod.rs

@@ -19,9 +19,11 @@
 //! ```
 
 pub mod credentials;
+pub mod fault_injection;
 
 use std::sync::Arc;
 use std::sync::Mutex;
+
 use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 
 use async_trait::async_trait;
@@ -84,6 +86,9 @@ pub struct StubLlm {
     call_count: AtomicU32,
     should_fail: AtomicBool,
     error_kind: StubErrorKind,
+    /// Optional fault injector for fine-grained failure control.
+    /// When set, takes precedence over the `should_fail` / `error_kind` fields.
+    fault_injector: Option<Arc<fault_injection::FaultInjector>>,
 }
 
 impl StubLlm {
@@ -95,6 +100,7 @@ impl StubLlm {
             call_count: AtomicU32::new(0),
             should_fail: AtomicBool::new(false),
             error_kind: StubErrorKind::Transient,
+            fault_injector: None,
         }
     }
 
@@ -106,6 +112,7 @@ impl StubLlm {
             call_count: AtomicU32::new(0),
             should_fail: AtomicBool::new(true),
             error_kind: StubErrorKind::Transient,
+            fault_injector: None,
         }
     }
 
@@ -117,6 +124,7 @@ impl StubLlm {
             call_count: AtomicU32::new(0),
             should_fail: AtomicBool::new(true),
             error_kind: StubErrorKind::NonTransient,
+            fault_injector: None,
         }
     }
 
@@ -131,11 +139,39 @@ impl StubLlm {
         self.call_count.load(Ordering::Relaxed)
     }
 
+    /// Attach a fault injector for fine-grained failure control.
+    ///
+    /// When set, the injector's `next_action()` is consulted on every call,
+    /// taking precedence over the `should_fail` / `error_kind` fields.
+    pub fn with_fault_injector(mut self, injector: Arc<fault_injection::FaultInjector>) -> Self {
+        self.fault_injector = Some(injector);
+        self
+    }
+
     /// Toggle whether calls should fail at runtime.
     pub fn set_failing(&self, fail: bool) {
         self.should_fail.store(fail, Ordering::Relaxed);
     }
 
+    /// Check the fault injector or should_fail flag, returning an error if
+    /// the call should fail, or None if it should succeed.
+    async fn check_faults(&self) -> Option<LlmError> {
+        if let Some(ref injector) = self.fault_injector {
+            match injector.next_action() {
+                fault_injection::FaultAction::Fail(fault) => {
+                    return Some(fault.to_llm_error(&self.model_name));
+                }
+                fault_injection::FaultAction::Delay(duration) => {
+                    tokio::time::sleep(duration).await;
+                }
+                fault_injection::FaultAction::Succeed => {}
+            }
+        } else if self.should_fail.load(Ordering::Relaxed) {
+            return Some(self.make_error());
+        }
+        None
+    }
+
     fn make_error(&self) -> LlmError {
         match self.error_kind {
             StubErrorKind::Transient => LlmError::RequestFailed {
@@ -168,8 +204,8 @@ impl LlmProvider for StubLlm {
 
     async fn complete(&self, _request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
         self.call_count.fetch_add(1, Ordering::Relaxed);
-        if self.should_fail.load(Ordering::Relaxed) {
-            return Err(self.make_error());
+        if let Some(err) = self.check_faults().await {
+            return Err(err);
         }
         Ok(CompletionResponse {
             content: self.response.clone(),
@@ -186,8 +222,8 @@ impl LlmProvider for StubLlm {
         _request: ToolCompletionRequest,
     ) -> Result<ToolCompletionResponse, LlmError> {
         self.call_count.fetch_add(1, Ordering::Relaxed);
-        if self.should_fail.load(Ordering::Relaxed) {
-            return Err(self.make_error());
+        if let Some(err) = self.check_faults().await {
+            return Err(err);
         }
         Ok(ToolCompletionResponse {
             content: Some(self.response.clone()),
@@ -1508,4 +1544,29 @@ mod tests {
         .await
         .expect("update actuals");
     }
+
+    #[tokio::test]
+    async fn stub_llm_fault_injector_sequence() {
+        use crate::llm::LlmProvider;
+        use crate::testing::fault_injection::{FaultAction, FaultInjector, FaultType};
+
+        let injector = Arc::new(FaultInjector::sequence([
+            FaultAction::Fail(FaultType::RateLimited { retry_after: None }),
+            FaultAction::Succeed,
+        ]));
+
+        let stub = StubLlm::new("hello").with_fault_injector(injector);
+
+        let req = crate::llm::CompletionRequest::new(vec![crate::llm::ChatMessage::user("hi")]);
+
+        // First call should fail with RateLimited
+        let result = stub.complete(req.clone()).await;
+        assert!(result.is_err());
+        assert!(matches!(result.unwrap_err(), LlmError::RateLimited { .. }));
+
+        // Second call should succeed
+        let result = stub.complete(req).await;
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap().content, "hello");
+    }
 }