fix(tests): replace hardcoded /tmp paths with tempdir + add 300 unit tests (#659)

* test: add unit tests across 20 modules for coverage push Add 300+ unit tests covering config, context, evaluation, extensions, LLM, secrets, tools/builder, and tools/mcp modules. All tests are pure unit tests (no mocks) exercising serde roundtrips, edge cases, error paths, and business logic. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix(tests): replace hardcoded /tmp paths with tempfile::tempdir The e2e_metrics_test::test_metrics_collected_from_tool_trace test was failing because setup_test_dir() created /tmp/ironclaw_metrics_test but the fixture referenced /tmp/ironclaw_e2e_test/hello.txt (path mismatch). Added LlmTrace::replace_paths() to substitute fixture paths at runtime, then converted all 12 test files from hardcoded /tmp/ironclaw_* paths to tempfile::tempdir(). Tests are now isolated, parallel-safe, and leave no debris on disk. Regression test: test_metrics_collected_from_tool_trace now passes consistently regardless of prior /tmp state. Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
2026-09-01 09:09:19 +00:00 · 2026-03-07 08:24:24 +00:00
parent 8fbb782090
commit cf96a3253c
33 changed files with 5904 additions and 119 deletions
@@ -238,4 +238,220 @@ mod tests {
        let rate = collector.success_rate();
        assert!((rate - 0.666).abs() < 0.01);
    }
+
+    // --- QualityMetrics default ---
+
+    #[test]
+    fn test_quality_metrics_default() {
+        let m = QualityMetrics::default();
+        assert_eq!(m.total_actions, 0);
+        assert_eq!(m.successful_actions, 0);
+        assert_eq!(m.failed_actions, 0);
+        assert_eq!(m.total_time, Duration::ZERO);
+        assert_eq!(m.total_cost, Decimal::ZERO);
+        assert!(m.tool_metrics.is_empty());
+        assert!(m.error_types.is_empty());
+    }
+
+    // --- ToolMetrics::success_rate ---
+
+    #[test]
+    fn test_tool_metrics_success_rate_zero_calls() {
+        let tm = ToolMetrics::default();
+        assert_eq!(tm.success_rate(), 0.0);
+    }
+
+    #[test]
+    fn test_tool_metrics_success_rate_mixed() {
+        let tm = ToolMetrics {
+            calls: 4,
+            successes: 3,
+            failures: 1,
+            ..Default::default()
+        };
+        assert!((tm.success_rate() - 0.75).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    fn test_tool_metrics_success_rate_all_failures() {
+        let tm = ToolMetrics {
+            calls: 5,
+            successes: 0,
+            failures: 5,
+            ..Default::default()
+        };
+        assert_eq!(tm.success_rate(), 0.0);
+    }
+
+    // --- MetricsCollector ---
+
+    #[test]
+    fn test_collector_default_is_new() {
+        let a = MetricsCollector::new();
+        let b = MetricsCollector::default();
+        assert_eq!(a.metrics().total_actions, b.metrics().total_actions);
+        assert_eq!(a.success_rate(), b.success_rate());
+    }
+
+    #[test]
+    fn test_success_rate_empty_collector() {
+        let collector = MetricsCollector::new();
+        assert_eq!(collector.success_rate(), 0.0);
+    }
+
+    #[test]
+    fn test_record_success_accumulates_cost() {
+        let mut c = MetricsCollector::new();
+        c.record_success("a", Duration::from_millis(100), Some(dec!(1.50)));
+        c.record_success("a", Duration::from_millis(200), Some(dec!(2.50)));
+        assert_eq!(c.metrics().total_cost, dec!(4.00));
+        let tool = c.tool_metrics("a").unwrap();
+        assert_eq!(tool.total_cost, dec!(4.00));
+    }
+
+    #[test]
+    fn test_record_success_none_cost_does_not_change_total() {
+        let mut c = MetricsCollector::new();
+        c.record_success("x", Duration::from_secs(1), Some(dec!(1.00)));
+        c.record_success("x", Duration::from_secs(1), None);
+        assert_eq!(c.metrics().total_cost, dec!(1.00));
+    }
+
+    #[test]
+    fn test_record_failure_does_not_add_cost() {
+        let mut c = MetricsCollector::new();
+        c.record_failure("t", "oops", Duration::from_secs(1));
+        assert_eq!(c.metrics().total_cost, Decimal::ZERO);
+    }
+
+    #[test]
+    fn test_tool_avg_time_updates() {
+        let mut c = MetricsCollector::new();
+        c.record_success("t", Duration::from_secs(2), None);
+        c.record_success("t", Duration::from_secs(4), None);
+        let tool = c.tool_metrics("t").unwrap();
+        // total 6s / 2 calls = 3s avg
+        assert_eq!(tool.avg_time, Duration::from_secs(3));
+    }
+
+    #[test]
+    fn test_total_time_across_success_and_failure() {
+        let mut c = MetricsCollector::new();
+        c.record_success("a", Duration::from_secs(3), None);
+        c.record_failure("b", "err", Duration::from_secs(7));
+        assert_eq!(c.metrics().total_time, Duration::from_secs(10));
+    }
+
+    #[test]
+    fn test_tool_metrics_returns_none_for_unknown() {
+        let c = MetricsCollector::new();
+        assert!(c.tool_metrics("nonexistent").is_none());
+    }
+
+    #[test]
+    fn test_reset_clears_everything() {
+        let mut c = MetricsCollector::new();
+        c.record_success("t", Duration::from_secs(1), Some(dec!(5.00)));
+        c.record_failure("t", "error", Duration::from_secs(1));
+        c.reset();
+        assert_eq!(c.metrics().total_actions, 0);
+        assert_eq!(c.metrics().successful_actions, 0);
+        assert_eq!(c.metrics().failed_actions, 0);
+        assert_eq!(c.metrics().total_cost, Decimal::ZERO);
+        assert!(c.metrics().tool_metrics.is_empty());
+        assert!(c.metrics().error_types.is_empty());
+        assert_eq!(c.success_rate(), 0.0);
+    }
+
+    #[test]
+    fn test_multiple_tools_tracked_independently() {
+        let mut c = MetricsCollector::new();
+        c.record_success("alpha", Duration::from_secs(1), None);
+        c.record_success("alpha", Duration::from_secs(1), None);
+        c.record_failure("beta", "bad", Duration::from_secs(1));
+        c.record_success("beta", Duration::from_secs(1), None);
+
+        let alpha = c.tool_metrics("alpha").unwrap();
+        assert_eq!(alpha.calls, 2);
+        assert_eq!(alpha.successes, 2);
+        assert_eq!(alpha.failures, 0);
+
+        let beta = c.tool_metrics("beta").unwrap();
+        assert_eq!(beta.calls, 2);
+        assert_eq!(beta.successes, 1);
+        assert_eq!(beta.failures, 1);
+    }
+
+    // --- categorize_error ---
+
+    #[test]
+    fn test_categorize_error_all_types() {
+        assert_eq!(categorize_error("Connection timeout"), "timeout");
+        assert_eq!(categorize_error("TIMEOUT exceeded"), "timeout");
+        assert_eq!(categorize_error("rate limit hit"), "rate_limit");
+        assert_eq!(categorize_error("Rate Limit 429"), "rate_limit");
+        assert_eq!(categorize_error("auth failure"), "auth");
+        assert_eq!(categorize_error("Unauthorized"), "auth");
+        assert_eq!(categorize_error("resource not found"), "not_found");
+        assert_eq!(categorize_error("HTTP 404"), "not_found");
+        assert_eq!(categorize_error("invalid parameter X"), "invalid_input");
+        assert_eq!(categorize_error("bad parameter"), "invalid_input");
+        assert_eq!(categorize_error("Invalid JSON"), "invalid_input");
+        assert_eq!(categorize_error("network error"), "network");
+        assert_eq!(categorize_error("connection refused"), "network");
+        assert_eq!(categorize_error("something else entirely"), "unknown");
+        assert_eq!(categorize_error(""), "unknown");
+    }
+
+    #[test]
+    fn test_error_types_accumulated_in_collector() {
+        let mut c = MetricsCollector::new();
+        c.record_failure("t", "timeout!", Duration::from_secs(1));
+        c.record_failure("t", "another timeout", Duration::from_secs(1));
+        c.record_failure("t", "auth denied", Duration::from_secs(1));
+
+        assert_eq!(c.metrics().error_types.get("timeout"), Some(&2));
+        assert_eq!(c.metrics().error_types.get("auth"), Some(&1));
+    }
+
+    // --- MetricsSummary ---
+
+    #[test]
+    fn test_summary_empty_collector() {
+        let c = MetricsCollector::new();
+        let s = c.summary();
+        assert_eq!(s.total_actions, 0);
+        assert_eq!(s.success_rate, 0.0);
+        assert_eq!(s.total_cost, Decimal::ZERO);
+        assert!(s.most_used_tool.is_none());
+        assert!(s.most_failed_tool.is_none());
+        assert!(s.top_errors.is_empty());
+    }
+
+    #[test]
+    fn test_summary_most_used_and_most_failed() {
+        let mut c = MetricsCollector::new();
+        // "alpha" gets 3 calls (all success)
+        c.record_success("alpha", Duration::from_secs(1), None);
+        c.record_success("alpha", Duration::from_secs(1), None);
+        c.record_success("alpha", Duration::from_secs(1), None);
+        // "beta" gets 2 calls (both failures)
+        c.record_failure("beta", "err", Duration::from_secs(1));
+        c.record_failure("beta", "err", Duration::from_secs(1));
+
+        let s = c.summary();
+        assert_eq!(s.most_used_tool.as_deref(), Some("alpha"));
+        assert_eq!(s.most_failed_tool.as_deref(), Some("beta"));
+        assert_eq!(s.total_actions, 5);
+    }
+
+    #[test]
+    fn test_summary_top_errors_populated() {
+        let mut c = MetricsCollector::new();
+        c.record_failure("t", "timeout", Duration::from_secs(1));
+        c.record_failure("t", "auth error", Duration::from_secs(1));
+        let s = c.summary();
+        assert!(!s.top_errors.is_empty());
+        assert!(s.top_errors.len() <= 3);
+    }
 }
@@ -331,6 +331,10 @@ mod tests {
    }

    fn create_action(success: bool) -> ActionRecord {
+        create_action_with_error(success, "Test error")
+    }
+
+    fn create_action_with_error(success: bool, error_msg: &str) -> ActionRecord {
        let mut action = ActionRecord::new(0, "test", serde_json::json!({}));
        if success {
            action = action.succeed(
@@ -339,8 +343,257 @@ mod tests {
                std::time::Duration::from_secs(1),
            );
        } else {
-            action = action.fail("Test error", std::time::Duration::from_secs(1));
+            action = action.fail(error_msg, std::time::Duration::from_secs(1));
        }
        action
    }
+
+    fn completed_job(title: &str) -> JobContext {
+        let mut job = JobContext::new(title, "test job");
+        job.transition_to(crate::context::JobState::InProgress, None)
+            .unwrap();
+        job.transition_to(crate::context::JobState::Completed, None)
+            .unwrap();
+        job
+    }
+
+    // --- EvaluationResult construction ---
+
+    #[test]
+    fn test_evaluation_result_success_defaults() {
+        let result = EvaluationResult::success("all good", 85);
+        assert!(result.success);
+        assert_eq!(result.confidence, 0.9);
+        assert_eq!(result.reasoning, "all good");
+        assert!(result.issues.is_empty());
+        assert!(result.suggestions.is_empty());
+        assert_eq!(result.quality_score, 85);
+    }
+
+    #[test]
+    fn test_evaluation_result_failure_defaults() {
+        let issues = vec!["bad thing".to_string(), "worse thing".to_string()];
+        let result = EvaluationResult::failure("went wrong", issues.clone());
+        assert!(!result.success);
+        assert_eq!(result.confidence, 0.9);
+        assert_eq!(result.reasoning, "went wrong");
+        assert_eq!(result.issues, issues);
+        assert_eq!(result.quality_score, 0);
+    }
+
+    #[test]
+    fn test_evaluation_result_serde_roundtrip() {
+        let result = EvaluationResult {
+            success: true,
+            confidence: 0.75,
+            reasoning: "looks fine".to_string(),
+            issues: vec!["minor".to_string()],
+            suggestions: vec!["try harder".to_string()],
+            quality_score: 60,
+        };
+        let json = serde_json::to_string(&result).unwrap();
+        let deserialized: EvaluationResult = serde_json::from_str(&json).unwrap();
+        assert_eq!(deserialized.success, result.success);
+        assert_eq!(deserialized.confidence, result.confidence);
+        assert_eq!(deserialized.reasoning, result.reasoning);
+        assert_eq!(deserialized.issues, result.issues);
+        assert_eq!(deserialized.suggestions, result.suggestions);
+        assert_eq!(deserialized.quality_score, result.quality_score);
+    }
+
+    // --- RuleBasedEvaluator builder ---
+
+    #[test]
+    fn test_rule_based_evaluator_default() {
+        let eval = RuleBasedEvaluator::default();
+        assert_eq!(eval.min_action_success_rate, 0.8);
+        assert_eq!(eval.max_failures, 3);
+    }
+
+    #[test]
+    fn test_rule_based_evaluator_builder_methods() {
+        let eval = RuleBasedEvaluator::new()
+            .with_min_success_rate(0.5)
+            .with_max_failures(10);
+        assert_eq!(eval.min_action_success_rate, 0.5);
+        assert_eq!(eval.max_failures, 10);
+    }
+
+    // --- RuleBasedEvaluator::evaluate edge cases ---
+
+    #[tokio::test]
+    async fn test_empty_actions_fails() {
+        let eval = RuleBasedEvaluator::new();
+        let job = completed_job("empty");
+        let result = eval.evaluate(&job, &[], None).await.unwrap();
+        assert!(!result.success);
+        assert!(result.issues.iter().any(|i| i.contains("No actions")));
+    }
+
+    #[tokio::test]
+    async fn test_all_actions_succeed_completed_job_gets_100() {
+        let eval = RuleBasedEvaluator::new();
+        let job = completed_job("perfect");
+        let actions = vec![
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+        ];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(result.success);
+        // 100% success rate -> base 80, completion bonus 20 -> 100
+        assert_eq!(result.quality_score, 100);
+    }
+
+    #[tokio::test]
+    async fn test_quality_score_no_completion_bonus_for_pending_job() {
+        // Even if all actions succeed, a non-completed job gets flagged
+        let eval = RuleBasedEvaluator::new();
+        let job = JobContext::new("pending", "still pending");
+        let actions = vec![create_action(true)];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        // Job not in completed state => issues present
+        assert!(!result.success);
+        assert!(
+            result
+                .issues
+                .iter()
+                .any(|i| i.contains("not in completed state"))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_submitted_state_counts_as_completed() {
+        let eval = RuleBasedEvaluator::new();
+        let mut job = JobContext::new("submitted", "test");
+        job.transition_to(crate::context::JobState::InProgress, None)
+            .unwrap();
+        job.transition_to(crate::context::JobState::Completed, None)
+            .unwrap();
+        job.transition_to(crate::context::JobState::Submitted, None)
+            .unwrap();
+        let actions = vec![create_action(true)];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        // Submitted is treated like completed for state check (no issue),
+        // but completion bonus only applies for Completed state
+        assert!(result.success);
+    }
+
+    #[tokio::test]
+    async fn test_success_rate_below_threshold_fails() {
+        let eval = RuleBasedEvaluator::new().with_min_success_rate(0.9);
+        let job = completed_job("threshold");
+        // 4 out of 5 = 80%, below 90% threshold
+        let actions = vec![
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(false),
+        ];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(!result.success);
+        assert!(
+            result
+                .issues
+                .iter()
+                .any(|i| i.contains("success rate") && i.contains("below threshold"))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_too_many_failures_flagged() {
+        let eval = RuleBasedEvaluator::new().with_max_failures(1);
+        let job = completed_job("failures");
+        // 8 successes, 2 failures: rate is 80% (passes default 0.8) but failures > max 1
+        let actions = vec![
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(false),
+            create_action(false),
+        ];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(!result.success);
+        assert!(
+            result
+                .issues
+                .iter()
+                .any(|i| i.contains("Too many failures"))
+        );
+    }
+
+    #[tokio::test]
+    async fn test_critical_error_detected() {
+        let eval = RuleBasedEvaluator::new().with_max_failures(10);
+        let job = completed_job("critical");
+        let actions = vec![
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action_with_error(false, "A CRITICAL system failure occurred"),
+        ];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(!result.success);
+        assert!(result.issues.iter().any(|i| i.contains("Critical error")));
+    }
+
+    #[tokio::test]
+    async fn test_fatal_error_detected() {
+        let eval = RuleBasedEvaluator::new().with_max_failures(10);
+        let job = completed_job("fatal");
+        let actions = vec![
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action(true),
+            create_action_with_error(false, "Fatal: disk full"),
+        ];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(result.issues.iter().any(|i| i.contains("Critical error")));
+    }
+
+    #[tokio::test]
+    async fn test_quality_score_capped_at_50_with_issues() {
+        let eval = RuleBasedEvaluator::new()
+            .with_min_success_rate(0.0)
+            .with_max_failures(100);
+        // Job not completed => issues present, quality capped
+        let job = JobContext::new("capped", "test");
+        let actions = vec![create_action(true)];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(!result.success);
+        assert!(result.quality_score <= 50);
+    }
+
+    #[tokio::test]
+    async fn test_failed_result_includes_suggestions() {
+        let eval = RuleBasedEvaluator::new().with_max_failures(0);
+        let job = completed_job("suggestions");
+        let actions = vec![create_action(false)];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(!result.success);
+        assert!(!result.suggestions.is_empty());
+        assert_eq!(result.confidence, 0.85);
+    }
+
+    #[tokio::test]
+    async fn test_single_successful_action_completed_job() {
+        let eval = RuleBasedEvaluator::new();
+        let job = completed_job("single");
+        let actions = vec![create_action(true)];
+        let result = eval.evaluate(&job, &actions, None).await.unwrap();
+        assert!(result.success);
+        // 100% rate -> base 80, + 20 completion = 100
+        assert_eq!(result.quality_score, 100);
+        assert!(result.reasoning.contains("1/1"));
+    }
 }