test: add 29 E2E trace tests for issues #571-575 (#593)

* test: add 29 E2E trace tests for worker, threading, tools, workspace, and routines (#571-575) Add comprehensive E2E test coverage across five test files: - e2e_worker_coverage (7 tests): parallel tool calls, error feedback, unknown tools, invalid params, rate limiting, iteration limits, planning mode - e2e_thread_scheduling (3 tests + 2 deferred): multi-turn state, undo/redo, concurrent dispatch - e2e_builtin_tool_coverage (8 tests): time parse/diff/invalid, routine CRUD/history, job create/status/list/cancel, HTTP replay - e2e_workspace_coverage (6 tests): chunked search, multi-doc search, hybrid search, directory tree, document lifecycle, identity in system prompt - e2e_routine_heartbeat (5 tests): cron triggers, event matching, cooldown enforcement, heartbeat findings, empty checklist skip Infrastructure: extend TestRig with database/workspace/trace_llm accessors, register job and routine tools by default, add with_extra_tools() for custom stub tools. Includes 24 JSON trace fixtures across worker/, threading/, tools/, and workspace/. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: use 6-field cron format in routine_create_list fixture The cron 0.13 crate accepts both 6 and 7 fields, but the routine_create tool documents 6-field format. Align the fixture to match. [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: eliminate vacuous passes and silently-skipped assertions in E2E tests - job_create_status: replace job_status (needs dynamic UUID) with list_jobs, assert both succeed via completed() not just started() - job_list_cancel: keep cancel_job but explicitly assert it fails with invalid canned job_id "latest", verify create_job + list_jobs succeed - unknown_tool_name: add !is_empty() guard before .all() to prevent vacuous pass on empty iterator - workspace tests: change `if let Some(ws)` to `.expect()` so assertions are never silently skipped when workspace/trace_llm is available [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * feat: add template substitution to TraceLlm for dynamic tool result forwarding Add {{call_id.json_path}} template syntax to trace fixtures, enabling tool results from one step to flow into subsequent steps' arguments. TraceLlm extracts variables from Role::Tool messages (stripping the safety layer's <tool_output> XML wrapper and unescaping entities) and substitutes them in canned tool_call arguments before returning. This fixes job_create_status and job_list_cancel tests to properly test job_status and cancel_job with real dynamic UUIDs from create_job, instead of using invalid canned IDs that silently failed. Also adds tool result content assertions to job_create_status to verify the actual tool output contains expected data (job_id, title). [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: address PR review feedback on E2E tests - undo_redo_cycle: assert exactly 3 turns instead of >= 2 - tool_error_feedback: use tempfile::tempdir() instead of hardcoded /tmp path, patch fixture path at runtime for CI portability - worker_timeout → iteration_limit: rename to accurately describe what's tested - post_plan_work_remaining → simple_echo_flow: rename, test doesn't exercise planning - identity_in_system_prompt: seed IDENTITY.md before test, assert system prompt contains the seeded content instead of just checking Role::System exists [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: strengthen workspace E2E test assertions per PR review - write_chunk_search: assert memory_search was called and returned payment/architecture-related results - multi_document_search: assert memory_search was called for cross-document search - hybrid_search_with_embeddings: assert both memory_write and memory_search were called to confirm write-then-search pipeline - directory_tree: assert tree output contains expected alpha/beta project paths [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
2026-08-25 14:53:34 +00:00 · 2026-03-06 08:12:56 +00:00
parent 2df9602d56
commit 37bba72397
31 changed files with 3073 additions and 4 deletions
@@ -0,0 +1,325 @@
+//! E2E trace tests: worker execution paths (#571).
+//!
+//! Covers parallel tool calls, error feedback loops, unknown tools,
+//! invalid parameters, rate limiting, iteration limits, and planning mode.
+
+#[cfg(feature = "libsql")]
+mod support;
+
+#[cfg(feature = "libsql")]
+mod tests {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use async_trait::async_trait;
+    use serde_json::json;
+
+    use ironclaw::context::JobContext;
+    use ironclaw::tools::{Tool, ToolError, ToolOutput};
+
+    use crate::support::test_rig::TestRigBuilder;
+    use crate::support::trace_llm::LlmTrace;
+
+    // -- Stub tools for rate-limit and timeout tests --------------------------
+
+    /// A tool that always returns RateLimited.
+    struct StubRateLimitTool;
+
+    #[async_trait]
+    impl Tool for StubRateLimitTool {
+        fn name(&self) -> &str {
+            "stub_rate_limit"
+        }
+        fn description(&self) -> &str {
+            "Always returns rate limited error"
+        }
+        fn parameters_schema(&self) -> serde_json::Value {
+            json!({ "type": "object", "properties": {} })
+        }
+        async fn execute(
+            &self,
+            _params: serde_json::Value,
+            _ctx: &JobContext,
+        ) -> Result<ToolOutput, ToolError> {
+            Err(ToolError::RateLimited(Some(Duration::from_secs(60))))
+        }
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 1: parallel_three_tools
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn parallel_three_tools() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/parallel_three_tools.json"
+        ))
+        .expect("failed to load parallel_three_tools.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Run three tools in parallel").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Verify all three tools were started.
+        let started = rig.tool_calls_started();
+        assert!(
+            started.contains(&"echo".to_string()),
+            "echo not started: {started:?}"
+        );
+        assert!(
+            started.contains(&"time".to_string()),
+            "time not started: {started:?}"
+        );
+        assert!(
+            started.contains(&"json".to_string()),
+            "json not started: {started:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 2: tool_error_feedback
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn tool_error_feedback() {
+        // Use a tempdir for the recovery file. The fixture's recovery path
+        // is updated to write here via the test_dir variable.
+        let tmp = tempfile::tempdir().expect("create temp dir");
+        let test_dir = tmp.path().to_str().expect("tempdir path");
+
+        // Patch the fixture's recovery path to use our tempdir.
+        let fixture_str = std::fs::read_to_string(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/tool_error_feedback.json"
+        ))
+        .expect("read fixture");
+        let fixture_str = fixture_str.replace(
+            "/tmp/ironclaw_error_feedback_test/recovered.txt",
+            &format!("{test_dir}/recovered.txt"),
+        );
+        let trace: LlmTrace = serde_json::from_str(&fixture_str).expect("parse patched fixture");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Write a file to a bad path then recover")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Verify the recovery file exists in the tempdir.
+        let content = std::fs::read_to_string(format!("{test_dir}/recovered.txt"))
+            .expect("recovered.txt should exist");
+        assert!(
+            content.contains("recovered"),
+            "Expected 'recovered' in file, got: {content:?}"
+        );
+
+        // At least one tool call should have failed (the bad path).
+        let completed = rig.tool_calls_completed();
+        let failures: Vec<_> = completed.iter().filter(|(_, ok)| !ok).collect();
+        assert!(
+            !failures.is_empty(),
+            "Expected at least one failed tool call, got: {completed:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 3: unknown_tool_name
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn unknown_tool_name() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/unknown_tool.json"
+        ))
+        .expect("failed to load unknown_tool.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Deploy to production").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // The deploy_to_production tool should have been attempted but failed.
+        let completed = rig.tool_calls_completed();
+        let deploy_results: Vec<_> = completed
+            .iter()
+            .filter(|(name, _)| name == "deploy_to_production")
+            .collect();
+        assert!(
+            !deploy_results.is_empty(),
+            "deploy_to_production should have been attempted: {completed:?}"
+        );
+        assert!(
+            deploy_results.iter().all(|(_, ok)| !ok),
+            "deploy_to_production should fail: {deploy_results:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 4: invalid_tool_params
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn invalid_tool_params() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/invalid_params.json"
+        ))
+        .expect("failed to load invalid_params.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Echo something with wrong params first")
+            .await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Echo should have been called at least twice (bad then good).
+        let started = rig.tool_calls_started();
+        let echo_count = started.iter().filter(|n| n.as_str() == "echo").count();
+        assert!(
+            echo_count >= 2,
+            "Expected >= 2 echo calls, got {echo_count}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 5: rate_limit_cascade
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn rate_limit_cascade() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/rate_limit_cascade.json"
+        ))
+        .expect("failed to load rate_limit_cascade.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .with_extra_tools(vec![Arc::new(StubRateLimitTool) as Arc<dyn Tool>])
+            .build()
+            .await;
+
+        rig.send_message("Call the rate limited tool").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Both calls should have failed due to rate limiting.
+        let completed = rig.tool_calls_completed();
+        let rl_calls: Vec<_> = completed
+            .iter()
+            .filter(|(name, _)| name == "stub_rate_limit")
+            .collect();
+        assert!(
+            !rl_calls.is_empty(),
+            "Expected stub_rate_limit calls: {completed:?}"
+        );
+        assert!(
+            rl_calls.iter().all(|(_, ok)| !ok),
+            "All stub_rate_limit calls should fail: {rl_calls:?}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 6: iteration_limit
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn iteration_limit() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/worker_timeout.json"
+        ))
+        .expect("failed to load worker_timeout.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .with_max_tool_iterations(2)
+            .build()
+            .await;
+
+        rig.send_message("Keep calling tools until the limit").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        // We should still get a response even with iteration limit.
+        assert!(
+            !responses.is_empty(),
+            "Expected at least one response with iteration limit"
+        );
+
+        // Metrics should show we hit the iteration limit.
+        let metrics = rig.collect_metrics().await;
+        assert!(
+            metrics.tool_calls.len() <= 2,
+            "Expected at most 2 tool calls with limit=2, got {}",
+            metrics.tool_calls.len()
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 7: simple_echo_flow
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn simple_echo_flow() {
+        let trace = LlmTrace::from_file(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/tests/fixtures/llm_traces/worker/plan_remaining_work.json"
+        ))
+        .expect("failed to load plan_remaining_work.json");
+
+        let rig = TestRigBuilder::new()
+            .with_trace(trace.clone())
+            .build()
+            .await;
+
+        rig.send_message("Plan and execute a task").await;
+        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
+
+        rig.verify_trace_expects(&trace, &responses);
+
+        // Verify echo was called during execution.
+        let started = rig.tool_calls_started();
+        assert!(
+            started.contains(&"echo".to_string()),
+            "echo should be called: {started:?}"
+        );
+
+        rig.shutdown();
+    }
+}