mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
* test: add 29 E2E trace tests for worker, threading, tools, workspace, and routines (#571-575) Add comprehensive E2E test coverage across five test files: - e2e_worker_coverage (7 tests): parallel tool calls, error feedback, unknown tools, invalid params, rate limiting, iteration limits, planning mode - e2e_thread_scheduling (3 tests + 2 deferred): multi-turn state, undo/redo, concurrent dispatch - e2e_builtin_tool_coverage (8 tests): time parse/diff/invalid, routine CRUD/history, job create/status/list/cancel, HTTP replay - e2e_workspace_coverage (6 tests): chunked search, multi-doc search, hybrid search, directory tree, document lifecycle, identity in system prompt - e2e_routine_heartbeat (5 tests): cron triggers, event matching, cooldown enforcement, heartbeat findings, empty checklist skip Infrastructure: extend TestRig with database/workspace/trace_llm accessors, register job and routine tools by default, add with_extra_tools() for custom stub tools. Includes 24 JSON trace fixtures across worker/, threading/, tools/, and workspace/. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: use 6-field cron format in routine_create_list fixture The cron 0.13 crate accepts both 6 and 7 fields, but the routine_create tool documents 6-field format. Align the fixture to match. [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: eliminate vacuous passes and silently-skipped assertions in E2E tests - job_create_status: replace job_status (needs dynamic UUID) with list_jobs, assert both succeed via completed() not just started() - job_list_cancel: keep cancel_job but explicitly assert it fails with invalid canned job_id "latest", verify create_job + list_jobs succeed - unknown_tool_name: add !is_empty() guard before .all() to prevent vacuous pass on empty iterator - workspace tests: change `if let Some(ws)` to `.expect()` so assertions are never silently skipped when workspace/trace_llm is available [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * feat: add template substitution to TraceLlm for dynamic tool result forwarding Add {{call_id.json_path}} template syntax to trace fixtures, enabling tool results from one step to flow into subsequent steps' arguments. TraceLlm extracts variables from Role::Tool messages (stripping the safety layer's <tool_output> XML wrapper and unescaping entities) and substitutes them in canned tool_call arguments before returning. This fixes job_create_status and job_list_cancel tests to properly test job_status and cancel_job with real dynamic UUIDs from create_job, instead of using invalid canned IDs that silently failed. Also adds tool result content assertions to job_create_status to verify the actual tool output contains expected data (job_id, title). [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: address PR review feedback on E2E tests - undo_redo_cycle: assert exactly 3 turns instead of >= 2 - tool_error_feedback: use tempfile::tempdir() instead of hardcoded /tmp path, patch fixture path at runtime for CI portability - worker_timeout → iteration_limit: rename to accurately describe what's tested - post_plan_work_remaining → simple_echo_flow: rename, test doesn't exercise planning - identity_in_system_prompt: seed IDENTITY.md before test, assert system prompt contains the seeded content instead of just checking Role::System exists [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: strengthen workspace E2E test assertions per PR review - write_chunk_search: assert memory_search was called and returned payment/architecture-related results - multi_document_search: assert memory_search was called for cross-document search - hybrid_search_with_embeddings: assert both memory_write and memory_search were called to confirm write-then-search pipeline - directory_tree: assert tree output contains expected alpha/beta project paths [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
156 lines
4.8 KiB
Rust
156 lines
4.8 KiB
Rust
//! E2E trace tests: thread/scheduler operations (#572).
|
|
//!
|
|
//! Covers multi-turn state persistence, undo/redo, and concurrent dispatch.
|
|
//! Tests for thread_interruption and max_parallel_exceeded are deferred.
|
|
|
|
#[cfg(feature = "libsql")]
|
|
mod support;
|
|
|
|
#[cfg(feature = "libsql")]
|
|
mod tests {
|
|
use std::time::Duration;
|
|
|
|
use crate::support::test_rig::TestRigBuilder;
|
|
use crate::support::trace_llm::LlmTrace;
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 1: multi_turn_state
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
async fn multi_turn_state() {
|
|
let trace = LlmTrace::from_file(concat!(
|
|
env!("CARGO_MANIFEST_DIR"),
|
|
"/tests/fixtures/llm_traces/threading/multi_turn_state.json"
|
|
))
|
|
.expect("failed to load multi_turn_state.json");
|
|
|
|
let rig = TestRigBuilder::new()
|
|
.with_trace(trace.clone())
|
|
.build()
|
|
.await;
|
|
|
|
let all_responses = rig
|
|
.run_and_verify_trace(&trace, Duration::from_secs(30))
|
|
.await;
|
|
|
|
// Should have 3 turns of responses.
|
|
assert_eq!(
|
|
all_responses.len(),
|
|
3,
|
|
"Expected 3 turns, got {}",
|
|
all_responses.len()
|
|
);
|
|
|
|
// Verify memory tools were used across turns.
|
|
let started = rig.tool_calls_started();
|
|
let mw_count = started
|
|
.iter()
|
|
.filter(|n| n.as_str() == "memory_write")
|
|
.count();
|
|
let ms_count = started
|
|
.iter()
|
|
.filter(|n| n.as_str() == "memory_search")
|
|
.count();
|
|
assert!(
|
|
mw_count >= 2,
|
|
"Expected >= 2 memory_write calls: {started:?}"
|
|
);
|
|
assert!(
|
|
ms_count >= 1,
|
|
"Expected >= 1 memory_search calls: {started:?}"
|
|
);
|
|
|
|
// Verify DB is accessible (conversation persistence is tested by
|
|
// the agent's internal session management).
|
|
let _db = rig.database();
|
|
|
|
rig.shutdown();
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 2: thread_interruption -- DEFERRED
|
|
// -----------------------------------------------------------------------
|
|
// Needs interrupt signaling infrastructure in TestChannel.
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 3: undo_redo_cycle
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
async fn undo_redo_cycle() {
|
|
let trace = LlmTrace::from_file(concat!(
|
|
env!("CARGO_MANIFEST_DIR"),
|
|
"/tests/fixtures/llm_traces/threading/undo_redo.json"
|
|
))
|
|
.expect("failed to load undo_redo.json");
|
|
|
|
let rig = TestRigBuilder::new()
|
|
.with_trace(trace.clone())
|
|
.build()
|
|
.await;
|
|
|
|
let all_responses = rig
|
|
.run_and_verify_trace(&trace, Duration::from_secs(30))
|
|
.await;
|
|
|
|
// Should get responses for all 3 turns (echo, /undo, /redo).
|
|
assert_eq!(
|
|
all_responses.len(),
|
|
3,
|
|
"Expected 3 turn responses, got {}",
|
|
all_responses.len()
|
|
);
|
|
|
|
rig.shutdown();
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 4: concurrent_dispatch
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
async fn concurrent_dispatch() {
|
|
let trace = LlmTrace::from_file(concat!(
|
|
env!("CARGO_MANIFEST_DIR"),
|
|
"/tests/fixtures/llm_traces/threading/concurrent_dispatch.json"
|
|
))
|
|
.expect("failed to load concurrent_dispatch.json");
|
|
|
|
let rig = TestRigBuilder::new()
|
|
.with_trace(trace.clone())
|
|
.build()
|
|
.await;
|
|
|
|
let all_responses = rig
|
|
.run_and_verify_trace(&trace, Duration::from_secs(30))
|
|
.await;
|
|
|
|
// Should have 2 turns.
|
|
assert_eq!(
|
|
all_responses.len(),
|
|
2,
|
|
"Expected 2 turns, got {}",
|
|
all_responses.len()
|
|
);
|
|
|
|
// Both echo calls should have succeeded.
|
|
let completed = rig.tool_calls_completed();
|
|
let echo_successes = completed
|
|
.iter()
|
|
.filter(|(name, ok)| name == "echo" && *ok)
|
|
.count();
|
|
assert!(
|
|
echo_successes >= 2,
|
|
"Expected >= 2 successful echo calls: {completed:?}"
|
|
);
|
|
|
|
rig.shutdown();
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 5: max_parallel_exceeded -- DEFERRED
|
|
// -----------------------------------------------------------------------
|
|
// Needs max_parallel config exposed through TestRigBuilder.
|
|
}
|