optimclaw/tests/e2e_tool_coverage.rs

//! E2E trace tests: tool coverage.
//!
//! Exercises tools that were previously untested: json, shell, list_dir,
//! apply_patch, memory_read, and memory_tree.

#[cfg(feature = "libsql")]
mod support;

#[cfg(feature = "libsql")]
mod tests {
    use std::time::Duration;

    use crate::support::cleanup::CleanupGuard;
    use crate::support::test_rig::TestRigBuilder;
    use crate::support::trace_llm::LlmTrace;

    const TEST_DIR_BASE: &str = "/tmp/ironclaw_coverage_test";

    fn setup_test_dir(suffix: &str) -> String {
        let dir = format!("{TEST_DIR_BASE}_{suffix}");
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).expect("failed to create test directory");
        dir
    }

    // -----------------------------------------------------------------------
    // json tool
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn test_json_operations() {
        let trace = LlmTrace::from_file(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/llm_traces/coverage/json_operations.json"
        ))
        .expect("failed to load json_operations.json");

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Parse and query this json data").await;
        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;

        rig.verify_trace_expects(&trace, &responses);

        // Extra: verify json tool was called at least 3 times.
        let started = rig.tool_calls_started();
        assert!(
            started.iter().filter(|n| n.as_str() == "json").count() >= 3,
            "Expected at least 3 json tool calls, got: {:?}",
            started
        );

        // Extra: metrics checks.
        let metrics = rig.collect_metrics().await;
        assert!(
            metrics.llm_calls >= 4,
            "Expected >= 4 LLM calls, got {}",
            metrics.llm_calls
        );

        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // shell tool
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn test_shell_echo() {
        let trace = LlmTrace::from_file(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/llm_traces/coverage/shell_echo.json"
        ))
        .expect("failed to load shell_echo.json");

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Run a shell command for me").await;
        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // list_dir tool
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn test_list_dir() {
        let test_dir = setup_test_dir("list_dir");
        let _cleanup = CleanupGuard::new().dir(&test_dir);
        std::fs::write(format!("{test_dir}/file_a.txt"), "content a").unwrap();
        std::fs::write(format!("{test_dir}/file_b.txt"), "content b").unwrap();

        let trace = LlmTrace::from_file(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/llm_traces/coverage/list_dir.json"
        ))
        .expect("failed to load list_dir.json");

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("List the test directory").await;
        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // apply_patch tool
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn test_apply_patch_chain() {
        let test_dir = setup_test_dir("apply_patch");
        let _cleanup = CleanupGuard::new().dir(&test_dir);

        let trace = LlmTrace::from_file(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/llm_traces/coverage/apply_patch_chain.json"
        ))
        .expect("failed to load apply_patch_chain.json");

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Write a file and patch it").await;
        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;

        rig.verify_trace_expects(&trace, &responses);

        // Extra: verify the patch was applied on disk.
        let content = std::fs::read_to_string(format!("{test_dir}/patch_target.txt"))
            .expect("patch_target.txt should exist");
        assert!(
            content.contains("PATCHED"),
            "Expected 'PATCHED' in file content, got: {content:?}"
        );
        assert!(
            !content.contains("original"),
            "Expected 'original' to be replaced, but it still exists in: {content:?}"
        );

        // Extra: metrics checks.
        let metrics = rig.collect_metrics().await;
        assert!(metrics.llm_calls >= 4, "Expected >= 4 LLM calls");
        assert!(metrics.total_tool_calls() >= 3, "Expected >= 3 tool calls");

        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // memory_read + memory_tree (full memory cycle)
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn test_memory_full_cycle() {
        let trace = LlmTrace::from_file(concat!(
            env!("CARGO_MANIFEST_DIR"),
            "/tests/fixtures/llm_traces/coverage/memory_full_cycle.json"
        ))
        .expect("failed to load memory_full_cycle.json");

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Exercise all four memory operations")
            .await;
        let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;

        rig.verify_trace_expects(&trace, &responses);

        // Extra: metrics checks.
        let metrics = rig.collect_metrics().await;
        assert!(metrics.llm_calls >= 5, "Expected >= 5 LLM calls");
        assert!(metrics.total_tool_calls() >= 4, "Expected >= 4 tool calls");

        rig.shutdown();
    }
}