optimclaw/tests/e2e_spot_checks.rs

//! E2E spot-check tests adapted from nearai/benchmarks SpotSuite tasks.jsonl.
//!
//! Each test replays an LLM trace through the real agent loop and validates
//! the result using declarative `expects` from the fixture JSON plus any
//! additional assertions that can't be expressed declaratively.

#[cfg(feature = "libsql")]
mod support;

#[cfg(feature = "libsql")]
mod spot_tests {
    use std::time::Duration;

    use crate::support::test_rig::TestRigBuilder;
    use crate::support::trace_llm::LlmTrace;

    const FIXTURES: &str = concat!(
        env!("CARGO_MANIFEST_DIR"),
        "/tests/fixtures/llm_traces/spot"
    );
    const TIMEOUT: Duration = Duration::from_secs(15);

    // -----------------------------------------------------------------------
    // Smoke tests -- no tools expected
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn spot_smoke_greeting() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/smoke_greeting.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Hello! Introduce yourself briefly.").await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    #[tokio::test]
    async fn spot_smoke_math() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/smoke_math.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("What is 47 * 23? Reply with just the number.")
            .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // Tool tests -- verify correct tool selection
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn spot_tool_echo() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/tool_echo.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Use the echo tool to repeat the message: 'Spot check passed'")
            .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    #[tokio::test]
    async fn spot_tool_json() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/tool_json.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Parse this json for me: {\"key\": \"value\"}")
            .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // Chain tests -- multi-tool sequences
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn spot_chain_write_read() {
        let tmp = tempfile::tempdir().unwrap();
        let test_file = tmp.path().join("ironclaw_spot_test.txt");

        let mut trace = LlmTrace::from_file(format!("{FIXTURES}/chain_write_read.json")).unwrap();
        trace.replace_paths("/tmp/ironclaw_spot_test.txt", test_file.to_str().unwrap());

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message(
            "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt \
             using the write_file tool, then read it back using read_file.",
        )
        .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);

        // Extra: verify file on disk (can't express in expects).
        let content = std::fs::read_to_string(&test_file).expect("file should exist");
        assert_eq!(content, "ironclaw spot check");

        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // Robustness tests -- correct behavior under constraints
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn spot_robust_no_tool() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/robust_no_tool.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("What is the capital of France? Answer directly without using any tools.")
            .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    #[tokio::test]
    async fn spot_robust_correct_tool() {
        let trace = LlmTrace::from_file(format!("{FIXTURES}/robust_correct_tool.json")).unwrap();
        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message("Please echo the word 'deterministic output'")
            .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }

    // -----------------------------------------------------------------------
    // Memory tests -- save and recall via file tools
    // -----------------------------------------------------------------------

    #[tokio::test]
    async fn spot_memory_save_recall() {
        let tmp = tempfile::tempdir().unwrap();
        let test_file = tmp.path().join("bench-meeting.md");

        let mut trace = LlmTrace::from_file(format!("{FIXTURES}/memory_save_recall.json")).unwrap();
        trace.replace_paths("/tmp/bench-meeting.md", test_file.to_str().unwrap());

        let rig = TestRigBuilder::new()
            .with_trace(trace.clone())
            .build()
            .await;

        rig.send_message(
            "Save these meeting notes to /tmp/bench-meeting.md:\n\
             Meeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\n\
             Decisions:\n- Launch date: April 15th\n- Budget: $50k approved\n\
             - Bob owns frontend, Carol owns backend\n\
             Then read it back and tell me who owns the frontend and what the launch date is.",
        )
        .await;
        let responses = rig.wait_for_responses(1, TIMEOUT).await;

        rig.verify_trace_expects(&trace, &responses);
        rig.shutdown();
    }
}