From 8fbb7820901a390a119291191f2418da30c7a0bc Mon Sep 17 00:00:00 2001 From: Illia Polosukhin Date: Sat, 7 Mar 2026 08:05:55 +0000 Subject: [PATCH] fix(llm): nudge LLM when it expresses tool intent without calling tools (#653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(llm): nudge LLM when it expresses tool intent without calling tools Non-Anthropic models (especially GLM-5 via NEAR AI) frequently output text like "Let me search for X" without including tool_calls, creating a frustrating loop where the user waits but nothing happens. Add llm_signals_tool_intent() detection that matches intent phrases ("let me search", "I'll fetch") while excluding conversational phrases ("let me explain", "let me know") and content inside code blocks. When detected, inject a nudge message telling the model to actually call the tool. Cap at 2 consecutive nudges to avoid infinite loops. Applied to all three agentic loops: dispatcher (interactive chat), agent/worker (background jobs), and worker/runtime (sandbox containers). Co-Authored-By: Claude Opus 4.6 * fix(nudge): address PR #653 review comments 1. Update doc comment to match implementation (code blocks only, not quotes) 2. Use match_indices() instead of find() to check all prefix occurrences 3. Add !available_tools.is_empty() guard in dispatcher nudge check 4. Reset consecutive_tool_intent_nudges on non-intent text responses 5. Add regression test for shadowed prefix detection Co-Authored-By: Claude Opus 4.6 * fix(nudge): address second round of PR #653 review comments 1. Strip double-quoted strings in tool-intent detection to avoid false positives on quoted prose like `"Let me search the database"`. 2. Only reset consecutive_tool_intent_nudges when text does NOT signal intent — preserves the 2-nudge cap when intent is detected but cap is already reached. 3. Fix assertion message in nudge_cap test to report correct call index. 4. Add regression test for quoted strings outside code blocks. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- src/agent/dispatcher.rs | 21 ++ src/agent/worker.rs | 30 ++- src/llm/mod.rs | 2 +- src/llm/reasoning.rs | 206 ++++++++++++++++++ src/worker/runtime.rs | 20 ++ tests/e2e_advanced_traces.rs | 132 +++++++++++ .../tool_intent_no_false_positive.json | 16 ++ .../advanced/tool_intent_nudge_cap.json | 34 +++ .../advanced/tool_intent_nudge_recovery.json | 41 ++++ 9 files changed, 496 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/llm_traces/advanced/tool_intent_no_false_positive.json create mode 100644 tests/fixtures/llm_traces/advanced/tool_intent_nudge_cap.json create mode 100644 tests/fixtures/llm_traces/advanced/tool_intent_nudge_recovery.json diff --git a/src/agent/dispatcher.rs b/src/agent/dispatcher.rs index 94f69ae6..e957d1c8 100644 --- a/src/agent/dispatcher.rs +++ b/src/agent/dispatcher.rs @@ -138,6 +138,8 @@ impl Agent { let force_text_at = max_tool_iterations; let nudge_at = max_tool_iterations.saturating_sub(1); let mut iteration = 0; + const MAX_TOOL_INTENT_NUDGES: u32 = 2; + let mut consecutive_tool_intent_nudges: u32 = 0; loop { iteration += 1; // Hard ceiling one past the forced-text iteration (should never be reached @@ -294,6 +296,24 @@ impl Agent { match output.result { RespondResult::Text(text) => { + // Nudge the LLM if it expressed tool intent without calling tools. + // This is common with non-Anthropic models (e.g. GLM-5 via NEAR AI) + // that output "Let me search…" but don't issue tool_calls. + if !force_text + && !context.available_tools.is_empty() + && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES + && crate::llm::llm_signals_tool_intent(&text) + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + iteration, + "LLM expressed tool intent without calling a tool, nudging" + ); + context_messages.push(ChatMessage::assistant(&text)); + context_messages.push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + continue; + } + // Strip internal "[Called tool ...]" text that can leak when // provider flattening (e.g. NEAR AI) converts tool_calls to // plain text and the LLM echoes it back. @@ -304,6 +324,7 @@ impl Agent { tool_calls, content, } => { + consecutive_tool_intent_nudges = 0; // Add the assistant message with tool_calls to context. // OpenAI protocol requires this before tool-result messages. context_messages.push(ChatMessage::assistant_with_tool_calls( diff --git a/src/agent/worker.rs b/src/agent/worker.rs index e3fa11e7..9298c0f2 100644 --- a/src/agent/worker.rs +++ b/src/agent/worker.rs @@ -305,6 +305,8 @@ Report when the job is complete or if you encounter issues you cannot resolve."# let mut iteration = 0; const MAX_CONSECUTIVE_RATE_LIMITS: usize = 10; let mut consecutive_rate_limits = 0usize; + const MAX_TOOL_INTENT_NUDGES: u32 = 2; + let mut consecutive_tool_intent_nudges: u32 = 0; // Initial tool definitions for planning (will be refreshed in loop) reason_ctx.available_tools = self.tools().tool_definitions().await; @@ -502,17 +504,34 @@ Report when the job is complete or if you encounter issues you cannot resolve."# }), ); - // Give it one more chance to select a tool - if iteration > 3 && iteration % 5 == 0 { - reason_ctx.messages.push(ChatMessage::user( - "Are you stuck? Do you need help completing this job?", - )); + // Nudge the LLM if it expressed tool intent without calling tools + let signals_intent = !reason_ctx.available_tools.is_empty() + && crate::llm::llm_signals_tool_intent(&response); + if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + job_id = %self.job_id, + "LLM expressed tool intent without calling a tool, nudging" + ); + reason_ctx + .messages + .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + } else if !signals_intent { + consecutive_tool_intent_nudges = 0; + if iteration > 3 && iteration % 5 == 0 { + // Generic fallback nudge + reason_ctx.messages.push(ChatMessage::user( + "Are you stuck? Do you need help completing this job?", + )); + } } } RespondResult::ToolCalls { tool_calls, content, } => { + consecutive_tool_intent_nudges = 0; // Model returned tool calls - execute them tracing::debug!( "Job {} respond_with_tools returned {} tool calls", @@ -558,6 +577,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# } } } else if selections.len() == 1 { + consecutive_tool_intent_nudges = 0; // Single tool: execute directly let selection = &selections[0]; tracing::debug!( diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 27083824..579c5e9f 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -30,7 +30,7 @@ pub use provider::{ }; pub use reasoning::{ ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, SILENT_REPLY_TOKEN, - TokenUsage, ToolSelection, is_silent_reply, + TOOL_INTENT_NUDGE, TokenUsage, ToolSelection, is_silent_reply, llm_signals_tool_intent, }; pub use recording::RecordingLlm; pub use registry::{ProviderDefinition, ProviderProtocol, ProviderRegistry}; diff --git a/src/llm/reasoning.rs b/src/llm/reasoning.rs index 0afa10d9..c3081ddb 100644 --- a/src/llm/reasoning.rs +++ b/src/llm/reasoning.rs @@ -16,6 +16,129 @@ use crate::safety::SafetyLayer; /// The dispatcher should check for this and suppress the message. pub const SILENT_REPLY_TOKEN: &str = "NO_REPLY"; +/// Nudge message injected when the LLM expresses intent to use a tool but +/// doesn't include any `tool_calls` in its response. +pub const TOOL_INTENT_NUDGE: &str = "\ +You said you would perform an action, but you did not include any tool calls.\n\ +Do NOT describe what you intend to do — actually call the tool now.\n\ +Use the tool_calls mechanism to invoke the appropriate tool."; + +/// Detect when an LLM response expresses intent to call a tool without +/// actually issuing tool calls. Returns `true` if the text contains phrases +/// like "Let me search …" or "I'll fetch …" outside of fenced/indented code blocks. +/// +/// Exclusion phrases (e.g. "let me explain") are checked first to avoid +/// false positives on conversational language. +pub fn llm_signals_tool_intent(response: &str) -> bool { + // Extract only non-code lines with quoted strings removed + let text = strip_code_blocks(response); + let lower = text.to_lowercase(); + + // Exclusion phrases — if any appear, bail out immediately + const EXCLUSIONS: &[&str] = &[ + "let me explain", + "let me know", + "let me think", + "let me summarize", + "let me clarify", + "let me describe", + "let me help", + "let me understand", + "let me break", + "let me outline", + "let me walk you", + "let me provide", + "let me suggest", + "let me elaborate", + "let me start by", + ]; + if EXCLUSIONS.iter().any(|e| lower.contains(e)) { + return false; + } + + const PREFIXES: &[&str] = &["let me ", "i'll ", "i will ", "i'm going to "]; + const ACTION_VERBS: &[&str] = &[ + "search", + "look up", + "check", + "fetch", + "find", + "read the", + "write the", + "create", + "run the", + "execute", + "query", + "retrieve", + "add it", + "add the", + "add this", + "add that", + "update the", + "delete", + "remove the", + "look into", + ]; + + for prefix in PREFIXES { + for (i, _) in lower.match_indices(prefix) { + let after = &lower[i + prefix.len()..]; + for verb in ACTION_VERBS { + if after.starts_with(verb) || after.contains(&format!(" {verb}")) { + return true; + } + } + } + } + + false +} + +/// Strip fenced code blocks (``` ... ```), indented code lines (4+ spaces / tab), +/// and double-quoted strings so that tool-intent detection only fires on prose. +fn strip_code_blocks(text: &str) -> String { + let mut result = String::new(); + let mut in_fence = false; + + for line in text.lines() { + let trimmed = line.trim_start(); + if trimmed.starts_with("```") { + in_fence = !in_fence; + continue; + } + if in_fence { + continue; + } + // Skip indented code lines (4+ spaces or tab) + if line.starts_with(" ") || line.starts_with('\t') { + continue; + } + // Strip double-quoted strings to avoid matching intent phrases inside quotes + let stripped = strip_quoted_strings(line); + result.push_str(&stripped); + result.push('\n'); + } + result +} + +/// Remove double-quoted string literals from a line. +fn strip_quoted_strings(line: &str) -> String { + let mut result = String::with_capacity(line.len()); + let mut in_quote = false; + let mut prev = '\0'; + for ch in line.chars() { + if ch == '"' && prev != '\\' { + in_quote = !in_quote; + continue; + } + if !in_quote { + result.push(ch); + } + prev = ch; + } + result +} + /// Check if a response is a silent reply (the agent has nothing to say). /// /// Returns true if the trimmed text is exactly the silent reply token or @@ -2008,4 +2131,87 @@ That's my plan."#; assert!(cleaned.contains("Let me fetch that.")); assert!(cleaned.contains("Here are the results.")); } + + // ---- Tool intent detection tests ---- + + #[test] + fn test_llm_signals_tool_intent_true_positives() { + assert!(llm_signals_tool_intent("Let me search for that file.")); + assert!(llm_signals_tool_intent("I'll fetch the data now.")); + assert!(llm_signals_tool_intent("I'm going to check the logs.")); + assert!(llm_signals_tool_intent("Let me add it now.")); + assert!(llm_signals_tool_intent("I will run the tests to verify.")); + assert!(llm_signals_tool_intent("I'll look up the documentation.")); + assert!(llm_signals_tool_intent("Let me read the file contents.")); + assert!(llm_signals_tool_intent("I'm going to execute the command.")); + } + + #[test] + fn test_llm_signals_tool_intent_true_negatives_conversational() { + assert!(!llm_signals_tool_intent("Let me explain how this works.")); + assert!(!llm_signals_tool_intent( + "Let me know if you need anything." + )); + assert!(!llm_signals_tool_intent("Let me think about this.")); + assert!(!llm_signals_tool_intent("Let me summarize the findings.")); + assert!(!llm_signals_tool_intent("Let me clarify what I mean.")); + } + + #[test] + fn test_llm_signals_tool_intent_exclusion_takes_precedence() { + // Exclusion phrase present alongside intent → false + assert!(!llm_signals_tool_intent( + "Let me explain the approach, then I'll search for the file." + )); + } + + #[test] + fn test_llm_signals_tool_intent_ignores_code_blocks() { + let with_code = "Here's the updated code:\n\n```\nfn main() {\n println!(\"Let me search the database\");\n}\n```"; + assert!(!llm_signals_tool_intent(with_code)); + } + + #[test] + fn test_llm_signals_tool_intent_ignores_indented_code() { + let with_indent = + "Here's the code:\n\n println!(\"I'll fetch the data\");\n\nThat's it."; + assert!(!llm_signals_tool_intent(with_indent)); + } + + #[test] + fn test_llm_signals_tool_intent_ignores_plain_text() { + assert!(!llm_signals_tool_intent("The task is complete.")); + assert!(!llm_signals_tool_intent( + "Here are the results you asked for." + )); + assert!(!llm_signals_tool_intent("I found 3 matching files.")); + } + + #[test] + fn test_llm_signals_tool_intent_quoted_string_in_code_block() { + let text = "The button text should say:\n```\n\"I will create your account\"\n```"; + assert!(!llm_signals_tool_intent(text)); + } + + #[test] + fn test_llm_signals_tool_intent_quoted_string_outside_code_block() { + // Quoted intent phrase in prose should not trigger. + let text = "The button says \"Let me search the database\" to the user."; + assert!(!llm_signals_tool_intent(text)); + // But unquoted intent in the same line should still trigger. + let text = "I'll fetch the results for you."; + assert!(llm_signals_tool_intent(text)); + } + + #[test] + fn test_llm_signals_tool_intent_shadowed_prefix() { + // An earlier non-intent "let me" should not shadow a later real intent. + let text = "Sure, let me think about it. Actually, let me search for the file."; + // "let me think" is an exclusion, so this returns false despite the second "let me search". + assert!(!llm_signals_tool_intent(text)); + + // But without an exclusion phrase, multiple prefixes should be checked. + let text = "I said let me be clear, then let me fetch the data."; + assert!(llm_signals_tool_intent(text)); + } } diff --git a/src/worker/runtime.rs b/src/worker/runtime.rs index 7de30d49..3f284db5 100644 --- a/src/worker/runtime.rs +++ b/src/worker/runtime.rs @@ -222,6 +222,8 @@ Work independently to complete this job. Report when done."#, ) -> Result { let max_iterations = self.config.max_iterations; let mut last_output = String::new(); + const MAX_TOOL_INTENT_NUDGES: u32 = 2; + let mut consecutive_tool_intent_nudges: u32 = 0; // Load tool definitions reason_ctx.available_tools = self.tools.tool_definitions().await; @@ -280,11 +282,28 @@ Work independently to complete this job. Report when done."#, return Ok(last_output); } reason_ctx.messages.push(ChatMessage::assistant(&response)); + + // Nudge the LLM if it expressed tool intent without calling tools + let signals_intent = !reason_ctx.available_tools.is_empty() + && crate::llm::llm_signals_tool_intent(&response); + if signals_intent && consecutive_tool_intent_nudges < MAX_TOOL_INTENT_NUDGES + { + consecutive_tool_intent_nudges += 1; + tracing::info!( + "LLM expressed tool intent without calling a tool, nudging" + ); + reason_ctx + .messages + .push(ChatMessage::user(crate::llm::TOOL_INTENT_NUDGE)); + } else if !signals_intent { + consecutive_tool_intent_nudges = 0; + } } RespondResult::ToolCalls { tool_calls, content, } => { + consecutive_tool_intent_nudges = 0; if let Some(ref text) = content { self.post_event( "message", @@ -344,6 +363,7 @@ Work independently to complete this job. Report when done."#, } } } else { + consecutive_tool_intent_nudges = 0; // Execute selected tools for selection in &selections { self.post_event( diff --git a/tests/e2e_advanced_traces.rs b/tests/e2e_advanced_traces.rs index 92dd81f4..54549382 100644 --- a/tests/e2e_advanced_traces.rs +++ b/tests/e2e_advanced_traces.rs @@ -390,4 +390,136 @@ mod advanced { rig.verify_trace_expects(&trace, &responses); rig.shutdown(); } + + // ----------------------------------------------------------------------- + // 7. Tool intent nudge — model recovers after nudge + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn tool_intent_nudge_recovery() { + let trace = + LlmTrace::from_file(format!("{FIXTURES}/tool_intent_nudge_recovery.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + rig.send_message("Search for the config file.").await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + rig.verify_trace_expects(&trace, &responses); + + // The nudge should have caused the model to actually call a tool. + let started = rig.tool_calls_started(); + assert!( + started.iter().any(|s| s == "echo"), + "expected echo tool call after nudge, got: {started:?}" + ); + + // Verify the nudge was injected: the TraceLlm request_hint on step 2 + // requires "tool_calls mechanism" in the last user message. If the hint + // didn't match, TraceLlm logs a warning but doesn't fail -- so also + // check captured requests directly. + let trace_llm = rig.trace_llm().expect("trace_llm should exist"); + assert_eq!( + trace_llm.hint_mismatches(), + 0, + "nudge message should have been injected before the tool-call step" + ); + + rig.shutdown(); + } + + // ----------------------------------------------------------------------- + // 8. Tool intent nudge — caps at 2 nudges + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn tool_intent_nudge_cap() { + let trace = LlmTrace::from_file(format!("{FIXTURES}/tool_intent_nudge_cap.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + rig.send_message("Fetch the project data for me.").await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + rig.verify_trace_expects(&trace, &responses); + + // Exactly 3 LLM calls: nudge after 1st, nudge after 2nd, 3rd text + // returned as-is (cap of 2 nudges reached). + let trace_llm = rig.trace_llm().expect("trace_llm should exist"); + let captured = trace_llm.captured_requests(); + assert_eq!( + captured.len(), + 3, + "expected exactly 3 LLM calls (2 nudged + 1 returned), got {}", + captured.len() + ); + + // Verify both nudges fired: calls 2 and 3 should have the nudge + // message as the last user message. + for call_idx in [1usize, 2] { + let msgs = &captured[call_idx]; + let last_user = msgs + .iter() + .rev() + .find(|m| matches!(m.role, ironclaw::llm::Role::User)); + assert!( + last_user.is_some_and(|m| m.content.contains("tool_calls mechanism")), + "call {} should have the nudge as last user message", + call_idx + 1 + ); + } + + // No tools should have been called (model never issued tool_calls). + let started = rig.tool_calls_started(); + assert!( + started.is_empty(), + "no tools should be called when model keeps narrating, got: {started:?}" + ); + + rig.shutdown(); + } + + // ----------------------------------------------------------------------- + // 9. Tool intent nudge — no false positive on conversational "let me explain" + // ----------------------------------------------------------------------- + + #[tokio::test] + async fn tool_intent_no_false_positive() { + let trace = + LlmTrace::from_file(format!("{FIXTURES}/tool_intent_no_false_positive.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + rig.send_message("How does auth work?").await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + rig.verify_trace_expects(&trace, &responses); + + // "Let me explain" should NOT trigger a nudge, so the TraceLlm should + // have been called exactly once (the text response) with no extra nudge + // messages injected. + let trace_llm = rig.trace_llm().expect("trace_llm should exist"); + let captured = trace_llm.captured_requests(); + assert_eq!( + captured.len(), + 1, + "expected exactly 1 LLM call (no nudge), got {}", + captured.len() + ); + + // No tools should have been called. + let started = rig.tool_calls_started(); + assert!( + started.is_empty(), + "no tools should be called for a conversational response, got: {started:?}" + ); + + rig.shutdown(); + } } diff --git a/tests/fixtures/llm_traces/advanced/tool_intent_no_false_positive.json b/tests/fixtures/llm_traces/advanced/tool_intent_no_false_positive.json new file mode 100644 index 00000000..b4a14e21 --- /dev/null +++ b/tests/fixtures/llm_traces/advanced/tool_intent_no_false_positive.json @@ -0,0 +1,16 @@ +{ + "model_name": "advanced-tool-intent-no-false-positive", + "steps": [ + { + "response": { + "type": "text", + "content": "Let me explain how the authentication system works. It uses JWT tokens with a 24-hour expiry. The job is complete.", + "input_tokens": 50, + "output_tokens": 30 + } + } + ], + "expects": { + "response_contains": ["authentication"] + } +} diff --git a/tests/fixtures/llm_traces/advanced/tool_intent_nudge_cap.json b/tests/fixtures/llm_traces/advanced/tool_intent_nudge_cap.json new file mode 100644 index 00000000..e76e6743 --- /dev/null +++ b/tests/fixtures/llm_traces/advanced/tool_intent_nudge_cap.json @@ -0,0 +1,34 @@ +{ + "model_name": "advanced-tool-intent-nudge-cap", + "steps": [ + { + "response": { + "type": "text", + "content": "I'll fetch the data right away.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "tool_calls mechanism" }, + "response": { + "type": "text", + "content": "I'm going to query the database now.", + "input_tokens": 100, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "tool_calls mechanism" }, + "response": { + "type": "text", + "content": "Let me run the search for you.", + "input_tokens": 150, + "output_tokens": 10 + } + } + ], + "expects": { + "response_contains": ["run the search"] + } +} diff --git a/tests/fixtures/llm_traces/advanced/tool_intent_nudge_recovery.json b/tests/fixtures/llm_traces/advanced/tool_intent_nudge_recovery.json new file mode 100644 index 00000000..1073ca41 --- /dev/null +++ b/tests/fixtures/llm_traces/advanced/tool_intent_nudge_recovery.json @@ -0,0 +1,41 @@ +{ + "model_name": "advanced-tool-intent-nudge-recovery", + "steps": [ + { + "response": { + "type": "text", + "content": "Let me search for that file now.", + "input_tokens": 50, + "output_tokens": 10 + } + }, + { + "request_hint": { "last_user_message_contains": "tool_calls mechanism" }, + "response": { + "type": "tool_calls", + "tool_calls": [ + { + "id": "call_echo_1", + "name": "echo", + "arguments": { "message": "found it" } + } + ], + "input_tokens": 100, + "output_tokens": 15 + } + }, + { + "response": { + "type": "text", + "content": "I found the file you were looking for. The job is complete.", + "input_tokens": 150, + "output_tokens": 20 + } + } + ], + "expects": { + "tools_used": ["echo"], + "response_contains": ["found"], + "all_tools_succeeded": true + } +}