From 6daa2f155f2683cf93669cac5844b6d85400b7a5 Mon Sep 17 00:00:00 2001 From: Jacob Lasky Date: Wed, 25 Mar 2026 03:31:44 -0400 Subject: [PATCH] fix: ensure LLM calls always end with user message (closes #763) (#1259) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: ensure LLM calls always end with user message (closes #763) Claude 4.6 models (claude-sonnet-4-6, claude-opus-4-6) no longer support assistant message prefill — any LLM call where the conversation ends on an assistant message is rejected with HTTP 400 "This model does not support assistant message prefill". The same root cause also triggers NEAR AI's "No user query found in messages" 400 error for the routine engine path. Two fixes: 1. src/worker/container.rs — before_llm_call() After poll_and_inject_prompt(), if no user follow-up arrived and handle_text_response() left an assistant message at the end of the conversation, inject a sentinel "Continue." user message before the next LLM call. 2. src/agent/routine_engine.rs — execute_lightweight_with_tools() Before the force_text final completion call, ensure messages end with a user-role message. Tool result messages (Role::Tool) satisfy Anthropic but not NEAR AI; assistant messages satisfy neither. Also updates the worker system prompt to instruct the agent to include the phrase "The job is complete" in its final message, so the agentic loop can detect termination reliably. Tested with claude-sonnet-4-6 and claude-opus-4-6. Workaround: ANTHROPIC_MODEL=claude-sonnet-4-20250514 (still supports prefill). * fix: broaden sentinel guard to any non-user message (per review) Gemini suggested the Role::Assistant check in before_llm_call() is too specific. Changed to !Role::User to match the routine_engine.rs fix and cover tool results too. * fix: address zmanian review — JobDelegate sentinel, shared helper, NearAI complete() flattening - Extract ensure_ends_with_user_message() to src/util.rs with 4 unit tests (empty list, after assistant, after tool result, no-op when already user) - Add sentinel guard to JobDelegate::before_llm_call() in src/worker/job.rs so scheduler jobs (CreateJob / /job path) no longer hit Claude 4.6 / NEAR AI 400s - Replace inline guards in ContainerDelegate and routine_engine.rs with the shared helper — all 3 call sites now use one implementation - Fix complete() in nearai_chat.rs to apply flatten_tool_messages when flatten_tool_messages=true — previously only complete_with_tools() flattened, so force_text paths could still send role:"tool" messages to NEAR AI - Update stale comment in container.rs: "assistant message" → "non-user message" - Add flatten tests in nearai_chat.rs covering the complete() path Co-Authored-By: Claude Sonnet 4.6 * ci: fix fmt and tar advisory --------- Co-authored-by: Jacob Lasky Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Illia Polosukhin Co-authored-by: firat.sertgoz --- src/agent/routine_engine.rs | 5 ++- src/llm/nearai_chat.rs | 70 +++++++++++++++++++++++++++++++++++-- src/util.rs | 52 ++++++++++++++++++++++++++- src/worker/container.rs | 6 +++- src/worker/job.rs | 5 +++ 5 files changed, 133 insertions(+), 5 deletions(-) diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index 39acb83d..9c55903f 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -1541,7 +1541,10 @@ async fn execute_lightweight_with_tools( let force_text = iteration >= max_iterations; if force_text { - // Final iteration: no tools, just get text response + // Final iteration: no tools, just get text response. + // Claude 4.6 rejects assistant prefill; NEAR AI rejects any non-user-ending + // conversation. Ensure the last message is user-role. + crate::util::ensure_ends_with_user_message(&mut messages); let request = CompletionRequest::new(messages) .with_max_tokens(effective_max_tokens) .with_temperature(0.3); diff --git a/src/llm/nearai_chat.rs b/src/llm/nearai_chat.rs index acbff6ad..5372d76d 100644 --- a/src/llm/nearai_chat.rs +++ b/src/llm/nearai_chat.rs @@ -463,8 +463,15 @@ impl LlmProvider for NearAiChatProvider { let model = req.model.unwrap_or_else(|| self.active_model_name()); let mut raw_messages = req.messages; crate::llm::provider::sanitize_tool_messages(&mut raw_messages); - let messages: Vec = - raw_messages.into_iter().map(|m| m.into()).collect(); + let raw: Vec = raw_messages.into_iter().map(|m| m.into()).collect(); + + // NEAR AI rejects `role:"tool"` messages even on text-only completion paths. + // Apply the same flattening used by complete_with_tools(). + let messages = if self.flatten_tool_messages { + flatten_tool_messages(raw) + } else { + raw + }; let request = ChatCompletionRequest { model, @@ -2193,6 +2200,65 @@ mod tests { assert_eq!(deserialized.function.arguments, r#"{"city":"London"}"#); } + // -- flatten_tool_messages in complete() path ---------------------------- + + #[test] + fn test_flatten_applied_on_text_only_path() { + // Verify that flatten_tool_messages converts tool-role messages to user + // messages (mirrors the complete_with_tools path). + let messages = vec![ + ChatCompletionMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("run it".to_string())), + tool_call_id: None, + name: None, + tool_calls: None, + }, + ChatCompletionMessage { + role: "tool".to_string(), + content: Some(MessageContent::Text("ok".to_string())), + tool_call_id: Some("call_1".to_string()), + name: Some("run_cmd".to_string()), + tool_calls: None, + }, + ]; + let flattened = flatten_tool_messages(messages); + assert_eq!(flattened.len(), 2); + assert_eq!(flattened[1].role, "user"); + let text = flattened[1] + .content + .as_ref() + .and_then(|c| c.as_text()) + .unwrap(); + assert!(text.contains("run_cmd"), "should reference tool name"); + assert!(text.contains("ok"), "should include tool result"); + } + + #[test] + fn test_no_flatten_when_no_tool_messages() { + // When there are no tool-role messages, flatten_tool_messages is a no-op. + let messages = vec![ + ChatCompletionMessage { + role: "user".to_string(), + content: Some(MessageContent::Text("hi".to_string())), + tool_call_id: None, + name: None, + tool_calls: None, + }, + ChatCompletionMessage { + role: "assistant".to_string(), + content: Some(MessageContent::Text("hello".to_string())), + tool_call_id: None, + name: None, + tool_calls: None, + }, + ]; + let result = flatten_tool_messages(messages); + // No tool messages → unchanged roles + assert_eq!(result[0].role, "user"); + assert_eq!(result[1].role, "assistant"); + } + // -- api_url edge cases --------------------------------------------------- #[test] diff --git a/src/util.rs b/src/util.rs index 866f623c..a76f3b27 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,5 +1,7 @@ //! Shared utility functions used across the codebase. +use crate::llm::{ChatMessage, Role}; + /// Find the largest valid UTF-8 char boundary at or before `pos`. /// /// Polyfill for `str::floor_char_boundary` (nightly-only). Use when @@ -16,6 +18,17 @@ pub fn floor_char_boundary(s: &str, pos: usize) -> usize { i } +/// Ensure the last message in `messages` is a user-role message. +/// +/// NEAR AI rejects conversations that don't end with a user message; +/// Claude 4.6 rejects assistant prefill. Call this before any LLM +/// completion request to satisfy both requirements. +pub fn ensure_ends_with_user_message(messages: &mut Vec) { + if !matches!(messages.last(), Some(m) if m.role == Role::User) { + messages.push(ChatMessage::user("Continue.")); + } +} + /// Check if an LLM response explicitly signals that a job/task is complete. /// /// Uses phrase-level matching to avoid false positives from bare words like @@ -72,7 +85,8 @@ pub fn llm_signals_completion(response: &str) -> bool { #[cfg(test)] mod tests { - use crate::util::{floor_char_boundary, llm_signals_completion}; + use crate::llm::ChatMessage; + use crate::util::{ensure_ends_with_user_message, floor_char_boundary, llm_signals_completion}; // ── floor_char_boundary ── @@ -103,6 +117,42 @@ mod tests { assert_eq!(floor_char_boundary("", 5), 0); } + // ── ensure_ends_with_user_message ── + + #[test] + fn ensure_user_message_injects_when_empty() { + let mut msgs: Vec = vec![]; + ensure_ends_with_user_message(&mut msgs); + assert_eq!(msgs.len(), 1); + assert_eq!(msgs[0].role, crate::llm::Role::User); + } + + #[test] + fn ensure_user_message_injects_after_assistant() { + let mut msgs = vec![ChatMessage::user("hi"), ChatMessage::assistant("hello")]; + ensure_ends_with_user_message(&mut msgs); + assert_eq!(msgs.len(), 3); + assert_eq!(msgs[2].role, crate::llm::Role::User); + } + + #[test] + fn ensure_user_message_injects_after_tool_result() { + let mut msgs = vec![ + ChatMessage::user("run tool"), + ChatMessage::tool_result("call_1", "my_tool", "result"), + ]; + ensure_ends_with_user_message(&mut msgs); + assert_eq!(msgs.len(), 3); + assert_eq!(msgs[2].role, crate::llm::Role::User); + } + + #[test] + fn ensure_user_message_no_op_when_already_user() { + let mut msgs = vec![ChatMessage::user("hello")]; + ensure_ends_with_user_message(&mut msgs); + assert_eq!(msgs.len(), 1); + } + // ── llm_signals_completion ── #[test] diff --git a/src/worker/container.rs b/src/worker/container.rs index e0933975..5d8e03b5 100644 --- a/src/worker/container.rs +++ b/src/worker/container.rs @@ -151,7 +151,7 @@ Job: {} Description: {} You have tools for shell commands, file operations, and code editing. -Work independently to complete this job. Report when done."#, +Work independently to complete this job. When finished, your final message MUST include the phrase "The job is complete" to signal termination."#, job.title, job.description ))); @@ -373,6 +373,10 @@ impl LoopDelegate for ContainerDelegate { // Poll for follow-up prompts from the user self.poll_and_inject_prompt(reason_ctx).await; + // Claude 4.6 rejects assistant prefill; NEAR AI rejects any non-user-ending + // conversation. Ensure the last message is user-role before calling the LLM. + crate::util::ensure_ends_with_user_message(&mut reason_ctx.messages); + // Refresh tools (in case WASM tools were built) reason_ctx.available_tools = self.tools.tool_definitions().await; diff --git a/src/worker/job.rs b/src/worker/job.rs index ed261039..9d5794ca 100644 --- a/src/worker/job.rs +++ b/src/worker/job.rs @@ -1232,6 +1232,11 @@ impl<'a> LoopDelegate for JobDelegate<'a> { ) -> Option { // Refresh tool definitions so newly built tools become visible reason_ctx.available_tools = self.worker.tools().tool_definitions().await; + + // Claude 4.6 rejects assistant prefill; NEAR AI rejects any non-user-ending + // conversation. Ensure the last message is user-role before calling the LLM. + crate::util::ensure_ends_with_user_message(&mut reason_ctx.messages); + None }