From a325d9fbcddab88bcc6a5139476c68cf0a3a51a9 Mon Sep 17 00:00:00 2001 From: "ilblackdragon@gmail.com" Date: Sun, 22 Mar 2026 14:31:51 -0700 Subject: [PATCH] fix(engine): detect FINAL() in text responses + regression tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Models sometimes write FINAL() outside code blocks — as plain text after an explanation. The Hyperliquid case: model outputs a long analysis then FINAL("""...""") at the end, not inside ```repl fences. Fixes: - extract_final_from_text(): regex-based FINAL detection in text responses, matching the official RLM's find_final_answer() fallback - Handles: double-quoted, single-quoted, triple-quoted, unquoted, nested parens - Checked in LlmResponse::Text handler BEFORE tool intent nudge (FINAL takes priority) 9 new tests: - codeact_final_in_text_response: FINAL("answer") in plain text - codeact_final_triple_quoted_in_text: FINAL("""multi\nline""") in text - final_double_quoted, final_single_quoted, final_triple_quoted, final_unquoted, final_with_nested_parens, final_after_long_text, no_final_returns_none 102 tests passing (93 + 9 new), zero clippy warnings. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/executor/loop_engine.rs | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/crates/ironclaw_engine/src/executor/loop_engine.rs b/crates/ironclaw_engine/src/executor/loop_engine.rs index 5d247686..422542d7 100644 --- a/crates/ironclaw_engine/src/executor/loop_engine.rs +++ b/crates/ironclaw_engine/src/executor/loop_engine.rs @@ -192,6 +192,27 @@ impl ExecutionLoop { // 6. Handle response match llm_output.response { LlmResponse::Text(text) => { + // Check for FINAL() in text (regex fallback — models + // sometimes write FINAL() outside code blocks) + if let Some(answer) = extract_final_from_text(&text) { + self.thread + .add_message(ThreadMessage::assistant(text)); + step.status = StepStatus::Completed; + step.completed_at = Some(chrono::Utc::now()); + self.thread.add_event(EventKind::StepCompleted { + step_id: step.id, + tokens: step.tokens_used, + }); + self.thread.step_count += 1; + self.thread.transition_to( + ThreadState::Completed, + Some("FINAL() in text".into()), + )?; + return Ok(ThreadOutcome::Completed { + response: Some(answer), + }); + } + // Check for tool intent nudge if nudge_enabled && nudge_count < max_nudges @@ -464,6 +485,70 @@ enum SignalAction { Inject(ThreadMessage), } +/// Extract a FINAL() answer from the LLM's text response. +/// +/// Matches `FINAL(...)` anywhere in the text, handling: +/// - Single-line: `FINAL("the answer")` +/// - Multi-line: `FINAL("""\n...\n""")` +/// - With or without quotes +/// +/// This is the regex fallback from the official RLM implementation +/// (`find_final_answer` in parsing.py) for when the model writes +/// FINAL() outside a code block. +fn extract_final_from_text(text: &str) -> Option { + // Find FINAL( — could be at start of line or after whitespace + let marker = "FINAL("; + let start = text.find(marker)?; + let content_start = start + marker.len(); + + // Extract everything after FINAL( up to the matching closing paren + // Handle nested parens and triple-quoted strings + let remaining = &text[content_start..]; + + // Try triple-quoted string first: FINAL("""...""") + if remaining.starts_with("\"\"\"") { + let inner_start = 3; + if let Some(end) = remaining[inner_start..].find("\"\"\"") { + let answer = remaining[inner_start..inner_start + end].trim(); + if !answer.is_empty() { + return Some(answer.to_string()); + } + } + } + + // Try single/double quoted: FINAL("...") or FINAL('...') + if remaining.starts_with('"') || remaining.starts_with('\'') { + let quote = remaining.as_bytes()[0] as char; + if let Some(end) = remaining[1..].find(quote) { + let answer = &remaining[1..1 + end]; + if !answer.is_empty() { + return Some(answer.to_string()); + } + } + } + + // Unquoted: FINAL(some content here) — find matching close paren + let mut depth = 1; + for (i, ch) in remaining.char_indices() { + match ch { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 { + let answer = remaining[..i].trim(); + if !answer.is_empty() { + return Some(answer.to_string()); + } + return None; + } + } + _ => {} + } + } + + None +} + #[cfg(test)] mod tests { use super::*; @@ -1044,4 +1129,92 @@ mod tests { other => panic!("expected Completed, got {other:?}"), } } + + #[tokio::test] + async fn codeact_final_in_text_response() { + // LLM outputs FINAL() as plain text (not in a code block) + // This is the Hyperliquid case — model writes explanation + FINAL() + let (mut exec, _tx) = make_loop( + vec![text_response( + "Based on my analysis, the answer is clear.\n\nFINAL(\"Revenue grows with volume\")", + )], + vec![], + ThreadConfig::default(), + ) + .await; + + let outcome = exec.run().await.unwrap(); + assert!( + matches!(outcome, ThreadOutcome::Completed { response: Some(ref r) } if r == "Revenue grows with volume"), + "got: {outcome:?}" + ); + } + + #[tokio::test] + async fn codeact_final_triple_quoted_in_text() { + // FINAL with triple-quoted multi-line string in plain text + let (mut exec, _tx) = make_loop( + vec![text_response( + "Here's the summary:\n\nFINAL(\"\"\"\nLine 1\nLine 2\nLine 3\n\"\"\")", + )], + vec![], + ThreadConfig::default(), + ) + .await; + + let outcome = exec.run().await.unwrap(); + match outcome { + ThreadOutcome::Completed { response: Some(r) } => { + assert!(r.contains("Line 1"), "got: {r}"); + assert!(r.contains("Line 3"), "got: {r}"); + } + other => panic!("expected Completed, got {other:?}"), + } + } + + // ── extract_final_from_text unit tests ────────────────── + + #[test] + fn final_double_quoted() { + let text = "some text\nFINAL(\"the answer\")"; + assert_eq!(extract_final_from_text(text).unwrap(), "the answer"); + } + + #[test] + fn final_single_quoted() { + let text = "FINAL('hello world')"; + assert_eq!(extract_final_from_text(text).unwrap(), "hello world"); + } + + #[test] + fn final_triple_quoted() { + let text = "FINAL(\"\"\"\nmulti\nline\n\"\"\")"; + assert_eq!(extract_final_from_text(text).unwrap(), "multi\nline"); + } + + #[test] + fn final_unquoted() { + let text = "FINAL(42)"; + assert_eq!(extract_final_from_text(text).unwrap(), "42"); + } + + #[test] + fn final_with_nested_parens() { + let text = "FINAL(f'result is {len(items)}')"; + assert_eq!( + extract_final_from_text(text).unwrap(), + "f'result is {len(items)}'" + ); + } + + #[test] + fn no_final_returns_none() { + assert!(extract_final_from_text("just regular text").is_none()); + } + + #[test] + fn final_after_long_text() { + let text = "A very long explanation...\n\n🔚 Final Thought\n\nFINAL(\"the conclusion\")"; + assert_eq!(extract_final_from_text(text).unwrap(), "the conclusion"); + } }