fix(engine): detect FINAL() in text responses + regression tests

Models sometimes write FINAL() outside code blocks — as plain text
after an explanation. The Hyperliquid case: model outputs a long
analysis then FINAL("""...""") at the end, not inside ```repl fences.

Fixes:
- extract_final_from_text(): regex-based FINAL detection in text
  responses, matching the official RLM's find_final_answer() fallback
- Handles: double-quoted, single-quoted, triple-quoted, unquoted,
  nested parens
- Checked in LlmResponse::Text handler BEFORE tool intent nudge
  (FINAL takes priority)

9 new tests:
- codeact_final_in_text_response: FINAL("answer") in plain text
- codeact_final_triple_quoted_in_text: FINAL("""multi\nline""") in text
- final_double_quoted, final_single_quoted, final_triple_quoted,
  final_unquoted, final_with_nested_parens, final_after_long_text,
  no_final_returns_none

102 tests passing (93 + 9 new), zero clippy warnings.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
2026-03-22 14:31:51 -07:00
co-authored by Claude Opus 4.6
parent 79120aa9d8
commit a325d9fbcd
@@ -192,6 +192,27 @@ impl ExecutionLoop {
// 6. Handle response
match llm_output.response {
LlmResponse::Text(text) => {
// Check for FINAL() in text (regex fallback — models
// sometimes write FINAL() outside code blocks)
if let Some(answer) = extract_final_from_text(&text) {
self.thread
.add_message(ThreadMessage::assistant(text));
step.status = StepStatus::Completed;
step.completed_at = Some(chrono::Utc::now());
self.thread.add_event(EventKind::StepCompleted {
step_id: step.id,
tokens: step.tokens_used,
});
self.thread.step_count += 1;
self.thread.transition_to(
ThreadState::Completed,
Some("FINAL() in text".into()),
)?;
return Ok(ThreadOutcome::Completed {
response: Some(answer),
});
}
// Check for tool intent nudge
if nudge_enabled
&& nudge_count < max_nudges
@@ -464,6 +485,70 @@ enum SignalAction {
Inject(ThreadMessage),
}
/// Extract a FINAL() answer from the LLM's text response.
///
/// Matches `FINAL(...)` anywhere in the text, handling:
/// - Single-line: `FINAL("the answer")`
/// - Multi-line: `FINAL("""\n...\n""")`
/// - With or without quotes
///
/// This is the regex fallback from the official RLM implementation
/// (`find_final_answer` in parsing.py) for when the model writes
/// FINAL() outside a code block.
fn extract_final_from_text(text: &str) -> Option<String> {
// Find FINAL( — could be at start of line or after whitespace
let marker = "FINAL(";
let start = text.find(marker)?;
let content_start = start + marker.len();
// Extract everything after FINAL( up to the matching closing paren
// Handle nested parens and triple-quoted strings
let remaining = &text[content_start..];
// Try triple-quoted string first: FINAL("""...""")
if remaining.starts_with("\"\"\"") {
let inner_start = 3;
if let Some(end) = remaining[inner_start..].find("\"\"\"") {
let answer = remaining[inner_start..inner_start + end].trim();
if !answer.is_empty() {
return Some(answer.to_string());
}
}
}
// Try single/double quoted: FINAL("...") or FINAL('...')
if remaining.starts_with('"') || remaining.starts_with('\'') {
let quote = remaining.as_bytes()[0] as char;
if let Some(end) = remaining[1..].find(quote) {
let answer = &remaining[1..1 + end];
if !answer.is_empty() {
return Some(answer.to_string());
}
}
}
// Unquoted: FINAL(some content here) — find matching close paren
let mut depth = 1;
for (i, ch) in remaining.char_indices() {
match ch {
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 {
let answer = remaining[..i].trim();
if !answer.is_empty() {
return Some(answer.to_string());
}
return None;
}
}
_ => {}
}
}
None
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1044,4 +1129,92 @@ mod tests {
other => panic!("expected Completed, got {other:?}"),
}
}
#[tokio::test]
async fn codeact_final_in_text_response() {
// LLM outputs FINAL() as plain text (not in a code block)
// This is the Hyperliquid case — model writes explanation + FINAL()
let (mut exec, _tx) = make_loop(
vec![text_response(
"Based on my analysis, the answer is clear.\n\nFINAL(\"Revenue grows with volume\")",
)],
vec![],
ThreadConfig::default(),
)
.await;
let outcome = exec.run().await.unwrap();
assert!(
matches!(outcome, ThreadOutcome::Completed { response: Some(ref r) } if r == "Revenue grows with volume"),
"got: {outcome:?}"
);
}
#[tokio::test]
async fn codeact_final_triple_quoted_in_text() {
// FINAL with triple-quoted multi-line string in plain text
let (mut exec, _tx) = make_loop(
vec![text_response(
"Here's the summary:\n\nFINAL(\"\"\"\nLine 1\nLine 2\nLine 3\n\"\"\")",
)],
vec![],
ThreadConfig::default(),
)
.await;
let outcome = exec.run().await.unwrap();
match outcome {
ThreadOutcome::Completed { response: Some(r) } => {
assert!(r.contains("Line 1"), "got: {r}");
assert!(r.contains("Line 3"), "got: {r}");
}
other => panic!("expected Completed, got {other:?}"),
}
}
// ── extract_final_from_text unit tests ──────────────────
#[test]
fn final_double_quoted() {
let text = "some text\nFINAL(\"the answer\")";
assert_eq!(extract_final_from_text(text).unwrap(), "the answer");
}
#[test]
fn final_single_quoted() {
let text = "FINAL('hello world')";
assert_eq!(extract_final_from_text(text).unwrap(), "hello world");
}
#[test]
fn final_triple_quoted() {
let text = "FINAL(\"\"\"\nmulti\nline\n\"\"\")";
assert_eq!(extract_final_from_text(text).unwrap(), "multi\nline");
}
#[test]
fn final_unquoted() {
let text = "FINAL(42)";
assert_eq!(extract_final_from_text(text).unwrap(), "42");
}
#[test]
fn final_with_nested_parens() {
let text = "FINAL(f'result is {len(items)}')";
assert_eq!(
extract_final_from_text(text).unwrap(),
"f'result is {len(items)}'"
);
}
#[test]
fn no_final_returns_none() {
assert!(extract_final_from_text("just regular text").is_none());
}
#[test]
fn final_after_long_text() {
let text = "A very long explanation...\n\n🔚 Final Thought\n\nFINAL(\"the conclusion\")";
assert_eq!(extract_final_from_text(text).unwrap(), "the conclusion");
}
}