Handle empty tool completions in autonomous jobs

src/worker/container.rs

@@ -21,11 +21,15 @@ use crate::agent::agentic_loop::{
 use crate::config::SafetyConfig;
 use crate::context::JobContext;
 use crate::error::WorkerError;
-use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext};
+use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, ResponseMetadata};
 use crate::safety::SafetyLayer;
 use crate::tools::ToolRegistry;
 use crate::tools::execute::{execute_tool_simple, process_tool_result};
 use crate::worker::api::{CompletionReport, JobEventPayload, StatusUpdate, WorkerHttpClient};
+use crate::worker::autonomous_recovery::{
+    AutonomousRecoveryAction, AutonomousRecoveryState, EMPTY_TOOL_COMPLETION_FAILURE,
+    EMPTY_TOOL_COMPLETION_NUDGE, FORCE_TEXT_RECOVERY_PROMPT,
+};
 use crate::worker::proxy_llm::ProxyLlmProvider;
 
 /// Configuration for the worker runtime.
@@ -170,6 +174,7 @@ Work independently to complete this job. When finished, your final message MUST
                 extra_env: self.extra_env.clone(),
                 last_output: Mutex::new(String::new()),
                 iteration_tracker: iteration_tracker.clone(),
+                recovery_state: Mutex::new(AutonomousRecoveryState::default()),
             };
 
             let config = AgenticLoopConfig {
@@ -228,6 +233,24 @@ Work independently to complete this job. When finished, your final message MUST
                     })
                     .await?;
             }
+            Ok(Ok(LoopOutcome::Failure(reason))) => {
+                tracing::warn!("Worker failed for job {}: {}", self.config.job_id, reason);
+                self.post_event(
+                    "result",
+                    serde_json::json!({
+                        "success": false,
+                        "message": reason,
+                    }),
+                )
+                .await;
+                self.client
+                    .report_complete(&CompletionReport {
+                        success: false,
+                        message: Some(reason),
+                        iterations,
+                    })
+                    .await?;
+            }
             Ok(Ok(LoopOutcome::Stopped | LoopOutcome::NeedApproval(_))) => {
                 tracing::info!("Worker for job {} stopped", self.config.job_id);
                 self.client
@@ -304,6 +327,7 @@ struct ContainerDelegate {
     /// Tracks the current iteration — shared with the outer `run` method so
     /// `CompletionReport` can include accurate iteration counts.
     iteration_tracker: Arc<Mutex<u32>>,
+    recovery_state: Mutex<AutonomousRecoveryState>,
 }
 
 impl ContainerDelegate {
@@ -377,8 +401,17 @@ impl LoopDelegate for ContainerDelegate {
         // conversation. Ensure the last message is user-role before calling the LLM.
         crate::util::ensure_ends_with_user_message(&mut reason_ctx.messages);
 
-        // Refresh tools (in case WASM tools were built)
-        reason_ctx.available_tools = self.tools.tool_definitions().await;
+        let force_text_recovery = {
+            let mut recovery = self.recovery_state.lock().await;
+            recovery.begin_iteration()
+        };
+        if force_text_recovery {
+            tracing::warn!("Switching to text-only recovery after malformed tool completions");
+            reason_ctx.available_tools.clear();
+        } else {
+            // Refresh tools (in case WASM tools were built)
+            reason_ctx.available_tools = self.tools.tool_definitions().await;
+        }
 
         None
     }
@@ -399,8 +432,53 @@ impl LoopDelegate for ContainerDelegate {
     async fn handle_text_response(
         &self,
         text: &str,
+        metadata: ResponseMetadata,
         reason_ctx: &mut ReasoningContext,
     ) -> TextAction {
+        let action = {
+            let mut recovery = self.recovery_state.lock().await;
+            recovery.on_text_response(metadata, text)
+        };
+        match action {
+            AutonomousRecoveryAction::ToolModeNudge => {
+                tracing::warn!("Malformed empty tool completion detected; retrying in tool mode");
+                self.post_event(
+                    "status",
+                    serde_json::json!({
+                        "message": "Model returned an empty tool-completion response; retrying with a stronger tool-use nudge.",
+                    }),
+                )
+                .await;
+                reason_ctx
+                    .messages
+                    .push(ChatMessage::user(EMPTY_TOOL_COMPLETION_NUDGE));
+                return TextAction::Continue;
+            }
+            AutonomousRecoveryAction::ForceTextRecovery => {
+                tracing::warn!(
+                    "Repeated malformed tool completions detected; switching to text-only recovery"
+                );
+                self.post_event(
+                    "status",
+                    serde_json::json!({
+                        "message": "Model returned repeated empty tool-completion responses; requesting a final status update without tools.",
+                    }),
+                )
+                .await;
+                reason_ctx
+                    .messages
+                    .push(ChatMessage::user(FORCE_TEXT_RECOVERY_PROMPT));
+                return TextAction::Continue;
+            }
+            AutonomousRecoveryAction::Fail => {
+                tracing::warn!("Failing fast after repeated malformed autonomous responses");
+                return TextAction::Return(LoopOutcome::Failure(
+                    EMPTY_TOOL_COMPLETION_FAILURE.to_string(),
+                ));
+            }
+            AutonomousRecoveryAction::Continue => {}
+        }
+
         self.post_event(
             "message",
             serde_json::json!({
@@ -431,6 +509,11 @@ impl LoopDelegate for ContainerDelegate {
         content: Option<String>,
         reason_ctx: &mut ReasoningContext,
     ) -> Result<Option<LoopOutcome>, crate::error::Error> {
+        {
+            let mut recovery = self.recovery_state.lock().await;
+            recovery.on_valid_tool_call();
+        }
+
         if let Some(ref text) = content {
             self.post_event(
                 "message",