//! Routine execution engine. //! //! Handles loading routines, checking triggers, enforcing guardrails, //! and executing both lightweight (single LLM call) and full-job routines. //! //! The engine runs two independent loops: //! - A **cron ticker** that polls the DB every N seconds for due cron routines //! - An **event matcher** called synchronously from the agent main loop //! //! Lightweight routines execute inline (single LLM call, no scheduler slot). //! Full-job routines are delegated to the existing `Scheduler`. use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; use chrono::Utc; use regex::Regex; use tokio::sync::{RwLock, mpsc}; use uuid::Uuid; use crate::agent::Scheduler; use crate::agent::routine::{ NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, next_cron_fire, }; use crate::channels::{IncomingMessage, OutgoingResponse}; use crate::config::RoutineConfig; use crate::db::Database; use crate::error::RoutineError; use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; use crate::tools::ApprovalContext; use crate::workspace::Workspace; /// The routine execution engine. pub struct RoutineEngine { config: RoutineConfig, store: Arc, llm: Arc, workspace: Arc, /// Sender for notifications (routed to channel manager). notify_tx: mpsc::Sender, /// Currently running routine count (across all routines). running_count: Arc, /// Compiled event regex cache: routine_id -> compiled regex. event_cache: Arc>>, /// Scheduler for dispatching jobs (FullJob mode). scheduler: Option>, } impl RoutineEngine { pub fn new( config: RoutineConfig, store: Arc, llm: Arc, workspace: Arc, notify_tx: mpsc::Sender, scheduler: Option>, ) -> Self { Self { config, store, llm, workspace, notify_tx, running_count: Arc::new(AtomicUsize::new(0)), event_cache: Arc::new(RwLock::new(Vec::new())), scheduler, } } /// Refresh the in-memory event trigger cache from DB. pub async fn refresh_event_cache(&self) { match self.store.list_event_routines().await { Ok(routines) => { let mut cache = Vec::new(); for routine in routines { if let Trigger::Event { ref pattern, .. } = routine.trigger { match Regex::new(pattern) { Ok(re) => cache.push((routine.id, routine.clone(), re)), Err(e) => { tracing::warn!( routine = %routine.name, "Invalid event regex '{}': {}", pattern, e ); } } } } let count = cache.len(); *self.event_cache.write().await = cache; tracing::debug!("Refreshed event cache: {} routines", count); } Err(e) => { tracing::error!("Failed to refresh event cache: {}", e); } } } /// Check incoming message against event triggers. Returns number of routines fired. /// /// Called synchronously from the main loop after handle_message(). The actual /// execution is spawned async so this returns quickly. pub async fn check_event_triggers(&self, message: &IncomingMessage) -> usize { let cache = self.event_cache.read().await; let mut fired = 0; for (_, routine, re) in cache.iter() { // Channel filter if let Trigger::Event { channel: Some(ch), .. } = &routine.trigger && ch != &message.channel { continue; } // Regex match if !re.is_match(&message.content) { continue; } // Cooldown check if !self.check_cooldown(routine) { tracing::debug!(routine = %routine.name, "Skipped: cooldown active"); continue; } // Concurrent run check if !self.check_concurrent(routine).await { tracing::debug!(routine = %routine.name, "Skipped: max concurrent reached"); continue; } // Global capacity check if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines { tracing::warn!(routine = %routine.name, "Skipped: global max concurrent reached"); continue; } let detail = truncate(&message.content, 200); self.spawn_fire(routine.clone(), "event", Some(detail)); fired += 1; } fired } /// Check all due cron routines and fire them. Called by the cron ticker. pub async fn check_cron_triggers(&self) { let routines = match self.store.list_due_cron_routines().await { Ok(r) => r, Err(e) => { tracing::error!("Failed to load due cron routines: {}", e); return; } }; for routine in routines { if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines { tracing::warn!("Global max concurrent routines reached, skipping remaining"); break; } if !self.check_cooldown(&routine) { continue; } if !self.check_concurrent(&routine).await { continue; } let detail = if let Trigger::Cron { ref schedule } = routine.trigger { Some(schedule.clone()) } else { None }; self.spawn_fire(routine, "cron", detail); } } /// Fire a routine manually (from tool call or CLI). /// /// Bypasses cooldown checks (those only apply to cron/event triggers). /// Still enforces enabled check and concurrent run limit. pub async fn fire_manual( &self, routine_id: Uuid, user_id: Option<&str>, ) -> Result { let routine = self .store .get_routine(routine_id) .await .map_err(|e| RoutineError::Database { reason: e.to_string(), })? .ok_or(RoutineError::NotFound { id: routine_id })?; // Enforce ownership when a user_id is provided (gateway calls). if let Some(uid) = user_id && routine.user_id != uid { return Err(RoutineError::NotAuthorized { id: routine_id }); } if !routine.enabled { return Err(RoutineError::Disabled { name: routine.name.clone(), }); } if !self.check_concurrent(&routine).await { return Err(RoutineError::MaxConcurrent { name: routine.name.clone(), }); } let run_id = Uuid::new_v4(); let run = RoutineRun { id: run_id, routine_id: routine.id, trigger_type: "manual".to_string(), trigger_detail: None, started_at: Utc::now(), completed_at: None, status: RunStatus::Running, result_summary: None, tokens_used: None, job_id: None, created_at: Utc::now(), }; if let Err(e) = self.store.create_routine_run(&run).await { return Err(RoutineError::Database { reason: format!("failed to create run record: {e}"), }); } // Execute inline for manual triggers (caller wants to wait) let engine = EngineContext { store: self.store.clone(), llm: self.llm.clone(), workspace: self.workspace.clone(), notify_tx: self.notify_tx.clone(), running_count: self.running_count.clone(), scheduler: self.scheduler.clone(), }; tokio::spawn(async move { execute_routine(engine, routine, run).await; }); Ok(run_id) } /// Spawn a fire in a background task. fn spawn_fire(&self, routine: Routine, trigger_type: &str, trigger_detail: Option) { let run = RoutineRun { id: Uuid::new_v4(), routine_id: routine.id, trigger_type: trigger_type.to_string(), trigger_detail, started_at: Utc::now(), completed_at: None, status: RunStatus::Running, result_summary: None, tokens_used: None, job_id: None, created_at: Utc::now(), }; let engine = EngineContext { store: self.store.clone(), llm: self.llm.clone(), workspace: self.workspace.clone(), notify_tx: self.notify_tx.clone(), running_count: self.running_count.clone(), scheduler: self.scheduler.clone(), }; // Record the run in DB, then spawn execution let store = self.store.clone(); tokio::spawn(async move { if let Err(e) = store.create_routine_run(&run).await { tracing::error!(routine = %routine.name, "Failed to record run: {}", e); return; } execute_routine(engine, routine, run).await; }); } fn check_cooldown(&self, routine: &Routine) -> bool { if let Some(last_run) = routine.last_run_at { let elapsed = Utc::now().signed_duration_since(last_run); let cooldown = chrono::Duration::from_std(routine.guardrails.cooldown) .unwrap_or(chrono::Duration::seconds(300)); if elapsed < cooldown { return false; } } true } async fn check_concurrent(&self, routine: &Routine) -> bool { match self.store.count_running_routine_runs(routine.id).await { Ok(count) => count < routine.guardrails.max_concurrent as i64, Err(e) => { tracing::error!( routine = %routine.name, "Failed to check concurrent runs: {}", e ); false } } } } /// Shared context passed to the execution function. struct EngineContext { store: Arc, llm: Arc, workspace: Arc, notify_tx: mpsc::Sender, running_count: Arc, scheduler: Option>, } /// Execute a routine run. Handles both lightweight and full_job modes. async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) { // Increment running count (atomic: survives panics in the execution below) ctx.running_count.fetch_add(1, Ordering::Relaxed); let result = match &routine.action { RoutineAction::Lightweight { prompt, context_paths, max_tokens, } => execute_lightweight(&ctx, &routine, prompt, context_paths, *max_tokens).await, RoutineAction::FullJob { title, description, max_iterations, tool_permissions, } => { execute_full_job( &ctx, &routine, &run, title, description, *max_iterations, tool_permissions, ) .await } }; // Decrement running count ctx.running_count.fetch_sub(1, Ordering::Relaxed); // Process result let (status, summary, tokens) = match result { Ok(execution) => execution, Err(e) => { tracing::error!(routine = %routine.name, "Execution failed: {}", e); (RunStatus::Failed, Some(e.to_string()), None) } }; // Complete the run record if let Err(e) = ctx .store .complete_routine_run(run.id, status, summary.as_deref(), tokens) .await { tracing::error!(routine = %routine.name, "Failed to complete run record: {}", e); } // Update routine runtime state let now = Utc::now(); let next_fire = if let Trigger::Cron { ref schedule } = routine.trigger { next_cron_fire(schedule).unwrap_or(None) } else { None }; let new_failures = if status == RunStatus::Failed { routine.consecutive_failures + 1 } else { 0 }; if let Err(e) = ctx .store .update_routine_runtime( routine.id, now, next_fire, routine.run_count + 1, new_failures, &routine.state, ) .await { tracing::error!(routine = %routine.name, "Failed to update runtime state: {}", e); } // Persist routine result to its dedicated conversation thread let thread_id = match ctx .store .get_or_create_routine_conversation(routine.id, &routine.name, &routine.user_id) .await { Ok(conv_id) => { tracing::debug!( routine = %routine.name, routine_id = %routine.id, conversation_id = %conv_id, "Resolved routine conversation thread" ); // Record the run result as a conversation message let msg = match (&summary, status) { (Some(s), _) => format!("[{}] {}: {}", run.trigger_type, status, s), (None, _) => format!("[{}] {}", run.trigger_type, status), }; if let Err(e) = ctx .store .add_conversation_message(conv_id, "assistant", &msg) .await { tracing::error!(routine = %routine.name, "Failed to persist routine message: {}", e); } Some(conv_id.to_string()) } Err(e) => { tracing::error!(routine = %routine.name, "Failed to get routine conversation: {}", e); None } }; // Send notifications based on config send_notification( &ctx.notify_tx, &routine.notify, &routine.name, status, summary.as_deref(), thread_id.as_deref(), ) .await; } /// Sanitize a routine name for use in workspace paths. /// Only keeps alphanumeric, dash, and underscore characters; replaces everything else. fn sanitize_routine_name(name: &str) -> String { name.chars() .map(|c| { if c.is_ascii_alphanumeric() || c == '-' || c == '_' { c } else { '_' } }) .collect() } /// Execute a full-job routine by dispatching to the scheduler. /// /// Fire-and-forget: creates a job via `Scheduler::dispatch_job` (which handles /// creation, metadata, persistence, and scheduling), links the routine run to /// the job, and returns immediately. The job runs independently via the /// existing Worker/Scheduler with full tool access. async fn execute_full_job( ctx: &EngineContext, routine: &Routine, run: &RoutineRun, title: &str, description: &str, max_iterations: u32, tool_permissions: &[String], ) -> Result<(RunStatus, Option, Option), RoutineError> { let scheduler = ctx .scheduler .as_ref() .ok_or_else(|| RoutineError::JobDispatchFailed { reason: "scheduler not available".to_string(), })?; // Set the message tool's default channel/target from the routine's notify config // so the LLM can send results without triggering cross-channel approval. // TODO: This mutates shared global state and can race with concurrent jobs. // Move notify config into JobContext metadata and apply per-job instead. if let Some(channel) = &routine.notify.channel { scheduler .tools() .set_message_tool_context(Some(channel.clone()), Some(routine.notify.user.clone())) .await; } let metadata = serde_json::json!({ "max_iterations": max_iterations }); // Build approval context: UnlessAutoApproved tools are auto-approved for routines; // Always tools require explicit listing in tool_permissions. let approval_context = ApprovalContext::autonomous_with_tools(tool_permissions.iter().cloned()); let job_id = scheduler .dispatch_job_with_context( &routine.user_id, title, description, Some(metadata), approval_context, ) .await .map_err(|e| RoutineError::JobDispatchFailed { reason: format!("failed to dispatch job: {e}"), })?; // Link the routine run to the dispatched job if let Err(e) = ctx.store.link_routine_run_to_job(run.id, job_id).await { tracing::error!( routine = %routine.name, "Failed to link run to job: {}", e ); } tracing::info!( routine = %routine.name, job_id = %job_id, max_iterations = max_iterations, "Dispatched full job for routine" ); let summary = format!( "Dispatched job {job_id} for full execution with tool access (max_iterations: {max_iterations})" ); Ok((RunStatus::Ok, Some(summary), None)) } /// Execute a lightweight routine (single LLM call). async fn execute_lightweight( ctx: &EngineContext, routine: &Routine, prompt: &str, context_paths: &[String], max_tokens: u32, ) -> Result<(RunStatus, Option, Option), RoutineError> { // Load context from workspace let mut context_parts = Vec::new(); for path in context_paths { match ctx.workspace.read(path).await { Ok(doc) => { context_parts.push(format!("## {}\n\n{}", path, doc.content)); } Err(e) => { tracing::debug!( routine = %routine.name, "Failed to read context path {}: {}", path, e ); } } } // Load routine state from workspace (name sanitized to prevent path traversal) let safe_name = sanitize_routine_name(&routine.name); let state_path = format!("routines/{safe_name}/state.md"); let state_content = match ctx.workspace.read(&state_path).await { Ok(doc) => Some(doc.content), Err(_) => None, }; // Build the prompt let mut full_prompt = String::new(); full_prompt.push_str(prompt); if !context_parts.is_empty() { full_prompt.push_str("\n\n---\n\n# Context\n\n"); full_prompt.push_str(&context_parts.join("\n\n")); } if let Some(state) = &state_content { full_prompt.push_str("\n\n---\n\n# Previous State\n\n"); full_prompt.push_str(state); } full_prompt.push_str( "\n\n---\n\nIf nothing needs attention, reply EXACTLY with: ROUTINE_OK\n\ If something needs attention, provide a concise summary.", ); // Get system prompt let system_prompt = match ctx.workspace.system_prompt().await { Ok(p) => p, Err(e) => { tracing::warn!(routine = %routine.name, "Failed to get system prompt: {}", e); String::new() } }; let messages = if system_prompt.is_empty() { vec![ChatMessage::user(&full_prompt)] } else { vec![ ChatMessage::system(&system_prompt), ChatMessage::user(&full_prompt), ] }; // Determine max_tokens from model metadata with fallback let effective_max_tokens = match ctx.llm.model_metadata().await { Ok(meta) => { let from_api = meta.context_length.map(|ctx| ctx / 2).unwrap_or(max_tokens); from_api.max(max_tokens) } Err(_) => max_tokens, }; let request = CompletionRequest::new(messages) .with_max_tokens(effective_max_tokens) .with_temperature(0.3); let response = ctx .llm .complete(request) .await .map_err(|e| RoutineError::LlmFailed { reason: e.to_string(), })?; let content = response.content.trim(); let tokens_used = Some((response.input_tokens + response.output_tokens) as i32); // Empty content guard (same as heartbeat) if content.is_empty() { return if response.finish_reason == FinishReason::Length { Err(RoutineError::TruncatedResponse) } else { Err(RoutineError::EmptyResponse) }; } // Check for the "nothing to do" sentinel if content == "ROUTINE_OK" || content.contains("ROUTINE_OK") { return Ok((RunStatus::Ok, None, tokens_used)); } Ok((RunStatus::Attention, Some(content.to_string()), tokens_used)) } /// Send a notification based on the routine's notify config and run status. async fn send_notification( tx: &mpsc::Sender, notify: &NotifyConfig, routine_name: &str, status: RunStatus, summary: Option<&str>, thread_id: Option<&str>, ) { let should_notify = match status { RunStatus::Ok => notify.on_success, RunStatus::Attention => notify.on_attention, RunStatus::Failed => notify.on_failure, RunStatus::Running => false, }; if !should_notify { return; } let icon = match status { RunStatus::Ok => "✅", RunStatus::Attention => "🔔", RunStatus::Failed => "❌", RunStatus::Running => "⏳", }; let message = match summary { Some(s) => format!("{} *Routine '{}'*: {}\n\n{}", icon, routine_name, status, s), None => format!("{} *Routine '{}'*: {}", icon, routine_name, status), }; let response = OutgoingResponse { content: message, thread_id: thread_id.map(String::from), attachments: Vec::new(), metadata: serde_json::json!({ "source": "routine", "routine_name": routine_name, "status": status.to_string(), "notify_user": notify.user, "notify_channel": notify.channel, }), }; if let Err(e) = tx.send(response).await { tracing::error!(routine = %routine_name, "Failed to send notification: {}", e); } } /// Spawn the cron ticker background task. pub fn spawn_cron_ticker( engine: Arc, interval: Duration, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); // Skip immediate first tick ticker.tick().await; loop { ticker.tick().await; engine.check_cron_triggers().await; } }) } fn truncate(s: &str, max: usize) -> String { if s.len() <= max { s.to_string() } else { let end = crate::util::floor_char_boundary(s, max); format!("{}...", &s[..end]) } } #[cfg(test)] mod tests { use crate::agent::routine::{NotifyConfig, RunStatus}; #[test] fn test_notification_gating() { let config = NotifyConfig { on_success: false, on_failure: true, on_attention: true, ..Default::default() }; // on_success = false means Ok status should not notify assert!(!config.on_success); assert!(config.on_failure); assert!(config.on_attention); } #[test] fn test_run_status_icons() { // Just verify the mapping doesn't panic for status in [ RunStatus::Ok, RunStatus::Attention, RunStatus::Failed, RunStatus::Running, ] { let _ = status.to_string(); } } }