diff --git a/src/agent/routine.rs b/src/agent/routine.rs index 26e769da..85a4a0f0 100644 --- a/src/agent/routine.rs +++ b/src/agent/routine.rs @@ -24,6 +24,7 @@ use std::time::Duration; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; use uuid::Uuid; use crate::error::RoutineError; @@ -52,6 +53,55 @@ pub struct Routine { pub updated_at: DateTime, } +const ROUTINE_VERIFICATION_STATE_KEY: &str = "_verification"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RoutineVerificationRecord { + current_fingerprint: String, + #[serde(default)] + verified_fingerprint: Option, + #[serde(default)] + last_verified_at: Option>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoutineVerificationStatus { + Verified, + Unverified, +} + +impl RoutineVerificationStatus { + pub fn as_str(self) -> &'static str { + match self { + RoutineVerificationStatus::Verified => "verified", + RoutineVerificationStatus::Unverified => "unverified", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoutineDisplayStatus { + Disabled, + Running, + Unverified, + Failing, + Attention, + Active, +} + +impl RoutineDisplayStatus { + pub fn as_str(self) -> &'static str { + match self { + RoutineDisplayStatus::Disabled => "disabled", + RoutineDisplayStatus::Running => "running", + RoutineDisplayStatus::Unverified => "unverified", + RoutineDisplayStatus::Failing => "failing", + RoutineDisplayStatus::Attention => "attention", + RoutineDisplayStatus::Active => "active", + } + } +} + /// When a routine should fire. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] @@ -517,6 +567,112 @@ pub fn content_hash(content: &str) -> u64 { hasher.finish() } +fn routine_state_as_object(state: &Value) -> Map { + state.as_object().cloned().unwrap_or_default() +} + +fn routine_verification_record(state: &Value) -> Option { + state + .as_object() + .and_then(|obj| obj.get(ROUTINE_VERIFICATION_STATE_KEY)) + .cloned() + .and_then(|value| serde_json::from_value(value).ok()) +} + +fn write_routine_verification_record( + state: &Value, + record: RoutineVerificationRecord, +) -> serde_json::Value { + let mut obj = routine_state_as_object(state); + if let Ok(value) = serde_json::to_value(record) { + obj.insert(ROUTINE_VERIFICATION_STATE_KEY.to_string(), value); + } + Value::Object(obj) +} + +pub fn routine_verification_fingerprint(routine: &Routine) -> String { + serde_json::json!({ + "trigger_type": routine.trigger.type_tag(), + "trigger": routine.trigger.to_config_json(), + "action_type": routine.action.type_tag(), + "action": routine.action.to_config_json(), + "guardrails": { + "cooldown_secs": routine.guardrails.cooldown.as_secs(), + "max_concurrent": routine.guardrails.max_concurrent, + "dedup_window_secs": routine.guardrails.dedup_window.map(|d| d.as_secs()), + }, + }) + .to_string() +} + +pub fn reset_routine_verification_state( + state: &Value, + current_fingerprint: String, +) -> serde_json::Value { + let mut record = routine_verification_record(state).unwrap_or(RoutineVerificationRecord { + current_fingerprint: current_fingerprint.clone(), + verified_fingerprint: None, + last_verified_at: None, + }); + record.current_fingerprint = current_fingerprint; + write_routine_verification_record(state, record) +} + +pub fn apply_routine_verification_result( + state: &Value, + current_fingerprint: String, + status: RunStatus, + now: DateTime, +) -> serde_json::Value { + let mut record = routine_verification_record(state).unwrap_or(RoutineVerificationRecord { + current_fingerprint: current_fingerprint.clone(), + verified_fingerprint: None, + last_verified_at: None, + }); + record.current_fingerprint = current_fingerprint.clone(); + if status == RunStatus::Ok { + record.verified_fingerprint = Some(current_fingerprint); + record.last_verified_at = Some(now); + } + write_routine_verification_record(state, record) +} + +pub fn routine_verification_status(routine: &Routine) -> RoutineVerificationStatus { + let fingerprint = routine_verification_fingerprint(routine); + let verified = + routine_verification_record(&routine.state).map_or(routine.run_count > 0, |record| { + record.current_fingerprint == fingerprint + && record.verified_fingerprint.as_deref() == Some(fingerprint.as_str()) + }); + if verified { + RoutineVerificationStatus::Verified + } else { + RoutineVerificationStatus::Unverified + } +} + +pub fn routine_display_status( + routine: &Routine, + last_run_status: Option, +) -> RoutineDisplayStatus { + if !routine.enabled { + return RoutineDisplayStatus::Disabled; + } + if last_run_status == Some(RunStatus::Running) { + return RoutineDisplayStatus::Running; + } + if routine_verification_status(routine) == RoutineVerificationStatus::Unverified { + return RoutineDisplayStatus::Unverified; + } + if routine.consecutive_failures > 0 { + return RoutineDisplayStatus::Failing; + } + if last_run_status == Some(RunStatus::Attention) { + return RoutineDisplayStatus::Attention; + } + RoutineDisplayStatus::Active +} + /// Normalize a cron expression to the 7-field format expected by the `cron` crate. /// /// The `cron` crate requires: `sec min hour day-of-month month day-of-week year`. @@ -725,9 +881,14 @@ pub fn describe_cron(schedule: &str, timezone: Option<&str>) -> String { #[cfg(test)] mod tests { use crate::agent::routine::{ - MAX_TOOL_ROUNDS_LIMIT, RoutineAction, RoutineGuardrails, RunStatus, Trigger, content_hash, - describe_cron, next_cron_fire, normalize_cron_expression, + MAX_TOOL_ROUNDS_LIMIT, NotifyConfig, Routine, RoutineAction, RoutineGuardrails, + RoutineVerificationStatus, RunStatus, Trigger, apply_routine_verification_result, + content_hash, describe_cron, next_cron_fire, normalize_cron_expression, + reset_routine_verification_state, routine_verification_fingerprint, + routine_verification_status, }; + use chrono::Utc; + use uuid::Uuid; #[test] fn test_trigger_roundtrip() { @@ -1117,4 +1278,155 @@ mod tests { _ => panic!("expected Lightweight"), } } + + fn make_verification_test_routine() -> Routine { + Routine { + id: Uuid::new_v4(), + name: "verify-me".to_string(), + description: "verification test".to_string(), + user_id: "test-user".to_string(), + enabled: true, + trigger: Trigger::Manual, + action: RoutineAction::Lightweight { + prompt: "Check routine output".to_string(), + context_paths: Vec::new(), + max_tokens: 1024, + use_tools: false, + max_tool_rounds: 1, + }, + guardrails: RoutineGuardrails::default(), + notify: NotifyConfig::default(), + last_run_at: None, + next_fire_at: None, + run_count: 0, + consecutive_failures: 0, + state: serde_json::json!({}), + created_at: Utc::now(), + updated_at: Utc::now(), + } + } + + #[test] + fn test_reset_verification_state_marks_new_routine_unverified() { + let mut routine = make_verification_test_routine(); + routine.state = reset_routine_verification_state( + &routine.state, + routine_verification_fingerprint(&routine), + ); + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Unverified + ); + } + + #[test] + fn test_successful_run_verifies_current_fingerprint() { + let mut routine = make_verification_test_routine(); + let fingerprint = routine_verification_fingerprint(&routine); + routine.state = reset_routine_verification_state(&routine.state, fingerprint.clone()); + routine.state = apply_routine_verification_result( + &routine.state, + fingerprint, + RunStatus::Ok, + Utc::now(), + ); + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Verified + ); + } + + #[test] + fn test_behavior_change_resets_prior_verification() { + let mut routine = make_verification_test_routine(); + let original_fingerprint = routine_verification_fingerprint(&routine); + routine.state = + reset_routine_verification_state(&routine.state, original_fingerprint.clone()); + routine.state = apply_routine_verification_result( + &routine.state, + original_fingerprint, + RunStatus::Ok, + Utc::now(), + ); + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Verified + ); + + if let RoutineAction::Lightweight { prompt, .. } = &mut routine.action { + *prompt = "Updated prompt".to_string(); + } + routine.state = reset_routine_verification_state( + &routine.state, + routine_verification_fingerprint(&routine), + ); + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Unverified + ); + } + + #[test] + fn test_failed_unverified_run_stays_unverified() { + let mut routine = make_verification_test_routine(); + let fingerprint = routine_verification_fingerprint(&routine); + routine.state = reset_routine_verification_state(&routine.state, fingerprint.clone()); + routine.state = apply_routine_verification_result( + &routine.state, + fingerprint, + RunStatus::Failed, + Utc::now(), + ); + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Unverified + ); + } + + #[test] + fn test_schedule_change_resets_verification() { + let mut routine = make_verification_test_routine(); + routine.trigger = Trigger::Cron { + schedule: "0 0 9 * * MON-FRI *".to_string(), + timezone: Some("UTC".to_string()), + }; + let original_fingerprint = routine_verification_fingerprint(&routine); + routine.state = + reset_routine_verification_state(&routine.state, original_fingerprint.clone()); + routine.state = apply_routine_verification_result( + &routine.state, + original_fingerprint, + RunStatus::Ok, + Utc::now(), + ); + + routine.trigger = Trigger::Cron { + schedule: "0 0 10 * * MON-FRI *".to_string(), + timezone: Some("UTC".to_string()), + }; + routine.state = reset_routine_verification_state( + &routine.state, + routine_verification_fingerprint(&routine), + ); + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Unverified + ); + } + + #[test] + fn test_legacy_routine_with_runs_is_treated_as_verified_without_metadata() { + let mut routine = make_verification_test_routine(); + routine.run_count = 3; + + assert_eq!( + routine_verification_status(&routine), + RoutineVerificationStatus::Verified + ); + } } diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index 64c3b94c..63e446fe 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -23,7 +23,8 @@ use uuid::Uuid; use crate::agent::Scheduler; use crate::agent::routine::{ - NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, next_cron_fire, + NotifyConfig, Routine, RoutineAction, RoutineRun, RunStatus, Trigger, + apply_routine_verification_result, next_cron_fire, routine_verification_fingerprint, }; use crate::channels::{IncomingMessage, OutgoingResponse}; use crate::config::RoutineConfig; @@ -621,7 +622,7 @@ impl RoutineEngine { ); // Load the routine to update consecutive_failures and send notification - let routine = match self.store.get_routine(run.routine_id).await { + let mut routine = match self.store.get_routine(run.routine_id).await { Ok(Some(r)) => r, Ok(None) => { tracing::warn!( @@ -649,6 +650,12 @@ impl RoutineEngine { }; let now = Utc::now(); + routine.state = apply_routine_verification_result( + &routine.state, + routine_verification_fingerprint(&routine), + status, + now, + ); let next_fire = if let Trigger::Cron { ref schedule, ref timezone, @@ -1085,7 +1092,7 @@ struct EngineContext { } /// Execute a routine run. Handles both lightweight and full_job modes. -async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) { +async fn execute_routine(ctx: EngineContext, mut routine: Routine, run: RoutineRun) { // Increment running count (atomic: survives panics in the execution below) ctx.running_count.fetch_add(1, Ordering::Relaxed); @@ -1143,8 +1150,15 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) tracing::error!(routine = %routine.name, "Failed to complete run record: {}", e); } - // Update routine runtime state let now = Utc::now(); + routine.state = apply_routine_verification_result( + &routine.state, + routine_verification_fingerprint(&routine), + status, + now, + ); + + // Update routine runtime state let next_fire = if let Trigger::Cron { ref schedule, ref timezone, diff --git a/src/channels/web/CLAUDE.md b/src/channels/web/CLAUDE.md index 8db9a6b7..8c610426 100644 --- a/src/channels/web/CLAUDE.md +++ b/src/channels/web/CLAUDE.md @@ -84,7 +84,7 @@ Browser-facing HTTP API and SSE/WebSocket real-time streaming. Axum-based, singl | Method | Path | Description | |--------|------|-------------| | GET | `/api/routines` | List routines | -| GET | `/api/routines/summary` | Aggregated stats (total/enabled/disabled/failing/runs_today) | +| GET | `/api/routines/summary` | Aggregated stats (total/enabled/disabled/unverified/failing/runs_today) | | GET | `/api/routines/{id}` | Routine detail with recent run history | | POST | `/api/routines/{id}/trigger` | Manually trigger a routine | | POST | `/api/routines/{id}/toggle` | Enable/disable a routine | diff --git a/src/channels/web/handlers/routines.rs b/src/channels/web/handlers/routines.rs index fc56b187..b139af21 100644 --- a/src/channels/web/handlers/routines.rs +++ b/src/channels/web/handlers/routines.rs @@ -10,7 +10,9 @@ use axum::{ use serde::Deserialize; use uuid::Uuid; -use crate::agent::routine::{Trigger, next_cron_fire}; +use crate::agent::routine::{ + RoutineDisplayStatus, Trigger, next_cron_fire, routine_display_status, +}; use crate::channels::web::auth::AuthenticatedUser; use crate::channels::web::server::GatewayState; use crate::channels::web::types::*; @@ -30,7 +32,18 @@ pub async fn routines_list_handler( .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - let items: Vec = routines.iter().map(RoutineInfo::from_routine).collect(); + let routine_ids: Vec = routines.iter().map(|routine| routine.id).collect(); + let last_run_statuses = store + .batch_get_last_run_status(&routine_ids) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + let items: Vec = routines + .iter() + .map(|routine| { + RoutineInfo::from_routine(routine, last_run_statuses.get(&routine.id).copied()) + }) + .collect(); Ok(Json(RoutineListResponse { routines: items })) } @@ -49,13 +62,31 @@ pub async fn routines_summary_handler( .await .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + let routine_ids: Vec = routines.iter().map(|routine| routine.id).collect(); + let last_run_statuses = store + .batch_get_last_run_status(&routine_ids) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + let total = routines.len() as u64; - let enabled = routines.iter().filter(|r| r.enabled).count() as u64; - let disabled = total - enabled; - let failing = routines - .iter() - .filter(|r| r.consecutive_failures > 0) - .count() as u64; + let mut enabled = 0u64; + let mut disabled = 0u64; + let mut unverified = 0u64; + let mut failing = 0u64; + + for routine in &routines { + if routine.enabled { + enabled += 1; + } else { + disabled += 1; + } + + match routine_display_status(routine, last_run_statuses.get(&routine.id).copied()) { + RoutineDisplayStatus::Unverified => unverified += 1, + RoutineDisplayStatus::Failing => failing += 1, + _ => {} + } + } let today_start = chrono::Utc::now() .date_naive() @@ -74,6 +105,7 @@ pub async fn routines_summary_handler( total, enabled, disabled, + unverified, failing, runs_today, })) @@ -120,7 +152,7 @@ pub async fn routines_detail_handler( job_id: run.job_id, }) .collect(); - let routine_info = RoutineInfo::from_routine(&routine); + let routine_info = RoutineInfo::from_routine(&routine, runs.first().map(|run| run.status)); Ok(Json(RoutineDetailResponse { id: routine.id, @@ -138,6 +170,8 @@ pub async fn routines_detail_handler( next_fire_at: routine.next_fire_at.map(|dt| dt.to_rfc3339()), run_count: routine.run_count, consecutive_failures: routine.consecutive_failures, + status: routine_info.status.clone(), + verification_status: routine_info.verification_status.clone(), created_at: routine.created_at.to_rfc3339(), recent_runs, })) diff --git a/src/channels/web/static/app.js b/src/channels/web/static/app.js index 6b366482..41e7ba25 100644 --- a/src/channels/web/static/app.js +++ b/src/channels/web/static/app.js @@ -4141,6 +4141,7 @@ function renderRoutinesSummary(s) { + summaryCard(I18n.t('routines.summary.total'), s.total, '') + summaryCard(I18n.t('routines.summary.enabled'), s.enabled, 'active') + summaryCard(I18n.t('routines.summary.disabled'), s.disabled, '') + + summaryCard(I18n.t('routines.summary.unverified'), s.unverified, 'pending') + summaryCard(I18n.t('routines.summary.failing'), s.failing, 'failed') + summaryCard(I18n.t('routines.summary.runsToday'), s.runs_today, 'completed'); } @@ -4159,6 +4160,8 @@ function renderRoutinesList(routines) { tbody.innerHTML = routines.map((r) => { const statusClass = r.status === 'active' ? 'completed' : r.status === 'failing' ? 'failed' + : r.status === 'attention' ? 'stuck' + : r.status === 'running' ? 'in_progress' : 'pending'; const toggleLabel = r.enabled ? 'Disable' : 'Enable'; @@ -4166,6 +4169,7 @@ function renderRoutinesList(routines) { const triggerTitle = (r.trigger_type === 'cron' && r.trigger_raw) ? ' title="' + escapeHtml(r.trigger_raw) + '"' : ''; + const runLabel = r.status === 'unverified' ? 'Verify now' : 'Run'; return '' + '' + escapeHtml(r.name) + '' @@ -4177,7 +4181,7 @@ function renderRoutinesList(routines) { + '' + escapeHtml(r.status) + '' + '' + ' ' - + ' ' + + ' ' + '' + '' + ''; @@ -4206,12 +4210,12 @@ function renderRoutineDetail(routine) { const detail = document.getElementById('routine-detail'); detail.style.display = 'block'; - const statusClass = !routine.enabled ? 'pending' - : routine.consecutive_failures > 0 ? 'failed' - : 'completed'; - const statusLabel = !routine.enabled ? 'disabled' - : routine.consecutive_failures > 0 ? 'failing' - : 'active'; + const statusClass = routine.status === 'active' ? 'completed' + : routine.status === 'failing' ? 'failed' + : routine.status === 'attention' ? 'stuck' + : routine.status === 'running' ? 'in_progress' + : 'pending'; + const statusLabel = routine.status || 'active'; let html = '
' + '' @@ -4236,6 +4240,20 @@ function renderRoutineDetail(routine) { + '
' + escapeHtml(routine.description) + '
'; } + if (routine.status === 'unverified') { + let verificationCopy = 'Created or updated, but not yet verified with a successful run.'; + if (routine.recent_runs && routine.recent_runs.length > 0) { + const latestRun = routine.recent_runs[0]; + if (latestRun.status === 'failed') { + verificationCopy = 'The latest verification attempt failed. Review the run details and verify again after fixing it.'; + } else if (latestRun.status === 'attention') { + verificationCopy = 'The latest verification attempt needs attention. Review the run details and verify again when ready.'; + } + } + html += '

Verification

' + + '
' + escapeHtml(verificationCopy) + '
'; + } + // Trigger config if (routine.trigger_type === 'cron') { const summary = routine.trigger_summary || 'cron'; diff --git a/src/channels/web/static/i18n/en.js b/src/channels/web/static/i18n/en.js index 761767fe..f0703c49 100644 --- a/src/channels/web/static/i18n/en.js +++ b/src/channels/web/static/i18n/en.js @@ -207,6 +207,7 @@ I18n.register('en', { 'routines.summary.total': 'Total', 'routines.summary.enabled': 'Enabled', 'routines.summary.disabled': 'Disabled', + 'routines.summary.unverified': 'Unverified', 'routines.summary.failing': 'Failing', 'routines.summary.runsToday': 'Runs Today', diff --git a/src/channels/web/static/i18n/zh-CN.js b/src/channels/web/static/i18n/zh-CN.js index 0fb1568a..0d1b342b 100644 --- a/src/channels/web/static/i18n/zh-CN.js +++ b/src/channels/web/static/i18n/zh-CN.js @@ -207,6 +207,7 @@ I18n.register('zh-CN', { 'routines.summary.total': '总计', 'routines.summary.enabled': '已启用', 'routines.summary.disabled': '已禁用', + 'routines.summary.unverified': '未验证', 'routines.summary.failing': '失败', 'routines.summary.runsToday': '今日运行', diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index 8698c030..fd2144b9 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -662,11 +662,15 @@ pub struct RoutineInfo { pub run_count: u64, pub consecutive_failures: u32, pub status: String, + pub verification_status: String, } impl RoutineInfo { /// Convert a `Routine` to the trimmed `RoutineInfo` for list display. - pub fn from_routine(r: &crate::agent::routine::Routine) -> Self { + pub fn from_routine( + r: &crate::agent::routine::Routine, + last_run_status: Option, + ) -> Self { let (trigger_type, trigger_raw, trigger_summary) = match &r.trigger { crate::agent::routine::Trigger::Cron { schedule, timezone } => ( "cron".to_string(), @@ -710,13 +714,8 @@ impl RoutineInfo { crate::agent::routine::RoutineAction::FullJob { .. } => "full_job", }; - let status = if !r.enabled { - "disabled" - } else if r.consecutive_failures > 0 { - "failing" - } else { - "active" - }; + let status = crate::agent::routine::routine_display_status(r, last_run_status).as_str(); + let verification_status = crate::agent::routine::routine_verification_status(r).as_str(); RoutineInfo { id: r.id, @@ -732,6 +731,7 @@ impl RoutineInfo { run_count: r.run_count, consecutive_failures: r.consecutive_failures, status: status.to_string(), + verification_status: verification_status.to_string(), } } } @@ -746,6 +746,7 @@ pub struct RoutineSummaryResponse { pub total: u64, pub enabled: u64, pub disabled: u64, + pub unverified: u64, pub failing: u64, pub runs_today: u64, } @@ -767,6 +768,8 @@ pub struct RoutineDetailResponse { pub next_fire_at: Option, pub run_count: u64, pub consecutive_failures: u32, + pub status: String, + pub verification_status: String, pub created_at: String, pub recent_runs: Vec, } @@ -823,6 +826,7 @@ pub struct HealthResponse { #[cfg(test)] mod tests { use super::*; + use chrono::Utc; // ---- WsClientMessage deserialization tests ---- @@ -1173,4 +1177,115 @@ mod tests { let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert!(parsed.get("channel").is_none()); } + + fn make_routine_for_status_tests() -> crate::agent::routine::Routine { + crate::agent::routine::Routine { + id: Uuid::new_v4(), + name: "status-check".to_string(), + description: "routine status test".to_string(), + user_id: "test-user".to_string(), + enabled: true, + trigger: crate::agent::routine::Trigger::Manual, + action: crate::agent::routine::RoutineAction::Lightweight { + prompt: "Check status".to_string(), + context_paths: Vec::new(), + max_tokens: 256, + use_tools: false, + max_tool_rounds: 1, + }, + guardrails: crate::agent::routine::RoutineGuardrails::default(), + notify: crate::agent::routine::NotifyConfig::default(), + last_run_at: None, + next_fire_at: None, + run_count: 0, + consecutive_failures: 0, + state: serde_json::json!({}), + created_at: Utc::now(), + updated_at: Utc::now(), + } + } + + #[test] + fn test_routine_info_marks_new_routine_unverified() { + let mut routine = make_routine_for_status_tests(); + routine.state = crate::agent::routine::reset_routine_verification_state( + &routine.state, + crate::agent::routine::routine_verification_fingerprint(&routine), + ); + + let info = RoutineInfo::from_routine(&routine, None); + + assert_eq!(info.status, "unverified"); + assert_eq!(info.verification_status, "unverified"); + } + + #[test] + fn test_routine_info_preserves_verified_state_for_description_only_changes() { + let mut routine = make_routine_for_status_tests(); + let fingerprint = crate::agent::routine::routine_verification_fingerprint(&routine); + routine.state = crate::agent::routine::reset_routine_verification_state( + &routine.state, + fingerprint.clone(), + ); + routine.state = crate::agent::routine::apply_routine_verification_result( + &routine.state, + fingerprint, + crate::agent::routine::RunStatus::Ok, + Utc::now(), + ); + routine.description = "Updated description".to_string(); + + let info = RoutineInfo::from_routine(&routine, Some(crate::agent::routine::RunStatus::Ok)); + + assert_eq!(info.status, "active"); + assert_eq!(info.verification_status, "verified"); + } + + #[test] + fn test_routine_info_surfaces_running_before_unverified() { + let mut routine = make_routine_for_status_tests(); + routine.state = crate::agent::routine::reset_routine_verification_state( + &routine.state, + crate::agent::routine::routine_verification_fingerprint(&routine), + ); + + let info = + RoutineInfo::from_routine(&routine, Some(crate::agent::routine::RunStatus::Running)); + + assert_eq!(info.status, "running"); + assert_eq!(info.verification_status, "unverified"); + } + + #[test] + fn test_routine_info_keeps_verified_state_when_disabled() { + let mut routine = make_routine_for_status_tests(); + let fingerprint = crate::agent::routine::routine_verification_fingerprint(&routine); + routine.state = crate::agent::routine::reset_routine_verification_state( + &routine.state, + fingerprint.clone(), + ); + routine.state = crate::agent::routine::apply_routine_verification_result( + &routine.state, + fingerprint, + crate::agent::routine::RunStatus::Ok, + Utc::now(), + ); + routine.enabled = false; + + let info = RoutineInfo::from_routine(&routine, Some(crate::agent::routine::RunStatus::Ok)); + + assert_eq!(info.status, "disabled"); + assert_eq!(info.verification_status, "verified"); + } + + #[test] + fn test_routine_info_treats_legacy_run_history_as_verified() { + let mut routine = make_routine_for_status_tests(); + routine.run_count = 2; + + let info = RoutineInfo::from_routine(&routine, Some(crate::agent::routine::RunStatus::Ok)); + + assert_eq!(info.status, "active"); + assert_eq!(info.verification_status, "verified"); + } } diff --git a/src/tools/builtin/routine.rs b/src/tools/builtin/routine.rs index bbc24139..71d8a368 100644 --- a/src/tools/builtin/routine.rs +++ b/src/tools/builtin/routine.rs @@ -20,7 +20,8 @@ use uuid::Uuid; use crate::agent::routine::{ NotifyConfig, Routine, RoutineAction, RoutineGuardrails, Trigger, next_cron_fire, - normalize_cron_expression, + normalize_cron_expression, reset_routine_verification_state, routine_display_status, + routine_verification_fingerprint, routine_verification_status, }; use crate::agent::routine_engine::RoutineEngine; use crate::context::JobContext; @@ -414,12 +415,30 @@ fn routine_create_tool_summary() -> ToolDiscoverySummary { "Set execution.use_tools=false to keep a new lightweight routine text-only.".into(), "Omitting delivery.user falls back to the owner's last-seen notification target.".into(), "advanced.cooldown_secs defaults to 300.".into(), + "Creating a routine only saves the configuration. It does not prove the routine can execute successfully.".into(), + "After routine_create, tell the user the routine is unverified and offer to test it now unless they asked not to.".into(), "Legacy flat aliases are still accepted for compatibility, but grouped fields are preferred.".into(), ], examples: routine_create_examples(), } } +fn verification_result_payload(routine: &Routine, verification_reset: bool) -> Value { + let verification_status = routine_verification_status(routine); + serde_json::json!({ + "verification_status": verification_status.as_str(), + "verification_reset": verification_reset, + "verification_hint": if verification_reset { + "The routine configuration changed and should be re-tested before being treated as reliable." + } else if verification_status == crate::agent::routine::RoutineVerificationStatus::Verified { + "The current routine configuration has already been verified with a successful run." + } else { + "The routine has been saved, but it has not been verified yet. Offer to test it now." + }, + "verification_fingerprint": routine_verification_fingerprint(routine), + }) +} + fn routine_create_schema(include_compatibility_aliases: bool) -> Value { let mut schema = serde_json::json!({ "type": "object", @@ -1063,7 +1082,8 @@ impl Tool for RoutineCreateTool { fn description(&self) -> &str { "Create a new routine (scheduled or event-driven task). \ Supports cron schedules, event pattern matching, system events, and manual triggers. \ - Use this when the user wants something to happen periodically or reactively." + Use this when the user wants something to happen periodically or reactively. \ + Creation saves the routine, but does not verify that it will execute successfully." } fn requires_approval(&self, params: &serde_json::Value) -> ApprovalRequirement { @@ -1108,7 +1128,7 @@ impl Tool for RoutineCreateTool { None }; - let routine = Routine { + let mut routine = Routine { id: Uuid::new_v4(), name: normalized.name.clone(), description: normalized.description.clone(), @@ -1134,6 +1154,10 @@ impl Tool for RoutineCreateTool { created_at: Utc::now(), updated_at: Utc::now(), }; + routine.state = reset_routine_verification_state( + &routine.state, + routine_verification_fingerprint(&routine), + ); self.store .create_routine(&routine) @@ -1148,12 +1172,14 @@ impl Tool for RoutineCreateTool { self.engine.refresh_event_cache().await; } + let verification = verification_result_payload(&routine, false); let result = serde_json::json!({ "id": routine.id.to_string(), - "name": routine.name, + "name": routine.name.clone(), "trigger_type": routine.trigger.type_tag(), "next_fire_at": routine.next_fire_at.map(|t| t.to_rfc3339()), "status": "created", + "verification": verification, }); Ok(ToolOutput::success(result, start.elapsed())) @@ -1206,10 +1232,19 @@ impl Tool for RoutineListTool { .list_routines(&ctx.user_id) .await .map_err(|e| ToolError::ExecutionFailed(format!("failed to list routines: {e}")))?; + let routine_ids: Vec = routines.iter().map(|routine| routine.id).collect(); + let last_run_statuses = self + .store + .batch_get_last_run_status(&routine_ids) + .await + .map_err(|e| { + ToolError::ExecutionFailed(format!("failed to read routine statuses: {e}")) + })?; let list: Vec = routines .iter() .map(|r| { + let status = routine_display_status(r, last_run_statuses.get(&r.id).copied()); serde_json::json!({ "id": r.id.to_string(), "name": r.name, @@ -1221,6 +1256,8 @@ impl Tool for RoutineListTool { "next_fire_at": r.next_fire_at.map(|t| t.to_rfc3339()), "run_count": r.run_count, "consecutive_failures": r.consecutive_failures, + "status": status.as_str(), + "verification_status": routine_verification_status(r).as_str(), }) }) .collect(); @@ -1259,7 +1296,8 @@ impl Tool for RoutineUpdateTool { fn description(&self) -> &str { "Update an existing routine. Can change prompt, description, enabled state, cron schedule/timezone, \ - Pass the routine name and only the fields you want to change. This does not convert trigger types." + Pass the routine name and only the fields you want to change. This does not convert trigger types. \ + Behavior-changing edits should leave the routine marked unverified until it is tested again." } fn parameters_schema(&self) -> serde_json::Value { @@ -1282,6 +1320,9 @@ impl Tool for RoutineUpdateTool { .map_err(|e| ToolError::ExecutionFailed(format!("DB error: {e}")))? .ok_or_else(|| ToolError::ExecutionFailed(format!("routine '{}' not found", name)))?; + let original_fingerprint = routine_verification_fingerprint(&routine); + let mut verification_reset = false; + // Apply updates if let Some(enabled) = params.get("enabled").and_then(|v| v.as_bool()) { routine.enabled = enabled; @@ -1293,8 +1334,18 @@ impl Tool for RoutineUpdateTool { if let Some(prompt) = params.get("prompt").and_then(|v| v.as_str()) { match &mut routine.action { - RoutineAction::Lightweight { prompt: p, .. } => *p = prompt.to_string(), - RoutineAction::FullJob { description: d, .. } => *d = prompt.to_string(), + RoutineAction::Lightweight { prompt: p, .. } => { + if p != prompt { + verification_reset = true; + *p = prompt.to_string(); + } + } + RoutineAction::FullJob { description: d, .. } => { + if d != prompt { + verification_reset = true; + *d = prompt.to_string(); + } + } } } @@ -1325,12 +1376,16 @@ impl Tool for RoutineUpdateTool { if let Some((old_schedule, old_tz)) = existing_cron { let effective_schedule = new_schedule.as_deref().unwrap_or(&old_schedule); - let effective_tz = new_timezone.or(old_tz); + let effective_tz = new_timezone.clone().or(old_tz.clone()); // Validate next_cron_fire(effective_schedule, effective_tz.as_deref()).map_err(|e| { ToolError::InvalidParameters(format!("invalid cron schedule: {e}")) })?; + if effective_schedule != old_schedule || effective_tz != old_tz { + verification_reset = true; + } + routine.trigger = Trigger::Cron { schedule: effective_schedule.to_string(), timezone: effective_tz.clone(), @@ -1344,6 +1399,12 @@ impl Tool for RoutineUpdateTool { } } + let updated_fingerprint = routine_verification_fingerprint(&routine); + if updated_fingerprint != original_fingerprint { + verification_reset = true; + routine.state = reset_routine_verification_state(&routine.state, updated_fingerprint); + } + self.store .update_routine(&routine) .await @@ -1352,12 +1413,14 @@ impl Tool for RoutineUpdateTool { // Refresh event cache in case trigger changed self.engine.refresh_event_cache().await; + let verification = verification_result_payload(&routine, verification_reset); let result = serde_json::json!({ - "name": routine.name, + "name": routine.name.clone(), "enabled": routine.enabled, "trigger_type": routine.trigger.type_tag(), "next_fire_at": routine.next_fire_at.map(|t| t.to_rfc3339()), "status": "updated", + "verification": verification, }); Ok(ToolOutput::success(result, start.elapsed())) diff --git a/tests/fixtures/llm_traces/advanced/routine_event_any_channel.json b/tests/fixtures/llm_traces/advanced/routine_event_any_channel.json index 61edf1e2..d83d9fbf 100644 --- a/tests/fixtures/llm_traces/advanced/routine_event_any_channel.json +++ b/tests/fixtures/llm_traces/advanced/routine_event_any_channel.json @@ -29,7 +29,7 @@ { "response": { "type": "text", - "content": "Created the any-channel-bug-watcher routine for bug messages.", + "content": "Created the any-channel-bug-watcher routine for bug messages, but it is not verified yet. It should stay unverified until it has a successful run.", "input_tokens": 170, "output_tokens": 18 } diff --git a/tests/fixtures/llm_traces/advanced/routine_event_telegram.json b/tests/fixtures/llm_traces/advanced/routine_event_telegram.json index b17c5382..39ebc805 100644 --- a/tests/fixtures/llm_traces/advanced/routine_event_telegram.json +++ b/tests/fixtures/llm_traces/advanced/routine_event_telegram.json @@ -30,7 +30,7 @@ { "response": { "type": "text", - "content": "Created the telegram-bug-watcher routine for Telegram bug messages.", + "content": "Created the telegram-bug-watcher routine for Telegram bug messages, but it is not verified yet. I can test it the next time you want to fire it.", "input_tokens": 180, "output_tokens": 20 } diff --git a/tests/fixtures/llm_traces/advanced/routine_news_digest.json b/tests/fixtures/llm_traces/advanced/routine_news_digest.json index 4c98b49f..5b7e1dfd 100644 --- a/tests/fixtures/llm_traces/advanced/routine_news_digest.json +++ b/tests/fixtures/llm_traces/advanced/routine_news_digest.json @@ -37,7 +37,7 @@ { "response": { "type": "text", - "content": "Created the **morning-tech-news** routine with manual trigger and full_job mode. The `message` and `http` tools are pre-authorized.", + "content": "Created the **morning-tech-news** routine with manual trigger and full_job mode. The `message` and `http` tools are available, but the routine is not verified yet.", "input_tokens": 200, "output_tokens": 50 } diff --git a/tests/fixtures/llm_traces/tools/routine_create_grouped.json b/tests/fixtures/llm_traces/tools/routine_create_grouped.json index ae4b6eb9..417c809d 100644 --- a/tests/fixtures/llm_traces/tools/routine_create_grouped.json +++ b/tests/fixtures/llm_traces/tools/routine_create_grouped.json @@ -57,7 +57,7 @@ { "response": { "type": "text", - "content": "Created the weekday-digest routine with a grouped cron request and listed the active routines.", + "content": "Created the weekday-digest routine with a grouped cron request and listed the routines. It is not verified yet, so it should stay unverified until it has a successful run.", "input_tokens": 250, "output_tokens": 24 } diff --git a/tests/fixtures/llm_traces/tools/routine_create_list.json b/tests/fixtures/llm_traces/tools/routine_create_list.json index 114bae16..05bce4a9 100644 --- a/tests/fixtures/llm_traces/tools/routine_create_list.json +++ b/tests/fixtures/llm_traces/tools/routine_create_list.json @@ -52,7 +52,7 @@ { "response": { "type": "text", - "content": "I created a daily-check routine that runs at 9 AM every day. The routine list shows it as active.", + "content": "I created the daily-check routine, but it is not verified yet. It is scheduled for 9 AM every day, and the routine list should show it as unverified until it has a successful run.", "input_tokens": 300, "output_tokens": 25 } diff --git a/tests/fixtures/llm_traces/tools/routine_history.json b/tests/fixtures/llm_traces/tools/routine_history.json index 0b397f9b..50e10b07 100644 --- a/tests/fixtures/llm_traces/tools/routine_history.json +++ b/tests/fixtures/llm_traces/tools/routine_history.json @@ -41,7 +41,7 @@ { "response": { "type": "text", - "content": "The history-test routine was created. Its run history is empty since it hasn't been triggered yet.", + "content": "The history-test routine was created, but it is not verified yet. Its run history is empty since it hasn't been triggered yet.", "input_tokens": 300, "output_tokens": 25 } diff --git a/tests/fixtures/llm_traces/tools/routine_manual_create.json b/tests/fixtures/llm_traces/tools/routine_manual_create.json index bf386263..c153c5d2 100644 --- a/tests/fixtures/llm_traces/tools/routine_manual_create.json +++ b/tests/fixtures/llm_traces/tools/routine_manual_create.json @@ -27,7 +27,7 @@ { "response": { "type": "text", - "content": "Created the manual-triage routine. It will only run when explicitly fired.", + "content": "Created the manual-triage routine, but it is not verified yet. It will only run when explicitly fired, so I can test it for you when you're ready.", "input_tokens": 140, "output_tokens": 18 } diff --git a/tests/fixtures/llm_traces/tools/routine_manual_create_no_tools.json b/tests/fixtures/llm_traces/tools/routine_manual_create_no_tools.json index 275f2269..41f98190 100644 --- a/tests/fixtures/llm_traces/tools/routine_manual_create_no_tools.json +++ b/tests/fixtures/llm_traces/tools/routine_manual_create_no_tools.json @@ -30,7 +30,7 @@ { "response": { "type": "text", - "content": "Created the manual-triage-no-tools routine. It will only run when explicitly fired and stay text-only.", + "content": "Created the manual-triage-no-tools routine, but it is not verified yet. It will only run when explicitly fired and stay text-only until you decide to test it.", "input_tokens": 140, "output_tokens": 18 } diff --git a/tests/fixtures/llm_traces/tools/routine_update_delete.json b/tests/fixtures/llm_traces/tools/routine_update_delete.json index 81f364f8..8479a3bf 100644 --- a/tests/fixtures/llm_traces/tools/routine_update_delete.json +++ b/tests/fixtures/llm_traces/tools/routine_update_delete.json @@ -59,7 +59,7 @@ { "response": { "type": "text", - "content": "Created, updated, and then deleted the temp-routine successfully.", + "content": "Created, updated, and then deleted the temp-routine successfully. The update would have left it unverified until it was tested again.", "input_tokens": 400, "output_tokens": 20 } diff --git a/tests/gateway_workflow_integration.rs b/tests/gateway_workflow_integration.rs index c955e5a1..534a202a 100644 --- a/tests/gateway_workflow_integration.rs +++ b/tests/gateway_workflow_integration.rs @@ -16,6 +16,7 @@ mod tests { use chrono::Utc; use ironclaw::agent::routine::{ NotifyConfig, Routine, RoutineAction, RoutineGuardrails, Trigger, + reset_routine_verification_state, routine_verification_fingerprint, }; use uuid::Uuid; @@ -338,4 +339,101 @@ mod tests { harness.shutdown().await; mock.shutdown().await; } + + #[tokio::test] + async fn routines_api_surfaces_unverified_status_for_new_routine() { + let mock = MockOpenAiServerBuilder::new() + .with_default_response(MockOpenAiResponse::Text("ack".to_string())) + .start() + .await; + + let harness = + GatewayWorkflowHarness::start_openai_compatible(&mock.openai_base_url(), "mock-model") + .await; + + let mut routine = Routine { + id: Uuid::new_v4(), + name: "wf-unverified".to_string(), + description: "Unverified status regression test".to_string(), + user_id: harness.user_id.clone(), + enabled: true, + trigger: Trigger::Manual, + action: RoutineAction::Lightweight { + prompt: "Check verification status".to_string(), + context_paths: Vec::new(), + max_tokens: 512, + use_tools: false, + max_tool_rounds: 1, + }, + guardrails: RoutineGuardrails { + cooldown: Duration::from_secs(0), + max_concurrent: 1, + dedup_window: None, + }, + notify: NotifyConfig::default(), + last_run_at: None, + next_fire_at: None, + run_count: 0, + consecutive_failures: 0, + state: serde_json::json!({}), + created_at: Utc::now(), + updated_at: Utc::now(), + }; + routine.state = reset_routine_verification_state( + &routine.state, + routine_verification_fingerprint(&routine), + ); + harness + .db + .create_routine(&routine) + .await + .expect("create routine"); + + let list = harness.list_routines().await; + let routine_id = routine.id.to_string(); + let listed = list["routines"] + .as_array() + .expect("routines array") + .iter() + .find(|item| item["id"].as_str() == Some(routine_id.as_str())) + .expect("routine should be listed"); + assert_eq!(listed["status"].as_str(), Some("unverified")); + assert_eq!(listed["verification_status"].as_str(), Some("unverified")); + + let summary = harness + .client + .get(format!("{}/api/routines/summary", harness.base_url())) + .bearer_auth(&harness.auth_token) + .send() + .await + .expect("summary request failed") + .error_for_status() + .expect("summary non-2xx") + .json::() + .await + .expect("invalid summary response"); + assert_eq!(summary["unverified"].as_u64(), Some(1)); + + let detail = harness + .client + .get(format!( + "{}/api/routines/{}", + harness.base_url(), + routine_id + )) + .bearer_auth(&harness.auth_token) + .send() + .await + .expect("detail request failed") + .error_for_status() + .expect("detail non-2xx") + .json::() + .await + .expect("invalid detail response"); + assert_eq!(detail["status"].as_str(), Some("unverified")); + assert_eq!(detail["verification_status"].as_str(), Some("unverified")); + + harness.shutdown().await; + mock.shutdown().await; + } }