mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
Illia PolosukhinandClaude Opus 4.6
274175184e
Introduce a unified EventBus as the single broadcast channel for all system events, replacing the 6 disconnected event mechanisms. Seal Thread/Turn/ContainerState fields behind private accessors with validated transitions to prevent invalid state mutations. Fix TOCTOU races in thread_ops and session_manager. Event bus (src/event_bus/): - SystemEvent envelope with EventPayload (Domain, StateChange, Telemetry, StateTransition, ToolExecution, AuthEvent, ConfigChange) - Four sinks: SSE (→SseManager), audit (→DB with JSONL fallback), state (→StateBus), metrics (→Observer) - AuditStore trait + implementations for PostgreSQL and libSQL - V13 audit_log migration for both backends - Wired into AppComponents and AgentDeps (Option<EventBus> for compat) - Worker dual-emit through bus alongside legacy SSE+DB paths Sealed state machines: - Thread.state private with state() accessor, can_transition_to(), set_processing(), reset_to_idle() - Turn.state private with state() accessor - ContainerHandle.state private with new() constructor, mark_running/stopped/failed(), can_transition_to() - TOCTOU fix: thread_ops moves safety validation before lock, then checks state + starts turn atomically under single lock - SessionManager TOCTOU fix: atomic check-and-insert with write lock held for entire UUID adoption sequence Startup verification: - AppComponents::verify_readiness() checks component presence vs config - ToolRegistry::verify_expected_tools() validates builtin registration - Config::validate() checks cross-field invariants (Docker, WASM dir) [skip-regression-check] Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
518 lines
18 KiB
Rust
518 lines
18 KiB
Rust
//! Tests for batch loading routine concurrent counts (N+1 query fix).
|
|
//!
|
|
//! Verifies:
|
|
//! 1. Batch query returns correct counts for multiple routines
|
|
//! 2. Concurrent limit enforcement uses batch counts correctly
|
|
|
|
#[cfg(feature = "libsql")]
|
|
mod tests {
|
|
use std::sync::Arc;
|
|
|
|
use chrono::Utc;
|
|
use uuid::Uuid;
|
|
|
|
use ironclaw::agent::routine::{
|
|
Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger,
|
|
};
|
|
use ironclaw::db::Database;
|
|
|
|
async fn create_test_db() -> (Arc<dyn Database>, tempfile::TempDir) {
|
|
use ironclaw::db::libsql::LibSqlBackend;
|
|
|
|
let temp_dir = tempfile::tempdir().expect("tempdir"); // safety: test-only
|
|
let db_path = temp_dir.path().join("test.db");
|
|
let backend = LibSqlBackend::new_local(&db_path)
|
|
.await
|
|
.expect("LibSqlBackend"); // safety: test-only
|
|
backend.run_migrations().await.expect("migrations"); // safety: test-only
|
|
let db: Arc<dyn Database> = Arc::new(backend);
|
|
(db, temp_dir)
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 1: Batch query returns correct counts for multiple routines
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
async fn batch_query_empty_list() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[])
|
|
.await
|
|
.expect("batch query should not fail"); // safety: test-only
|
|
assert!(counts.is_empty(), "Empty input should return empty map"); // safety: test-only
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn batch_query_single_routine() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
let routine_id = Uuid::new_v4();
|
|
|
|
// Create routine
|
|
let routine = Routine {
|
|
id: routine_id,
|
|
name: "test-routine".to_string(),
|
|
description: "Test".to_string(),
|
|
user_id: "default".to_string(),
|
|
enabled: true,
|
|
trigger: Trigger::Cron {
|
|
schedule: "* * * * *".to_string(),
|
|
timezone: None,
|
|
},
|
|
action: RoutineAction::Lightweight {
|
|
prompt: "test".to_string(),
|
|
context_paths: vec![],
|
|
max_tokens: 1000,
|
|
use_tools: false,
|
|
max_tool_rounds: 3,
|
|
},
|
|
guardrails: RoutineGuardrails {
|
|
cooldown: std::time::Duration::from_secs(0),
|
|
max_concurrent: 5,
|
|
dedup_window: None,
|
|
},
|
|
notify: Default::default(),
|
|
last_run_at: None,
|
|
next_fire_at: None,
|
|
run_count: 0,
|
|
consecutive_failures: 0,
|
|
state: serde_json::json!({}),
|
|
created_at: Utc::now(),
|
|
updated_at: Utc::now(),
|
|
};
|
|
db.create_routine(&routine).await.expect("create routine"); // safety: test-only
|
|
|
|
// Create 3 running runs
|
|
for _ in 0..3 {
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
}
|
|
|
|
// Batch query for single routine
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[routine_id])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
assert_eq!(counts.len(), 1, "Should return 1 routine"); // safety: test-only
|
|
assert_eq!(counts[&routine_id], 3, "Should count 3 running runs"); // safety: test-only
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn batch_query_multiple_routines_different_counts() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
|
|
let r1 = Uuid::new_v4();
|
|
let r2 = Uuid::new_v4();
|
|
let r3 = Uuid::new_v4();
|
|
|
|
// Create 3 routines
|
|
for routine_id in [r1, r2, r3] {
|
|
let routine = Routine {
|
|
id: routine_id,
|
|
name: format!("routine-{}", routine_id),
|
|
description: "Test".to_string(),
|
|
user_id: "default".to_string(),
|
|
enabled: true,
|
|
trigger: Trigger::Cron {
|
|
schedule: "* * * * *".to_string(),
|
|
timezone: None,
|
|
},
|
|
action: RoutineAction::Lightweight {
|
|
prompt: "test".to_string(),
|
|
context_paths: vec![],
|
|
max_tokens: 1000,
|
|
use_tools: false,
|
|
max_tool_rounds: 3,
|
|
},
|
|
guardrails: RoutineGuardrails {
|
|
cooldown: std::time::Duration::from_secs(0),
|
|
max_concurrent: 5,
|
|
dedup_window: None,
|
|
},
|
|
notify: Default::default(),
|
|
last_run_at: None,
|
|
next_fire_at: None,
|
|
run_count: 0,
|
|
consecutive_failures: 0,
|
|
state: serde_json::json!({}),
|
|
created_at: Utc::now(),
|
|
updated_at: Utc::now(),
|
|
};
|
|
db.create_routine(&routine).await.expect("create routine"); // safety: test-only
|
|
}
|
|
|
|
// r1: 2 running
|
|
for _ in 0..2 {
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r1,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
}
|
|
|
|
// r2: 1 running
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r2,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
|
|
// r3: 0 running (but has 1 Ok result)
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r3,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: Some(Utc::now()),
|
|
status: RunStatus::Ok,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
|
|
// Single batch query for all 3
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[r1, r2, r3])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
assert_eq!(counts.len(), 3, "Should return 3 routines"); // safety: test-only
|
|
assert_eq!(counts[&r1], 2, "r1 should have 2 running"); // safety: test-only
|
|
assert_eq!(counts[&r2], 1, "r2 should have 1 running"); // safety: test-only
|
|
assert_eq!(
|
|
// safety: test-only
|
|
counts[&r3],
|
|
0,
|
|
"r3 should have 0 running (Ok status is not running)"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn batch_query_missing_routines_default_to_zero() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
|
|
let r1 = Uuid::new_v4();
|
|
let r2 = Uuid::new_v4();
|
|
let r3 = Uuid::new_v4(); // This one won't exist
|
|
|
|
// Only create r1
|
|
let routine = Routine {
|
|
id: r1,
|
|
name: "routine-1".to_string(),
|
|
description: "Test".to_string(),
|
|
user_id: "default".to_string(),
|
|
enabled: true,
|
|
trigger: Trigger::Cron {
|
|
schedule: "* * * * *".to_string(),
|
|
timezone: None,
|
|
},
|
|
action: RoutineAction::Lightweight {
|
|
prompt: "test".to_string(),
|
|
context_paths: vec![],
|
|
max_tokens: 1000,
|
|
use_tools: false,
|
|
max_tool_rounds: 3,
|
|
},
|
|
guardrails: RoutineGuardrails {
|
|
cooldown: std::time::Duration::from_secs(0),
|
|
max_concurrent: 5,
|
|
dedup_window: None,
|
|
},
|
|
notify: Default::default(),
|
|
last_run_at: None,
|
|
next_fire_at: None,
|
|
run_count: 0,
|
|
consecutive_failures: 0,
|
|
state: serde_json::json!({}),
|
|
created_at: Utc::now(),
|
|
updated_at: Utc::now(),
|
|
};
|
|
db.create_routine(&routine).await.expect("create routine"); // safety: test-only
|
|
|
|
// r1 has 1 running
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r1,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
|
|
// Query for r1, r2 (doesn't exist), r3 (doesn't exist)
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[r1, r2, r3])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
assert_eq!(counts.len(), 3, "Should have all 3 routine IDs"); // safety: test-only
|
|
assert_eq!(counts[&r1], 1, "r1 should have 1 running"); // safety: test-only
|
|
assert_eq!(counts[&r2], 0, "r2 should default to 0"); // safety: test-only
|
|
assert_eq!(counts[&r3], 0, "r3 should default to 0"); // safety: test-only
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn batch_query_only_counts_running_status() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
let routine_id = Uuid::new_v4();
|
|
|
|
// Create routine
|
|
let routine = Routine {
|
|
id: routine_id,
|
|
name: "test-routine".to_string(),
|
|
description: "Test".to_string(),
|
|
user_id: "default".to_string(),
|
|
enabled: true,
|
|
trigger: Trigger::Cron {
|
|
schedule: "* * * * *".to_string(),
|
|
timezone: None,
|
|
},
|
|
action: RoutineAction::Lightweight {
|
|
prompt: "test".to_string(),
|
|
context_paths: vec![],
|
|
max_tokens: 1000,
|
|
use_tools: false,
|
|
max_tool_rounds: 3,
|
|
},
|
|
guardrails: RoutineGuardrails {
|
|
cooldown: std::time::Duration::from_secs(0),
|
|
max_concurrent: 5,
|
|
dedup_window: None,
|
|
},
|
|
notify: Default::default(),
|
|
last_run_at: None,
|
|
next_fire_at: None,
|
|
run_count: 0,
|
|
consecutive_failures: 0,
|
|
state: serde_json::json!({}),
|
|
created_at: Utc::now(),
|
|
updated_at: Utc::now(),
|
|
};
|
|
db.create_routine(&routine).await.expect("create routine"); // safety: test-only
|
|
|
|
// Create 5 runs with mixed statuses
|
|
let statuses = [
|
|
RunStatus::Running,
|
|
RunStatus::Running,
|
|
RunStatus::Ok,
|
|
RunStatus::Failed,
|
|
RunStatus::Attention,
|
|
];
|
|
|
|
for status in statuses.iter() {
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: Some(Utc::now()),
|
|
status: *status,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
}
|
|
|
|
// Batch query should only count Running status
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[routine_id])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
assert_eq!(
|
|
// safety: test-only
|
|
counts[&routine_id],
|
|
2,
|
|
"Should only count 2 Running status runs"
|
|
);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Test 2: Concurrent limit enforcement uses batch counts
|
|
// -----------------------------------------------------------------------
|
|
|
|
#[tokio::test]
|
|
async fn concurrent_limit_enforcement_with_batch_counts() {
|
|
let (db, _tmp) = create_test_db().await;
|
|
|
|
let r1 = Uuid::new_v4();
|
|
let r2 = Uuid::new_v4();
|
|
|
|
// Create 2 routines with max_concurrent=1 (r1) and max_concurrent=2 (r2)
|
|
for (routine_id, max_concurrent) in [(r1, 1), (r2, 2)] {
|
|
let routine = Routine {
|
|
id: routine_id,
|
|
name: format!("routine-{}", routine_id),
|
|
description: "Test".to_string(),
|
|
user_id: "default".to_string(),
|
|
enabled: true,
|
|
trigger: Trigger::Cron {
|
|
schedule: "* * * * *".to_string(),
|
|
timezone: None,
|
|
},
|
|
action: RoutineAction::Lightweight {
|
|
prompt: "test".to_string(),
|
|
context_paths: vec![],
|
|
max_tokens: 1000,
|
|
use_tools: false,
|
|
max_tool_rounds: 3,
|
|
},
|
|
guardrails: RoutineGuardrails {
|
|
cooldown: std::time::Duration::from_secs(0),
|
|
max_concurrent,
|
|
dedup_window: None,
|
|
},
|
|
notify: Default::default(),
|
|
last_run_at: None,
|
|
next_fire_at: None,
|
|
run_count: 0,
|
|
consecutive_failures: 0,
|
|
state: serde_json::json!({}),
|
|
created_at: Utc::now(),
|
|
updated_at: Utc::now(),
|
|
};
|
|
db.create_routine(&routine).await.expect("create routine"); // safety: test-only
|
|
}
|
|
|
|
// r1: create 1 running run (will hit max_concurrent=1)
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r1,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
|
|
// r2: create 2 running runs (will hit max_concurrent=2)
|
|
for _ in 0..2 {
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r2,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
}
|
|
|
|
// Batch query should return correct counts
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[r1, r2])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
// Verify counts match the limits
|
|
assert_eq!(
|
|
// safety: test-only
|
|
counts[&r1],
|
|
1,
|
|
"r1 should have 1 running (at max_concurrent=1)"
|
|
);
|
|
assert_eq!(
|
|
// safety: test-only
|
|
counts[&r2],
|
|
2,
|
|
"r2 should have 2 running (at max_concurrent=2)"
|
|
);
|
|
|
|
// Now verify the limit enforcement logic
|
|
let r1_routine = db
|
|
.get_routine(r1)
|
|
.await
|
|
.expect("get routine") // safety: test-only
|
|
.expect("routine exists"); // safety: test-only
|
|
let r2_routine = db
|
|
.get_routine(r2)
|
|
.await
|
|
.expect("get routine") // safety: test-only
|
|
.expect("routine exists"); // safety: test-only
|
|
|
|
let r1_at_limit = counts[&r1] >= r1_routine.guardrails.max_concurrent as i64;
|
|
let r2_at_limit = counts[&r2] >= r2_routine.guardrails.max_concurrent as i64;
|
|
|
|
assert!(r1_at_limit, "r1 should be detected as at limit"); // safety: test-only
|
|
assert!(r2_at_limit, "r2 should be detected as at limit"); // safety: test-only
|
|
|
|
// If we add one more run to r2, it should exceed limit
|
|
let run = RoutineRun {
|
|
id: Uuid::new_v4(),
|
|
routine_id: r2,
|
|
trigger_type: "cron".to_string(),
|
|
trigger_detail: None,
|
|
started_at: Utc::now(),
|
|
completed_at: None,
|
|
status: RunStatus::Running,
|
|
result_summary: None,
|
|
tokens_used: None,
|
|
job_id: None,
|
|
created_at: Utc::now(),
|
|
};
|
|
db.create_routine_run(&run).await.expect("create run"); // safety: test-only
|
|
|
|
// Re-query to get updated counts
|
|
let counts = db
|
|
.count_running_routine_runs_batch(&[r1, r2])
|
|
.await
|
|
.expect("batch query should work"); // safety: test-only
|
|
|
|
let r2_exceeded_limit = counts[&r2] > r2_routine.guardrails.max_concurrent as i64;
|
|
assert!(r2_exceeded_limit, "r2 should have exceeded its limit"); // safety: test-only
|
|
}
|
|
}
|