mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-31 08:39:24 +00:00
Add event-driven workflow orchestration skill, webhook flow, and trace coverage
This commit is contained in:
Illia Polosukhin
@@ -27,6 +27,8 @@ mod tests {
|
||||
|
||||
let rig = TestRigBuilder::new()
|
||||
.with_trace(trace.clone())
|
||||
.with_auto_approve_tools(true)
|
||||
.with_skills()
|
||||
.build()
|
||||
.await;
|
||||
|
||||
@@ -60,6 +62,8 @@ mod tests {
|
||||
|
||||
let rig = TestRigBuilder::new()
|
||||
.with_trace(trace.clone())
|
||||
.with_auto_approve_tools(true)
|
||||
.with_skills()
|
||||
.build()
|
||||
.await;
|
||||
|
||||
@@ -97,6 +101,8 @@ mod tests {
|
||||
|
||||
let rig = TestRigBuilder::new()
|
||||
.with_trace(trace.clone())
|
||||
.with_auto_approve_tools(true)
|
||||
.with_skills()
|
||||
.build()
|
||||
.await;
|
||||
|
||||
@@ -197,7 +203,107 @@ mod tests {
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Test 6: job_create_status
|
||||
// Test 6: routine_system_event_emit
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn routine_system_event_emit() {
|
||||
let trace = LlmTrace::from_file(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/llm_traces/tools/routine_system_event_emit.json"
|
||||
))
|
||||
.expect("failed to load routine_system_event_emit.json");
|
||||
|
||||
let rig = TestRigBuilder::new()
|
||||
.with_trace(trace.clone())
|
||||
.build()
|
||||
.await;
|
||||
|
||||
rig.send_message("Create a system-event routine and emit an event")
|
||||
.await;
|
||||
let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
|
||||
|
||||
rig.verify_trace_expects(&trace, &responses);
|
||||
|
||||
let completed = rig.tool_calls_completed();
|
||||
assert!(
|
||||
completed.iter().any(|(n, ok)| n == "event_emit" && *ok),
|
||||
"event_emit should succeed: {completed:?}"
|
||||
);
|
||||
|
||||
let results = rig.tool_results();
|
||||
let emit_result = results
|
||||
.iter()
|
||||
.find(|(n, _)| n == "event_emit")
|
||||
.expect("event_emit result missing");
|
||||
assert!(
|
||||
emit_result.1.contains("fired_routines"),
|
||||
"event_emit should report fired routine count: {:?}",
|
||||
emit_result.1
|
||||
);
|
||||
|
||||
rig.shutdown();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Test 7: skill_install_routine_webhook_sim
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn skill_install_routine_webhook_sim() {
|
||||
let trace = LlmTrace::from_file(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/tests/fixtures/llm_traces/tools/skill_install_routine_webhook_sim.json"
|
||||
))
|
||||
.expect("failed to load skill_install_routine_webhook_sim.json");
|
||||
|
||||
let rig = TestRigBuilder::new()
|
||||
.with_trace(trace.clone())
|
||||
.build()
|
||||
.await;
|
||||
|
||||
rig.send_message("Install the workflow skill template and simulate a webhook routine run")
|
||||
.await;
|
||||
// `skill_install` is approval-gated in the interactive loop.
|
||||
// Approve once so the trace can proceed through the remaining steps.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
rig.send_message("always").await;
|
||||
let responses = rig.wait_for_responses(1, Duration::from_secs(20)).await;
|
||||
rig.verify_trace_expects(&trace, &responses);
|
||||
|
||||
let completed = rig.tool_calls_completed();
|
||||
assert!(
|
||||
completed.iter().any(|(n, _)| n == "skill_install"),
|
||||
"skill_install should be called: {completed:?}"
|
||||
);
|
||||
for tool in &["routine_create", "event_emit", "routine_history"] {
|
||||
assert!(
|
||||
completed.iter().any(|(n, ok)| n == tool && *ok),
|
||||
"{tool} should succeed: {completed:?}"
|
||||
);
|
||||
}
|
||||
|
||||
let results = rig.tool_results();
|
||||
let emit_result = results
|
||||
.iter()
|
||||
.find(|(n, _)| n == "event_emit")
|
||||
.expect("event_emit result missing");
|
||||
assert!(
|
||||
emit_result.1.contains("fired_routines"),
|
||||
"event_emit should include fired_routines: {:?}",
|
||||
emit_result.1
|
||||
);
|
||||
|
||||
let _history_result = results
|
||||
.iter()
|
||||
.find(|(n, _)| n == "routine_history")
|
||||
.expect("routine_history result missing");
|
||||
|
||||
rig.shutdown();
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Test 8: job_create_status
|
||||
// -----------------------------------------------------------------------
|
||||
// Uses {{call_cj_1.job_id}} template to forward the dynamic UUID from
|
||||
// create_job's result into job_status's arguments.
|
||||
@@ -256,7 +362,7 @@ mod tests {
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Test 7: job_list_cancel
|
||||
// Test 9: job_list_cancel
|
||||
// -----------------------------------------------------------------------
|
||||
// Uses {{call_cj_lc.job_id}} template to forward the dynamic UUID from
|
||||
// create_job into cancel_job.
|
||||
|
||||
@@ -234,6 +234,105 @@ mod tests {
|
||||
// Test 3: routine_cooldown
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn system_event_trigger_matches_and_filters() {
|
||||
let (db, _tmp) = create_test_db().await;
|
||||
let ws = create_workspace(&db);
|
||||
|
||||
let trace = LlmTrace::single_turn(
|
||||
"test-system-event-match",
|
||||
"event",
|
||||
vec![TraceStep {
|
||||
request_hint: None,
|
||||
response: TraceResponse::Text {
|
||||
content: "System event handled".to_string(),
|
||||
input_tokens: 40,
|
||||
output_tokens: 8,
|
||||
},
|
||||
expected_tool_results: vec![],
|
||||
}],
|
||||
);
|
||||
let llm = Arc::new(TraceLlm::from_trace(trace));
|
||||
let (notify_tx, _notify_rx) = tokio::sync::mpsc::channel(16);
|
||||
|
||||
let engine = Arc::new(RoutineEngine::new(
|
||||
RoutineConfig::default(),
|
||||
db.clone(),
|
||||
llm,
|
||||
ws,
|
||||
notify_tx,
|
||||
None,
|
||||
));
|
||||
|
||||
let mut filters = std::collections::HashMap::new();
|
||||
filters.insert("repository".to_string(), "nearai/ironclaw".to_string());
|
||||
|
||||
let routine = make_routine(
|
||||
"github-issue-opened",
|
||||
Trigger::SystemEvent {
|
||||
source: "github".to_string(),
|
||||
event_type: "issue.opened".to_string(),
|
||||
filters,
|
||||
},
|
||||
"Summarize the issue and propose an implementation plan.",
|
||||
);
|
||||
db.create_routine(&routine).await.expect("create_routine");
|
||||
engine.refresh_event_cache().await;
|
||||
|
||||
// Matching event should fire.
|
||||
let fired = engine
|
||||
.emit_system_event(
|
||||
"github",
|
||||
"issue.opened",
|
||||
&serde_json::json!({
|
||||
"repository": "nearai/ironclaw",
|
||||
"issue_number": 42
|
||||
}),
|
||||
Some("default"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(fired, 1, "Expected one routine to fire for matching event");
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
let runs = db
|
||||
.list_routine_runs(routine.id, 10)
|
||||
.await
|
||||
.expect("list runs");
|
||||
assert!(
|
||||
!runs.is_empty(),
|
||||
"Expected run history after matching event"
|
||||
);
|
||||
|
||||
// Wrong event type should not fire.
|
||||
let fired_wrong_type = engine
|
||||
.emit_system_event(
|
||||
"github",
|
||||
"issue.closed",
|
||||
&serde_json::json!({"repository": "nearai/ironclaw"}),
|
||||
Some("default"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
fired_wrong_type, 0,
|
||||
"Expected no routine for wrong event type"
|
||||
);
|
||||
|
||||
// Wrong filter value should not fire.
|
||||
let fired_wrong_filter = engine
|
||||
.emit_system_event(
|
||||
"github",
|
||||
"issue.opened",
|
||||
&serde_json::json!({"repository": "other/repo"}),
|
||||
Some("default"),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
fired_wrong_filter, 0,
|
||||
"Expected no routine for filter mismatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routine_cooldown() {
|
||||
let (db, _tmp) = create_test_db().await;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"model_name": "test-routine-system-event-emit",
|
||||
"expects": {
|
||||
"tools_used": ["event_emit"],
|
||||
"all_tools_succeeded": true,
|
||||
"tool_results_contain": {
|
||||
"event_emit": "fired_routines"
|
||||
}
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"response": {
|
||||
"type": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_ee_1",
|
||||
"name": "event_emit",
|
||||
"arguments": {
|
||||
"source": "github",
|
||||
"event_type": "issue.opened",
|
||||
"payload": {
|
||||
"repository": "nearai/ironclaw",
|
||||
"issue_number": 123,
|
||||
"title": "Support event-driven project workflow"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"input_tokens": 90,
|
||||
"output_tokens": 28
|
||||
}
|
||||
},
|
||||
{
|
||||
"response": {
|
||||
"type": "text",
|
||||
"content": "Emitted a GitHub system event successfully.",
|
||||
"input_tokens": 140,
|
||||
"output_tokens": 14
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"model_name": "test-skill-install-routine-webhook-sim",
|
||||
"expects": {
|
||||
"tools_used": ["skill_install", "routine_create", "event_emit", "routine_history"],
|
||||
"tool_results_contain": {
|
||||
"event_emit": "fired_routines"
|
||||
}
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"response": {
|
||||
"type": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_skill_install_1",
|
||||
"name": "skill_install",
|
||||
"arguments": {
|
||||
"name": "wf-orchestrator-trace-install-1",
|
||||
"content": "---\nname: wf-orchestrator-trace-install-1\ndescription: Minimal workflow skill for trace install validation\nactivation:\n keywords: [\"workflow\", \"orchestrator\"]\n---\n\nYou are a minimal workflow skill used for trace install validation.\n"
|
||||
}
|
||||
}
|
||||
],
|
||||
"input_tokens": 120,
|
||||
"output_tokens": 32
|
||||
}
|
||||
},
|
||||
{
|
||||
"response": {
|
||||
"type": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_routine_create_1",
|
||||
"name": "routine_create",
|
||||
"arguments": {
|
||||
"name": "wf-webhook-sim-trace",
|
||||
"description": "Trace routine to simulate webhook event flow",
|
||||
"trigger_type": "system_event",
|
||||
"event_source": "github",
|
||||
"event_type": "issue.opened",
|
||||
"event_filters": {
|
||||
"repository": "nearai/ironclaw"
|
||||
},
|
||||
"action_type": "full_job",
|
||||
"prompt": "When issue webhook event arrives, start implementation loop and create branch/PR updates."
|
||||
}
|
||||
}
|
||||
],
|
||||
"input_tokens": 170,
|
||||
"output_tokens": 36
|
||||
}
|
||||
},
|
||||
{
|
||||
"response": {
|
||||
"type": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_event_emit_1",
|
||||
"name": "event_emit",
|
||||
"arguments": {
|
||||
"source": "github",
|
||||
"event_type": "issue.opened",
|
||||
"payload": {
|
||||
"repository": "nearai/ironclaw",
|
||||
"issue_number": 4242,
|
||||
"sender": "trace-bot"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"input_tokens": 210,
|
||||
"output_tokens": 28
|
||||
}
|
||||
},
|
||||
{
|
||||
"response": {
|
||||
"type": "tool_calls",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_routine_history_1",
|
||||
"name": "routine_history",
|
||||
"arguments": {
|
||||
"name": "wf-webhook-sim-trace",
|
||||
"limit": 5
|
||||
}
|
||||
}
|
||||
],
|
||||
"input_tokens": 240,
|
||||
"output_tokens": 22
|
||||
}
|
||||
},
|
||||
{
|
||||
"response": {
|
||||
"type": "text",
|
||||
"content": "Installed the skill template, created a system-event routine, emitted a webhook-equivalent event, and verified the routine run history.",
|
||||
"input_tokens": 280,
|
||||
"output_tokens": 25
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -379,6 +379,8 @@ pub struct TestRigBuilder {
|
||||
llm: Option<Arc<dyn LlmProvider>>,
|
||||
max_tool_iterations: usize,
|
||||
injection_check: bool,
|
||||
auto_approve_tools: Option<bool>,
|
||||
enable_skills: bool,
|
||||
enable_routines: bool,
|
||||
http_exchanges: Vec<HttpExchange>,
|
||||
extra_tools: Vec<Arc<dyn Tool>>,
|
||||
@@ -392,6 +394,8 @@ impl TestRigBuilder {
|
||||
llm: None,
|
||||
max_tool_iterations: 10,
|
||||
injection_check: false,
|
||||
auto_approve_tools: None,
|
||||
enable_skills: false,
|
||||
enable_routines: false,
|
||||
http_exchanges: Vec::new(),
|
||||
extra_tools: Vec::new(),
|
||||
@@ -432,6 +436,18 @@ impl TestRigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Override agent-level automatic approval of `UnlessAutoApproved` tools.
|
||||
pub fn with_auto_approve_tools(mut self, enable: bool) -> Self {
|
||||
self.auto_approve_tools = Some(enable);
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable skill discovery and registration for this test rig.
|
||||
pub fn with_skills(mut self) -> Self {
|
||||
self.enable_skills = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable the routines system so the scheduler is wired with a `RoutineEngine`,
|
||||
/// allowing routine jobs to actually execute. Routine tools are always registered
|
||||
/// but require the engine to dispatch jobs.
|
||||
@@ -466,6 +482,8 @@ impl TestRigBuilder {
|
||||
llm,
|
||||
max_tool_iterations,
|
||||
injection_check,
|
||||
auto_approve_tools,
|
||||
enable_skills,
|
||||
enable_routines,
|
||||
http_exchanges: explicit_http_exchanges,
|
||||
extra_tools,
|
||||
@@ -491,6 +509,10 @@ impl TestRigBuilder {
|
||||
let mut config = Config::for_testing(db_path, skills_dir, installed_skills_dir);
|
||||
config.agent.max_tool_iterations = max_tool_iterations;
|
||||
config.safety.injection_check_enabled = injection_check;
|
||||
config.skills.enabled = enable_skills;
|
||||
if let Some(v) = auto_approve_tools {
|
||||
config.agent.auto_approve_tools = v;
|
||||
}
|
||||
|
||||
// 3. Create SessionManager + LogBroadcaster.
|
||||
let session = Arc::new(SessionManager::new(SessionConfig::default()));
|
||||
@@ -540,7 +562,7 @@ impl TestRigBuilder {
|
||||
);
|
||||
builder.with_database(Arc::clone(&db));
|
||||
builder.with_llm(llm);
|
||||
let components = builder
|
||||
let mut components = builder
|
||||
.build_all()
|
||||
.await
|
||||
.expect("AppBuilder::build_all() failed in test rig");
|
||||
@@ -583,6 +605,21 @@ impl TestRigBuilder {
|
||||
.register_routine_tools(Arc::clone(db_arc), engine);
|
||||
}
|
||||
|
||||
// Skills tools: ensure tests use temp skill dirs (sandbox-safe) even if
|
||||
// AppBuilder did not wire them for this environment.
|
||||
if enable_skills {
|
||||
let registry = Arc::new(std::sync::RwLock::new(
|
||||
ironclaw::skills::SkillRegistry::new(temp_dir.path().join("skills"))
|
||||
.with_installed_dir(temp_dir.path().join("installed_skills")),
|
||||
));
|
||||
let catalog = ironclaw::skills::catalog::shared_catalog();
|
||||
components
|
||||
.tools
|
||||
.register_skill_tools(Arc::clone(®istry), Arc::clone(&catalog));
|
||||
components.skill_registry = Some(registry);
|
||||
components.skill_catalog = Some(catalog);
|
||||
}
|
||||
|
||||
// Register any extra test-specific tools.
|
||||
for tool in extra_tools {
|
||||
components.tools.register(tool).await;
|
||||
|
||||
Reference in New Issue
Block a user