gitops/optimclaw
1
0
Fork 0
mirror of https://github.com/outbackdingo/optimclaw.git synced 2026-08-26 15:40:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Redesign routine create requests for LLMs (#1147)

Browse Source 
* Redesign routine create requests for LLMs

* Fix panic-check false positives in routine tests

* Tighten routine schema requirements

* Tighten routine schema tests

* Mark test assertions safe for CI scan

* Align test assertions with panic scan

* Polish routine schema metadata

* Simplify routine test assertions

* Improve tool discovery guidance

* Clarify lightweight routine delivery prompts

* Fix routine delivery target defaults
This commit is contained in:
Henry Park
2026-03-18 09:04:00 -07:00
committed by GitHub
parent 2784cef4d7
commit 428303af11
11 changed files with 2334 additions and 474 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e_builtin_tool_coverage.rs
+176 -21
View File
@@ -142,16 +142,18 @@ mod tests {
match &routine.action {
RoutineAction::Lightweight {
prompt,
context_paths,
use_tools,
max_tool_rounds,
..
} => {
assert!(prompt.contains("Check system status"));
assert_eq!(context_paths, &vec!["context/priorities.md".to_string()]);
assert!(*use_tools, "lightweight routine should keep use_tools=true");
assert_eq!(*max_tool_rounds, 2);
}
other => panic!("expected lightweight action, got {other:?}"),
other => panic!("expected lightweight routine action, got {other:?}"),
}
assert_eq!(routine.notify.channel.as_deref(), Some("telegram"));
@@ -369,7 +371,132 @@ mod tests {
}
// -----------------------------------------------------------------------
// Test 8: skill_install_routine_webhook_sim
// Test 8: routine_create_grouped
// -----------------------------------------------------------------------
#[tokio::test]
async fn routine_create_grouped() {
let trace = LlmTrace::from_file(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/llm_traces/tools/routine_create_grouped.json"
))
.expect("failed to load routine_create_grouped.json");
let rig = TestRigBuilder::new()
.with_trace(trace.clone())
.with_auto_approve_tools(true)
.build()
.await;
rig.send_message("Create a grouped cron routine with delivery settings")
.await;
let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
rig.verify_trace_expects(&trace, &responses);
let routine = rig
.database()
.get_routine_by_name("test-user", "weekday-digest")
.await
.expect("get_routine_by_name")
.expect("weekday-digest should exist");
match &routine.trigger {
Trigger::Cron { schedule, timezone } => {
assert_eq!(schedule, "0 0 9 * * MON-FRI");
assert_eq!(timezone.as_deref(), Some("UTC"));
}
other => panic!("expected cron trigger, got {other:?}"),
}
match &routine.action {
RoutineAction::FullJob {
description,
tool_permissions,
..
} => {
assert!(description.contains("Prepare the morning digest"));
assert_eq!(
tool_permissions,
&vec!["message".to_string(), "http".to_string()]
);
}
other => panic!("expected full_job action, got {other:?}"),
}
assert_eq!(routine.notify.channel.as_deref(), Some("telegram"));
assert_eq!(routine.notify.user.as_deref(), Some("ops-team"));
assert_eq!(routine.guardrails.cooldown.as_secs(), 30);
rig.shutdown();
}
// -----------------------------------------------------------------------
// Test 9: routine_system_event_emit_grouped
// -----------------------------------------------------------------------
#[tokio::test]
async fn routine_system_event_emit_grouped() {
let trace = LlmTrace::from_file(concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/llm_traces/tools/routine_system_event_emit_grouped.json"
))
.expect("failed to load routine_system_event_emit_grouped.json");
let rig = TestRigBuilder::new()
.with_trace(trace.clone())
.with_auto_approve_tools(true)
.build()
.await;
rig.send_message("Create a grouped system-event routine and emit a matching event")
.await;
let responses = rig.wait_for_responses(1, Duration::from_secs(15)).await;
rig.verify_trace_expects(&trace, &responses);
let routine = rig
.database()
.get_routine_by_name("test-user", "grouped-gh-issue-watch")
.await
.expect("get_routine_by_name")
.expect("grouped-gh-issue-watch should exist");
match &routine.trigger {
Trigger::SystemEvent {
source,
event_type,
filters,
} => {
assert_eq!(source, "github");
assert_eq!(event_type, "issue.opened");
assert_eq!(
filters.get("repository").map(String::as_str),
Some("nearai/ironclaw")
);
assert_eq!(filters.get("priority").map(String::as_str), Some("p1"));
}
other => panic!("expected system_event trigger, got {other:?}"),
}
let results = rig.tool_results();
let emit_result = results
.iter()
.find(|(n, _)| n == "event_emit")
.expect("event_emit result missing");
let emit_json: serde_json::Value =
serde_json::from_str(&emit_result.1).expect("event_emit result should be valid JSON");
assert!(
emit_json["fired_routines"].as_u64().unwrap_or(0) > 0,
"event_emit should have fired at least one grouped routine: {:?}",
emit_result.1
);
rig.shutdown();
}
// -----------------------------------------------------------------------
// Test 10: skill_install_routine_webhook_sim
// -----------------------------------------------------------------------
#[tokio::test]
@@ -571,10 +698,11 @@ mod tests {
}
// -----------------------------------------------------------------------
// Test: tool_info_discovery (two-level detail)
// Test: tool_info_discovery (three-level detail)
// -----------------------------------------------------------------------
// Verifies the tool_info built-in returns:
// - Default (no include_schema): name, description, parameter names array
// - `detail: "summary"`: curated summary guidance
// - With include_schema: true: adds full typed JSON Schema
#[tokio::test]
@@ -597,13 +725,13 @@ mod tests {
rig.verify_trace_expects(&trace, &responses);
// tool_info should have been called twice (echo + time), both succeeding.
// tool_info should have been called three times (echo + routine_create + time), all succeeding.
let completed = rig.tool_calls_completed();
let tool_info_calls: Vec<_> = completed.iter().filter(|(n, _)| n == "tool_info").collect();
assert_eq!(
tool_info_calls.len(),
2,
"Expected 2 tool_info calls, got {tool_info_calls:?}"
3,
"Expected 3 tool_info calls, got {tool_info_calls:?}"
);
assert!(
tool_info_calls.iter().all(|(_, ok)| *ok),
@@ -613,44 +741,71 @@ mod tests {
// Verify the results contain expected fields.
let results = rig.tool_results();
let info_results: Vec<_> = results.iter().filter(|(n, _)| n == "tool_info").collect();
let info_json: Vec<serde_json::Value> = info_results
.iter()
.map(|(_, preview)| {
serde_json::from_str(preview)
.expect("tool_info result preview should be valid JSON")
})
.collect();
// First call was for "echo" (default, no include_schema) — result should
// contain "echo" and "parameters" as an array of names (not full schema).
let echo_result = info_results
let echo_json = info_json
.iter()
.find(|(_, preview)| preview.contains("echo"))
.find(|info| info["name"] == "echo")
.expect("tool_info result should contain 'echo'");
assert!(
echo_result.1.contains("message"),
echo_json["parameters"]
.as_array()
.is_some_and(|params| params.iter().any(|param| param == "message")),
"echo default result should list 'message' parameter name: {:?}",
echo_result.1
echo_json
);
// Default mode should NOT include the full "schema" key
let echo_json: serde_json::Value = serde_json::from_str(&echo_result.1)
.expect("echo tool_info result should be valid JSON");
assert!(
echo_json.get("schema").is_none(),
"Default tool_info should not include schema field: {:?}",
echo_result.1
echo_json
);
// Second call was for "time" with include_schema: true — result should
// contain "time", "schema" field with full object.
let time_result = info_results
// Second call was for "routine_create" with detail: "summary" — result
// should contain a summary object with rules/examples.
let routine_json = info_json
.iter()
.find(|(_, preview)| preview.contains("time"))
.find(|info| info["name"] == "routine_create")
.expect("tool_info result should contain 'routine_create'");
assert!(
routine_json.get("summary").is_some(),
"detail: summary should include summary field: {:?}",
routine_json
);
assert!(
routine_json["summary"]["conditional_requirements"]
.as_array()
.is_some_and(|rules| rules.iter().any(|rule| {
rule.as_str()
.is_some_and(|rule| rule.contains("request.kind='cron'"))
})),
"routine_create summary should mention cron requirement: {:?}",
routine_json
);
// Third call was for "time" with include_schema: true — result should
// contain "time", "schema" field with full object.
let time_json = info_json
.iter()
.find(|info| info["name"] == "time")
.expect("tool_info result should contain 'time'");
let time_json: serde_json::Value = serde_json::from_str(&time_result.1)
.expect("time tool_info result should be valid JSON");
assert!(
time_json.get("schema").is_some(),
"include_schema: true should include schema field: {:?}",
time_result.1
time_json
);
assert!(
time_json["schema"]["properties"].is_object(),
"schema should have properties: {:?}",
time_result.1
time_json
);
rig.shutdown();
tests/fixtures/llm_traces/tools/routine_create_grouped.json
Vendored
+66
View File
@@ -0,0 +1,66 @@
{
"model_name": "test-routine-create-grouped",
"expects": {
"tools_used": ["routine_create", "routine_list"],
"all_tools_succeeded": true,
"min_responses": 1
},
"steps": [
{
"response": {
"type": "tool_calls",
"tool_calls": [
{
"id": "call_rc_grouped_1",
"name": "routine_create",
"arguments": {
"name": "weekday-digest",
"prompt": "Prepare the morning digest for the ops team.",
"description": "Weekday digest for morning operations",
"request": {
"kind": "cron",
"schedule": "0 0 9 * * MON-FRI",
"timezone": "UTC"
},
"execution": {
"mode": "full_job",
"tool_permissions": ["message", "http"]
},
"delivery": {
"channel": "telegram",
"user": "ops-team"
},
"advanced": {
"cooldown_secs": 30
}
}
}
],
"input_tokens": 130,
"output_tokens": 44
}
},
{
"response": {
"type": "tool_calls",
"tool_calls": [
{
"id": "call_rl_grouped_1",
"name": "routine_list",
"arguments": {}
}
],
"input_tokens": 190,
"output_tokens": 20
}
},
{
"response": {
"type": "text",
"content": "Created the weekday-digest routine with a grouped cron request and listed the active routines.",
"input_tokens": 250,
"output_tokens": 24
}
}
]
}
tests/fixtures/llm_traces/tools/routine_system_event_emit_grouped.json
Vendored
+74
View File
@@ -0,0 +1,74 @@
{
"model_name": "test-routine-system-event-emit-grouped",
"expects": {
"tools_used": ["routine_create", "event_emit"],
"all_tools_succeeded": true,
"tool_results_contain": {
"event_emit": "fired_routines"
}
},
"steps": [
{
"response": {
"type": "tool_calls",
"tool_calls": [
{
"id": "call_rc_grouped_system_1",
"name": "routine_create",
"arguments": {
"name": "grouped-gh-issue-watch",
"prompt": "Summarize the new issue and propose next steps.",
"description": "React to important GitHub issue.opened events",
"request": {
"kind": "system_event",
"source": "github",
"event_type": "issue.opened",
"filters": {
"repository": "nearai/ironclaw",
"priority": "p1"
}
},
"execution": {
"mode": "full_job",
"tool_permissions": ["shell"]
}
}
}
],
"input_tokens": 120,
"output_tokens": 40
}
},
{
"response": {
"type": "tool_calls",
"tool_calls": [
{
"id": "call_ee_grouped_1",
"name": "event_emit",
"arguments": {
"event_source": "github",
"event_type": "issue.opened",
"payload": {
"repository": "nearai/ironclaw",
"priority": "p1",
"issue_number": 123,
"title": "Support grouped routine create requests"
}
}
}
],
"input_tokens": 180,
"output_tokens": 30
}
},
{
"response": {
"type": "text",
"content": "Created the grouped system-event routine and emitted a matching GitHub event.",
"input_tokens": 230,
"output_tokens": 18
}
}
]
}
tests/fixtures/llm_traces/tools/tool_info_discovery.json
Vendored
+18 -4
View File
@@ -24,6 +24,20 @@
"output_tokens": 20
}
},
{
"response": {
"type": "tool_calls",
"tool_calls": [
{
"id": "call_tool_info_routine_create",
"name": "tool_info",
"arguments": { "name": "routine_create", "detail": "summary" }
}
],
"input_tokens": 160,
"output_tokens": 25
}
},
{
"response": {
"type": "tool_calls",
@@ -34,16 +48,16 @@
"arguments": { "name": "time", "include_schema": true }
}
],
"input_tokens": 200,
"input_tokens": 240,
"output_tokens": 20
}
},
{
"response": {
"type": "text",
"content": "I found the info for both tools. The echo tool has a 'message' parameter. The time tool accepts an 'operation' parameter with options like 'now', 'parse', and 'diff'.",
"input_tokens": 400,
"output_tokens": 40
"content": "I found the info for all three tools. The echo tool has a 'message' parameter. routine_create's summary explains that cron needs request.schedule, message_event needs request.pattern, and system_event needs request.source plus request.event_type. The time tool accepts an 'operation' parameter with options like 'now', 'parse', and 'diff'.",
"input_tokens": 520,
"output_tokens": 60
}
}
]