mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-30 01:19:34 +00:00
feat: Add benchmarking harness for agent evaluation
Introduces ironclaw-bench, a Rust-native benchmarking crate that drives the real agent loop headlessly. Supports standard benchmarks (GAIA, Tau-bench, SWE-bench Pro) and custom JSONL task sets with parallel execution, resume support, and incremental JSONL output. Key components: - BenchChannel: headless Channel impl with auto-approval and response capture - InstrumentedLlm: LlmProvider wrapper recording per-call token/cost metrics - BenchRunner: task orchestration with parallel execution and JSONL resume - Scoring utilities: exact match, contains, regex (all with normalization) - CLI: run, results, compare, list subcommands via clap - Four suite adapters: custom, gaia, tau_bench, swe_bench Also fixes a pre-existing missing SseEvent::ToolResult match arm in the web gateway and adds FinishReason to the LLM module's public re-exports. Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
a35db4d32d
commit
285616d1cd
@@ -314,6 +314,7 @@ impl WsServerMessage {
|
||||
SseEvent::Thinking { .. } => "thinking",
|
||||
SseEvent::ToolStarted { .. } => "tool_started",
|
||||
SseEvent::ToolCompleted { .. } => "tool_completed",
|
||||
SseEvent::ToolResult { .. } => "tool_result",
|
||||
SseEvent::StreamChunk { .. } => "stream_chunk",
|
||||
SseEvent::Status { .. } => "status",
|
||||
SseEvent::ApprovalNeeded { .. } => "approval_needed",
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ pub mod session;
|
||||
pub use nearai::{ModelInfo, NearAiProvider};
|
||||
pub use nearai_chat::NearAiChatProvider;
|
||||
pub use provider::{
|
||||
ChatMessage, CompletionRequest, CompletionResponse, LlmProvider, Role, ToolCall,
|
||||
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, Role, ToolCall,
|
||||
ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult,
|
||||
};
|
||||
pub use reasoning::{ActionPlan, Reasoning, ReasoningContext, RespondResult, ToolSelection};
|
||||
|
||||
Reference in New Issue
Block a user