mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-09-01 00:59:33 +00:00
Illia PolosukhinandClaude Opus 4.6
285616d1cd
Introduces ironclaw-bench, a Rust-native benchmarking crate that drives the real agent loop headlessly. Supports standard benchmarks (GAIA, Tau-bench, SWE-bench Pro) and custom JSONL task sets with parallel execution, resume support, and incremental JSONL output. Key components: - BenchChannel: headless Channel impl with auto-approval and response capture - InstrumentedLlm: LlmProvider wrapper recording per-call token/cost metrics - BenchRunner: task orchestration with parallel execution and JSONL resume - Scoring utilities: exact match, contains, regex (all with normalization) - CLI: run, results, compare, list subcommands via clap - Four suite adapters: custom, gaia, tau_bench, swe_bench Also fixes a pre-existing missing SseEvent::ToolResult match arm in the web gateway and adds FinishReason to the LLM module's public re-exports. Co-Authored-By: Claude Opus 4.6 <[email protected]>
50 lines
988 B
TOML
50 lines
988 B
TOML
[package]
|
|
name = "ironclaw-bench"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.85"
|
|
description = "Benchmarking harness for IronClaw agent"
|
|
license = "MIT OR Apache-2.0"
|
|
|
|
[[bin]]
|
|
name = "ironclaw-bench"
|
|
path = "src/main.rs"
|
|
|
|
[dependencies]
|
|
ironclaw = { path = ".." }
|
|
|
|
# Async runtime
|
|
tokio = { version = "1", features = ["full"] }
|
|
tokio-stream = { version = "0.1", features = ["sync"] }
|
|
futures = "0.3"
|
|
|
|
# Serialization
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
toml = "0.8"
|
|
|
|
# CLI
|
|
clap = { version = "4", features = ["derive"] }
|
|
|
|
# Core types
|
|
uuid = { version = "1", features = ["v4", "serde"] }
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
rust_decimal = { version = "1", features = ["serde", "serde-with-str"] }
|
|
|
|
# Error handling
|
|
thiserror = "2"
|
|
anyhow = "1"
|
|
|
|
# Async traits
|
|
async-trait = "0.1"
|
|
|
|
# Logging
|
|
tracing = "0.1"
|
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
|
|
|
# Scoring
|
|
regex = "1"
|
|
|
|
[dev-dependencies]
|
|
tempfile = "3"
|