mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-26 15:40:18 +00:00
Compare commits
124
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f18422b88 | ||
|
|
448383cfb0 | ||
|
|
7df356c109 | ||
|
|
3829d81269 | ||
|
|
8a4f3b6f88 | ||
|
|
140f29decf | ||
|
|
5725a62c83 | ||
|
|
bfe393eb38 | ||
|
|
9906190de7 | ||
|
|
9349a3baca | ||
|
|
3f135bdde9 | ||
|
|
97a7637f30 | ||
|
|
dae26d640e | ||
|
|
fa64df05ff | ||
|
|
356f56f77c | ||
|
|
17434d6499 | ||
|
|
3f58ed6232 | ||
|
|
097a26ace6 | ||
|
|
e87d7bd066 | ||
|
|
e42b1e5ec1 | ||
|
|
ccf60055f4 | ||
|
|
89fdd81420 | ||
|
|
fd46cbd30d | ||
|
|
8dbb0996da | ||
|
|
ae714b5003 | ||
|
|
5416866bcf | ||
|
|
c18f6730f8 | ||
|
|
479ca888a2 | ||
|
|
5c9546602b | ||
|
|
ffb1cc9be8 | ||
|
|
6330f1b27a | ||
|
|
9e6e1471ab | ||
|
|
2d3eb4de9a | ||
|
|
913073d83d | ||
|
|
d46ab3a1d7 | ||
|
|
05cb01816b | ||
|
|
750a94030b | ||
|
|
c3340c60ef | ||
|
|
3669a7b1cd | ||
|
|
96d5fc0d39 | ||
|
|
c1926c83d9 | ||
|
|
a1b0e34b3b | ||
|
|
cfb579a4bb | ||
|
|
bac2d75713 | ||
|
|
8e6e84a08d | ||
|
|
a158eee1b0 | ||
|
|
436dda0f2f | ||
|
|
c1ca3bb91c | ||
|
|
e499795b8c | ||
|
|
5e1da4827a | ||
|
|
d04af5cd75 | ||
|
|
956037c4d3 | ||
|
|
68a1851c19 | ||
|
|
dfa105539b | ||
|
|
8929baf76a | ||
|
|
e07dfab449 | ||
|
|
6783cba4e4 | ||
|
|
63302ab406 | ||
|
|
7c553b0973 | ||
|
|
5e44185e48 | ||
|
|
72623c9e5b | ||
|
|
6895adbcc9 | ||
|
|
f1480f471b | ||
|
|
9db949746f | ||
|
|
61a123a746 | ||
|
|
0e981429ee | ||
|
|
1b38a64e15 | ||
|
|
2e5f8b60d5 | ||
|
|
f0a0642e7d | ||
|
|
ca8d5c6b5e | ||
|
|
9fed8453c7 | ||
|
|
eaef335db6 | ||
|
|
225af29db2 | ||
|
|
a53b2c10b5 | ||
|
|
408ae8a29a | ||
|
|
d9ff86d7e0 | ||
|
|
e843c18141 | ||
|
|
54e9206f0b | ||
|
|
5df0d13b59 | ||
|
|
bbb68f7490 | ||
|
|
b3dee13954 | ||
|
|
33ef0a6ea5 | ||
|
|
e0a43c81f9 | ||
|
|
bada79ba4a | ||
|
|
a70c89d9e3 | ||
|
|
247445f819 | ||
|
|
14254a699f | ||
|
|
2039442885 | ||
|
|
e796b838fa | ||
|
|
54ce7434fb | ||
|
|
e9e0374c85 | ||
|
|
5582a0dfbf | ||
|
|
1cf08a4b42 | ||
|
|
d55b302b39 | ||
|
|
517be42ccc | ||
|
|
09198c68ab | ||
|
|
115b7f38fe | ||
|
|
bb228f6315 | ||
|
|
45f547c711 | ||
|
|
23de75d75b | ||
|
|
ced83d5b4d | ||
|
|
810ba58fd2 | ||
|
|
202665a55c | ||
|
|
a35db4d32d | ||
|
|
e6725eb6d9 | ||
|
|
91e27b7395 | ||
|
|
642c320b13 | ||
|
|
6831a54793 | ||
|
|
6bcc168ec5 | ||
|
|
bf3b8b339f | ||
|
|
2cdd04a359 | ||
|
|
9b729795fb | ||
|
|
a351711312 | ||
|
|
f34a80191e | ||
|
|
9d156411fc | ||
|
|
8439293df3 | ||
|
|
8be390afab | ||
|
|
4d0fe7d37e | ||
|
|
cb987321a9 | ||
|
|
a93c7ed893 | ||
|
|
3c54e692a5 | ||
|
|
4f8fd4ad5f | ||
|
|
7fcc2279cc | ||
|
|
0ca05e3de3 |
@@ -0,0 +1,95 @@
|
||||
---
|
||||
description: Scaffold a new SSE event end-to-end (Rust backend to web frontend)
|
||||
allowed-tools: Read, Edit, Write, Glob, Grep, Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*)
|
||||
argument-hint: <event_name> [description]
|
||||
model: opus
|
||||
---
|
||||
|
||||
Add a new SSE event called `$ARGUMENTS` to the IronClaw web gateway. This involves changes across 5 files in a specific order. Follow each step exactly.
|
||||
|
||||
## Step 1: Add `StatusUpdate` variant
|
||||
|
||||
**File**: `src/channels/channel.rs`
|
||||
|
||||
Find the `StatusUpdate` enum and add a new variant. Use the event name in PascalCase. Include any fields the event needs as named fields (not a generic String).
|
||||
|
||||
Example for reference (existing variants):
|
||||
```rust
|
||||
pub enum StatusUpdate {
|
||||
Thinking(String),
|
||||
ToolStarted { name: String },
|
||||
ToolCompleted { name: String, success: bool },
|
||||
Status(String),
|
||||
ApprovalNeeded {
|
||||
request_id: String,
|
||||
tool_name: String,
|
||||
description: String,
|
||||
parameters: serde_json::Value,
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
## Step 2: Map to `SseEvent` in web channel
|
||||
|
||||
**File**: `src/channels/web/mod.rs`
|
||||
|
||||
Find the `send_status` method in the `Channel` impl for `WebChannel`. Add a match arm for the new `StatusUpdate` variant that maps it to an `SseEvent`. The SSE event name should be snake_case.
|
||||
|
||||
Look at existing match arms for the pattern. The event data is serialized as JSON.
|
||||
|
||||
## Step 3: Add types if needed
|
||||
|
||||
**File**: `src/channels/web/types.rs`
|
||||
|
||||
If the event carries structured data beyond a simple string, add a serializable DTO struct here. Use `#[derive(Debug, Clone, Serialize, Deserialize)]`. Follow the existing patterns in the file.
|
||||
|
||||
## Step 4: Add frontend handler
|
||||
|
||||
**File**: `src/channels/web/static/app.js`
|
||||
|
||||
In the `connectSSE()` function, add a new `eventSource.addEventListener()` for the snake_case event name. Parse the JSON data and call a handler function.
|
||||
|
||||
Create the handler function that updates the DOM. Follow existing patterns:
|
||||
- `showApproval(data)` for complex card-style UI
|
||||
- `addMessage(role, content)` for simple text
|
||||
- `setStatus(text, spinning)` for status bar updates
|
||||
|
||||
## Step 5: Add CSS if needed
|
||||
|
||||
**File**: `src/channels/web/static/style.css`
|
||||
|
||||
If the event needs custom UI (cards, badges, etc.), add styles. Follow the existing naming conventions (`.approval-card`, `.log-entry`, etc.).
|
||||
|
||||
## Step 6: Send the event from Rust
|
||||
|
||||
Identify where in the backend this event should be triggered. Common locations:
|
||||
- `src/agent/agent_loop.rs` - During message processing or tool execution
|
||||
- `src/agent/worker.rs` - During job execution
|
||||
- `src/agent/heartbeat.rs` - During periodic execution
|
||||
|
||||
Use the existing pattern:
|
||||
```rust
|
||||
let _ = self.channels.send_status(
|
||||
&message.channel,
|
||||
StatusUpdate::YourNewVariant { ... },
|
||||
&message.metadata,
|
||||
).await;
|
||||
```
|
||||
|
||||
## Step 7: Quality gate
|
||||
|
||||
Run `cargo fmt` and `cargo clippy --all --benches --tests --examples --all-features` to verify the changes compile cleanly.
|
||||
|
||||
## Checklist
|
||||
|
||||
Before finishing, verify:
|
||||
- [ ] `StatusUpdate` variant added in `channel.rs`
|
||||
- [ ] Match arm added in `web/mod.rs` `send_status`
|
||||
- [ ] DTO added in `types.rs` (if needed)
|
||||
- [ ] `addEventListener` added in `app.js`
|
||||
- [ ] Handler function created in `app.js`
|
||||
- [ ] CSS styles added (if needed)
|
||||
- [ ] Event sent from appropriate backend location
|
||||
- [ ] `cargo fmt` clean
|
||||
- [ ] `cargo clippy` clean
|
||||
- [ ] Non-web channels unaffected (they ignore unknown StatusUpdate variants)
|
||||
@@ -0,0 +1,382 @@
|
||||
---
|
||||
description: Scaffold a new tool (WASM or built-in Rust) with all boilerplate wired up
|
||||
allowed-tools: Read, Edit, Write, Glob, Grep, Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*), Bash(cargo component:*), Bash(ls:*), Bash(mkdir:*)
|
||||
argument-hint: <tool_name> [description]
|
||||
model: opus
|
||||
---
|
||||
|
||||
Scaffold a new tool called `$ARGUMENTS` for the IronClaw agent. First, determine the tool type and then follow the appropriate path.
|
||||
|
||||
## Step 0: Determine tool type
|
||||
|
||||
Ask the user which type of tool to create:
|
||||
|
||||
- **WASM tool** (recommended) - Sandboxed, dynamically loadable, external API integrations. Lives in `tools-src/<name>/`. This is the right choice for anything that talks to an external service (Notion, GitHub, Discord, etc.).
|
||||
- **Built-in tool** - Compiled into the main binary. Only for core agent infrastructure (e.g., memory, file ops, shell). Lives in `src/tools/builtin/<name>.rs`.
|
||||
|
||||
If the description clearly implies an external service integration, default to WASM. If it's a core agent capability, default to built-in.
|
||||
|
||||
---
|
||||
|
||||
## Path A: WASM Tool
|
||||
|
||||
### A1: Create directory structure
|
||||
|
||||
Create `tools-src/<name>/` with:
|
||||
|
||||
```
|
||||
tools-src/<name>/
|
||||
├── Cargo.toml
|
||||
├── <name>-tool.capabilities.json
|
||||
└── src/
|
||||
├── lib.rs
|
||||
├── types.rs
|
||||
└── api.rs
|
||||
```
|
||||
|
||||
### A2: Write `Cargo.toml`
|
||||
|
||||
Follow this exact pattern (adjust name and description):
|
||||
|
||||
```toml
|
||||
[package]
|
||||
name = "<name>-tool"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "<Description> tool for IronClaw (WASM component)"
|
||||
license = "MIT OR Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
wit-bindgen = "=0.36"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
[profile.release]
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
```
|
||||
|
||||
### A3: Write `<name>-tool.capabilities.json`
|
||||
|
||||
Declare the tool's security requirements. Determine what APIs it needs and create the allowlist. Reference `tools-src/slack/slack-tool.capabilities.json` for the format.
|
||||
|
||||
Key sections to include:
|
||||
- `http.allowlist` - API endpoints (host, path_prefix, methods)
|
||||
- `http.credentials` - Secret injection config (secret_name, location type: bearer/header/query)
|
||||
- `http.rate_limit` - requests_per_minute, requests_per_hour
|
||||
- `http.timeout_secs`
|
||||
- `secrets.allowed_names` - Which secrets the tool can check existence of
|
||||
- `auth` - Authentication setup (OAuth or manual token entry)
|
||||
|
||||
If the tool needs OAuth, include:
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "<service>_token",
|
||||
"display_name": "<Service>",
|
||||
"oauth": {
|
||||
"authorization_url": "https://...",
|
||||
"token_url": "https://...",
|
||||
"client_id_env": "<SERVICE>_OAUTH_CLIENT_ID",
|
||||
"client_secret_env": "<SERVICE>_OAUTH_CLIENT_SECRET",
|
||||
"scopes": [],
|
||||
"use_pkce": false
|
||||
},
|
||||
"env_var": "<SERVICE>_TOKEN"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If no OAuth, include manual setup instructions:
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "<service>_api_key",
|
||||
"display_name": "<Service>",
|
||||
"instructions": "Get your API key from <url>",
|
||||
"setup_url": "https://...",
|
||||
"token_hint": "Starts with '<prefix>'",
|
||||
"env_var": "<SERVICE>_API_KEY"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### A4: Write `src/types.rs`
|
||||
|
||||
Define the action enum using serde's tagged enum pattern:
|
||||
|
||||
```rust
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "action", rename_all = "snake_case")]
|
||||
pub enum <Name>Action {
|
||||
// Add variants based on the tool's capabilities.
|
||||
// Each variant maps to one API operation.
|
||||
}
|
||||
```
|
||||
|
||||
Add result structs with `#[derive(Debug, Serialize)]`. Use `#[serde(skip_serializing_if = "Option::is_none")]` for optional fields.
|
||||
|
||||
### A5: Write `src/api.rs`
|
||||
|
||||
Implement the API calls using the host HTTP capability:
|
||||
|
||||
```rust
|
||||
use crate::near::agent::host;
|
||||
use crate::types::*;
|
||||
|
||||
const API_BASE: &str = "https://api.example.com";
|
||||
|
||||
fn api_call(method: &str, endpoint: &str, body: Option<&str>) -> Result<String, String> {
|
||||
let url = format!("{}/{}", API_BASE, endpoint);
|
||||
let headers = if body.is_some() {
|
||||
r#"{"Content-Type": "application/json"}"#
|
||||
} else {
|
||||
"{}"
|
||||
};
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec());
|
||||
|
||||
host::log(host::LogLevel::Debug, &format!("API: {} {}", method, endpoint));
|
||||
|
||||
let response = host::http_request(method, &url, headers, body_bytes.as_deref())?;
|
||||
|
||||
if response.status < 200 || response.status >= 300 {
|
||||
return Err(format!(
|
||||
"API returned status {}: {}",
|
||||
response.status,
|
||||
String::from_utf8_lossy(&response.body)
|
||||
));
|
||||
}
|
||||
|
||||
String::from_utf8(response.body).map_err(|e| format!("Invalid UTF-8: {}", e))
|
||||
}
|
||||
```
|
||||
|
||||
Add one function per action variant that calls `api_call` and parses the response into the result structs.
|
||||
|
||||
### A6: Write `src/lib.rs`
|
||||
|
||||
Wire everything together:
|
||||
|
||||
```rust
|
||||
mod api;
|
||||
mod types;
|
||||
|
||||
use types::<Name>Action;
|
||||
|
||||
wit_bindgen::generate!({
|
||||
world: "sandboxed-tool",
|
||||
path: "../../wit/tool.wit",
|
||||
});
|
||||
|
||||
struct <Name>Tool;
|
||||
|
||||
impl exports::near::agent::tool::Guest for <Name>Tool {
|
||||
fn execute(req: exports::near::agent::tool::Request) -> exports::near::agent::tool::Response {
|
||||
match execute_inner(&req.params) {
|
||||
Ok(result) => exports::near::agent::tool::Response {
|
||||
output: Some(result),
|
||||
error: None,
|
||||
},
|
||||
Err(e) => exports::near::agent::tool::Response {
|
||||
output: None,
|
||||
error: Some(e),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn schema() -> String {
|
||||
// Return JSON Schema matching the action enum
|
||||
todo!("Fill in JSON Schema")
|
||||
}
|
||||
|
||||
fn description() -> String {
|
||||
"<Description>".to_string()
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_inner(params: &str) -> Result<String, String> {
|
||||
// Check required secrets
|
||||
if !crate::near::agent::host::secret_exists("<secret_name>") {
|
||||
return Err("<Secret> not configured. Please add the '<secret_name>' secret.".to_string());
|
||||
}
|
||||
|
||||
let action: <Name>Action =
|
||||
serde_json::from_str(params).map_err(|e| format!("Invalid parameters: {}", e))?;
|
||||
|
||||
crate::near::agent::host::log(
|
||||
crate::near::agent::host::LogLevel::Info,
|
||||
&format!("Executing action: {:?}", action),
|
||||
);
|
||||
|
||||
let result = match action {
|
||||
// Dispatch to api:: functions for each variant
|
||||
};
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
export!(<Name>Tool);
|
||||
```
|
||||
|
||||
Fill in the `schema()` with a proper JSON Schema using `oneOf` for each action variant. Reference `tools-src/slack/src/lib.rs` for the exact pattern.
|
||||
|
||||
### A7: Verify
|
||||
|
||||
Run `cargo fmt` in the tool directory. If `cargo-component` is available, run `cargo component build --release` to verify the WASM compiles.
|
||||
|
||||
---
|
||||
|
||||
## Path B: Built-in Tool
|
||||
|
||||
### B1: Create the tool file
|
||||
|
||||
Create `src/tools/builtin/<name>.rs` implementing the `Tool` trait:
|
||||
|
||||
```rust
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::context::JobContext;
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
|
||||
pub struct <Name>Tool;
|
||||
|
||||
#[async_trait]
|
||||
impl Tool for <Name>Tool {
|
||||
fn name(&self) -> &str {
|
||||
"<snake_case_name>"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"<Description>"
|
||||
}
|
||||
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
// Define parameters here
|
||||
},
|
||||
"required": []
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute(
|
||||
&self,
|
||||
params: serde_json::Value,
|
||||
_ctx: &JobContext,
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
// Extract and validate parameters
|
||||
// Do the work
|
||||
// Return result
|
||||
|
||||
Ok(ToolOutput::text("result", start.elapsed()))
|
||||
}
|
||||
|
||||
fn requires_sanitization(&self) -> bool {
|
||||
false // Set true if tool processes external data
|
||||
}
|
||||
|
||||
fn requires_approval(&self) -> bool {
|
||||
false // Set true if tool is destructive or contacts external services
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If the tool needs shared state (HTTP client, config), add a struct field and `new()` constructor:
|
||||
|
||||
```rust
|
||||
pub struct <Name>Tool {
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl <Name>Tool {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(30))
|
||||
.build()
|
||||
.expect("Failed to create HTTP client"),
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### B2: Update `src/tools/builtin/mod.rs`
|
||||
|
||||
Add the module declaration and pub use, keeping alphabetical order:
|
||||
|
||||
```rust
|
||||
mod <name>;
|
||||
pub use <name>::<Name>Tool;
|
||||
```
|
||||
|
||||
### B3: Update `src/tools/registry.rs`
|
||||
|
||||
Add the import to the `use crate::tools::builtin::{...}` block and register the tool in the appropriate registration method:
|
||||
|
||||
- If it's a core tool: add to `register_builtin_tools()`
|
||||
- If it needs shared state (workspace, context_manager, etc.): create a new `register_<category>_tools()` method or add to an existing one
|
||||
- Wire the new registration call in `src/main.rs` if a new method was created
|
||||
|
||||
### B4: Add tests
|
||||
|
||||
Add a `mod tests {}` block at the bottom of the tool file:
|
||||
|
||||
```rust
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::context::JobContext;
|
||||
|
||||
fn test_context() -> JobContext {
|
||||
JobContext::test_default()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_<name>_basic() {
|
||||
let tool = <Name>Tool::new();
|
||||
let params = serde_json::json!({ /* test params */ });
|
||||
let result = tool.execute(params, &test_context()).await;
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_<name>_missing_params() {
|
||||
let tool = <Name>Tool::new();
|
||||
let params = serde_json::json!({});
|
||||
let result = tool.execute(params, &test_context()).await;
|
||||
assert!(matches!(result, Err(ToolError::InvalidParameters(_))));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### B5: Quality gate
|
||||
|
||||
Run `cargo fmt` and `cargo clippy --all --benches --tests --examples --all-features`. Fix any issues.
|
||||
|
||||
Run the new tests: `cargo test --lib -- builtin::<name>::tests`
|
||||
|
||||
---
|
||||
|
||||
## Checklist
|
||||
|
||||
Before finishing, verify:
|
||||
- [ ] Tool type chosen (WASM or built-in) and confirmed with user
|
||||
- [ ] All files created with correct structure
|
||||
- [ ] For WASM: capabilities.json declares all needed permissions (HTTP, secrets, auth)
|
||||
- [ ] For WASM: JSON Schema in `schema()` matches the action enum variants
|
||||
- [ ] For built-in: mod.rs updated with module + pub use
|
||||
- [ ] For built-in: registry.rs imports and registers the tool
|
||||
- [ ] For built-in: tests added and passing
|
||||
- [ ] `cargo fmt` clean
|
||||
- [ ] `cargo clippy` clean (for built-in) or `cargo component build` clean (for WASM)
|
||||
@@ -0,0 +1,97 @@
|
||||
---
|
||||
description: Fetch a GitHub issue, create a branch, research the codebase, plan the fix, implement with tests, and commit
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(gh issue view:*), Bash(gh repo view:*), Bash(git fetch:*), Bash(git checkout:*), Bash(git status:*), Bash(git branch:*), Bash(git add:*), Bash(git commit:*), Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*), Read, Edit, Write, Grep, Glob
|
||||
argument-hint: "<issue-number or github-issue-url>"
|
||||
---
|
||||
|
||||
# Fix GitHub Issue
|
||||
|
||||
## Step 1: Resolve the issue
|
||||
|
||||
Parse `$ARGUMENTS` to extract the issue number:
|
||||
- If it's a URL like `https://github.com/owner/repo/issues/42`, extract `42`.
|
||||
- If it's a bare number, use it directly.
|
||||
- If empty, stop and ask the user for an issue number.
|
||||
|
||||
Fetch the issue:
|
||||
|
||||
```
|
||||
gh issue view {number} --json title,body,labels,assignees,comments,state
|
||||
```
|
||||
|
||||
If the issue is closed, warn the user and ask if they still want to proceed.
|
||||
|
||||
## Step 2: Create a branch
|
||||
|
||||
Create a fresh branch off the latest main:
|
||||
|
||||
1. Fetch latest: `git fetch origin`
|
||||
2. Detect default branch: `gh repo view --json defaultBranchRef --jq .defaultBranchRef.name`
|
||||
3. Create and switch to a new branch: `git checkout -b fix/{number}-{short-slug} origin/{default-branch}`
|
||||
- `{short-slug}` is 3-5 words from the issue title, lowercase, hyphenated (e.g. `fix/42-idor-workspace-check`)
|
||||
|
||||
If the working tree has uncommitted changes, warn the user and stop. Do not stash or discard their work.
|
||||
|
||||
## Step 3: Understand the issue
|
||||
|
||||
Summarize the issue in 2-3 sentences. Identify:
|
||||
- **What's broken or missing** (the symptom or feature request)
|
||||
- **Acceptance criteria** (what "done" looks like, from the issue body or comments)
|
||||
- **Constraints** (mentioned technologies, backward compatibility, performance requirements)
|
||||
|
||||
If the issue is unclear or ambiguous, list the open questions. These will be addressed during planning.
|
||||
|
||||
## Step 4: Research the codebase
|
||||
|
||||
Before planning, gather context:
|
||||
|
||||
1. **Find relevant code** - Search for files, functions, types, and patterns mentioned in the issue. Read them in full.
|
||||
2. **Trace the flow** - If the issue is about a specific behavior, trace the code path from the entry point (route handler, CLI command, etc.) through to the relevant logic.
|
||||
3. **Check existing tests** - Find tests related to the affected code. Understand what's already covered.
|
||||
4. **Check for prior art** - Look for similar patterns in the codebase that solve analogous problems. Prefer consistency with existing patterns.
|
||||
|
||||
## Step 5: Enter planning mode
|
||||
|
||||
Enter planning mode to design the implementation. The plan MUST cover:
|
||||
|
||||
1. **Root cause** (for bugs) or **design approach** (for features)
|
||||
2. **Files to modify** with specific descriptions of what changes in each
|
||||
3. **New files** (if any) with justification for why they're needed
|
||||
4. **Tests to add** - every code path introduced or changed needs a test:
|
||||
- Happy path (expected input produces expected output)
|
||||
- Error paths (invalid input, missing data, permission denied)
|
||||
- Edge cases (empty collections, boundary values, concurrent access)
|
||||
5. **IronClaw-specific concerns**:
|
||||
- If the change touches persistence, both database backends must be updated (`postgres.rs` and `libsql_backend.rs`)
|
||||
- New `Database` trait methods need implementations in both backends
|
||||
- No `.unwrap()` or `.expect()` in production code
|
||||
- Use `crate::` imports, not `super::`
|
||||
- Error types via `thiserror` in `error.rs`
|
||||
6. **Migration or compatibility concerns** (if any)
|
||||
|
||||
Follow the project's CLAUDE.md guidance for architecture decisions.
|
||||
|
||||
Wait for user approval before implementing.
|
||||
|
||||
## Step 6: Implement
|
||||
|
||||
After the plan is approved:
|
||||
|
||||
1. Implement each change from the plan.
|
||||
2. Write all planned tests.
|
||||
3. Run IronClaw's full quality gate:
|
||||
- `cargo fmt`
|
||||
- `cargo clippy --all --benches --tests --examples --all-features` (zero warnings)
|
||||
- `cargo test --lib` (all tests pass)
|
||||
4. If any check fails, fix it before proceeding.
|
||||
|
||||
Note: Integration tests (`--test workspace_integration`) require PostgreSQL and are expected to fail locally. Only `--lib` test failures are blocking.
|
||||
|
||||
## Step 7: Commit and summarize
|
||||
|
||||
1. Commit with a descriptive message referencing the issue (e.g. `fix: prevent IDOR in function call outputs (#42)`).
|
||||
2. Summarize what was done:
|
||||
- Files changed with line references
|
||||
- Tests added and what they cover
|
||||
- Any follow-up work or open questions
|
||||
@@ -0,0 +1,81 @@
|
||||
---
|
||||
description: Respond to PR review comments — triage, plan fixes, implement after confirmation, push, and reply to reviewers
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(gh pr list:*), Bash(gh pr comment:*), Bash(gh api:*), Bash(gh repo view:*), Bash(git branch:*), Bash(git status:*), Bash(git add:*), Bash(git commit:*), Bash(git push:*), Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*), Read, Edit, Write, Grep, Glob
|
||||
argument-hint: "[pr-number (optional, auto-detects from branch)]"
|
||||
---
|
||||
|
||||
# Review and Address PR Comments
|
||||
|
||||
## Step 1: Find the PR
|
||||
|
||||
If `$ARGUMENTS` is provided, use that as the PR number. Otherwise, detect the PR for the current branch:
|
||||
|
||||
```
|
||||
gh pr list --head $(git branch --show-current) --json number,title,url --jq '.[0]'
|
||||
```
|
||||
|
||||
If no PR is found, tell the user and stop.
|
||||
|
||||
## Step 2: Fetch all review comments
|
||||
|
||||
Resolve the repo owner and name:
|
||||
|
||||
```
|
||||
gh repo view --json owner,name --jq '"\(.owner.login)/\(.name)"'
|
||||
```
|
||||
|
||||
Fetch the full set of review comments (not issue-level comments):
|
||||
|
||||
```
|
||||
gh api --paginate repos/{owner}/{repo}/pulls/{number}/comments
|
||||
```
|
||||
|
||||
Also fetch the review summaries:
|
||||
|
||||
```
|
||||
gh api --paginate repos/{owner}/{repo}/pulls/{number}/reviews
|
||||
```
|
||||
|
||||
Deduplicate comments that appear multiple times (bots sometimes post the same finding under different IDs). Group by the actual issue being raised, not by comment ID.
|
||||
|
||||
## Step 3: Triage and plan
|
||||
|
||||
For each unique issue raised in the comments:
|
||||
|
||||
1. **Check if already addressed** - Read the current code at the referenced location. If a prior commit already fixed it, note it as "already resolved".
|
||||
2. **Assess validity** - Determine if the comment identifies a real problem or is a false positive. Be honest about false positives but explain why.
|
||||
3. **Classify severity** - Critical (security/data loss), High (bugs/broken behavior), Medium (correctness/robustness), Low (style/naming/nits).
|
||||
4. **Plan the fix** - For each valid unresolved issue, describe the specific code change needed.
|
||||
|
||||
Present the plan as a table to the user:
|
||||
|
||||
| # | Issue | File:Line | Severity | Status | Planned Fix |
|
||||
|---|-------|-----------|----------|--------|-------------|
|
||||
|
||||
Wait for user confirmation before proceeding to implementation.
|
||||
|
||||
## Step 4: Implement fixes
|
||||
|
||||
After user confirms:
|
||||
|
||||
1. Implement each fix in the plan.
|
||||
2. Run IronClaw's quality gate to verify nothing breaks:
|
||||
- `cargo fmt`
|
||||
- `cargo clippy --all --benches --tests --examples --all-features`
|
||||
- `cargo test --lib`
|
||||
3. Commit with a descriptive message referencing the PR review.
|
||||
4. Push to the branch.
|
||||
|
||||
## Step 5: Reply to comments
|
||||
|
||||
For each comment addressed, reply on the PR with a short message stating what was fixed and the commit SHA. For false positives or already-resolved items, reply explaining why no change was needed.
|
||||
|
||||
## Rules
|
||||
|
||||
- Never guess at code you haven't read. Always read the referenced file and line before assessing a comment.
|
||||
- Group duplicate comments (same issue reported by multiple bots) and reply to all of them.
|
||||
- Do not make changes beyond what the review comments ask for. Stay focused.
|
||||
- If a comment suggests a change you disagree with, present your reasoning to the user during the planning phase rather than silently ignoring it.
|
||||
- Follow IronClaw conventions: no `.unwrap()` in production code, use `crate::` imports, `thiserror` errors.
|
||||
- If changes touch persistence, verify both database backends are updated.
|
||||
@@ -0,0 +1,245 @@
|
||||
---
|
||||
description: Deep audit of the IronClaw crate for vulnerabilities, bugs, unfinished work, inconsistencies, and oversights
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*), Bash(cargo audit:*), Bash(git diff:*), Bash(git log:*), Bash(git show:*), Bash(wc:*), Read, Grep, Glob, Task
|
||||
argument-hint: "[path/to/crate]"
|
||||
---
|
||||
|
||||
# Rust Crate Audit
|
||||
|
||||
You are performing a thorough audit of a Rust crate. Your goal is to find every vulnerability, bug, unfinished piece of work, inconsistency, and oversight before it ships. Leave no stone unturned.
|
||||
|
||||
## Step 1: Locate the crate
|
||||
|
||||
Parse `$ARGUMENTS`:
|
||||
- If a path is provided, use it as the crate root.
|
||||
- If empty, use the current working directory.
|
||||
|
||||
Verify it's a valid Rust crate by checking for `Cargo.toml`. If not found, stop and ask the user.
|
||||
|
||||
## Step 2: Understand the crate
|
||||
|
||||
Read `Cargo.toml` to understand:
|
||||
- Crate name, version, edition
|
||||
- Dependencies (look for outdated, unmaintained, or suspicious crates)
|
||||
- Feature flags and their implications
|
||||
- Build scripts (`build.rs`) if any
|
||||
|
||||
Read `CLAUDE.md`, `README.md`, or top-level documentation if present to understand intent and architecture.
|
||||
|
||||
Read `src/lib.rs` or `src/main.rs` to get the module tree. Then read each module's `mod.rs` or top-level file to build a mental map of the crate's structure before diving into details.
|
||||
Read all Rust files (`src/*.rs`) to make sure everything is in context when you are reasoning.
|
||||
|
||||
## Step 3: Run the compiler's checks
|
||||
|
||||
Run these commands and capture output. Do NOT fix anything, just collect findings:
|
||||
|
||||
```
|
||||
cargo fmt --check 2>&1
|
||||
```
|
||||
|
||||
```
|
||||
cargo clippy --all --benches --tests --examples --all-features -- -W clippy::all -W clippy::pedantic -W clippy::nursery 2>&1
|
||||
```
|
||||
|
||||
```
|
||||
cargo test --lib 2>&1
|
||||
```
|
||||
|
||||
If any of these fail, record the failures as findings. If `cargo test` has ignored tests, note which ones and why.
|
||||
|
||||
Note: Integration tests (`--test workspace_integration`) require a PostgreSQL database and are expected to fail locally. Only report `--lib` test failures as blocking.
|
||||
|
||||
## Step 4: Scan for unfinished work
|
||||
|
||||
Search the entire `src/` tree for:
|
||||
|
||||
```
|
||||
todo!
|
||||
unimplemented!
|
||||
fixme
|
||||
FIXME
|
||||
TODO
|
||||
HACK
|
||||
XXX
|
||||
SAFETY:
|
||||
stub
|
||||
placeholder
|
||||
temporary
|
||||
```
|
||||
|
||||
For each match:
|
||||
- Is it in production code or test code?
|
||||
- Is it a genuine incomplete feature or a deliberate placeholder?
|
||||
- Is there a tracking issue referenced?
|
||||
- Could this panic at runtime?
|
||||
|
||||
Any `todo!()` or `unimplemented!()` in non-test code is **High severity** (runtime panic).
|
||||
|
||||
## Step 5: Audit for vulnerabilities and unsafe code
|
||||
|
||||
### 5a. Unsafe code
|
||||
|
||||
Search for all `unsafe` blocks. For each one:
|
||||
- Is the safety invariant documented with a `// SAFETY:` comment?
|
||||
- Is the invariant actually upheld by the surrounding code?
|
||||
- Could the unsafe block be replaced with a safe alternative?
|
||||
- Are there any pointer dereferences, transmutes, or FFI calls?
|
||||
|
||||
### 5b. Unwrap and panic paths
|
||||
|
||||
Search for `.unwrap()`, `.expect(`, `panic!`, `unreachable!` in non-test code. For each:
|
||||
- Can this actually panic in production?
|
||||
- Is there a code path that reaches this with None/Err?
|
||||
- Should it be replaced with proper error handling (`?`, `.ok()`, `.unwrap_or_default()`)?
|
||||
|
||||
IronClaw convention: `.unwrap()` and `.expect()` are banned in production code. Any occurrence outside `#[cfg(test)]` blocks is a **High severity** finding.
|
||||
|
||||
### 5c. SQL and injection vectors
|
||||
|
||||
Search for string formatting used in SQL queries, shell commands, or HTML:
|
||||
- `format!` used near `.execute(`, `.query(`, `Command::new(`
|
||||
- String interpolation in query construction vs parameterized queries
|
||||
- User input flowing into file paths (`Path::new`, `std::fs::`)
|
||||
|
||||
IronClaw has two database backends (PostgreSQL and libSQL). Check both for injection vectors.
|
||||
|
||||
### 5d. Cryptographic issues
|
||||
|
||||
If the crate uses crypto:
|
||||
- Are comparisons constant-time? (look for `==` on secrets/hashes vs `subtle::ConstantTimeEq`)
|
||||
- Is randomness from `OsRng` / `thread_rng` and not a fixed seed?
|
||||
- Are keys/secrets zeroized after use? (`secrecy`, `zeroize` crates)
|
||||
- Are deprecated algorithms used? (MD5, SHA1 for security, RC4, DES)
|
||||
|
||||
### 5e. Resource exhaustion
|
||||
|
||||
- Are there unbounded allocations? (`Vec` growing from user input without limits)
|
||||
- Are there unbounded loops? (retry loops without max attempts)
|
||||
- Are file reads bounded? (`std::fs::read_to_string` on user-provided paths)
|
||||
- Are timeouts set on all network operations?
|
||||
- Are there connection/resource leaks? (opened but never closed, missing `Drop`)
|
||||
|
||||
### 5f. Error handling
|
||||
|
||||
- Are errors swallowed silently? (`let _ = ...`, `.ok()` discarding errors that matter)
|
||||
- Do error types carry enough context to debug in production?
|
||||
- Are there error type mismatches? (returning generic `anyhow::Error` where a typed error would prevent confusion)
|
||||
- Is `thiserror` used consistently for error types (IronClaw convention)?
|
||||
|
||||
## Step 6: Check for inconsistencies
|
||||
|
||||
### 6a. Naming conventions
|
||||
|
||||
- Are types, functions, modules named consistently? (e.g., mixing `get_` and `fetch_`, `create_` and `new_`)
|
||||
- Do similar operations follow the same patterns?
|
||||
|
||||
### 6b. Duplicate or near-duplicate code
|
||||
|
||||
Look for:
|
||||
- Functions that do nearly the same thing with minor variations (candidates for generics or shared helpers)
|
||||
- Repeated error mapping patterns that should be extracted
|
||||
- Copy-pasted SQL queries or string templates with slight differences
|
||||
- Identical struct definitions or conversion logic in different modules
|
||||
|
||||
### 6c. API consistency
|
||||
|
||||
- Do similar functions take arguments in the same order?
|
||||
- Are return types consistent? (e.g., some functions return `Option<T>`, similar ones return `Result<T, E>`)
|
||||
- Are visibility modifiers consistent? (`pub` where it should be `pub(crate)`, or vice versa)
|
||||
|
||||
### 6d. Dead code and unused items
|
||||
|
||||
- Are there functions, structs, or modules that nothing references?
|
||||
- Are there `#[allow(dead_code)]` annotations that should be investigated?
|
||||
- Are there feature-gated items where the feature is never enabled?
|
||||
|
||||
### 6e. Import style
|
||||
|
||||
IronClaw convention: use `crate::` imports, not `super::`. Flag any `super::` imports in non-test code.
|
||||
|
||||
## Step 7: Inspect for change oversights
|
||||
|
||||
### 7a. Partial refactors
|
||||
|
||||
- Are there old patterns coexisting with new patterns?
|
||||
- Are there renamed types/functions where some call sites still use the old name via a compatibility alias?
|
||||
- Are there comments referencing behavior that no longer exists?
|
||||
|
||||
### 7b. Trait implementation gaps
|
||||
|
||||
- If a trait is defined, do all intended types implement it?
|
||||
- Are there `impl` blocks that look incomplete?
|
||||
- Are `Default` implementations sensible?
|
||||
|
||||
IronClaw key traits: `Database` (~60 methods), `Channel`, `Tool`, `LlmProvider`, `SuccessEvaluator`, `EmbeddingProvider`. If any new methods were added to `Database`, verify both `postgres.rs` and `libsql_backend.rs` implement them.
|
||||
|
||||
### 7c. Test coverage gaps
|
||||
|
||||
- Are there public functions without any test?
|
||||
- Are there error paths without tests?
|
||||
- Are there recently-changed functions where the tests still assert old behavior?
|
||||
|
||||
### 7d. Documentation drift
|
||||
|
||||
- Do doc comments match actual function behavior?
|
||||
- Are examples in doc comments still valid and compilable?
|
||||
|
||||
## Step 8: Dependency audit
|
||||
|
||||
Review `Cargo.toml` and `Cargo.lock`:
|
||||
- Are there duplicate versions of the same crate in the lock file? (potential version conflicts)
|
||||
- Are there dependencies with known security advisories? Run `cargo audit` to check (install with `cargo install cargo-audit` if not present).
|
||||
- Are there heavy dependencies used for trivial functionality?
|
||||
- Are dependency features minimal?
|
||||
|
||||
## Step 9: Present findings
|
||||
|
||||
Compile all findings into a structured report. Group by severity, then by category.
|
||||
|
||||
### Format
|
||||
|
||||
For each finding:
|
||||
|
||||
```
|
||||
### [Severity] Category: One-line summary
|
||||
|
||||
**Location:** `file_path:line_number`
|
||||
**Category:** Vulnerability | Bug | Unfinished | Inconsistency | Duplicate | Oversight | Style
|
||||
|
||||
**Description:**
|
||||
Detailed explanation of the issue, why it matters, and how it could manifest.
|
||||
|
||||
**Suggested fix:**
|
||||
Concrete suggestion with code if applicable.
|
||||
```
|
||||
|
||||
### Severity levels
|
||||
|
||||
- **Critical**: Security vulnerability, data loss, or crash in production
|
||||
- **High**: Bug that causes incorrect behavior, `todo!()`/`unimplemented!()` in prod code, or missing validation on trust boundaries
|
||||
- **Medium**: Inconsistency, duplicate code, incomplete error handling, missing tests for important paths
|
||||
- **Low**: Naming inconsistency, unnecessary complexity, documentation drift, minor dead code
|
||||
- **Nit**: Style preference, optional improvement
|
||||
|
||||
### Summary table
|
||||
|
||||
End with a summary table:
|
||||
|
||||
| # | Severity | Category | File:Line | Finding |
|
||||
|---|----------|----------|-----------|---------|
|
||||
|
||||
And a final tally: X Critical, Y High, Z Medium, W Low, V Nit.
|
||||
|
||||
## Rules
|
||||
|
||||
- Read every file before reporting on it. Never guess about code you haven't seen.
|
||||
- Be specific. "This might have issues" is worthless. "Line 42 calls `.unwrap()` on a `Result` that returns `Err` when the DB connection is dropped" is useful.
|
||||
- Distinguish certainty levels: "this IS a bug" vs "this COULD be a bug if X".
|
||||
- Don't invent problems to look thorough. If the code is solid, say so.
|
||||
- Focus on substance over style. Don't flag formatting unless it causes real confusion.
|
||||
- Respect existing project conventions (check CLAUDE.md). Don't flag patterns the project explicitly endorses.
|
||||
- When in doubt about severity, round up.
|
||||
- For large crates (>50 files), prioritize: core logic > public API > internal utilities > tests > examples.
|
||||
- Use the Task tool to parallelize file reading across modules when the crate is large.
|
||||
- Do NOT fix anything. This is a read-only audit. Report findings for the user to action.
|
||||
@@ -0,0 +1,170 @@
|
||||
---
|
||||
description: Paranoid architect review of a PR — fetches diff, reads changed files, deep review across 6 lenses, posts findings as GitHub comments
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(gh pr view:*), Bash(gh pr diff:*), Bash(gh pr comment:*), Bash(gh api:*), Bash(gh repo view:*), Bash(git diff:*), Bash(git log:*), Read, Grep, Glob
|
||||
argument-hint: "<pr-number or github-pr-url>"
|
||||
---
|
||||
|
||||
# Paranoid Architect Code Review
|
||||
|
||||
You are reviewing this PR as a paranoid architect. Your job is to find every bug, vulnerability, race condition, edge case, and undocumented assumption before it ships. Assume adversarial users, concurrent access, and Murphy's law.
|
||||
|
||||
## Step 1: Resolve the PR
|
||||
|
||||
Parse `$ARGUMENTS` to extract the PR number:
|
||||
- If it's a URL like `https://github.com/owner/repo/pull/123`, extract `123`.
|
||||
- If it's a bare number, use it directly.
|
||||
- If empty, stop and ask the user for a PR number.
|
||||
|
||||
Fetch PR metadata (including head commit SHA for posting line comments later):
|
||||
|
||||
```
|
||||
gh pr view {number} --json title,body,baseRefName,headRefName,headRefOid,files,additions,deletions
|
||||
```
|
||||
|
||||
Save the `headRefOid` value, you'll need it as `commit_id` in Step 6.
|
||||
|
||||
## Step 2: Load the full diff
|
||||
|
||||
```
|
||||
gh pr diff {number}
|
||||
```
|
||||
|
||||
Also get the list of changed files:
|
||||
|
||||
```
|
||||
gh pr diff {number} --name-only
|
||||
```
|
||||
|
||||
## Step 3: Read every changed file in full
|
||||
|
||||
For each changed file, read the ENTIRE current file (not just the diff hunks). You need surrounding context to catch:
|
||||
- Callers of modified functions that now behave differently
|
||||
- Trait/interface contracts that the change may violate
|
||||
- Invariants established elsewhere that the diff breaks
|
||||
|
||||
If the PR touches more than 20 files, still read all of them, but process in this priority order: service logic > routes/handlers > models/types > tests > docs. Batch reads in groups of ~20 if needed.
|
||||
|
||||
## Step 4: Deep review
|
||||
|
||||
Go through the changes with each of these lenses. For every finding, note the file, line range, severity, and a concrete description.
|
||||
|
||||
### IronClaw-specific checks
|
||||
|
||||
In addition to the general lenses below, check IronClaw conventions (see CLAUDE.md):
|
||||
- No `.unwrap()` or `.expect()` in production code (tests are fine)
|
||||
- Use `crate::` imports, not `super::`
|
||||
- Error types use `thiserror` in `error.rs`
|
||||
- If the change touches persistence, verify both database backends are updated (PostgreSQL in `postgres.rs` AND libSQL in `libsql_backend.rs`)
|
||||
- New tools must implement the `Tool` trait correctly and be registered in `registry.rs`
|
||||
- External tool output must pass through the safety layer
|
||||
|
||||
### 4a. Correctness and bugs
|
||||
|
||||
- Off-by-one errors, wrong comparison operators, inverted conditions
|
||||
- Unreachable code, dead branches, impossible match arms
|
||||
- Type confusion (mixing up IDs, using wrong enum variant)
|
||||
- Incorrect error propagation (swallowed errors, wrong error type/status code)
|
||||
- Broken invariants (e.g. uniqueness assumptions violated, ordering assumptions wrong)
|
||||
- Concurrency issues (TOCTOU, missing locks, race conditions between check and use)
|
||||
|
||||
### 4b. Edge cases and failure handling
|
||||
|
||||
- What happens with empty input, None/null, zero-length collections?
|
||||
- What happens when external services fail (DB down, HTTP timeout, malformed response)?
|
||||
- What happens at integer boundaries (overflow, underflow, i64::MAX)?
|
||||
- What happens with malformed or adversarial input (invalid UTF-8, huge payloads, deeply nested JSON)?
|
||||
- Are all error paths tested? Does every `?` propagation make sense?
|
||||
- Are partial failures handled (e.g. wrote to DB but failed to emit event)?
|
||||
|
||||
### 4c. Security (assume a malicious actor)
|
||||
|
||||
- **Authentication/Authorization bypass**: Can an unauthenticated user reach this? Can workspace A's user access workspace B's data? Are there IDOR vulnerabilities?
|
||||
- **Injection**: SQL injection via string interpolation? Command injection? Log injection? Header injection?
|
||||
- **Data leakage**: Are secrets, PII, or conversation content logged? Returned in error messages? Exposed in API responses?
|
||||
- **Resource exhaustion / DoS**: Can an attacker send unbounded input? Trigger expensive operations without rate limits? Cause OOM via large allocations?
|
||||
- **Financial abuse**: Can tokens/credits be consumed without being tracked? Can usage limits be bypassed?
|
||||
- **Replay / race conditions**: Can the same request be replayed for double-spend? Can concurrent requests bypass limits?
|
||||
- **Cryptographic issues**: Timing attacks on comparisons? Weak randomness? Missing HMAC verification?
|
||||
|
||||
### 4d. Test coverage
|
||||
|
||||
- Is every new public function/method tested?
|
||||
- Are error paths tested (not just happy paths)?
|
||||
- Are edge cases covered (empty input, boundary values, concurrent access)?
|
||||
- Do existing tests still make sense with the new changes, or do they assert stale behavior?
|
||||
- Are there integration/e2e tests for the full flow?
|
||||
- If a test is missing, describe exactly what test should be written.
|
||||
|
||||
### 4e. Documentation and assumptions
|
||||
|
||||
- Are new assumptions documented in comments? (e.g. "this field is always non-empty because X")
|
||||
- Are non-obvious algorithms or business rules explained?
|
||||
- Are API contracts (request/response shapes, error codes, status codes) documented?
|
||||
- Are there TODO/FIXME/HACK comments that should be tracked as issues?
|
||||
|
||||
### 4f. Architectural concerns
|
||||
|
||||
- Does this change follow existing patterns in the codebase, or does it introduce a new one without justification?
|
||||
- Are there unnecessary abstractions or premature generalizations?
|
||||
- Is there duplicated logic that should be extracted?
|
||||
- Are dependencies between modules clean, or does this create circular/tight coupling?
|
||||
- Will this change make future work harder?
|
||||
|
||||
## Step 5: Present findings
|
||||
|
||||
Summarize findings to the user as a table:
|
||||
|
||||
| # | Severity | Category | File:Line | Finding | Suggested Fix |
|
||||
|---|----------|----------|-----------|---------|---------------|
|
||||
|
||||
Severity levels:
|
||||
- **Critical**: Security vulnerability, data loss, or financial exploit
|
||||
- **High**: Bug that will cause incorrect behavior in production
|
||||
- **Medium**: Robustness issue, missing validation, or incomplete error handling
|
||||
- **Low**: Style, naming, documentation, or minor improvement
|
||||
- **Nit**: Optional suggestion, take-it-or-leave-it
|
||||
|
||||
Ask the user which findings to post as PR comments. Default: all Critical, High, and Medium.
|
||||
|
||||
## Step 6: Post comments on GitHub
|
||||
|
||||
Resolve the repo owner and name if not already known:
|
||||
|
||||
```
|
||||
gh repo view --json owner,name --jq '"\(.owner.login)/\(.name)"'
|
||||
```
|
||||
|
||||
For each approved finding, post a review comment on the PR at the specific file and line. Use the `headRefOid` from Step 1 as the `commit_id`:
|
||||
|
||||
```
|
||||
gh api repos/{owner}/{repo}/pulls/{number}/comments \
|
||||
-f body="..." \
|
||||
-f path="..." \
|
||||
-f commit_id="{headRefOid}" \
|
||||
-F line=... \
|
||||
-f side="RIGHT"
|
||||
```
|
||||
|
||||
For findings that span multiple locations or are architectural, post as a regular PR comment:
|
||||
|
||||
```
|
||||
gh pr comment {number} --body "..."
|
||||
```
|
||||
|
||||
Format each comment clearly:
|
||||
- Severity tag (e.g. `**High Severity**`)
|
||||
- One-line summary
|
||||
- Detailed explanation of the issue
|
||||
- Concrete suggestion for the fix (with code if possible)
|
||||
|
||||
## Rules
|
||||
|
||||
- Read every changed file in full before writing a single finding. Context matters.
|
||||
- Never post a comment about code you haven't actually read. Verify line numbers against the actual file.
|
||||
- Be specific. "This might have issues" is useless. "Line 42 returns 404 but should return 400 because X" is useful.
|
||||
- Distinguish between "this IS a bug" and "this COULD be a bug if X". Be honest about certainty.
|
||||
- Don't nitpick formatting or style unless it causes actual confusion. Focus on substance.
|
||||
- If the code is good and you find nothing, say so. Don't invent problems to look thorough.
|
||||
- Respect the project's CLAUDE.md privacy rules: never include customer data, secrets, or PII in comments.
|
||||
- When in doubt about severity, round up. It's cheaper to dismiss a false alarm than to miss a real bug.
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
description: Run the full Rust quality gate (fmt, clippy, tests) before shipping changes
|
||||
allowed-tools: Bash(cargo fmt:*), Bash(cargo clippy:*), Bash(cargo test:*)
|
||||
---
|
||||
|
||||
Run the IronClaw shipping checklist. This is the mandatory quality gate before any change is considered done.
|
||||
|
||||
## Steps
|
||||
|
||||
1. **Format**: Run `cargo fmt` to normalize formatting.
|
||||
|
||||
2. **Lint**: Run `cargo clippy --all --benches --tests --examples --all-features` and report any warnings or errors. ALL clippy warnings must be resolved before proceeding.
|
||||
|
||||
3. **Test**: Run `cargo test --lib` to execute the full library test suite. Report the total pass/fail count.
|
||||
|
||||
4. **Summary**: Report results for all three steps. If any step failed, list the specific errors and suggest fixes. Do NOT proceed past a failing step.
|
||||
|
||||
If `$ARGUMENTS` is provided, treat it as a specific test filter and run `cargo test --lib -- $ARGUMENTS` instead of the full suite in step 3.
|
||||
|
||||
The expected outcome for a clean ship is:
|
||||
- `cargo fmt` produces no changes
|
||||
- `cargo clippy` has zero warnings
|
||||
- All tests pass
|
||||
|
||||
Note: Integration tests (`--test workspace_integration`) require a PostgreSQL database and are expected to fail locally. Only report `--lib` test failures as blocking.
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
description: Trace a data flow or bug through the IronClaw codebase end-to-end
|
||||
allowed-tools: Read, Glob, Grep, Bash(cargo test:*)
|
||||
argument-hint: <symptom or feature name>
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
Trace the flow of `$ARGUMENTS` through the IronClaw codebase. Your job is to map every file and function involved, identify where data transforms or could break, and report the full chain.
|
||||
|
||||
## Architecture Reference
|
||||
|
||||
IronClaw has three main data flow paths. Identify which one(s) are relevant and trace through them:
|
||||
|
||||
### Message Flow (user input to LLM response)
|
||||
```
|
||||
Channel (cli/web/wasm) → IncomingMessage
|
||||
→ Agent::run() message loop (agent_loop.rs)
|
||||
→ handle_message() dispatches by Submission type
|
||||
→ SubmissionParser::parse() (submission.rs) classifies input
|
||||
→ process_user_input() for new turns
|
||||
→ process_approval() for tool approval responses
|
||||
→ handle_command() for /commands
|
||||
→ run_agentic_loop() iterates LLM calls
|
||||
→ Reasoning::respond_with_tools() (reasoning.rs)
|
||||
→ LlmProvider::complete_with_tools() (nearai_chat.rs or nearai.rs)
|
||||
→ Tool execution with approval gating
|
||||
→ Context message accumulation
|
||||
→ Response flows back through Channel::send_response()
|
||||
```
|
||||
|
||||
### SSE Event Flow (backend status to web UI)
|
||||
```
|
||||
StatusUpdate variant (channel.rs)
|
||||
→ Channel::send_status() trait method
|
||||
→ WebChannel::send_status() (web/mod.rs) maps to SseEvent
|
||||
→ broadcast via tokio::broadcast channel
|
||||
→ SSE endpoint streams events (web/server.rs)
|
||||
→ Browser EventSource listener (app.js)
|
||||
→ DOM update function
|
||||
→ CSS styling (style.css)
|
||||
```
|
||||
|
||||
### Tool Flow (tool definition to execution)
|
||||
```
|
||||
Tool trait impl (tools/builtin/*.rs or tools/mcp/client.rs or tools/wasm/wrapper.rs)
|
||||
→ ToolRegistry::register() (tools/registry.rs)
|
||||
→ tool_definitions() builds Vec<ToolDefinition> for LLM
|
||||
→ ToolDefinition { name, description, parameters } (llm/provider.rs)
|
||||
→ Serialized to ChatCompletionTool (nearai_chat.rs)
|
||||
→ LLM returns ToolCall { id, name, arguments }
|
||||
→ agent_loop.rs executes via execute_chat_tool()
|
||||
→ Safety layer sanitizes output
|
||||
→ Result added as ChatMessage::tool_result()
|
||||
```
|
||||
|
||||
## Tracing Instructions
|
||||
|
||||
1. **Read** each file in the relevant flow path, focusing on the functions that handle the data.
|
||||
2. **Identify transforms**: Where does the data change shape? (e.g., `McpTool.input_schema` → `ToolDefinition.parameters` → `ChatCompletionTool.function.parameters`)
|
||||
3. **Identify failure points**: Where could the data be lost, malformed, or misrouted?
|
||||
4. **Report the chain**: List every file:line involved, what happens at each step, and where the issue (if any) is.
|
||||
|
||||
## Key Files Quick Reference
|
||||
|
||||
| Area | File | Key Functions |
|
||||
|------|------|---------------|
|
||||
| Message dispatch | `src/agent/agent_loop.rs` | `handle_message`, `process_user_input`, `process_approval`, `run_agentic_loop` |
|
||||
| Input parsing | `src/agent/submission.rs` | `SubmissionParser::parse` |
|
||||
| LLM reasoning | `src/llm/reasoning.rs` | `respond_with_tools`, `select_tools`, `plan` |
|
||||
| Chat completions | `src/llm/nearai_chat.rs` | `complete_with_tools`, `From<ChatMessage>` |
|
||||
| Responses API | `src/llm/nearai.rs` | `complete_with_tools`, `split_messages` |
|
||||
| Channel trait | `src/channels/channel.rs` | `Channel`, `StatusUpdate`, `IncomingMessage` |
|
||||
| Web gateway | `src/channels/web/mod.rs` | `send_status`, `send_response` |
|
||||
| Web server | `src/channels/web/server.rs` | Route handlers, SSE endpoints |
|
||||
| Web frontend | `src/channels/web/static/app.js` | SSE listeners, DOM builders |
|
||||
| Tool registry | `src/tools/registry.rs` | `tool_definitions`, `get`, `register` |
|
||||
| MCP tools | `src/tools/mcp/client.rs` | `McpToolWrapper`, `list_tools`, `call_tool` |
|
||||
| MCP protocol | `src/tools/mcp/protocol.rs` | `McpTool`, `inputSchema` |
|
||||
| Safety | `src/safety/sanitizer.rs` | `sanitize_tool_output`, `wrap_for_llm` |
|
||||
| Session state | `src/agent/session.rs` | `ThreadState`, `Turn`, `PendingApproval` |
|
||||
|
||||
## Output Format
|
||||
|
||||
Report your findings as:
|
||||
|
||||
1. **Flow path**: The specific chain of files and functions involved
|
||||
2. **Data transforms**: How the data changes at each step
|
||||
3. **Findings**: Any bugs, missing data, or suspicious patterns
|
||||
4. **Recommendation**: What to fix or investigate further
|
||||
@@ -0,0 +1,257 @@
|
||||
---
|
||||
description: Triage open GitHub issues — split into bugs vs features, rank by severity/opportunity, and flag under-specified issues
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(gh issue list:*), Bash(gh issue view:*), Bash(gh api:*), Bash(git log:*), Read, Grep, Glob, Task
|
||||
argument-hint: "[--label=<filter>] [--milestone=<filter>]"
|
||||
---
|
||||
|
||||
# Issue Triage
|
||||
|
||||
You are triaging all open issues on this repository. Your job is to split them into **bugs** and **feature requests**, rank each group, assess how well-specified each issue is, and produce an actionable triage report.
|
||||
|
||||
## Step 1: Fetch all open issues
|
||||
|
||||
Fetch every open issue with metadata:
|
||||
|
||||
```
|
||||
gh issue list --state open --limit 200 --json number,title,author,labels,assignees,createdAt,updatedAt,body,commentsCount,reactionGroups,milestone
|
||||
```
|
||||
|
||||
If `$ARGUMENTS` contains `--label=<X>`, append `--label '<X>'` to the command. If it contains `--milestone=<X>`, append `--milestone '<X>'` to the command.
|
||||
|
||||
Also fetch recently closed issues (last 14 days) to detect duplicates and already-resolved work:
|
||||
|
||||
```
|
||||
gh issue list --state closed --search "closed:>=$(date -v-14d +%Y-%m-%d)" --limit 100 --json number,title,body,labels,closedAt
|
||||
```
|
||||
|
||||
**Exclude pull requests** — `gh issue list` may include PRs. Fetch open PR numbers to filter them out:
|
||||
|
||||
```
|
||||
gh pr list --state open --json number --jq '.[].number'
|
||||
```
|
||||
|
||||
Remove any issue whose number appears in this list.
|
||||
|
||||
## Step 2: Classify each issue as Bug or Feature
|
||||
|
||||
Read each issue's title, body, and labels to classify it into one of these categories:
|
||||
|
||||
### Bugs
|
||||
Issues that describe **broken existing behavior** — something that worked or should work but doesn't. Signals:
|
||||
- Labels: `bug`, `defect`, `regression`, `crash`, `error`
|
||||
- Title/body keywords: "broken", "fails", "crash", "panic", "error", "regression", "doesn't work", "unexpected behavior"
|
||||
- Includes reproduction steps or error output
|
||||
- References existing functionality not working as documented
|
||||
|
||||
### Feature Requests
|
||||
Issues that describe **new or enhanced behavior** — something that doesn't exist yet. Signals:
|
||||
- Labels: `enhancement`, `feature`, `feature-request`, `improvement`, `proposal`
|
||||
- Title/body keywords: "add", "support", "implement", "would be nice", "proposal", "RFC", "new"
|
||||
- Describes a capability the project doesn't have
|
||||
- Proposes a design or API change
|
||||
|
||||
### Ambiguous
|
||||
If an issue doesn't clearly fit either category (e.g., "improve X performance" could be a bug or a feature), classify it as **Ambiguous** and note why.
|
||||
|
||||
## Step 3: Rate issue detail level
|
||||
|
||||
For each issue, assess how well-specified it is on a 3-tier scale:
|
||||
|
||||
| Detail Level | Criteria |
|
||||
|-------------|----------|
|
||||
| **Well-specified** | Has clear description of what/why, reproduction steps (bugs) or user story (features), acceptance criteria or expected behavior, and enough context to start working immediately |
|
||||
| **Adequate** | Describes the problem or request clearly, but missing some detail — no repro steps, vague acceptance criteria, or unclear scope. Needs 1-2 clarifying questions before work can start |
|
||||
| **Under-specified** | Vague title-only or single-sentence body, no context on why it matters, no clear definition of done. Needs significant discussion before it's actionable |
|
||||
|
||||
Indicators of good specification:
|
||||
- Code snippets, error logs, or screenshots
|
||||
- Steps to reproduce (bugs)
|
||||
- Proposed API/behavior (features)
|
||||
- Links to related issues or discussions
|
||||
- Clear "done when" criteria
|
||||
|
||||
## Step 4: Rank bugs by severity
|
||||
|
||||
Score each bug on these dimensions and compute an overall severity rank:
|
||||
|
||||
### Impact (1-4)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 4 | **Critical** | Data loss, security vulnerability, complete feature broken, crash in common path |
|
||||
| 3 | **High** | Major feature degraded, workaround exists but painful, affects many users |
|
||||
| 2 | **Medium** | Minor feature broken, easy workaround, affects subset of users |
|
||||
| 1 | **Low** | Cosmetic, edge case, documentation error, minor inconvenience |
|
||||
|
||||
### Urgency (1-3)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 3 | **Urgent** | Security issue, regression in recent release, blocking other work |
|
||||
| 2 | **Normal** | Should be fixed in next release cycle |
|
||||
| 1 | **Low** | Fix when convenient, backlog-worthy |
|
||||
|
||||
### Scope (1-3)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 3 | **Broad** | Affects core path, multiple modules, or all users |
|
||||
| 2 | **Moderate** | Affects one module or a specific configuration |
|
||||
| 1 | **Narrow** | Affects edge case or single obscure path |
|
||||
|
||||
**Bug severity score** = Impact × 2 + Urgency + Scope (base max 14)
|
||||
|
||||
Apply a one-time +2 boost if any of the following are true (max 16):
|
||||
- Has a linked PR already (someone is working on it — fast-track review)
|
||||
- Is labeled `security`
|
||||
- Is a regression (worked before, broken now)
|
||||
|
||||
## Step 5: Rank features by opportunity
|
||||
|
||||
Score each feature request on these dimensions:
|
||||
|
||||
### Value (1-4)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 4 | **High** | Unlocks new use cases, frequently requested, strategic alignment |
|
||||
| 3 | **Medium-High** | Significant quality-of-life improvement, good user demand signals |
|
||||
| 2 | **Medium** | Nice to have, modest improvement to existing workflow |
|
||||
| 1 | **Low** | Marginal value, niche use case, unclear demand |
|
||||
|
||||
Look for value signals in the issue:
|
||||
- Number of thumbs-up reactions or "+1" comments
|
||||
- Multiple people asking for the same thing
|
||||
- Alignment with project roadmap (check CLAUDE.md TODOs)
|
||||
- Unblocks other features or simplifies architecture
|
||||
|
||||
### Effort estimate (1-3, inverted — lower effort = higher score)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 3 | **Small** | <1 day, isolated change, clear implementation path |
|
||||
| 2 | **Medium** | 1-3 days, touches a few modules, some design needed |
|
||||
| 1 | **Large** | 3+ days, cross-cutting, needs RFC or architectural discussion |
|
||||
|
||||
### Readiness (1-3)
|
||||
| Score | Level | Description |
|
||||
|-------|-------|-------------|
|
||||
| 3 | **Ready** | Well-specified, implementation path clear, no blockers |
|
||||
| 2 | **Almost ready** | Needs minor clarification, but scope is understood |
|
||||
| 1 | **Not ready** | Needs design discussion, has open questions, blocked by other work |
|
||||
|
||||
**Opportunity score** = Value × 2 + Effort + Readiness (base max 14)
|
||||
|
||||
Apply a one-time +2 boost if any of the following are true (max 16):
|
||||
- A community member offered to implement it
|
||||
- It has a linked draft PR
|
||||
- It closes a gap listed in the project's "Current Limitations / TODOs"
|
||||
|
||||
## Step 6: Detect duplicates and relationships
|
||||
|
||||
Check for:
|
||||
- **Duplicates** — Issues describing the same bug or requesting the same feature (compare titles and bodies)
|
||||
- **Related clusters** — Groups of issues around the same area (e.g., multiple workspace issues, multiple CLI issues)
|
||||
- **Already fixed** — Open issues that may have been resolved by recently closed issues or merged PRs
|
||||
- **Blockers** — Issues that reference other issues as prerequisites ("depends on #N", "blocked by #N")
|
||||
- **Epic candidates** — Multiple small issues that could be grouped under a single tracking issue
|
||||
|
||||
## Step 7: Produce the triage report
|
||||
|
||||
Present the output in this format:
|
||||
|
||||
### Quick Stats
|
||||
|
||||
```
|
||||
Open: N | Bugs: N | Features: N | Ambiguous: N
|
||||
Well-specified: N | Adequate: N | Under-specified: N
|
||||
Unassigned: N | Stale (>30d): N
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Critical Bugs (Severity 12+)
|
||||
|
||||
Bugs that need immediate attention. For each:
|
||||
|
||||
| # | Title | Severity | Impact | Detail | Age | Assignee |
|
||||
|---|-------|----------|--------|--------|-----|----------|
|
||||
|
||||
Include a 1-line summary of the root cause if discernible from the issue.
|
||||
|
||||
### High-Priority Bugs (Severity 8-12)
|
||||
|
||||
Same table format. These should be addressed in the next release cycle.
|
||||
|
||||
### Medium/Low Bugs (Severity <8)
|
||||
|
||||
Compact table, sorted by severity descending.
|
||||
|
||||
---
|
||||
|
||||
### Quick Wins (Opportunity 12+ AND Effort = Small)
|
||||
|
||||
Features that are high-value and low-effort — do these first. For each:
|
||||
|
||||
| # | Title | Opportunity | Value | Effort | Detail | Age |
|
||||
|---|-------|-------------|-------|--------|--------|-----|
|
||||
|
||||
### High-Opportunity Features (Opportunity 10+)
|
||||
|
||||
Same table format. Worth investing in.
|
||||
|
||||
### Backlog Features (Opportunity <10)
|
||||
|
||||
Compact table, sorted by opportunity descending.
|
||||
|
||||
---
|
||||
|
||||
### Under-Specified Issues (Need Clarification)
|
||||
|
||||
Issues rated "Under-specified" that can't be triaged effectively. For each, suggest 1-2 specific questions to ask the author to make it actionable.
|
||||
|
||||
| # | Title | Type | What's missing |
|
||||
|---|-------|------|---------------|
|
||||
|
||||
### Ambiguous Issues (Bug or Feature?)
|
||||
|
||||
Issues that couldn't be clearly classified. For each, explain the ambiguity and suggest which category it likely belongs in.
|
||||
|
||||
---
|
||||
|
||||
### Duplicates & Overlaps
|
||||
|
||||
Groups of issues that appear to be duplicates or closely related. Recommend which to keep and which to close.
|
||||
|
||||
### Already Fixed?
|
||||
|
||||
Open issues that may have been resolved by recently closed issues or merged PRs.
|
||||
|
||||
### Stale Issues (>30 days, no activity)
|
||||
|
||||
Issues with no updates in 30+ days. Recommend: close, ping author, or keep.
|
||||
|
||||
---
|
||||
|
||||
### By Area
|
||||
|
||||
Group all issues by the area of the codebase they affect (infer from title/body/labels):
|
||||
|
||||
| Area | Bugs | Features | Top Priority |
|
||||
|------|------|----------|-------------|
|
||||
|
||||
### Suggested Next Actions
|
||||
|
||||
Based on the triage, provide 3-5 concrete recommendations:
|
||||
1. Which bugs to fix first and why
|
||||
2. Which quick-win features to pick up
|
||||
3. Which under-specified issues to clarify
|
||||
4. Which stale issues to close
|
||||
5. Any clusters that suggest a larger initiative
|
||||
|
||||
## Rules
|
||||
|
||||
- Use `gh` CLI for all GitHub operations. Never guess issue state — always check.
|
||||
- For large issue lists (>20), use the Task tool to parallelize fetching issue details and comments.
|
||||
- Be concise in summaries. One line per issue in tables.
|
||||
- When scoring, be honest about uncertainty. If you can't tell severity from the description, say so and rate it conservatively.
|
||||
- Factor in issue age — older unresolved bugs may indicate they're less critical than they seem, or that they're hard to fix. Note this in your assessment.
|
||||
- Check comment threads for additional context that the original body may lack. An under-specified issue with rich discussion may actually be well-understood.
|
||||
- Do NOT post comments, close issues, or take any action. This skill is read-only analysis.
|
||||
- If the repo has >100 open issues, focus the detailed analysis on the top 30 by recency and engagement (comments + reactions), and provide a summary table for the rest.
|
||||
@@ -0,0 +1,161 @@
|
||||
---
|
||||
description: Classify all open PRs by module, review state, scope, and architectural impact — produces a prioritized triage dashboard
|
||||
disable-model-invocation: true
|
||||
allowed-tools: Bash(gh pr list:*), Bash(gh pr view:*), Bash(gh pr diff:*), Bash(gh api:*), Bash(gh pr checks:*), Bash(git log:*), Read, Grep, Glob, Task
|
||||
argument-hint: "[--label=<filter>] [--author=<filter>]"
|
||||
---
|
||||
|
||||
# PR Triage Dashboard
|
||||
|
||||
You are triaging all open PRs on this repository. Your job is to produce a prioritized, module-grouped dashboard that tells the maintainer exactly which PRs need attention and in what order.
|
||||
|
||||
## Step 1: Fetch all open PRs
|
||||
|
||||
Fetch every open PR with metadata:
|
||||
|
||||
```
|
||||
gh pr list --state open --limit 100 --json number,title,author,labels,additions,deletions,headRefName,createdAt,updatedAt,isDraft,reviewRequests,reviews,files,body
|
||||
```
|
||||
|
||||
If `$ARGUMENTS` contains `--label=<X>`, append `--label '<X>'` to the `gh pr list` command. If it contains `--author=<X>`, append `--author '<X>'` to the command.
|
||||
|
||||
Also fetch recently merged PRs (last 7 days) to detect superseded/conflicting work:
|
||||
|
||||
```
|
||||
gh pr list --state merged --search "merged:>=$(date -v-7d +%Y-%m-%d)" --limit 100 --json number,title,body,mergedAt
|
||||
```
|
||||
|
||||
## Step 2: Classify each PR by module
|
||||
|
||||
For each open PR, determine the primary module it touches by examining the `files` field. Classify into these categories based on the dominant `src/` subdirectory:
|
||||
|
||||
| Category | Directories |
|
||||
|----------|------------|
|
||||
| **LLM & Inference** | `src/llm/` |
|
||||
| **Agent Core** | `src/agent/`, `src/skills/` |
|
||||
| **Tools** | `src/tools/`, `tools-src/` |
|
||||
| **Channels** | `src/channels/`, `channels-src/` |
|
||||
| **Storage & Memory** | `src/db/`, `src/workspace/`, `migrations/` |
|
||||
| **Security** | `src/safety/`, `src/secrets/` |
|
||||
| **Config & Setup** | `src/config.rs`, `src/setup/`, `src/cli/` |
|
||||
| **Sandbox & Orchestration** | `src/sandbox/`, `src/orchestrator/`, `src/worker/` |
|
||||
| **Hooks & Extensions** | `src/hooks/`, `src/extensions/` |
|
||||
| **Context & History** | `src/context/`, `src/history/`, `src/estimation/`, `src/evaluation/` |
|
||||
| **Web Gateway** | `src/channels/web/` |
|
||||
| **CI/CD & Docs** | `.github/`, `README.md`, `CLAUDE.md`, `*.md` (no src) |
|
||||
| **Other** | Anything else |
|
||||
|
||||
If a PR touches multiple modules, assign it to the **primary** module (most files changed) but note the cross-cutting modules.
|
||||
|
||||
## Step 3: Assess review state
|
||||
|
||||
For each PR, determine its review status:
|
||||
|
||||
- **Approved** — At least one human APPROVED review, no outstanding CHANGES_REQUESTED
|
||||
- **Changes requested** — At least one CHANGES_REQUESTED review still unresolved
|
||||
- **Reviewed (comments only)** — Human comments but no formal approve/reject
|
||||
- **Automated only** — Only bot reviews (gemini-code-assist, copilot, etc.)
|
||||
- **No review** — No reviews at all
|
||||
|
||||
Also check:
|
||||
- CI status: `gh pr checks {number}` — PASS / FAIL / NONE
|
||||
- Draft status: is the PR marked as draft?
|
||||
- Staleness: how many days since `updatedAt`?
|
||||
|
||||
## Step 4: Determine scope and risk
|
||||
|
||||
Classify each PR by scope:
|
||||
|
||||
| Scope | Criteria |
|
||||
|-------|----------|
|
||||
| **Tiny** | <50 lines changed (additions + deletions), 1-2 files |
|
||||
| **Small** | 50-200 lines, 1-5 files |
|
||||
| **Medium** | 200-500 lines, 3-10 files |
|
||||
| **Large** | 500-2000 lines, 5-20 files |
|
||||
| **XL** | 2000+ lines or 20+ files |
|
||||
|
||||
## Step 5: Classify as fix vs. architectural
|
||||
|
||||
For each PR, determine its nature:
|
||||
|
||||
### Fixes (merge fast)
|
||||
- Bug fixes with clear root cause
|
||||
- Security patches
|
||||
- Crash/panic prevention
|
||||
- Typo/doc corrections
|
||||
- Code quality (removing .unwrap(), etc.)
|
||||
|
||||
### Features (standard review)
|
||||
- New functionality within existing patterns
|
||||
- New tool implementations
|
||||
- Configuration additions
|
||||
- Test additions
|
||||
|
||||
### Architectural (deep review needed)
|
||||
- New modules or subsystems
|
||||
- Changes to core traits or interfaces
|
||||
- New database backends or storage engines
|
||||
- New provider abstractions
|
||||
- Changes touching 5+ modules
|
||||
- Anything modifying the agent loop, session model, or security layer
|
||||
- New dependencies (check Cargo.toml changes)
|
||||
|
||||
## Step 6: Detect conflicts and superseded PRs
|
||||
|
||||
Check for:
|
||||
- Multiple PRs fixing the same issue (look at "Closes #N" / "Fixes #N" in PR bodies)
|
||||
- PRs touching the same files (potential merge conflicts)
|
||||
- PRs that are follow-ups to other open PRs (dependency chains)
|
||||
- PRs superseded by recently merged work
|
||||
|
||||
## Step 7: Produce the dashboard
|
||||
|
||||
Present the output in this format:
|
||||
|
||||
### Quick Stats
|
||||
```
|
||||
Open: N | Draft: N | Needs review: N | Changes requested: N | Ready to merge: N
|
||||
```
|
||||
|
||||
### Ready to Merge
|
||||
PRs that are approved, CI passing, and non-draft. List with one-line summary.
|
||||
|
||||
### Needs Human Review (Fixes)
|
||||
Fixes that have no human review yet, sorted by severity (security > crash > bug > quality).
|
||||
|
||||
### Needs Human Review (Features)
|
||||
Features with no human review, sorted by scope (smallest first).
|
||||
|
||||
### Needs Deep Architectural Review
|
||||
Large/XL PRs, new modules, or cross-cutting changes. For each, include:
|
||||
- Which modules are affected
|
||||
- What new patterns or abstractions are introduced
|
||||
- Key risk areas to focus review on
|
||||
|
||||
### Changes Requested (Waiting on Author)
|
||||
PRs where a reviewer asked for changes. Include who requested and a 1-line summary of what's needed.
|
||||
|
||||
### Stale / Blocked
|
||||
PRs with no activity >7 days, or blocked by other PRs.
|
||||
|
||||
### Conflicts & Overlaps
|
||||
Any detected conflicts, superseded PRs, or dependency chains.
|
||||
|
||||
### By Module
|
||||
Group all PRs by their primary module in a compact table:
|
||||
|
||||
| Module | PRs | Key PR to review first |
|
||||
|--------|-----|----------------------|
|
||||
|
||||
### Superseded PRs (recommend closing)
|
||||
PRs that are clearly superseded by merged work. Include reasoning.
|
||||
|
||||
## Rules
|
||||
|
||||
- Use `gh` CLI for all GitHub operations. Never guess PR state — always check.
|
||||
- For large PR lists (>15), use the Task tool to parallelize fetching PR details and diffs.
|
||||
- Be concise in summaries. One line per PR in tables.
|
||||
- When assessing "ready to merge", be conservative. If there's any unresolved concern from a repo member, it's not ready.
|
||||
- Flag any PR that has been open >14 days with no review as needing attention.
|
||||
- If a PR description says "Closes #N" but #N was already closed by another merged PR, flag it as potentially superseded.
|
||||
- Do NOT post comments or take any action on PRs. This skill is read-only analysis.
|
||||
@@ -0,0 +1,8 @@
|
||||
target/
|
||||
.git/
|
||||
.env
|
||||
.env.*
|
||||
*.md
|
||||
!CLAUDE.md
|
||||
node_modules/
|
||||
tools-src/
|
||||
+43
-8
@@ -1,15 +1,44 @@
|
||||
# Database Configuration
|
||||
DATABASE_URL=postgres://ironclaw:password@localhost:5432/ironclaw
|
||||
DATABASE_URL=postgres://localhost/ironclaw
|
||||
DATABASE_POOL_SIZE=10
|
||||
|
||||
# LLM Provider (NEAR AI)
|
||||
# NEAR AI provides a unified interface to all models with user authentication
|
||||
# Session token is stored in ~/.near-agent/session.json and managed automatically.
|
||||
# On first run, the agent will open a browser for OAuth authentication.
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||
# LLM Provider
|
||||
# LLM_BACKEND=nearai # default
|
||||
# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil
|
||||
|
||||
# === NEAR AI (Chat Completions API) ===
|
||||
# Two auth modes:
|
||||
# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run.
|
||||
# Session token stored in ~/.ironclaw/session.json automatically.
|
||||
# Base URL defaults to https://private.near.ai
|
||||
# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai.
|
||||
# Base URL defaults to https://cloud-api.near.ai
|
||||
NEARAI_MODEL=zai-org/GLM-5-FP8
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
NEARAI_AUTH_URL=https://private.near.ai
|
||||
# NEARAI_SESSION_PATH=~/.near-agent/session.json # optional, default shown
|
||||
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
|
||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||
|
||||
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
|
||||
|
||||
# === Ollama ===
|
||||
# OLLAMA_MODEL=llama3.2
|
||||
# LLM_BACKEND=ollama
|
||||
# OLLAMA_BASE_URL=http://localhost:11434 # default
|
||||
|
||||
# === OpenAI-compatible (LM Studio, vLLM, Anything-LLM) ===
|
||||
# LLM_MODEL=llama-3.2-3b-instruct-q4_K_M
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=http://localhost:1234/v1
|
||||
# LLM_API_KEY=sk-... # optional for local servers
|
||||
|
||||
# === OpenRouter (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=anthropic/claude-sonnet-4
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
# LLM_API_KEY=sk-or-...
|
||||
|
||||
|
||||
# Channel Configuration
|
||||
# CLI is always enabled
|
||||
@@ -46,6 +75,12 @@ HEARTBEAT_INTERVAL_SECS=1800
|
||||
HEARTBEAT_NOTIFY_CHANNEL=cli
|
||||
HEARTBEAT_NOTIFY_USER=default
|
||||
|
||||
# Memory hygiene settings (automatic cleanup of stale workspace documents)
|
||||
# Runs on each heartbeat tick; identity files (IDENTITY.md, SOUL.md) are never deleted
|
||||
# MEMORY_HYGIENE_ENABLED=true
|
||||
# MEMORY_HYGIENE_RETENTION_DAYS=30 # delete daily/ docs older than this many days
|
||||
# MEMORY_HYGIENE_CADENCE_HOURS=12 # minimum hours between cleanup passes
|
||||
|
||||
# Safety settings
|
||||
SAFETY_MAX_OUTPUT_LENGTH=100000
|
||||
SAFETY_INJECTION_CHECK_ENABLED=true
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# Scope labels for actions/labeler@v6
|
||||
# Maps file path globs to scope labels. Multiple labels can apply per PR.
|
||||
|
||||
"scope: agent":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/agent/**
|
||||
|
||||
"scope: channel":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/channel.rs
|
||||
- src/channels/manager.rs
|
||||
- src/channels/mod.rs
|
||||
|
||||
"scope: channel/cli":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/cli/**
|
||||
- src/cli/**
|
||||
|
||||
"scope: channel/web":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/web/**
|
||||
|
||||
"scope: channel/wasm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/channels/wasm/**
|
||||
|
||||
"scope: tool":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/tool.rs
|
||||
- src/tools/registry.rs
|
||||
- src/tools/mod.rs
|
||||
- src/tools/sandbox.rs
|
||||
|
||||
"scope: tool/builtin":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/builtin/**
|
||||
|
||||
"scope: tool/wasm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/wasm/**
|
||||
|
||||
"scope: tool/mcp":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/mcp/**
|
||||
|
||||
"scope: tool/builder":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/tools/builder/**
|
||||
|
||||
"scope: db":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/mod.rs
|
||||
|
||||
"scope: db/postgres":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/postgres.rs
|
||||
- migrations/**
|
||||
|
||||
"scope: db/libsql":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/db/libsql_backend.rs
|
||||
- src/db/libsql_migrations.rs
|
||||
|
||||
"scope: safety":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/safety/**
|
||||
|
||||
"scope: llm":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/llm/**
|
||||
|
||||
"scope: workspace":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/workspace/**
|
||||
|
||||
"scope: orchestrator":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/orchestrator/**
|
||||
|
||||
"scope: worker":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/worker/**
|
||||
|
||||
"scope: secrets":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/secrets/**
|
||||
|
||||
"scope: config":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/config.rs
|
||||
- src/settings.rs
|
||||
|
||||
"scope: extensions":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/extensions/**
|
||||
|
||||
"scope: setup":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/setup/**
|
||||
|
||||
"scope: evaluation":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/evaluation/**
|
||||
|
||||
"scope: estimation":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/estimation/**
|
||||
|
||||
"scope: sandbox":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/sandbox/**
|
||||
- Dockerfile*
|
||||
|
||||
"scope: hooks":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/hooks/**
|
||||
|
||||
"scope: pairing":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- src/pairing/**
|
||||
|
||||
"scope: ci":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- .github/workflows/**
|
||||
- .github/scripts/**
|
||||
|
||||
"scope: docs":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- "**/*.md"
|
||||
- docs/**
|
||||
- LICENSE*
|
||||
|
||||
"scope: dependencies":
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- Cargo.toml
|
||||
- Cargo.lock
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# Idempotent label bootstrap for IronClaw PR automation.
|
||||
# Uses `gh label create --force` so it can be re-run safely.
|
||||
#
|
||||
# Usage: bash .github/scripts/create-labels.sh
|
||||
# Requires: gh CLI authenticated with repo scope
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v gh &>/dev/null; then
|
||||
echo "Error: gh CLI is required. Install from https://cli.github.com" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create() {
|
||||
local name="$1" color="$2" description="$3"
|
||||
gh label create "$name" --color "$color" --description "$description" --force
|
||||
}
|
||||
|
||||
echo "==> Creating size labels..."
|
||||
create "size: XS" "F9D0C4" "< 10 changed lines (excluding docs)"
|
||||
create "size: S" "F5A3A3" "10-49 changed lines"
|
||||
create "size: M" "E57373" "50-199 changed lines"
|
||||
create "size: L" "D32F2F" "200-499 changed lines"
|
||||
create "size: XL" "B71C1C" "500+ changed lines"
|
||||
|
||||
echo "==> Creating risk labels..."
|
||||
create "risk: low" "4CAF50" "Changes to docs, tests, or low-risk modules"
|
||||
create "risk: medium" "FFC107" "Business logic, config, or moderate-risk modules"
|
||||
create "risk: high" "F44336" "Safety, secrets, auth, or critical infrastructure"
|
||||
create "risk: manual" "9E9E9E" "Risk level set manually (sticky, not overwritten)"
|
||||
|
||||
echo "==> Creating scope labels..."
|
||||
create "scope: agent" "006B75" "Agent core (agent loop, router, scheduler)"
|
||||
create "scope: channel" "00838F" "Channel infrastructure"
|
||||
create "scope: channel/cli" "00897B" "TUI / CLI channel"
|
||||
create "scope: channel/web" "00796B" "Web gateway channel"
|
||||
create "scope: channel/wasm" "00695C" "WASM channel runtime"
|
||||
create "scope: tool" "1565C0" "Tool infrastructure"
|
||||
create "scope: tool/builtin" "1976D2" "Built-in tools"
|
||||
create "scope: tool/wasm" "1E88E5" "WASM tool sandbox"
|
||||
create "scope: tool/mcp" "2196F3" "MCP client"
|
||||
create "scope: tool/builder" "42A5F5" "Dynamic tool builder"
|
||||
create "scope: db" "4A148C" "Database trait / abstraction"
|
||||
create "scope: db/postgres" "6A1B9A" "PostgreSQL backend"
|
||||
create "scope: db/libsql" "7B1FA2" "libSQL / Turso backend"
|
||||
create "scope: safety" "880E4F" "Prompt injection defense"
|
||||
create "scope: llm" "4527A0" "LLM integration"
|
||||
create "scope: workspace" "283593" "Persistent memory / workspace"
|
||||
create "scope: orchestrator" "0D47A1" "Container orchestrator"
|
||||
create "scope: worker" "01579B" "Container worker"
|
||||
create "scope: secrets" "BF360C" "Secrets management"
|
||||
create "scope: config" "E65100" "Configuration"
|
||||
create "scope: extensions" "33691E" "Extension management"
|
||||
create "scope: setup" "827717" "Onboarding / setup"
|
||||
create "scope: evaluation" "558B2F" "Success evaluation"
|
||||
create "scope: estimation" "9E9D24" "Cost/time estimation"
|
||||
create "scope: sandbox" "00BFA5" "Docker sandbox"
|
||||
create "scope: hooks" "6D4C41" "Git/event hooks"
|
||||
create "scope: pairing" "4E342E" "Pairing mode"
|
||||
create "scope: ci" "546E7A" "CI/CD workflows"
|
||||
create "scope: docs" "78909C" "Documentation"
|
||||
create "scope: dependencies" "90A4AE" "Dependency updates"
|
||||
|
||||
echo "==> Creating contributor labels..."
|
||||
create "contributor: new" "FFF9C4" "First-time contributor"
|
||||
create "contributor: regular" "FFE082" "2-5 merged PRs"
|
||||
create "contributor: experienced" "FFB74D" "6-19 merged PRs"
|
||||
create "contributor: core" "FF8A65" "20+ merged PRs"
|
||||
|
||||
echo "Done. All labels created/updated."
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# Classify a PR by size, risk, and contributor tier.
|
||||
# Called by the pr-label-classify workflow.
|
||||
#
|
||||
# Inputs (env vars):
|
||||
# PR_NUMBER — pull request number
|
||||
# REPO — owner/repo (e.g. "user/ironclaw")
|
||||
#
|
||||
# Requires: gh CLI, jq
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PR_NUMBER="${PR_NUMBER:?PR_NUMBER is required}"
|
||||
REPO="${REPO:?REPO is required}"
|
||||
|
||||
# ─── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Remove all labels in a dimension except the desired one.
|
||||
# Usage: set_exclusive_label "size" "size: M"
|
||||
set_exclusive_label() {
|
||||
local prefix="$1" desired="$2"
|
||||
|
||||
# Fetch current labels on the PR
|
||||
local current
|
||||
current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name')
|
||||
|
||||
# Remove any existing label with the same prefix
|
||||
while IFS= read -r label; do
|
||||
[[ -z "$label" ]] && continue
|
||||
if [[ "$label" == "${prefix}:"* && "$label" != "$desired" ]]; then
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO" --remove-label "$label" 2>/dev/null || true
|
||||
fi
|
||||
done <<< "$current"
|
||||
|
||||
# Add the desired label
|
||||
gh pr edit "$PR_NUMBER" --repo "$REPO" --add-label "$desired"
|
||||
}
|
||||
|
||||
# ─── size ───────────────────────────────────────────────────────────────────
|
||||
|
||||
classify_size() {
|
||||
# Sum changed lines across non-doc files
|
||||
local total
|
||||
total=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \
|
||||
--paginate --jq '
|
||||
[.[] | select(.filename | test("\\.(md|txt|rst|adoc)$") | not) | .changes]
|
||||
| add // 0
|
||||
')
|
||||
|
||||
local label
|
||||
if (( total < 10 )); then label="size: XS"
|
||||
elif (( total < 50 )); then label="size: S"
|
||||
elif (( total < 200 )); then label="size: M"
|
||||
elif (( total < 500 )); then label="size: L"
|
||||
else label="size: XL"
|
||||
fi
|
||||
|
||||
echo "Size: ${total} changed lines -> ${label}"
|
||||
set_exclusive_label "size" "$label"
|
||||
}
|
||||
|
||||
# ─── risk ───────────────────────────────────────────────────────────────────
|
||||
|
||||
classify_risk() {
|
||||
# If "risk: manual" is present, skip — it's a sticky override
|
||||
local current
|
||||
current=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json labels --jq '.labels[].name')
|
||||
if echo "$current" | grep -qx "risk: manual"; then
|
||||
echo "Risk: skipped (manual override)"
|
||||
return
|
||||
fi
|
||||
|
||||
# Fetch changed file paths
|
||||
local files
|
||||
files=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" \
|
||||
--paginate --jq '.[].filename')
|
||||
|
||||
local risk="low"
|
||||
|
||||
while IFS= read -r file; do
|
||||
[[ -z "$file" ]] && continue
|
||||
|
||||
case "$file" in
|
||||
# High risk: safety, secrets, auth, crypto, setup, orchestrator auth
|
||||
src/safety/*|src/secrets/*|src/llm/session.rs|src/orchestrator/auth.rs|\
|
||||
src/channels/web/auth.rs|src/setup/*)
|
||||
risk="high"
|
||||
break # can't go higher
|
||||
;;
|
||||
|
||||
# Medium risk: agent core, config, database, worker, tools, channels
|
||||
src/agent/*|src/config.rs|src/settings.rs|src/db/*|src/worker/*|\
|
||||
src/tools/*|src/channels/*|src/orchestrator/*|src/context/*|\
|
||||
src/hooks/*|src/sandbox/*|src/extensions/*|Cargo.toml|\
|
||||
.github/workflows/*)
|
||||
# Only upgrade, never downgrade
|
||||
[[ "$risk" != "high" ]] && risk="medium"
|
||||
;;
|
||||
|
||||
# Low risk: docs, tests, estimation, evaluation, history, etc.
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
done <<< "$files"
|
||||
|
||||
echo "Risk: ${risk}"
|
||||
set_exclusive_label "risk" "risk: ${risk}"
|
||||
}
|
||||
|
||||
# ─── contributor tier ───────────────────────────────────────────────────────
|
||||
|
||||
classify_contributor() {
|
||||
# Get PR author
|
||||
local author
|
||||
author=$(gh pr view "$PR_NUMBER" --repo "$REPO" --json author --jq '.author.login')
|
||||
|
||||
# Count merged PRs by this author in this repo
|
||||
local count
|
||||
count=$(gh pr list --repo "$REPO" --state merged --author "$author" \
|
||||
--limit 100 --json number --jq 'length')
|
||||
|
||||
local label
|
||||
if (( count == 0 )); then label="contributor: new"
|
||||
elif (( count < 6 )); then label="contributor: regular"
|
||||
elif (( count < 20 )); then label="contributor: experienced"
|
||||
else label="contributor: core"
|
||||
fi
|
||||
|
||||
echo "Contributor: ${author} has ${count} merged PRs -> ${label}"
|
||||
set_exclusive_label "contributor" "$label"
|
||||
}
|
||||
|
||||
# ─── main ───────────────────────────────────────────────────────────────────
|
||||
|
||||
echo "Classifying PR #${PR_NUMBER} in ${REPO}..."
|
||||
classify_size
|
||||
classify_risk
|
||||
classify_contributor
|
||||
echo "Done."
|
||||
@@ -0,0 +1,22 @@
|
||||
name: Code Style
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
codestyle:
|
||||
name: Code Style (fmt + clippy)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
profile: minimal
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Check formatting
|
||||
run: |
|
||||
cargo fmt --all -- --check
|
||||
- name: Check lints (cargo clippy)
|
||||
run: cargo clippy -- -D warnings
|
||||
@@ -0,0 +1,26 @@
|
||||
name: "PR: Classify (Size, Risk, Contributor)"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: read # needed for search/issues API (contributor count)
|
||||
|
||||
jobs:
|
||||
classify:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout base branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
|
||||
- name: Classify PR
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: bash .github/scripts/pr-labeler.sh
|
||||
@@ -0,0 +1,18 @@
|
||||
name: "PR: Scope Labels"
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
scope:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/labeler@v5
|
||||
with:
|
||||
configuration-path: .github/labeler.yml
|
||||
sync-labels: false # additive only — never remove scope labels
|
||||
@@ -0,0 +1,67 @@
|
||||
name: Release-plz
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
|
||||
# Release unpublished packages.
|
||||
release-plz-release:
|
||||
if: ${{ github.repository_owner == 'nearai' }}
|
||||
name: Release-plz release
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- &checkout
|
||||
name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- &install-rust
|
||||
name: Install Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
# Generating a GitHub token, so that PRs and tags created by
|
||||
# the release-plz-action can trigger actions workflows.
|
||||
- name: Generate GitHub token
|
||||
uses: actions/create-github-app-token@v2
|
||||
id: generate-token
|
||||
with:
|
||||
# GitHub App ID secret name
|
||||
app-id: ${{ secrets.GH_RELEASES_MANAGER_APP_ID }}
|
||||
# GitHub App private key secret name
|
||||
private-key: ${{ secrets.GH_RELEASES_MANAGER_APP_PRIVATE_KEY }}
|
||||
- name: Run release-plz
|
||||
uses: release-plz/[email protected]
|
||||
with:
|
||||
command: release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
|
||||
# Create a PR with the new versions and changelog, preparing the next release.
|
||||
release-plz-pr:
|
||||
if: ${{ github.repository_owner == 'nearai' }}
|
||||
name: Release-plz PR
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
concurrency:
|
||||
group: release-plz-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- *checkout
|
||||
- *install-rust
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run release-plz
|
||||
uses: release-plz/[email protected]
|
||||
with:
|
||||
command: release-pr
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
@@ -0,0 +1,299 @@
|
||||
# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist
|
||||
#
|
||||
# Copyright 2022-2024, axodotdev
|
||||
# SPDX-License-Identifier: MIT or Apache-2.0
|
||||
#
|
||||
# CI that:
|
||||
#
|
||||
# * checks for a Git Tag that looks like a release
|
||||
# * builds artifacts with dist (archives, installers, hashes)
|
||||
# * uploads those artifacts to temporary workflow zip
|
||||
# * on success, uploads the artifacts to a GitHub Release
|
||||
#
|
||||
# Note that the GitHub Release will be created with a generated
|
||||
# title/body based on your changelogs.
|
||||
|
||||
name: Release
|
||||
permissions:
|
||||
"contents": "write"
|
||||
|
||||
# This task will run whenever you push a git tag that looks like a version
|
||||
# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
|
||||
# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where
|
||||
# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION
|
||||
# must be a Cargo-style SemVer Version (must have at least major.minor.patch).
|
||||
#
|
||||
# If PACKAGE_NAME is specified, then the announcement will be for that
|
||||
# package (erroring out if it doesn't have the given version or isn't dist-able).
|
||||
#
|
||||
# If PACKAGE_NAME isn't specified, then the announcement will be for all
|
||||
# (dist-able) packages in the workspace with that version (this mode is
|
||||
# intended for workspaces with only one dist-able package, or with all dist-able
|
||||
# packages versioned/released in lockstep).
|
||||
#
|
||||
# If you push multiple tags at once, separate instances of this workflow will
|
||||
# spin up, creating an independent announcement for each one. However, GitHub
|
||||
# will hard limit this to 3 tags per commit, as it will assume more tags is a
|
||||
# mistake.
|
||||
#
|
||||
# If there's a prerelease-style suffix to the version, then the release(s)
|
||||
# will be marked as a prerelease.
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '**[0-9]+.[0-9]+.[0-9]+*'
|
||||
|
||||
jobs:
|
||||
# Run 'dist plan' (or host) to determine what tasks we need to do
|
||||
plan:
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.plan.outputs.manifest }}
|
||||
tag: ${{ !github.event.pull_request && github.ref_name || '' }}
|
||||
tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }}
|
||||
publishing: ${{ !github.event.pull_request }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install dist
|
||||
# we specify bash to get pipefail; it guards against the `curl` command
|
||||
# failing. otherwise `sh` won't catch that `curl` returned non-0
|
||||
shell: bash
|
||||
run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.30.3/cargo-dist-installer.sh | sh"
|
||||
- name: Cache dist
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/dist
|
||||
# sure would be cool if github gave us proper conditionals...
|
||||
# so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
|
||||
# functionality based on whether this is a pull_request, and whether it's from a fork.
|
||||
# (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
|
||||
# but also really annoying to build CI around when it needs secrets to work right.)
|
||||
- id: plan
|
||||
run: |
|
||||
dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
cat plan-dist-manifest.json
|
||||
echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-plan-dist-manifest
|
||||
path: plan-dist-manifest.json
|
||||
|
||||
# Build and packages all the platform-specific things
|
||||
build-local-artifacts:
|
||||
name: build-local-artifacts (${{ join(matrix.targets, ', ') }})
|
||||
# Let the initial task tell us to not run (currently very blunt)
|
||||
needs:
|
||||
- plan
|
||||
if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# Target platforms/runners are computed by dist in create-release.
|
||||
# Each member of the matrix has the following arguments:
|
||||
#
|
||||
# - runner: the github runner
|
||||
# - dist-args: cli flags to pass to dist
|
||||
# - install-dist: expression to run to install dist on the runner
|
||||
#
|
||||
# Typically there will be:
|
||||
# - 1 "global" task that builds universal installers
|
||||
# - N "local" tasks that build each platform's binaries and platform-specific installers
|
||||
matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
container: ${{ matrix.container && matrix.container.image || null }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
|
||||
steps:
|
||||
- name: enable windows longpaths
|
||||
run: |
|
||||
git config --global core.longpaths true
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install Rust non-interactively if not already installed
|
||||
if: ${{ matrix.container }}
|
||||
run: |
|
||||
if ! command -v cargo > /dev/null 2>&1; then
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
fi
|
||||
- uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
key: ${{ join(matrix.targets, '-') }}
|
||||
cache-provider: ${{ matrix.cache_provider }}
|
||||
- name: Install dist
|
||||
run: ${{ matrix.install_dist.run }}
|
||||
# Get the dist-manifest
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
${{ matrix.packages_install }}
|
||||
- name: Build artifacts
|
||||
run: |
|
||||
# Actually do builds and make zips and whatnot
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
- id: cargo-dist
|
||||
name: Post-build
|
||||
# We force bash here just because github makes it really hard to get values up
|
||||
# to "real" actions without writing to env-vars, and writing to env-vars has
|
||||
# inconsistent syntax between shell and powershell.
|
||||
shell: bash
|
||||
run: |
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-local-${{ join(matrix.targets, '_') }}
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
|
||||
# Build and package all the platform-agnostic(ish) things
|
||||
build-global-artifacts:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Get all the local artifacts for the global tasks to use (for e.g. checksums)
|
||||
- name: Fetch local artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: cargo-dist
|
||||
shell: bash
|
||||
run: |
|
||||
dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
|
||||
echo "dist ran successfully"
|
||||
|
||||
# Parse out what we just built and upload it to scratch storage
|
||||
echo "paths<<EOF" >> "$GITHUB_OUTPUT"
|
||||
jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
cp dist-manifest.json "$BUILD_MANIFEST_NAME"
|
||||
- name: "Upload artifacts"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-build-global
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
# Determines if we should publish/announce
|
||||
host:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
outputs:
|
||||
val: ${{ steps.host.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install cached dist
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: cargo-dist-cache
|
||||
path: ~/.cargo/bin/
|
||||
- run: chmod +x ~/.cargo/bin/dist
|
||||
# Fetch artifacts from scratch-storage
|
||||
- name: Fetch artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: target/distrib/
|
||||
merge-multiple: true
|
||||
- id: host
|
||||
shell: bash
|
||||
run: |
|
||||
dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
|
||||
echo "artifacts uploaded and released successfully"
|
||||
cat dist-manifest.json
|
||||
echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
|
||||
- name: "Upload dist-manifest.json"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
# Overwrite the previous copy
|
||||
name: artifacts-dist-manifest
|
||||
path: dist-manifest.json
|
||||
# Create a GitHub Release while uploading all files to it
|
||||
- name: "Download GitHub Artifacts"
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: artifacts-*
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
- name: Cleanup
|
||||
run: |
|
||||
# Remove the granular manifests
|
||||
rm -f artifacts/*-dist-manifest.json
|
||||
- name: Create GitHub Release
|
||||
env:
|
||||
PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}"
|
||||
ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}"
|
||||
ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}"
|
||||
RELEASE_COMMIT: "${{ github.sha }}"
|
||||
run: |
|
||||
# Write and read notes from a file to avoid quoting breaking things
|
||||
echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt
|
||||
|
||||
gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/*
|
||||
|
||||
announce:
|
||||
needs:
|
||||
- plan
|
||||
- host
|
||||
# use "always() && ..." to allow us to wait for all publish jobs while
|
||||
# still allowing individual publish jobs to skip themselves (for prereleases).
|
||||
# "host" however must run to completion, no skipping allowed!
|
||||
if: ${{ always() && needs.host.result == 'success' }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
@@ -0,0 +1,21 @@
|
||||
name: Run Tests
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
tests:
|
||||
name: Run Tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
profile: minimal
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run Tests
|
||||
run: cargo test --all-features -- --nocapture
|
||||
+15
@@ -1,6 +1,21 @@
|
||||
|
||||
.env
|
||||
.env.local
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Claude Code worktrees
|
||||
.claude/worktrees/
|
||||
|
||||
# Sidecar tool data
|
||||
.sidecar/
|
||||
.todos/
|
||||
|
||||
target/
|
||||
|
||||
# Benchmark results (local runs, not committed)
|
||||
bench-results/
|
||||
|
||||
# WASM build artifacts (loaded from disk, not bundled)
|
||||
*.wasm
|
||||
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Agent Rules
|
||||
|
||||
## Feature Parity Update Policy
|
||||
|
||||
- If you change implementation status for any feature tracked in `FEATURE_PARITY.md`, update that file in the same branch.
|
||||
- Do not open a PR that changes feature behavior without checking `FEATURE_PARITY.md` for needed status updates (`❌`, `🚧`, `✅`, notes, and priorities).
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.8.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.7.0...ironclaw-v0.8.0) - 2026-02-20
|
||||
|
||||
### Added
|
||||
|
||||
- extension registry with metadata catalog and onboarding integration ([#238](https://github.com/nearai/ironclaw/pull/238))
|
||||
- *(models)* add GPT-5.3 Codex, full GPT-5.x family, Claude 4.x series, o4-mini ([#197](https://github.com/nearai/ironclaw/pull/197))
|
||||
- wire memory hygiene into the heartbeat loop ([#195](https://github.com/nearai/ironclaw/pull/195))
|
||||
|
||||
### Fixed
|
||||
|
||||
- persist WASM channel workspace writes across callbacks ([#264](https://github.com/nearai/ironclaw/pull/264))
|
||||
- consolidate per-module ENV_MUTEX into crate-wide test lock ([#246](https://github.com/nearai/ironclaw/pull/246))
|
||||
- remove auto-proceed fake user message injection from agent loop ([#255](https://github.com/nearai/ironclaw/pull/255))
|
||||
- onboarding errors reset flow and remote server auth (#185, #186) ([#248](https://github.com/nearai/ironclaw/pull/248))
|
||||
- parallelize tool call execution via JoinSet ([#219](https://github.com/nearai/ironclaw/pull/219)) ([#252](https://github.com/nearai/ironclaw/pull/252))
|
||||
- prevent pipe deadlock in shell command execution ([#140](https://github.com/nearai/ironclaw/pull/140))
|
||||
- persist turns after approval and add agent-level tests ([#250](https://github.com/nearai/ironclaw/pull/250))
|
||||
|
||||
### Other
|
||||
|
||||
- add automated PR labeling system ([#253](https://github.com/nearai/ironclaw/pull/253))
|
||||
- update CLAUDE.md for recently merged features ([#183](https://github.com/nearai/ironclaw/pull/183))
|
||||
|
||||
## [0.7.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.6.0...ironclaw-v0.7.0) - 2026-02-19
|
||||
|
||||
### Added
|
||||
|
||||
- extend lifecycle hooks with declarative bundles ([#176](https://github.com/nearai/ironclaw/pull/176))
|
||||
- support per-request model override in /v1/chat/completions ([#103](https://github.com/nearai/ironclaw/pull/103))
|
||||
|
||||
### Fixed
|
||||
|
||||
- harden openai-compatible provider, approval replay, and embeddings defaults ([#237](https://github.com/nearai/ironclaw/pull/237))
|
||||
- Network Security Findings ([#201](https://github.com/nearai/ironclaw/pull/201))
|
||||
|
||||
### Added
|
||||
|
||||
- Refactored OpenAI-compatible chat completion routing to use the rig adapter and `RetryProvider` composition for custom base URL usage.
|
||||
- Added Ollama embeddings provider support (`EMBEDDING_PROVIDER=ollama`, `OLLAMA_BASE_URL`) in workspace embeddings.
|
||||
- Added migration `V9__flexible_embedding_dimension.sql` for flexible embedding vector dimensions.
|
||||
|
||||
### Changed
|
||||
|
||||
- Changed default sandbox image to `ironclaw-worker:latest` in config/settings/sandbox defaults.
|
||||
- Improved tool-message sanitization and provider compatibility handling across NEAR AI, rig adapter, and shared LLM provider code.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed approval-input aliases (`a`, `/approve`, `/always`, `/deny`, etc.) in submission parsing.
|
||||
- Fixed multi-tool approval resume flow by preserving and replaying deferred tool calls so all prior `tool_use` IDs receive matching `tool_result` messages.
|
||||
- Fixed REPL quit/exit handling to route shutdown through the agent loop for graceful termination.
|
||||
|
||||
## [0.6.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.5.0...ironclaw-v0.6.0) - 2026-02-19
|
||||
|
||||
### Added
|
||||
|
||||
- add issue triage skill ([#200](https://github.com/nearai/ironclaw/pull/200))
|
||||
- add PR triage dashboard skill ([#196](https://github.com/nearai/ironclaw/pull/196))
|
||||
- add OpenRouter usage examples ([#189](https://github.com/nearai/ironclaw/pull/189))
|
||||
- add Tinfoil private inference provider ([#62](https://github.com/nearai/ironclaw/pull/62))
|
||||
- shell env scrubbing and command injection detection ([#164](https://github.com/nearai/ironclaw/pull/164))
|
||||
- Add PR review tools, job monitor, and channel injection for E2E sandbox workflows ([#57](https://github.com/nearai/ironclaw/pull/57))
|
||||
- Secure prompt-based skills system (Phases 1-4) ([#51](https://github.com/nearai/ironclaw/pull/51))
|
||||
- Add benchmarking harness with spot suite ([#10](https://github.com/nearai/ironclaw/pull/10))
|
||||
- 10 infrastructure improvements from zeroclaw ([#126](https://github.com/nearai/ironclaw/pull/126))
|
||||
|
||||
### Fixed
|
||||
|
||||
- *(rig)* prevent OpenAI Responses API panic on tool call IDs ([#182](https://github.com/nearai/ironclaw/pull/182))
|
||||
- *(docs)* correct settings storage path in README ([#194](https://github.com/nearai/ironclaw/pull/194))
|
||||
- OpenAI tool calling — schema normalization, missing types, and Responses API panic ([#132](https://github.com/nearai/ironclaw/pull/132))
|
||||
- *(security)* prevent path traversal bypass in WASM HTTP allowlist ([#137](https://github.com/nearai/ironclaw/pull/137))
|
||||
- persist OpenAI-compatible provider and respect embeddings disable ([#177](https://github.com/nearai/ironclaw/pull/177))
|
||||
- remove .expect() calls in FailoverProvider::try_providers ([#156](https://github.com/nearai/ironclaw/pull/156))
|
||||
- sentinel value collision in FailoverProvider cooldown ([#125](https://github.com/nearai/ironclaw/pull/125)) ([#154](https://github.com/nearai/ironclaw/pull/154))
|
||||
- skills module audit cleanup ([#173](https://github.com/nearai/ironclaw/pull/173))
|
||||
|
||||
### Other
|
||||
|
||||
- Fix division by zero panic in ValueEstimator::is_profitable ([#139](https://github.com/nearai/ironclaw/pull/139))
|
||||
- audit feature parity matrix against codebase and recent commits ([#202](https://github.com/nearai/ironclaw/pull/202))
|
||||
- architecture improvements for contributor velocity ([#198](https://github.com/nearai/ironclaw/pull/198))
|
||||
- fix rustfmt formatting from PR #137
|
||||
- add .env.example examples for Ollama and OpenAI-compatible ([#110](https://github.com/nearai/ironclaw/pull/110))
|
||||
|
||||
## [0.5.0](https://github.com/nearai/ironclaw/compare/v0.4.0...v0.5.0) - 2026-02-17
|
||||
|
||||
### Added
|
||||
|
||||
- add cooldown management to FailoverProvider ([#114](https://github.com/nearai/ironclaw/pull/114))
|
||||
|
||||
## [0.4.0](https://github.com/nearai/ironclaw/compare/v0.3.0...v0.4.0) - 2026-02-17
|
||||
|
||||
### Added
|
||||
|
||||
- move per-invocation approval check into Tool trait ([#119](https://github.com/nearai/ironclaw/pull/119))
|
||||
- add polished boot screen on CLI startup ([#118](https://github.com/nearai/ironclaw/pull/118))
|
||||
- Add lifecycle hooks system with 6 interception points ([#18](https://github.com/nearai/ironclaw/pull/18))
|
||||
|
||||
### Other
|
||||
|
||||
- remove accidentally committed .sidecar and .todos directories ([#123](https://github.com/nearai/ironclaw/pull/123))
|
||||
|
||||
## [0.3.0](https://github.com/nearai/ironclaw/compare/v0.2.0...v0.3.0) - 2026-02-17
|
||||
|
||||
### Added
|
||||
|
||||
- direct api key and cheap model ([#116](https://github.com/nearai/ironclaw/pull/116))
|
||||
|
||||
## [0.2.0](https://github.com/nearai/ironclaw/compare/v0.1.3...v0.2.0) - 2026-02-16
|
||||
|
||||
### Added
|
||||
|
||||
- mark Ollama + OpenAI-compatible as implemented ([#102](https://github.com/nearai/ironclaw/pull/102))
|
||||
- multi-provider inference + libSQL onboarding selection ([#92](https://github.com/nearai/ironclaw/pull/92))
|
||||
- add multi-provider LLM failover with retry backoff ([#28](https://github.com/nearai/ironclaw/pull/28))
|
||||
- add libSQL/Turso embedded database backend ([#47](https://github.com/nearai/ironclaw/pull/47))
|
||||
- Move debug log truncation from agent loop to REPL channel ([#65](https://github.com/nearai/ironclaw/pull/65))
|
||||
|
||||
### Fixed
|
||||
|
||||
- shell destructive-command check bypassed by Value::Object arguments ([#72](https://github.com/nearai/ironclaw/pull/72))
|
||||
- propagate real tool_call_id instead of hardcoded placeholder ([#73](https://github.com/nearai/ironclaw/pull/73))
|
||||
- Fix wasm tool schemas and runtime ([#42](https://github.com/nearai/ironclaw/pull/42))
|
||||
- flatten tool messages for NEAR AI cloud-api compatibility ([#41](https://github.com/nearai/ironclaw/pull/41))
|
||||
- security hardening across all layers ([#35](https://github.com/nearai/ironclaw/pull/35))
|
||||
|
||||
### Other
|
||||
|
||||
- Explicitly enable cargo-dist caching for binary artifacts building
|
||||
- Skip building binary artifacts on every PR
|
||||
- add module specification rules to CLAUDE.md
|
||||
- add setup/onboarding specification (src/setup/README.md)
|
||||
- deduplicate tool code and remove dead stubs ([#98](https://github.com/nearai/ironclaw/pull/98))
|
||||
- Reformat architecture diagram in README ([#64](https://github.com/nearai/ironclaw/pull/64))
|
||||
- Add review discipline guidelines to CLAUDE.md ([#68](https://github.com/nearai/ironclaw/pull/68))
|
||||
- Bump MSRV to 1.92, add GCP deployment files ([#40](https://github.com/nearai/ironclaw/pull/40))
|
||||
- Add OpenAI-compatible HTTP API (/v1/chat/completions, /v1/models) ([#31](https://github.com/nearai/ironclaw/pull/31))
|
||||
|
||||
|
||||
## [0.1.3](https://github.com/nearai/ironclaw/compare/v0.1.2...v0.1.3) - 2026-02-12
|
||||
|
||||
### Other
|
||||
|
||||
- Enabled builds caching during CI/CD
|
||||
- Disabled npm publishing as the name is already taken
|
||||
|
||||
## [0.1.2](https://github.com/nearai/ironclaw/compare/v0.1.1...v0.1.2) - 2026-02-12
|
||||
|
||||
### Other
|
||||
|
||||
- Added Installation instructions for the pre-built binaries
|
||||
- Disabled Windows ARM64 builds as auto-updater [provided by cargo-dist] does not support this platform yet and it is not a common platform for us to support
|
||||
|
||||
## [0.1.1](https://github.com/nearai/ironclaw/compare/v0.1.0...v0.1.1) - 2026-02-12
|
||||
|
||||
### Other
|
||||
|
||||
- Renamed the secrets in release-plz.yml to match the configuration
|
||||
- Make sure that the binaries release CD it kicking in after release-plz
|
||||
|
||||
## [0.1.0](https://github.com/nearai/ironclaw/releases/tag/v0.1.0) - 2026-02-12
|
||||
|
||||
### Added
|
||||
|
||||
- Add multi-provider LLM support via rig-core adapter ([#36](https://github.com/nearai/ironclaw/pull/36))
|
||||
- Sandbox jobs ([#4](https://github.com/nearai/ironclaw/pull/4))
|
||||
- Add Google Suite & Telegram WASM tools ([#9](https://github.com/nearai/ironclaw/pull/9))
|
||||
- Improve CLI ([#5](https://github.com/nearai/ironclaw/pull/5))
|
||||
|
||||
### Fixed
|
||||
|
||||
- resolve runtime panic in Linux keychain integration ([#32](https://github.com/nearai/ironclaw/pull/32))
|
||||
|
||||
### Other
|
||||
|
||||
- Skip release-plz on forks
|
||||
- Upgraded release-plz CD pipeline
|
||||
- Added CI/CD and release pipelines ([#45](https://github.com/nearai/ironclaw/pull/45))
|
||||
- DM pairing + Telegram channel improvements ([#17](https://github.com/nearai/ironclaw/pull/17))
|
||||
- Fixes build, adds missing sse event and correct command ([#11](https://github.com/nearai/ironclaw/pull/11))
|
||||
- Codex/feature parity pr hook ([#6](https://github.com/nearai/ironclaw/pull/6))
|
||||
- Add WebSocket gateway and control plane ([#8](https://github.com/nearai/ironclaw/pull/8))
|
||||
- select bundled Telegram channel and auto-install ([#3](https://github.com/nearai/ironclaw/pull/3))
|
||||
- Adding skills for reusable work
|
||||
- Fix MCP tool calls, approval loop, shutdown, and improve web UI
|
||||
- Add auth mode, fix MCP token handling, and parallelize startup loading
|
||||
- Merge remote-tracking branch 'origin/main' into ui
|
||||
- Adding web UI
|
||||
- Rename `setup` CLI command to `onboard` for compatibility
|
||||
- Add in-chat extension discovery, auth, and activation system
|
||||
- Add Telegram typing indicator via WIT on-status callback
|
||||
- Add proactivity features: memory CLI, session pruning, self-repair notifications, slash commands, status diagnostics, context warnings
|
||||
- Add hosted MCP server support with OAuth 2.1 and token refresh
|
||||
- Add interactive setup wizard and persistent settings
|
||||
- Rebrand to IronClaw with security-first mission
|
||||
- Fix build_software tool stuck in planning mode loop
|
||||
- Enable sandbox by default
|
||||
- Fix Telegram Markdown formatting and clarify tool/memory distinctions
|
||||
- Simplify Telegram channel config with host-injected tunnel/webhook settings
|
||||
- Apply Telegram channel learnings to WhatsApp implementation
|
||||
- Merge remote-tracking branch 'origin/main'
|
||||
- Docker file for sandbox
|
||||
- Replace hardcoded intent patterns with job tools
|
||||
- Fix router test to match intentional job creation patterns
|
||||
- Add Docker execution sandbox for secure shell command isolation
|
||||
- Move setup wizard credentials to database storage
|
||||
- Add interactive setup wizard for first-run configuration
|
||||
- Add Telegram Bot API channel as WASM module
|
||||
- Add OpenClaw feature parity tracking matrix
|
||||
- Add Chat Completions API support and expand REPL debugging
|
||||
- Implementing channels to be handled in wasm
|
||||
- Support non interactive mode and model selection
|
||||
- Implement tool approval, fix tool definition refresh, and wire embeddings
|
||||
- Tool use
|
||||
- Wiring more
|
||||
- Add heartbeat integration, planning phase, and auto-repair
|
||||
- Login flow
|
||||
- Extend support for session management
|
||||
- Adding builder capability
|
||||
- Load tools at launch
|
||||
- Fix multiline message rendering in TUI
|
||||
- Parse NEAR AI alternative response format with output field
|
||||
- Handle NEAR AI plain text responses
|
||||
- Disable mouse capture to allow text selection in TUI
|
||||
- Add verbose logging to debug empty NEAR AI responses
|
||||
- Improve NEAR AI response parsing for varying response formats
|
||||
- Show status/thinking messages in chat window, debug empty responses
|
||||
- Add timeout and logging to NEAR AI provider
|
||||
- Add status updates to show agent thinking/processing state
|
||||
- Add CLI subcommands for WASM tool management
|
||||
- Fix TUI shutdown: send /shutdown message and handle in agent loop
|
||||
- Remove SimpleCliChannel, add Ctrl+D twice quit, redirect logs to TUI
|
||||
- Fix TuiChannel integration and enable in main.rs
|
||||
- Integrate Codex patterns: task scheduler, TUI, sessions, compaction
|
||||
- Adding LICENSE
|
||||
- Add README with IronClaw branding
|
||||
- Add WASM sandbox secure API extension
|
||||
- Wire database Store into agent loop
|
||||
- Implementing WASM runtime
|
||||
- Add workspace integration tests
|
||||
- Compact memory_tree output format
|
||||
- Replace memory_list with memory_tree tool
|
||||
- Simplify workspace to path-based storage, remove legacy code
|
||||
- Add NEAR AI chat-api as default LLM provider
|
||||
- Add CLAUDE.md project documentation
|
||||
- Add workspace and memory system (OpenClaw-inspired)
|
||||
- Initial implementation of the agent framework
|
||||
@@ -11,11 +11,19 @@
|
||||
- **Always available** - Multi-channel access with proactive background execution
|
||||
|
||||
### Features
|
||||
- **Multi-channel input**: TUI (Ratatui), HTTP webhooks, Telegram, WhatsApp, Slack (WASM channels)
|
||||
- **Multi-channel input**: TUI (Ratatui), HTTP webhooks, WASM channels (Telegram, Slack), web gateway
|
||||
- **Parallel job execution** with state machine and self-repair for stuck jobs
|
||||
- **Sandbox execution**: Docker container isolation with network proxy and credential injection
|
||||
- **Claude Code mode**: Delegate jobs to Claude CLI inside containers
|
||||
- **Skills system**: SKILL.md prompt extensions with trust model, tool attenuation, and ClawHub registry
|
||||
- **Routines**: Scheduled (cron) and reactive (event, webhook) task execution
|
||||
- **Web gateway**: Browser UI with SSE/WebSocket real-time streaming
|
||||
- **Extension management**: Install, auth, activate MCP/WASM extensions
|
||||
- **Extensible tools**: Built-in tools, WASM sandbox, MCP client, dynamic builder
|
||||
- **Persistent memory**: Workspace with hybrid search (FTS + vector via RRF)
|
||||
- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection
|
||||
- **Prompt injection defense**: Sanitizer, validator, policy rules, leak detection, shell env scrubbing
|
||||
- **Multi-provider LLM**: NEAR AI, OpenAI, Anthropic, Ollama, OpenAI-compatible, Tinfoil private inference
|
||||
- **Setup wizard**: 7-step interactive onboarding for first-run configuration
|
||||
- **Heartbeat system**: Proactive periodic execution with checklist
|
||||
|
||||
## Build & Test
|
||||
@@ -59,7 +67,10 @@ src/
|
||||
│ ├── context_monitor.rs # Memory pressure detection
|
||||
│ ├── undo.rs # Turn-based undo/redo with checkpoints
|
||||
│ ├── submission.rs # Submission parsing (undo, redo, compact, clear, etc.)
|
||||
│ └── task.rs # Sub-task execution framework
|
||||
│ ├── dispatcher.rs # Skill-aware job dispatching
|
||||
│ ├── task.rs # Sub-task execution framework
|
||||
│ ├── routine.rs # Routine types (Trigger, Action, Guardrails)
|
||||
│ └── routine_engine.rs # Routine execution (cron ticker, event matcher)
|
||||
│
|
||||
├── channels/ # Multi-channel input
|
||||
│ ├── channel.rs # Channel trait, IncomingMessage, OutgoingResponse
|
||||
@@ -72,8 +83,33 @@ src/
|
||||
│ │ ├── overlay.rs # Approval overlays
|
||||
│ │ └── composer.rs # Message composition
|
||||
│ ├── http.rs # HTTP webhook (axum) with secret validation
|
||||
│ ├── slack.rs # Stub
|
||||
│ └── telegram.rs # Stub
|
||||
│ ├── repl.rs # Simple REPL (for testing)
|
||||
│ ├── web/ # Web gateway (browser UI)
|
||||
│ │ ├── mod.rs # Gateway builder, startup
|
||||
│ │ ├── server.rs # Axum router, 40+ API endpoints
|
||||
│ │ ├── sse.rs # SSE broadcast manager
|
||||
│ │ ├── ws.rs # WebSocket gateway + connection tracking
|
||||
│ │ ├── types.rs # Request/response types, SseEvent enum
|
||||
│ │ ├── auth.rs # Bearer token auth middleware
|
||||
│ │ ├── log_layer.rs # Tracing layer for log streaming
|
||||
│ │ └── static/ # HTML, CSS, JS (single-page app)
|
||||
│ └── wasm/ # WASM channel runtime
|
||||
│ ├── mod.rs
|
||||
│ ├── bundled.rs # Bundled channel discovery
|
||||
│ └── wrapper.rs # Channel trait wrapper for WASM modules
|
||||
│
|
||||
├── orchestrator/ # Internal HTTP API for sandbox containers
|
||||
│ ├── mod.rs
|
||||
│ ├── api.rs # Axum endpoints (LLM proxy, events, prompts)
|
||||
│ ├── auth.rs # Per-job bearer token store
|
||||
│ └── job_manager.rs # Container lifecycle (create, stop, cleanup)
|
||||
│
|
||||
├── worker/ # Runs inside Docker containers
|
||||
│ ├── mod.rs
|
||||
│ ├── runtime.rs # Worker execution loop (tool calls, LLM)
|
||||
│ ├── claude_bridge.rs # Claude Code bridge (spawns claude CLI)
|
||||
│ ├── api.rs # HTTP client to orchestrator
|
||||
│ └── proxy_llm.rs # LlmProvider that proxies through orchestrator
|
||||
│
|
||||
├── safety/ # Prompt injection defense
|
||||
│ ├── sanitizer.rs # Pattern detection, content escaping
|
||||
@@ -81,11 +117,18 @@ src/
|
||||
│ ├── policy.rs # PolicyRule system with severity/actions
|
||||
│ └── leak_detector.rs # Secret detection (API keys, tokens, etc.)
|
||||
│
|
||||
├── llm/ # LLM integration (NEAR AI only)
|
||||
├── llm/ # LLM integration (multi-provider)
|
||||
│ ├── mod.rs # Provider factory, LlmBackend enum
|
||||
│ ├── provider.rs # LlmProvider trait, message types
|
||||
│ ├── nearai.rs # NEAR AI chat-api implementation
|
||||
│ ├── nearai_chat.rs # NEAR AI Chat Completions provider (session token + API key auth)
|
||||
│ ├── reasoning.rs # Planning, tool selection, evaluation
|
||||
│ └── session.rs # Session token management with auto-renewal
|
||||
│ ├── session.rs # Session token management with auto-renewal
|
||||
│ ├── circuit_breaker.rs # Circuit breaker for provider failures
|
||||
│ ├── retry.rs # Retry with exponential backoff
|
||||
│ ├── failover.rs # Multi-provider failover chain
|
||||
│ ├── response_cache.rs # LLM response caching
|
||||
│ ├── costs.rs # Token cost tracking
|
||||
│ └── rig_adapter.rs # Rig framework adapter
|
||||
│
|
||||
├── tools/ # Extensible tool system
|
||||
│ ├── tool.rs # Tool trait, ToolOutput, ToolError
|
||||
@@ -96,6 +139,10 @@ src/
|
||||
│ │ ├── file.rs # ReadFile, WriteFile, ListDir, ApplyPatch
|
||||
│ │ ├── shell.rs # Shell command execution
|
||||
│ │ ├── memory.rs # Memory tools (search, write, read, tree)
|
||||
│ │ ├── job.rs # CreateJob, ListJobs, JobStatus, CancelJob
|
||||
│ │ ├── routine.rs # routine_create/list/update/delete/history
|
||||
│ │ ├── extension_tools.rs # Extension install/auth/activate/remove
|
||||
│ │ ├── skill_tools.rs # skill_list/search/install/remove tools
|
||||
│ │ └── marketplace.rs, ecommerce.rs, taskrabbit.rs, restaurant.rs (stubs)
|
||||
│ ├── builder/ # Dynamic tool building
|
||||
│ │ ├── core.rs # BuildRequirement, SoftwareType, Language
|
||||
@@ -116,6 +163,12 @@ src/
|
||||
│ ├── rate_limiter.rs # Per-tool rate limiting
|
||||
│ └── storage.rs # Linear memory persistence
|
||||
│
|
||||
├── db/ # Database abstraction layer
|
||||
│ ├── mod.rs # Database trait (~60 async methods)
|
||||
│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository)
|
||||
│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite)
|
||||
│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent)
|
||||
│
|
||||
├── workspace/ # Persistent memory system (OpenClaw-inspired)
|
||||
│ ├── mod.rs # Workspace struct, memory operations
|
||||
│ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry
|
||||
@@ -139,11 +192,38 @@ src/
|
||||
│ ├── success.rs # SuccessEvaluator trait, RuleBasedEvaluator, LlmEvaluator
|
||||
│ └── metrics.rs # MetricsCollector, QualityMetrics
|
||||
│
|
||||
├── sandbox/ # Docker execution sandbox
|
||||
│ ├── mod.rs # Public API, default allowlist
|
||||
│ ├── config.rs # SandboxConfig, SandboxPolicy enum
|
||||
│ ├── manager.rs # SandboxManager orchestration
|
||||
│ ├── container.rs # ContainerRunner, Docker lifecycle
|
||||
│ ├── error.rs # SandboxError types
|
||||
│ └── proxy/ # Network proxy for containers
|
||||
│ ├── mod.rs # NetworkProxyBuilder
|
||||
│ ├── http.rs # HttpProxy, CredentialResolver trait
|
||||
│ ├── policy.rs # NetworkPolicyDecider trait
|
||||
│ └── allowlist.rs # DomainAllowlist validation
|
||||
│
|
||||
├── secrets/ # Secrets management
|
||||
│ ├── crypto.rs # AES-256-GCM encryption
|
||||
│ ├── store.rs # Secret storage
|
||||
│ └── types.rs # Credential types
|
||||
│
|
||||
├── setup/ # Onboarding wizard (spec: src/setup/README.md)
|
||||
│ ├── mod.rs # Entry point, check_onboard_needed()
|
||||
│ ├── wizard.rs # 7-step interactive wizard
|
||||
│ ├── channels.rs # Channel setup helpers
|
||||
│ └── prompts.rs # Terminal prompts (select, confirm, secret)
|
||||
│
|
||||
├── skills/ # SKILL.md prompt extension system
|
||||
│ ├── mod.rs # Core types (SkillTrust, LoadedSkill)
|
||||
│ ├── registry.rs # SkillRegistry: discover, install, remove
|
||||
│ ├── selector.rs # Deterministic scoring prefilter
|
||||
│ ├── attenuation.rs # Trust-based tool ceiling
|
||||
│ ├── gating.rs # Requirement checks (bins, env, config)
|
||||
│ ├── parser.rs # SKILL.md frontmatter + markdown parser
|
||||
│ └── catalog.rs # ClawHub registry client
|
||||
│
|
||||
└── history/ # Persistence
|
||||
├── store.rs # PostgreSQL repositories
|
||||
└── analytics.rs # Aggregation queries (JobStats, ToolStats)
|
||||
@@ -151,10 +231,15 @@ src/
|
||||
|
||||
## Key Patterns
|
||||
|
||||
### Architecture
|
||||
|
||||
When designing new features or systems, always prefer generic/extensible architectures over hardcoding specific integrations. Ask clarifying questions about the desired abstraction level before implementing.
|
||||
|
||||
### Error Handling
|
||||
- Use `thiserror` for error types in `error.rs`
|
||||
- Never use `.unwrap()` in production code (tests are fine)
|
||||
- Never use `.unwrap()` or `.expect()` in production code (tests are fine)
|
||||
- Map errors with context: `.map_err(|e| SomeError::Variant { reason: e.to_string() })?`
|
||||
- Before committing, grep for `.unwrap()` and `.expect(` in changed files to catch violations mechanically
|
||||
|
||||
### Async
|
||||
- All I/O is async with tokio
|
||||
@@ -162,11 +247,13 @@ src/
|
||||
- Use `RwLock` for concurrent read/write access
|
||||
|
||||
### Traits for Extensibility
|
||||
- `Database` - Add new database backends (must implement all ~60 methods)
|
||||
- `Channel` - Add new input sources
|
||||
- `Tool` - Add new capabilities
|
||||
- `LlmProvider` - Add new LLM backends
|
||||
- `SuccessEvaluator` - Custom evaluation logic
|
||||
- `EmbeddingProvider` - Add embedding backends (workspace search)
|
||||
- `NetworkPolicyDecider` - Custom network access policies for sandbox containers
|
||||
|
||||
### Tool Implementation
|
||||
```rust
|
||||
@@ -205,16 +292,59 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted
|
||||
\-> Failed
|
||||
```
|
||||
|
||||
### Code Style
|
||||
|
||||
- Use `crate::` imports, not `super::`
|
||||
- No `pub use` re-exports unless exposing to downstream consumers
|
||||
- Prefer strong types over strings (enums, newtypes)
|
||||
- Keep functions focused, extract helpers when logic is reused
|
||||
- Comments for non-obvious logic only
|
||||
|
||||
### Review & Fix Discipline
|
||||
|
||||
Hard-won lessons from code review -- follow these when fixing bugs or addressing review feedback.
|
||||
|
||||
**Fix the pattern, not just the instance:** When a reviewer flags a bug (e.g., TOCTOU race in INSERT + SELECT-back), search the entire codebase for all instances of that same pattern. A fix in `SecretsStore::create()` that doesn't also fix `WasmToolStore::store()` is half a fix.
|
||||
|
||||
**Propagate architectural fixes to satellite types:** If a core type changes its concurrency model (e.g., `LibSqlBackend` switches to connection-per-operation), every type that was handed a resource from the old model (e.g., `LibSqlSecretsStore`, `LibSqlWasmToolStore` holding a single `Connection`) must also be updated. Grep for the old type across the codebase.
|
||||
|
||||
**Schema translation is more than DDL:** When translating a database schema between backends (PostgreSQL to libSQL, etc.), check for:
|
||||
- **Indexes** -- diff `CREATE INDEX` statements between the two schemas
|
||||
- **Seed data** -- check for `INSERT INTO` in migrations (e.g., `leak_detection_patterns`)
|
||||
- **Semantic differences** -- document where SQL functions behave differently (e.g., `json_patch` vs `jsonb_set`)
|
||||
|
||||
**Feature flag testing:** When adding feature-gated code, test compilation with each feature in isolation:
|
||||
```bash
|
||||
cargo check # default features
|
||||
cargo check --no-default-features --features libsql # libsql only
|
||||
cargo check --all-features # all features
|
||||
```
|
||||
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||
|
||||
**Mechanical verification before committing:** Run these checks on changed files before committing:
|
||||
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables (see `.env.example`):
|
||||
```bash
|
||||
# Database backend (default: postgres)
|
||||
DATABASE_BACKEND=postgres # or "libsql" / "turso"
|
||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||
LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
|
||||
# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional)
|
||||
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
|
||||
|
||||
# NEAR AI (required)
|
||||
NEARAI_SESSION_TOKEN=sess_...
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
# NEAR AI (when LLM_BACKEND=nearai, the default)
|
||||
# Two auth modes: session token (default) or API key
|
||||
# Session token auth (default): uses browser OAuth on first run
|
||||
NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
# API key auth: set NEARAI_API_KEY, base URL defaults to cloud-api.near.ai
|
||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
# Agent settings
|
||||
AGENT_NAME=ironclaw
|
||||
@@ -232,20 +362,101 @@ HEARTBEAT_ENABLED=true
|
||||
HEARTBEAT_INTERVAL_SECS=1800 # 30 minutes
|
||||
HEARTBEAT_NOTIFY_CHANNEL=tui
|
||||
HEARTBEAT_NOTIFY_USER=default
|
||||
|
||||
# Web gateway
|
||||
GATEWAY_ENABLED=true
|
||||
GATEWAY_HOST=127.0.0.1
|
||||
GATEWAY_PORT=3001
|
||||
GATEWAY_AUTH_TOKEN=changeme # Required for API access
|
||||
GATEWAY_USER_ID=default
|
||||
|
||||
# Docker sandbox
|
||||
SANDBOX_ENABLED=true
|
||||
SANDBOX_IMAGE=ironclaw-worker:latest
|
||||
SANDBOX_MEMORY_LIMIT_MB=512
|
||||
SANDBOX_TIMEOUT_SECS=1800
|
||||
SANDBOX_CPU_LIMIT=1.0 # CPU cores per container
|
||||
SANDBOX_NETWORK_PROXY=true # Enable network proxy for containers
|
||||
SANDBOX_PROXY_PORT=8080 # Proxy listener port
|
||||
SANDBOX_DEFAULT_POLICY=workspace_write # ReadOnly, WorkspaceWrite, FullAccess
|
||||
|
||||
# Claude Code mode (runs inside sandbox containers)
|
||||
CLAUDE_CODE_ENABLED=false
|
||||
CLAUDE_CODE_MODEL=claude-sonnet-4-20250514
|
||||
CLAUDE_CODE_MAX_TURNS=50
|
||||
CLAUDE_CODE_CONFIG_DIR=/home/worker/.claude
|
||||
|
||||
# Routines (scheduled/reactive execution)
|
||||
ROUTINES_ENABLED=true
|
||||
ROUTINES_CRON_INTERVAL=60 # Tick interval in seconds
|
||||
ROUTINES_MAX_CONCURRENT=3
|
||||
|
||||
# Skills system
|
||||
SKILLS_ENABLED=true
|
||||
SKILLS_MAX_TOKENS=4000 # Max prompt budget per turn
|
||||
SKILLS_CATALOG_URL=https://clawhub.dev # ClawHub registry URL
|
||||
SKILLS_AUTO_DISCOVER=true # Scan skill directories on startup
|
||||
|
||||
# Tinfoil private inference
|
||||
TINFOIL_API_KEY=... # Required when LLM_BACKEND=tinfoil
|
||||
TINFOIL_MODEL=kimi-k2-5 # Default model
|
||||
```
|
||||
|
||||
### NEAR AI Provider
|
||||
### LLM Providers
|
||||
|
||||
Uses the NEAR AI chat-api (`https://api.near.ai/v1/responses`) which provides:
|
||||
- Unified access to multiple models (OpenAI, Anthropic, etc.)
|
||||
- User authentication via session tokens
|
||||
- Usage tracking and billing through NEAR AI
|
||||
IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, and `tinfoil`.
|
||||
|
||||
Session tokens have the format `sess_xxx` (37 characters). They are authenticated against the NEAR AI auth service.
|
||||
**NEAR AI** -- Uses the Chat Completions API with dual auth support. Session token auth (default): authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google), base URL defaults to `https://private.near.ai`. API key auth: set `NEARAI_API_KEY` (from `cloud.near.ai`), base URL defaults to `https://cloud-api.near.ai`. Both modes use the same Chat Completions endpoint. Tool messages are flattened to plain text for compatibility. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment.
|
||||
|
||||
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API. Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
|
||||
|
||||
## Database
|
||||
|
||||
Single migration in `migrations/V1__initial.sql`. Tables:
|
||||
IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable.
|
||||
|
||||
**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL).
|
||||
|
||||
### Backends
|
||||
|
||||
| Backend | Feature Flag | Default | Use Case |
|
||||
|---------|-------------|---------|----------|
|
||||
| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments |
|
||||
| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud |
|
||||
|
||||
```bash
|
||||
# Build with PostgreSQL only (default)
|
||||
cargo build
|
||||
|
||||
# Build with libSQL only
|
||||
cargo build --no-default-features --features libsql
|
||||
|
||||
# Build with both backends available
|
||||
cargo build --features "postgres,libsql"
|
||||
```
|
||||
|
||||
### Database Trait
|
||||
|
||||
The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence:
|
||||
- Conversations, messages, metadata
|
||||
- Jobs, actions, LLM calls, estimation snapshots
|
||||
- Sandbox jobs, job events
|
||||
- Routines, routine runs
|
||||
- Tool failures, settings
|
||||
- Workspace: documents, chunks, hybrid search
|
||||
|
||||
Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL.
|
||||
|
||||
### Schema
|
||||
|
||||
**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`.
|
||||
|
||||
**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types:
|
||||
- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT`
|
||||
- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx`
|
||||
- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers
|
||||
- PL/pgSQL functions -> SQLite triggers
|
||||
|
||||
**Tables (both backends):**
|
||||
|
||||
**Core:**
|
||||
- `conversations` - Multi-channel conversation tracking
|
||||
@@ -257,12 +468,26 @@ Single migration in `migrations/V1__initial.sql`. Tables:
|
||||
|
||||
**Workspace/Memory:**
|
||||
- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md")
|
||||
- `memory_chunks` - Chunked content with FTS (tsvector) and vector (pgvector) indexes
|
||||
- `memory_chunks` - Chunked content with FTS and vector indexes
|
||||
- `heartbeat_state` - Periodic execution tracking
|
||||
|
||||
Requires pgvector extension: `CREATE EXTENSION IF NOT EXISTS vector;`
|
||||
**Other:**
|
||||
- `routines`, `routine_runs` - Scheduled/reactive execution
|
||||
- `settings` - Per-user key-value settings
|
||||
- `tool_failures` - Self-repair tracking
|
||||
- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure
|
||||
|
||||
Run migrations: `refinery migrate -c refinery.toml`
|
||||
Database configuration: see Configuration section above.
|
||||
|
||||
### Current Limitations (libSQL backend)
|
||||
|
||||
- **Workspace/memory system** not yet wired through Database trait (requires Store migration)
|
||||
- **Secrets store** not yet available (still requires PostgresSecretsStore)
|
||||
- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented)
|
||||
- **Settings reload from DB** skipped (Config::from_db requires Store)
|
||||
- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet)
|
||||
- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage.
|
||||
- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields.
|
||||
|
||||
## Safety Layer
|
||||
|
||||
@@ -270,6 +495,7 @@ All external tool output passes through `SafetyLayer`:
|
||||
1. **Sanitizer** - Detects injection patterns, escapes dangerous content
|
||||
2. **Validator** - Checks length, encoding, forbidden patterns
|
||||
3. **Policy** - Rules with severity (Critical/High/Medium/Low) and actions (Block/Warn/Review/Sanitize)
|
||||
4. **Leak Detector** - Scans for 15+ secret patterns (API keys, tokens, private keys, connection strings) at two points: tool output before it reaches the LLM, and LLM responses before they reach the user. Actions per pattern: Block (reject entirely), Redact (mask the secret), or Warn (flag but allow)
|
||||
|
||||
Tool outputs are wrapped before reaching LLM:
|
||||
```xml
|
||||
@@ -278,6 +504,95 @@ Tool outputs are wrapped before reaching LLM:
|
||||
</tool_output>
|
||||
```
|
||||
|
||||
### Shell Environment Scrubbing
|
||||
|
||||
The shell tool (`src/tools/builtin/shell.rs`) scrubs sensitive environment variables before executing commands, preventing secrets from leaking through `env`, `printenv`, or `$VAR` expansion. The sanitizer (`src/safety/sanitizer.rs`) also detects command injection patterns (chained commands, subshells, path traversal) and blocks or escapes them based on policy rules.
|
||||
|
||||
## Skills System
|
||||
|
||||
Skills are SKILL.md files that extend the agent's prompt with domain-specific instructions. Each skill is a YAML frontmatter block (metadata, activation criteria, required tools) followed by a markdown body that gets injected into the LLM context when the skill activates.
|
||||
|
||||
### Trust Model
|
||||
|
||||
| Trust Level | Source | Tool Access |
|
||||
|-------------|--------|-------------|
|
||||
| **Trusted** | User-placed in `~/.ironclaw/skills/` or workspace `skills/` | All tools available to the agent |
|
||||
| **Installed** | Downloaded from ClawHub registry | Read-only tools only (no shell, file write, HTTP) |
|
||||
|
||||
### SKILL.md Format
|
||||
|
||||
```yaml
|
||||
---
|
||||
name: my-skill
|
||||
version: 0.1.0
|
||||
description: Does something useful
|
||||
activation:
|
||||
patterns:
|
||||
- "deploy to.*production"
|
||||
keywords:
|
||||
- "deployment"
|
||||
max_context_tokens: 2000
|
||||
metadata:
|
||||
openclaw:
|
||||
requires:
|
||||
bins: [docker, kubectl]
|
||||
env: [KUBECONFIG]
|
||||
---
|
||||
|
||||
# Deployment Skill
|
||||
|
||||
Instructions for the agent when this skill activates...
|
||||
```
|
||||
|
||||
### Selection Pipeline
|
||||
|
||||
1. **Gating** -- Check binary/env/config requirements; skip skills whose prerequisites are missing
|
||||
2. **Scoring** -- Deterministic scoring against message content using keywords, tags, and regex patterns
|
||||
3. **Budget** -- Select top-scoring skills that fit within `SKILLS_MAX_TOKENS` prompt budget
|
||||
4. **Attenuation** -- Apply trust-based tool ceiling; installed skills lose access to dangerous tools
|
||||
|
||||
### Skill Tools
|
||||
|
||||
Four built-in tools for managing skills at runtime:
|
||||
- **`skill_list`** -- List all discovered skills with trust level and status
|
||||
- **`skill_search`** -- Search ClawHub registry for available skills
|
||||
- **`skill_install`** -- Download and install a skill from ClawHub
|
||||
- **`skill_remove`** -- Remove an installed skill
|
||||
|
||||
### Skill Directories
|
||||
|
||||
- `~/.ironclaw/skills/` -- User's global skills (trusted)
|
||||
- `<workspace>/skills/` -- Per-workspace skills (trusted)
|
||||
- `~/.ironclaw/installed_skills/` -- Registry-installed skills (installed trust)
|
||||
|
||||
Skills configuration: see Configuration section above.
|
||||
|
||||
## Docker Sandbox
|
||||
|
||||
The `src/sandbox/` module provides Docker-based isolation for job execution with a network proxy that controls outbound access and injects credentials.
|
||||
|
||||
### Sandbox Policies
|
||||
|
||||
| Policy | Filesystem | Network | Use Case |
|
||||
|--------|-----------|---------|----------|
|
||||
| **ReadOnly** | Read-only workspace mount | Allowlisted domains only | Analysis, code review |
|
||||
| **WorkspaceWrite** | Read-write workspace mount | Allowlisted domains only | Code generation, file edits |
|
||||
| **FullAccess** | Full filesystem | Unrestricted | Trusted admin tasks |
|
||||
|
||||
### Network Proxy
|
||||
|
||||
Containers route all HTTP/HTTPS traffic through a host-side proxy (`src/sandbox/proxy/`):
|
||||
- **Domain allowlist** -- Only allowlisted domains are reachable (default: package registries, docs sites, GitHub, common APIs)
|
||||
- **Credential injection** -- The `CredentialResolver` trait injects auth headers into proxied requests so secrets never enter the container environment
|
||||
- **CONNECT tunnel** -- HTTPS traffic uses CONNECT method; the proxy validates the target domain against the allowlist before establishing the tunnel
|
||||
- **Policy decisions** -- The `NetworkPolicyDecider` trait allows custom logic for allow/deny/inject decisions per request
|
||||
|
||||
### Zero-Exposure Credential Model
|
||||
|
||||
Secrets (API keys, tokens) are stored encrypted on the host and injected into HTTP requests by the proxy at transit time. Container processes never have access to raw credential values, preventing exfiltration even if container code is compromised.
|
||||
|
||||
Sandbox configuration: see Configuration section above.
|
||||
|
||||
## Testing
|
||||
|
||||
Tests are in `mod tests {}` blocks at the bottom of each file. Run specific module tests:
|
||||
@@ -293,131 +608,22 @@ Key test patterns:
|
||||
|
||||
## Current Limitations / TODOs
|
||||
|
||||
1. **Slack/Telegram channels** - Stubs only, need implementation
|
||||
2. **Domain-specific tools** - `marketplace.rs`, `restaurant.rs`, `taskrabbit.rs`, `ecommerce.rs` return placeholder responses; need real API integrations
|
||||
3. **Integration tests** - Need testcontainers setup for PostgreSQL
|
||||
4. **MCP stdio transport** - Only HTTP transport implemented
|
||||
5. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed)
|
||||
6. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access
|
||||
7. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools
|
||||
1. **Domain-specific tools** - `marketplace.rs`, `restaurant.rs`, `taskrabbit.rs`, `ecommerce.rs` return placeholder responses; need real API integrations
|
||||
2. **Integration tests** - Need testcontainers setup for PostgreSQL
|
||||
3. **MCP stdio transport** - Only HTTP transport implemented
|
||||
4. **WIT bindgen integration** - Auto-extract tool description/schema from WASM modules (stubbed)
|
||||
5. **Capability granting after tool build** - Built tools get empty capabilities; need UX for granting HTTP/secrets access
|
||||
6. **Tool versioning workflow** - No version tracking or rollback for dynamically built tools
|
||||
7. **Webhook trigger endpoint** - Routines webhook trigger not yet exposed in web gateway
|
||||
8. **Full channel status view** - Gateway status widget exists, but no per-channel connection dashboard
|
||||
|
||||
### Completed
|
||||
## Tool Architecture
|
||||
|
||||
- ✅ **Workspace integration** - Memory tools registered, workspace passed to Agent and heartbeat
|
||||
- ✅ **WASM sandboxing** - Full implementation in `tools/wasm/` with fuel metering, memory limits, capabilities
|
||||
- ✅ **Dynamic tool building** - `tools/builder/` has LlmSoftwareBuilder with iterative build loop
|
||||
- ✅ **HTTP webhook security** - Secret validation implemented, proper error handling (no panics)
|
||||
- ✅ **Embeddings integration** - OpenAI and NEAR AI providers wired to workspace for semantic search
|
||||
- ✅ **Workspace system prompt** - Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) injected into LLM context
|
||||
- ✅ **Heartbeat notifications** - Route through channel manager (broadcast API) instead of logging-only
|
||||
- ✅ **Auto-context compaction** - Triggers automatically when context exceeds threshold
|
||||
- ✅ **Embedding backfill** - Runs on startup when embeddings provider is enabled
|
||||
- ✅ **Clippy clean** - All warnings addressed via config struct refactoring
|
||||
- ✅ **Tool approval enforcement** - Tools with `requires_approval()` (shell, http, file write/patch, build_software) now gate execution, track auto-approved tools per session
|
||||
- ✅ **Tool definition refresh** - Tool definitions refreshed each iteration so newly built tools become visible in same session
|
||||
- ✅ **Worker tool call handling** - Uses `respond_with_tools()` to properly execute tool calls when `select_tools()` returns empty
|
||||
**Keep tool-specific logic out of the main agent codebase.** The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through `capabilities.json` files (API endpoints, credentials, rate limits, auth setup). Service-specific auth flows, CLI commands, and configuration do not belong in the main agent.
|
||||
|
||||
## Adding a New Tool
|
||||
Tools can be built as **WASM** (sandboxed, credential-injected, single binary) or **MCP servers** (ecosystem of pre-built servers, any language, but no sandbox). Both are first-class via `ironclaw tool install`. Auth is declared in capabilities files with OAuth and manual token entry support.
|
||||
|
||||
### Built-in Tools (Rust)
|
||||
|
||||
1. Create `src/tools/builtin/my_tool.rs`
|
||||
2. Implement the `Tool` trait
|
||||
3. Add `mod my_tool;` and `pub use` in `src/tools/builtin/mod.rs`
|
||||
4. Register in `ToolRegistry::register_builtin_tools()` in `registry.rs`
|
||||
5. Add tests
|
||||
|
||||
### WASM Tools (Recommended)
|
||||
|
||||
WASM tools are the preferred way to add new capabilities. They run in a sandboxed environment with explicit capabilities.
|
||||
|
||||
1. Create a new crate in `examples/wasm-tools/<name>/`
|
||||
2. Implement the WIT interface (`wit/tool.wit`)
|
||||
3. Create `<name>.capabilities.json` declaring required permissions
|
||||
4. Build with `cargo build --target wasm32-wasip2 --release`
|
||||
5. Install with `ironclaw tool install path/to/tool.wasm`
|
||||
|
||||
See `examples/wasm-tools/` for examples.
|
||||
|
||||
## Tool Architecture Principles
|
||||
|
||||
**CRITICAL: Keep tool-specific logic out of the main agent codebase.**
|
||||
|
||||
The main agent provides generic infrastructure; tools are self-contained units that declare their requirements through capabilities files.
|
||||
|
||||
### What Goes in Tools (capabilities.json)
|
||||
|
||||
- API endpoints the tool needs (HTTP allowlist)
|
||||
- Credentials required (secret names, injection locations)
|
||||
- Rate limits and timeouts
|
||||
- Auth setup instructions (see below)
|
||||
- Workspace paths the tool can read
|
||||
|
||||
### What Does NOT Go in Main Agent
|
||||
|
||||
- Service-specific auth flows (OAuth for Notion, Slack, etc.)
|
||||
- Service-specific CLI commands (`auth notion`, `auth slack`)
|
||||
- Service-specific configuration handling
|
||||
- Hardcoded API URLs or token formats
|
||||
|
||||
### Tool Authentication
|
||||
|
||||
Tools declare their auth requirements in `<tool>.capabilities.json` under the `auth` section. Two methods are supported:
|
||||
|
||||
#### OAuth (Browser-based login)
|
||||
|
||||
For services that support OAuth, users just click through browser login:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "notion_api_token",
|
||||
"display_name": "Notion",
|
||||
"oauth": {
|
||||
"authorization_url": "https://api.notion.com/v1/oauth/authorize",
|
||||
"token_url": "https://api.notion.com/v1/oauth/token",
|
||||
"client_id_env": "NOTION_OAUTH_CLIENT_ID",
|
||||
"client_secret_env": "NOTION_OAUTH_CLIENT_SECRET",
|
||||
"scopes": [],
|
||||
"use_pkce": false,
|
||||
"extra_params": { "owner": "user" }
|
||||
},
|
||||
"env_var": "NOTION_TOKEN"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
To enable OAuth for a tool:
|
||||
1. Register a public OAuth app with the service (e.g., notion.so/my-integrations)
|
||||
2. Configure redirect URIs: `http://localhost:9876/callback` through `http://localhost:9886/callback`
|
||||
3. Set environment variables for client_id and client_secret
|
||||
|
||||
#### Manual Token Entry (Fallback)
|
||||
|
||||
For services without OAuth or when OAuth isn't configured:
|
||||
|
||||
```json
|
||||
{
|
||||
"auth": {
|
||||
"secret_name": "openai_api_key",
|
||||
"display_name": "OpenAI",
|
||||
"instructions": "Get your API key from platform.openai.com/api-keys",
|
||||
"setup_url": "https://platform.openai.com/api-keys",
|
||||
"token_hint": "Starts with 'sk-'",
|
||||
"env_var": "OPENAI_API_KEY"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Auth Flow Priority
|
||||
|
||||
When running `ironclaw tool auth <tool>`:
|
||||
|
||||
1. Check `env_var` - if set in environment, use it directly
|
||||
2. Check `oauth` - if configured, open browser for OAuth flow
|
||||
3. Fall back to `instructions` + manual token entry
|
||||
|
||||
The agent reads auth config from the tool's capabilities file and provides the appropriate flow. No service-specific code in the main agent.
|
||||
See `src/tools/README.md` for full tool architecture, adding new tools (built-in Rust and WASM), auth JSON examples, and WASM vs MCP decision guide.
|
||||
|
||||
## Adding a New Channel
|
||||
|
||||
@@ -439,118 +645,30 @@ RUST_LOG=ironclaw::agent=debug cargo run
|
||||
RUST_LOG=ironclaw=debug,tower_http=debug cargo run
|
||||
```
|
||||
|
||||
## Code Style
|
||||
## Module Specifications
|
||||
|
||||
- Use `crate::` imports, not `super::`
|
||||
- No `pub use` re-exports unless exposing to downstream consumers
|
||||
- Prefer strong types over strings (enums, newtypes)
|
||||
- Keep functions focused, extract helpers when logic is reused
|
||||
- Comments for non-obvious logic only
|
||||
Some modules have a `README.md` that serves as the authoritative specification
|
||||
for that module's behavior. When modifying code in a module that has a spec:
|
||||
|
||||
1. **Read the spec first** before making changes
|
||||
2. **Code follows spec**: if the spec says X, the code must do X
|
||||
3. **Update both sides**: if you change behavior, update the spec to match;
|
||||
if you're implementing a spec change, update the code to match
|
||||
4. **Spec is the tiebreaker**: when code and spec disagree, the spec is correct
|
||||
(unless the spec is clearly outdated, in which case fix the spec first)
|
||||
|
||||
| Module | Spec File |
|
||||
|--------|-----------|
|
||||
| `src/setup/` | `src/setup/README.md` |
|
||||
| `src/workspace/` | `src/workspace/README.md` |
|
||||
| `src/tools/` | `src/tools/README.md` |
|
||||
|
||||
## Workspace & Memory System
|
||||
|
||||
Inspired by [OpenClaw](https://github.com/openclaw/openclaw), the workspace provides persistent memory for agents with a flexible filesystem-like structure.
|
||||
OpenClaw-inspired persistent memory with a flexible filesystem-like structure. Principle: "Memory is database, not RAM" -- if you want to remember something, write it explicitly. Uses hybrid search combining FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion.
|
||||
|
||||
### Key Principles
|
||||
Four memory tools for LLM use: `memory_search` (hybrid search -- call before answering questions about prior work), `memory_write`, `memory_read`, `memory_tree`. Identity files (AGENTS.md, SOUL.md, USER.md, IDENTITY.md) are injected into the LLM system prompt.
|
||||
|
||||
1. **"Memory is database, not RAM"** - If you want to remember something, write it explicitly
|
||||
2. **Flexible structure** - Create any directory/file hierarchy you need
|
||||
3. **Self-documenting** - Use README.md files to describe directory structure
|
||||
4. **Hybrid search** - Combines FTS (keyword) + vector (semantic) via Reciprocal Rank Fusion
|
||||
The heartbeat system runs proactive periodic execution (default: 30 minutes), reading `HEARTBEAT.md` and notifying via channel if findings are detected.
|
||||
|
||||
### Filesystem Structure
|
||||
|
||||
```
|
||||
workspace/
|
||||
├── README.md <- Root runbook/index
|
||||
├── MEMORY.md <- Long-term curated memory
|
||||
├── HEARTBEAT.md <- Periodic checklist
|
||||
├── IDENTITY.md <- Agent name, nature, vibe
|
||||
├── SOUL.md <- Core values
|
||||
├── AGENTS.md <- Behavior instructions
|
||||
├── USER.md <- User context
|
||||
├── context/ <- Identity-related docs
|
||||
│ ├── vision.md
|
||||
│ └── priorities.md
|
||||
├── daily/ <- Daily logs
|
||||
│ ├── 2024-01-15.md
|
||||
│ └── 2024-01-16.md
|
||||
├── projects/ <- Arbitrary structure
|
||||
│ └── alpha/
|
||||
│ ├── README.md
|
||||
│ └── notes.md
|
||||
└── ...
|
||||
```
|
||||
|
||||
### Using the Workspace
|
||||
|
||||
```rust
|
||||
use crate::workspace::{Workspace, OpenAiEmbeddings, paths};
|
||||
|
||||
// Create workspace for a user
|
||||
let workspace = Workspace::new("user_123", pool)
|
||||
.with_embeddings(Arc::new(OpenAiEmbeddings::new(api_key)));
|
||||
|
||||
// Read/write any path
|
||||
let doc = workspace.read("projects/alpha/notes.md").await?;
|
||||
workspace.write("context/priorities.md", "# Priorities\n\n1. Feature X").await?;
|
||||
workspace.append("daily/2024-01-15.md", "Completed task X").await?;
|
||||
|
||||
// Convenience methods for well-known files
|
||||
workspace.append_memory("User prefers dark mode").await?;
|
||||
workspace.append_daily_log("Session note").await?;
|
||||
|
||||
// List directory contents
|
||||
let entries = workspace.list("projects/").await?;
|
||||
|
||||
// Search (hybrid FTS + vector)
|
||||
let results = workspace.search("dark mode preference", 5).await?;
|
||||
|
||||
// Get system prompt from identity files
|
||||
let prompt = workspace.system_prompt().await?;
|
||||
```
|
||||
|
||||
### Memory Tools
|
||||
|
||||
Four tools for LLM use:
|
||||
|
||||
- **`memory_search`** - Hybrid search, MUST be called before answering questions about prior work
|
||||
- **`memory_write`** - Write to any path (memory, daily_log, or custom paths)
|
||||
- **`memory_read`** - Read any file by path
|
||||
- **`memory_tree`** - View workspace structure as a tree (depth parameter, default 1)
|
||||
|
||||
### Hybrid Search (RRF)
|
||||
|
||||
Combines full-text search (PostgreSQL `ts_rank_cd`) and vector similarity (pgvector cosine) using Reciprocal Rank Fusion:
|
||||
|
||||
```
|
||||
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||
```
|
||||
|
||||
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
||||
|
||||
### Heartbeat System
|
||||
|
||||
Proactive periodic execution (default: 30 minutes):
|
||||
|
||||
1. Reads `HEARTBEAT.md` checklist
|
||||
2. Runs agent turn with checklist prompt
|
||||
3. If findings, notifies via channel
|
||||
4. If nothing, agent replies "HEARTBEAT_OK" (no notification)
|
||||
|
||||
```rust
|
||||
use crate::agent::{HeartbeatConfig, spawn_heartbeat};
|
||||
|
||||
let config = HeartbeatConfig::default()
|
||||
.with_interval(Duration::from_secs(60 * 30))
|
||||
.with_notify("user_123", "telegram");
|
||||
|
||||
spawn_heartbeat(config, workspace, llm, response_tx);
|
||||
```
|
||||
|
||||
### Chunking Strategy
|
||||
|
||||
Documents are chunked for search indexing:
|
||||
- Default: 800 words per chunk (roughly 800 tokens for English)
|
||||
- 15% overlap between chunks for context preservation
|
||||
- Minimum chunk size: 50 words (tiny trailing chunks merge with previous)
|
||||
See `src/workspace/README.md` for full API documentation, filesystem structure, hybrid search details, chunking strategy, and heartbeat system.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# Contributing
|
||||
|
||||
## Feature Parity Requirement
|
||||
|
||||
When your change affects a tracked capability, update `FEATURE_PARITY.md` in the same branch.
|
||||
|
||||
### Required before opening a PR
|
||||
|
||||
1. Review the relevant parity rows in `FEATURE_PARITY.md`.
|
||||
2. Update status/notes if behavior changed.
|
||||
3. Include the `FEATURE_PARITY.md` diff in your commit when applicable.
|
||||
Generated
+1433
-233
File diff suppressed because it is too large
Load Diff
+117
-18
@@ -1,29 +1,60 @@
|
||||
[workspace]
|
||||
members = [".", "benchmarks"]
|
||||
exclude = [
|
||||
"channels-src/discord",
|
||||
"channels-src/telegram",
|
||||
"channels-src/slack",
|
||||
"channels-src/whatsapp",
|
||||
"tools-src/github",
|
||||
"tools-src/gmail",
|
||||
"tools-src/google-calendar",
|
||||
"tools-src/google-docs",
|
||||
"tools-src/google-drive",
|
||||
"tools-src/google-sheets",
|
||||
"tools-src/google-slides",
|
||||
"tools-src/okta",
|
||||
"tools-src/slack",
|
||||
"tools-src/telegram",
|
||||
]
|
||||
|
||||
[package]
|
||||
name = "ironclaw"
|
||||
version = "0.1.0"
|
||||
version = "0.8.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
rust-version = "1.92"
|
||||
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
||||
authors = ["NEAR AI <[email protected]>"]
|
||||
license = "MIT OR Apache-2.0"
|
||||
homepage = "https://github.com/nearai/ironclaw"
|
||||
repository = "https://github.com/nearai/ironclaw"
|
||||
|
||||
[package.metadata.wix]
|
||||
upgrade-guid = "D0156E61-BA37-451E-8AB9-1A2ECCCFA48F"
|
||||
path-guid = "F90B6EA6-87F7-499B-BB19-CF55DE1EB339"
|
||||
license = false
|
||||
eula = false
|
||||
|
||||
[dependencies]
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
# Database
|
||||
deadpool-postgres = "0.14"
|
||||
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"] }
|
||||
postgres-types = { version = "0.2", features = ["with-serde_json-1"] }
|
||||
refinery = { version = "0.8", features = ["tokio-postgres"] }
|
||||
# Database - PostgreSQL (default, feature-gated)
|
||||
deadpool-postgres = { version = "0.14", optional = true }
|
||||
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true }
|
||||
postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true }
|
||||
refinery = { version = "0.8", features = ["tokio-postgres"], optional = true }
|
||||
|
||||
# Database - libSQL/Turso (optional embedded database)
|
||||
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2"
|
||||
@@ -35,11 +66,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
|
||||
|
||||
# Configuration
|
||||
dotenvy = "0.15"
|
||||
toml = "0.8"
|
||||
|
||||
# Core types
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "db-tokio-postgres", "maths"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] }
|
||||
rust_decimal_macros = "1"
|
||||
|
||||
# Async traits
|
||||
@@ -48,26 +80,35 @@ async-trait = "0.1"
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
# TUI
|
||||
ratatui = "0.29"
|
||||
crossterm = { version = "0.28", features = ["event-stream"] }
|
||||
# Terminal
|
||||
crossterm = "0.28"
|
||||
rustyline = { version = "17", features = ["derive", "with-file-history"] }
|
||||
termimad = "0.34"
|
||||
|
||||
# Channel integrations
|
||||
axum = "0.8"
|
||||
axum = { version = "0.8", features = ["ws"] }
|
||||
tower = "0.5"
|
||||
tower-http = { version = "0.6", features = ["trace", "cors"] }
|
||||
tower-http = { version = "0.6", features = ["trace", "cors", "set-header"] }
|
||||
|
||||
# Cron scheduling for routines
|
||||
cron = "0.13"
|
||||
|
||||
# Safety/sanitization
|
||||
regex = "1"
|
||||
aho-corasick = "1"
|
||||
|
||||
# YAML parsing for SKILL.md frontmatter
|
||||
serde_yml = "0.0.12"
|
||||
|
||||
# Filesystem paths
|
||||
dirs = "6"
|
||||
fs4 = "0.6"
|
||||
|
||||
# Secrecy for sensitive values
|
||||
secrecy = { version = "0.10", features = ["serde"] }
|
||||
|
||||
# URL encoding for OAuth flow
|
||||
# URL parsing and encoding
|
||||
url = "2"
|
||||
urlencoding = "2"
|
||||
|
||||
# Open URLs in browser
|
||||
@@ -75,7 +116,7 @@ open = "5"
|
||||
|
||||
# Vector embeddings for semantic search
|
||||
# The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres)
|
||||
pgvector = { version = "0.4", features = ["postgres"] }
|
||||
pgvector = { version = "0.4", features = ["postgres"], optional = true }
|
||||
|
||||
# WASM sandbox for untrusted tool execution
|
||||
wasmtime = { version = "28", features = ["component-model"] }
|
||||
@@ -88,6 +129,10 @@ hkdf = "0.12"
|
||||
sha2 = "0.10"
|
||||
blake3 = "1"
|
||||
rand = "0.8"
|
||||
subtle = "2" # Constant-time comparisons for token validation
|
||||
|
||||
# Multi-provider LLM support
|
||||
rig-core = "0.30"
|
||||
|
||||
# Docker sandbox
|
||||
bollard = "0.18"
|
||||
@@ -98,6 +143,7 @@ hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"]
|
||||
http-body-util = "0.1"
|
||||
bytes = "1"
|
||||
base64 = "0.22.1"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
# macOS keychain
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
@@ -110,10 +156,63 @@ zbus = "4"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
tokio-tungstenite = "0.26"
|
||||
testcontainers-modules = { version = "0.11", features = ["postgres"] }
|
||||
pretty_assertions = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
default = ["postgres", "libsql"]
|
||||
postgres = [
|
||||
"dep:deadpool-postgres",
|
||||
"dep:tokio-postgres",
|
||||
"dep:postgres-types",
|
||||
"dep:refinery",
|
||||
"dep:pgvector",
|
||||
"rust_decimal/db-tokio-postgres",
|
||||
]
|
||||
libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
|
||||
# The profile that 'cargo dist' will build with
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
lto = "thin"
|
||||
|
||||
# Config for 'dist'
|
||||
[workspace.metadata.dist]
|
||||
# The preferred dist version to use in CI (Cargo.toml SemVer syntax)
|
||||
cargo-dist-version = "0.30.3"
|
||||
# CI backends to support
|
||||
ci = "github"
|
||||
# The installers to generate for each app
|
||||
installers = ["shell", "powershell", "npm", "msi"]
|
||||
# Publish jobs to run in CI
|
||||
publish-jobs = []
|
||||
# Target platforms to build apps for (Rust target-triple syntax)
|
||||
targets = [
|
||||
"aarch64-apple-darwin",
|
||||
"aarch64-unknown-linux-gnu",
|
||||
"x86_64-apple-darwin",
|
||||
"x86_64-unknown-linux-gnu",
|
||||
"x86_64-pc-windows-msvc",
|
||||
]
|
||||
# The archive format to use for windows builds (defaults .zip)
|
||||
windows-archive = ".tar.gz"
|
||||
# The archive format to use for non-windows builds (defaults .tar.xz)
|
||||
unix-archive = ".tar.gz"
|
||||
# Which actions to run on pull requests
|
||||
pr-run-mode = "skip"
|
||||
# Path that installers should place binaries in
|
||||
install-path = "CARGO_HOME"
|
||||
# Whether to install an updater program
|
||||
install-updater = true
|
||||
# Cache intermediate build artifacts to speed up the release pipelines
|
||||
cache-builds = true
|
||||
|
||||
[workspace.metadata.dist.github-custom-runners]
|
||||
aarch64-unknown-linux-gnu = "ubuntu-24.04-arm"
|
||||
x86_64-unknown-linux-gnu = "ubuntu-22.04"
|
||||
x86_64-pc-windows-msvc = "windows-2022"
|
||||
x86_64-apple-darwin = "macos-15-intel"
|
||||
aarch64-apple-darwin = "macos-14"
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
# Multi-stage Dockerfile for the IronClaw agent (cloud deployment).
|
||||
#
|
||||
# Build:
|
||||
# docker build --platform linux/amd64 -t ironclaw:latest .
|
||||
#
|
||||
# Run:
|
||||
# docker run --env-file .env -p 3000:3000 ironclaw:latest
|
||||
|
||||
# Stage 1: Build
|
||||
FROM rust:1.92-slim-bookworm AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config libssl-dev cmake gcc g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy manifests first for layer caching
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
|
||||
# Copy source and build artifacts
|
||||
COPY src/ src/
|
||||
COPY migrations/ migrations/
|
||||
COPY wit/ wit/
|
||||
|
||||
RUN cargo build --release --bin ironclaw
|
||||
|
||||
# Stage 2: Runtime
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates libssl3 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /app/target/release/ironclaw /usr/local/bin/ironclaw
|
||||
COPY --from=builder /app/migrations /app/migrations
|
||||
|
||||
# Non-root user
|
||||
RUN useradd -m -u 1000 -s /bin/bash ironclaw
|
||||
USER ironclaw
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
ENV RUST_LOG=ironclaw=info
|
||||
|
||||
ENTRYPOINT ["ironclaw"]
|
||||
@@ -0,0 +1,69 @@
|
||||
# Multi-stage Dockerfile for the IronClaw worker container.
|
||||
#
|
||||
# This image runs the ironclaw binary in worker mode inside Docker containers.
|
||||
# The orchestrator creates instances of this image for sandboxed job execution.
|
||||
#
|
||||
# Build:
|
||||
# docker build -f Dockerfile.worker -t ironclaw-worker .
|
||||
#
|
||||
# The image includes common development tools so workers can build software,
|
||||
# run tests, and execute shell commands.
|
||||
|
||||
FROM rust:1.92-bookworm AS builder
|
||||
|
||||
WORKDIR /build
|
||||
COPY . .
|
||||
|
||||
# Build only the ironclaw binary (release mode)
|
||||
RUN cargo build --release --bin ironclaw
|
||||
|
||||
# ---
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
# Install curl first (needed to fetch the GitHub CLI GPG key), then add the
|
||||
# gh CLI apt repository, then install all remaining dev tools in one layer.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
|
||||
| dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
|
||||
> /etc/apt/sources.list.d/github-cli.list \
|
||||
&& apt-get update && apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
build-essential \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
python3 \
|
||||
python3-pip \
|
||||
python3-venv \
|
||||
gh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Rust toolchain for the sandbox user
|
||||
ENV RUSTUP_HOME=/usr/local/rustup \
|
||||
CARGO_HOME=/usr/local/cargo \
|
||||
PATH=/usr/local/cargo/bin:$PATH
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.92.0 \
|
||||
&& chmod -R a+r /usr/local/rustup /usr/local/cargo
|
||||
|
||||
# Install Claude Code CLI (for claude-bridge mode)
|
||||
RUN npm install -g @anthropic-ai/claude-code@latest
|
||||
|
||||
# Copy the binary
|
||||
COPY --from=builder /build/target/release/ironclaw /usr/local/bin/ironclaw
|
||||
|
||||
# Create non-root user (UID 1000 matches the orchestrator's container config)
|
||||
RUN useradd -m -u 1000 -s /bin/bash sandbox \
|
||||
&& mkdir -p /workspace \
|
||||
&& chown sandbox:sandbox /workspace \
|
||||
&& mkdir -p /home/sandbox/.claude \
|
||||
&& chown sandbox:sandbox /home/sandbox/.claude
|
||||
|
||||
USER sandbox
|
||||
WORKDIR /workspace
|
||||
|
||||
# The orchestrator passes the full command via Docker cmd.
|
||||
ENTRYPOINT ["ironclaw"]
|
||||
+206
-83
@@ -16,8 +16,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Hub-and-spoke architecture | ✅ | 🚧 | IronClaw has channels but no central gateway |
|
||||
| WebSocket control plane | ✅ | ❌ | Gateway with ws://127.0.0.1:18789 |
|
||||
| Hub-and-spoke architecture | ✅ | ✅ | Web gateway as central hub |
|
||||
| WebSocket control plane | ✅ | ✅ | Gateway with WebSocket + SSE |
|
||||
| Single-user system | ✅ | ✅ | |
|
||||
| Multi-agent routing | ✅ | ❌ | Workspace isolation per-agent |
|
||||
| Session-based messaging | ✅ | ✅ | Per-sender sessions |
|
||||
@@ -31,20 +31,27 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Gateway control plane | ✅ | ❌ | Central WebSocket server |
|
||||
| HTTP endpoints for Control UI | ✅ | ❌ | Web dashboard |
|
||||
| Channel connection lifecycle | ✅ | 🚧 | ChannelManager handles streams |
|
||||
| Gateway control plane | ✅ | ✅ | Web gateway with 40+ API endpoints |
|
||||
| HTTP endpoints for Control UI | ✅ | ✅ | Web dashboard with chat, memory, jobs, logs, extensions |
|
||||
| Channel connection lifecycle | ✅ | ✅ | ChannelManager + WebSocket tracker |
|
||||
| Session management/routing | ✅ | ✅ | SessionManager exists |
|
||||
| Configuration hot-reload | ✅ | ❌ | |
|
||||
| Network modes (loopback/LAN/remote) | ✅ | 🚧 | HTTP only |
|
||||
| OpenAI-compatible HTTP API | ✅ | ❌ | /v1/chat/completions |
|
||||
| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions, per-request `model` override |
|
||||
| Canvas hosting | ✅ | ❌ | Agent-driven UI |
|
||||
| Gateway lock (PID-based) | ✅ | ❌ | |
|
||||
| launchd/systemd integration | ✅ | ❌ | |
|
||||
| Bonjour/mDNS discovery | ✅ | ❌ | |
|
||||
| Tailscale integration | ✅ | ❌ | |
|
||||
| Health check endpoints | ✅ | ❌ | |
|
||||
| Health check endpoints | ✅ | ✅ | /api/health + /api/gateway/status |
|
||||
| `doctor` diagnostics | ✅ | ❌ | |
|
||||
| Agent event broadcast | ✅ | 🚧 | SSE broadcast manager exists (SseManager) but tool/job-state events not fully wired |
|
||||
| Channel health monitor | ✅ | ❌ | Auto-restart with configurable interval |
|
||||
| Presence system | ✅ | ❌ | Beacons on connect, system presence for agents |
|
||||
| Trusted-proxy auth mode | ✅ | ❌ | Header-based auth for reverse proxies |
|
||||
| APNs push pipeline | ✅ | ❌ | Wake disconnected iOS nodes via push |
|
||||
| Oversized payload guard | ✅ | 🚧 | HTTP webhook has 64KB body limit + Content-Length check; no chat.history cap |
|
||||
| Pre-prompt context diagnostics | ✅ | ❌ | Context size logging before prompt |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -58,35 +65,65 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| HTTP webhook | ✅ | ✅ | - | axum with secret validation |
|
||||
| REPL (simple) | ✅ | ✅ | - | For testing |
|
||||
| WASM channels | ❌ | ✅ | - | IronClaw innovation |
|
||||
| WhatsApp | ✅ | ❌ | P1 | Baileys (Web) |
|
||||
| Telegram | ✅ | ❌ | P1 | grammY (Bot API) |
|
||||
| Discord | ✅ | ❌ | P2 | discord.js |
|
||||
| WhatsApp | ✅ | ❌ | P1 | Baileys (Web), same-phone mode with echo detection |
|
||||
| Telegram | ✅ | ✅ | - | WASM channel(MTProto), DM pairing, caption, /start, bot_username |
|
||||
| Discord | ✅ | ❌ | P2 | discord.js, thread parent binding inheritance |
|
||||
| Signal | ✅ | ❌ | P2 | signal-cli |
|
||||
| Slack | ✅ | 🚧 | P1 | Stub exists, needs implementation |
|
||||
| iMessage | ✅ | ❌ | P3 | BlueBubbles recommended |
|
||||
| Feishu/Lark | ✅ | ❌ | P3 | |
|
||||
| Slack | ✅ | ✅ | - | WASM tool |
|
||||
| iMessage | ✅ | ❌ | P3 | BlueBubbles or Linq recommended |
|
||||
| Linq | ✅ | ❌ | P3 | Real iMessage via API, no Mac required |
|
||||
| Feishu/Lark | ✅ | ❌ | P3 | Bitable create app/field tools |
|
||||
| LINE | ✅ | ❌ | P3 | |
|
||||
| WebChat | ✅ | ❌ | P2 | Browser-based chat |
|
||||
| WebChat | ✅ | ✅ | - | Web gateway chat |
|
||||
| Matrix | ✅ | ❌ | P3 | E2EE support |
|
||||
| Mattermost | ✅ | ❌ | P3 | |
|
||||
| Mattermost | ✅ | ❌ | P3 | Emoji reactions |
|
||||
| Google Chat | ✅ | ❌ | P3 | |
|
||||
| MS Teams | ✅ | ❌ | P3 | |
|
||||
| Twitch | ✅ | ❌ | P3 | |
|
||||
| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx |
|
||||
| Voice Call | ✅ | ❌ | P3 | Twilio/Telnyx, stale call reaper, pre-cached greeting |
|
||||
| Nostr | ✅ | ❌ | P3 | |
|
||||
|
||||
### Telegram-Specific Features (since Feb 2025)
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Forum topic creation | ✅ | ❌ | Create topics in forum groups |
|
||||
| channel_post support | ✅ | ❌ | Bot-to-bot communication |
|
||||
| User message reactions | ✅ | ❌ | Surface inbound reactions |
|
||||
| sendPoll | ✅ | ❌ | Poll creation via agent |
|
||||
| Cron/heartbeat topic targeting | ✅ | ❌ | Messages land in correct topic |
|
||||
|
||||
### Discord-Specific Features (since Feb 2025)
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Forwarded attachment downloads | ✅ | ❌ | Fetch media from forwarded messages |
|
||||
| Faster reaction state machine | ✅ | ❌ | Watchdog + debounce |
|
||||
| Thread parent binding inheritance | ✅ | ❌ | Threads inherit parent routing |
|
||||
|
||||
### Slack-Specific Features (since Feb 2025)
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Streaming draft replies | ✅ | ❌ | Partial replies via draft message updates |
|
||||
| Configurable stream modes | ✅ | ❌ | Per-channel stream behavior |
|
||||
| Thread ownership | ✅ | ❌ | Thread-level ownership tracking |
|
||||
|
||||
### Channel Features
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| DM pairing codes | ✅ | ❌ | Verification for unknown senders |
|
||||
| Allowlist/blocklist | ✅ | ❌ | Per-channel access control |
|
||||
| DM pairing codes | ✅ | ✅ | `ironclaw pairing list/approve`, host APIs |
|
||||
| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store |
|
||||
| Self-message bypass | ✅ | ❌ | Own messages skip pairing |
|
||||
| Mention-based activation | ✅ | ❌ | Configurable patterns |
|
||||
| Mention-based activation | ✅ | ✅ | bot_username + respond_to_all_group_messages |
|
||||
| Per-group tool policies | ✅ | ❌ | Allow/deny specific tools |
|
||||
| Thread isolation | ✅ | ✅ | Separate sessions per thread |
|
||||
| Per-channel media limits | ✅ | ❌ | |
|
||||
| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits |
|
||||
| Typing indicators | ✅ | 🚧 | TUI shows status |
|
||||
| Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions |
|
||||
| Group session priming | ✅ | ❌ | Member roster injected for context |
|
||||
| Sender_id in trusted metadata | ✅ | ❌ | Exposed in system metadata |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -99,21 +136,21 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| `run` (agent) | ✅ | ✅ | - | Default command |
|
||||
| `tool install/list/remove` | ✅ | ✅ | - | WASM tools |
|
||||
| `gateway start/stop` | ✅ | ❌ | P2 | |
|
||||
| `onboard` (wizard) | ✅ | ❌ | P2 | Interactive setup |
|
||||
| `onboard` (wizard) | ✅ | ✅ | - | Interactive setup |
|
||||
| `tui` | ✅ | ✅ | - | Ratatui TUI |
|
||||
| `config` | ✅ | ❌ | P2 | Read/write config |
|
||||
| `config` | ✅ | ✅ | - | Read/write config |
|
||||
| `channels` | ✅ | ❌ | P2 | Channel management |
|
||||
| `models` | ✅ | 🚧 | - | Model selector in TUI |
|
||||
| `status` | ✅ | ❌ | P2 | System status |
|
||||
| `status` | ✅ | ✅ | - | System status (enriched session details) |
|
||||
| `agents` | ✅ | ❌ | P3 | Multi-agent management |
|
||||
| `sessions` | ✅ | ❌ | P3 | Session listing |
|
||||
| `memory` | ✅ | ❌ | P2 | Memory search CLI |
|
||||
| `skills` | ✅ | ❌ | P3 | Agent skills |
|
||||
| `pairing` | ✅ | ❌ | P3 | Node pairing |
|
||||
| `nodes` | ✅ | ❌ | P3 | Device management |
|
||||
| `sessions` | ✅ | ❌ | P3 | Session listing (shows subagent models) |
|
||||
| `memory` | ✅ | ✅ | - | Memory search CLI |
|
||||
| `skills` | ✅ | ✅ | - | Skills tools + web API endpoints (install, list, activate) |
|
||||
| `pairing` | ✅ | ✅ | - | list/approve, account selector |
|
||||
| `nodes` | ✅ | ❌ | P3 | Device management, remove/clear flows |
|
||||
| `plugins` | ✅ | ❌ | P3 | Plugin management |
|
||||
| `hooks` | ✅ | ❌ | P2 | Lifecycle hooks |
|
||||
| `cron` | ✅ | ❌ | P2 | Scheduled jobs |
|
||||
| `hooks` | ✅ | ✅ | P2 | Lifecycle hooks |
|
||||
| `cron` | ✅ | ❌ | P2 | Scheduled jobs (model/thinking fields in edit) |
|
||||
| `webhooks` | ✅ | ❌ | P3 | Webhook config |
|
||||
| `message send` | ✅ | ❌ | P2 | Send to channels |
|
||||
| `browser` | ✅ | ❌ | P3 | Browser automation |
|
||||
@@ -122,6 +159,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| `logs` | ✅ | ❌ | P3 | Query logs |
|
||||
| `update` | ✅ | ❌ | P3 | Self-update |
|
||||
| `completion` | ✅ | ❌ | P3 | Shell completion |
|
||||
| `/subagents spawn` | ✅ | ❌ | P3 | Spawn subagents from chat |
|
||||
| `/export-session` | ✅ | ❌ | P3 | Export current session transcript |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -132,23 +171,38 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Pi agent runtime | ✅ | ➖ | IronClaw uses custom runtime |
|
||||
| RPC-based execution | ✅ | 🚧 | Worker isolation |
|
||||
| Multi-provider failover | ✅ | ❌ | Provider fallback chains |
|
||||
| RPC-based execution | ✅ | ✅ | Orchestrator/worker pattern |
|
||||
| Multi-provider failover | ✅ | ✅ | `FailoverProvider` tries providers sequentially on retryable errors |
|
||||
| Per-sender sessions | ✅ | ✅ | |
|
||||
| Global sessions | ✅ | ❌ | Optional shared context |
|
||||
| Session pruning | ✅ | ❌ | Auto cleanup old sessions |
|
||||
| Context compaction | ✅ | ✅ | Auto summarization |
|
||||
| Custom system prompts | ✅ | ✅ | Template variables |
|
||||
| Skills (modular capabilities) | ✅ | ❌ | Capability bundles |
|
||||
| Post-compaction read audit | ✅ | ❌ | Layer 3: workspace rules appended to summaries |
|
||||
| Post-compaction context injection | ✅ | ❌ | Workspace context as system event |
|
||||
| Custom system prompts | ✅ | ✅ | Template variables, safety guardrails |
|
||||
| Skills (modular capabilities) | ✅ | ✅ | Prompt-based skills with trust gating, attenuation, activation criteria, catalog, selector |
|
||||
| Skill routing blocks | ✅ | 🚧 | ActivationCriteria (keywords, patterns, tags) but no "Use when / Don't use when" blocks |
|
||||
| Skill path compaction | ✅ | ❌ | ~ prefix to reduce prompt tokens |
|
||||
| Thinking modes (low/med/high) | ✅ | ❌ | Configurable reasoning depth |
|
||||
| Per-model thinkingDefault override | ✅ | ❌ | Override thinking level per model |
|
||||
| Block-level streaming | ✅ | ❌ | |
|
||||
| Tool-level streaming | ✅ | ❌ | |
|
||||
| Z.AI tool_stream | ✅ | ❌ | Real-time tool call streaming |
|
||||
| Plugin tools | ✅ | ✅ | WASM tools |
|
||||
| Tool policies (allow/deny) | ✅ | ✅ | |
|
||||
| Exec approvals (`/approve`) | ✅ | ✅ | TUI approval overlay |
|
||||
| Elevated mode | ✅ | ❌ | Privileged execution |
|
||||
| Subagent support | ✅ | ✅ | Task framework |
|
||||
| `/subagents spawn` command | ✅ | ❌ | Spawn from chat |
|
||||
| Auth profiles | ✅ | ❌ | Multiple auth strategies |
|
||||
| Generic API key rotation | ✅ | ❌ | Rotate keys across providers |
|
||||
| Stuck loop detection | ✅ | ❌ | Exponential backoff on stuck agent loops |
|
||||
| llms.txt discovery | ✅ | ❌ | Auto-discover site metadata |
|
||||
| Multiple images per tool call | ✅ | ❌ | Single tool call, multiple images |
|
||||
| URL allowlist (web_search/fetch) | ✅ | ❌ | Restrict web tool targets |
|
||||
| suppressToolErrors config | ✅ | ❌ | Hide tool errors from user |
|
||||
| Intent-first tool display | ✅ | ❌ | Details and exec summaries |
|
||||
| Transcript file size in status | ✅ | ❌ | Show size in session status |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -159,12 +213,18 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Provider | OpenClaw | IronClaw | Priority | Notes |
|
||||
|----------|----------|----------|----------|-------|
|
||||
| NEAR AI | ✅ | ✅ | - | Primary provider |
|
||||
| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy |
|
||||
| Anthropic (Claude) | ✅ | 🚧 | - | Via NEAR AI proxy; Opus 4.5, Sonnet 4, Sonnet 4.6 |
|
||||
| OpenAI | ✅ | 🚧 | - | Via NEAR AI proxy |
|
||||
| AWS Bedrock | ✅ | ❌ | P3 | |
|
||||
| Google Gemini | ✅ | ❌ | P3 | |
|
||||
| OpenRouter | ✅ | ❌ | P3 | |
|
||||
| Ollama (local) | ✅ | ❌ | P2 | Local models |
|
||||
| NVIDIA API | ✅ | ❌ | P3 | New provider |
|
||||
| OpenRouter | ✅ | ✅ | - | Via OpenAI-compatible provider (RigAdapter) |
|
||||
| Tinfoil | ❌ | ✅ | - | Private inference provider (IronClaw-only) |
|
||||
| OpenAI-compatible | ❌ | ✅ | - | Generic OpenAI-compatible endpoint (RigAdapter) |
|
||||
| Ollama (local) | ✅ | ✅ | - | via `rig::providers::ollama` (full support) |
|
||||
| Perplexity | ✅ | ❌ | P3 | Freshness parameter for web_search |
|
||||
| MiniMax | ✅ | ❌ | P3 | Regional endpoint selection |
|
||||
| GLM-5 | ✅ | ❌ | P3 | |
|
||||
| node-llama-cpp | ✅ | ➖ | - | N/A for Rust |
|
||||
| llama.cpp (native) | ❌ | 🔮 | P3 | Rust bindings |
|
||||
|
||||
@@ -173,10 +233,12 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Auto-discovery | ✅ | ❌ | |
|
||||
| Failover chains | ✅ | ❌ | Provider fallback |
|
||||
| Cooldown management | ✅ | ❌ | Skip failed providers |
|
||||
| Failover chains | ✅ | ✅ | `FailoverProvider` with configurable `fallback_model` |
|
||||
| Cooldown management | ✅ | ✅ | Lock-free per-provider cooldown in `FailoverProvider` |
|
||||
| Per-session model override | ✅ | ✅ | Model selector in TUI |
|
||||
| Model selection UI | ✅ | ✅ | TUI keyboard shortcut |
|
||||
| Per-model thinkingDefault | ✅ | ❌ | Override thinking level per model in config |
|
||||
| 1M context beta header | ✅ | ❌ | Anthropic extended context support |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -187,6 +249,8 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Feature | OpenClaw | IronClaw | Priority | Notes |
|
||||
|---------|----------|----------|----------|-------|
|
||||
| Image processing (Sharp) | ✅ | ❌ | P2 | Resize, format convert |
|
||||
| Configurable image resize dims | ✅ | ❌ | P2 | Per-agent dimension config |
|
||||
| Multiple images per tool call | ✅ | ❌ | P2 | Single tool invocation, multiple images |
|
||||
| Audio transcription | ✅ | ❌ | P2 | |
|
||||
| Video support | ✅ | ❌ | P3 | |
|
||||
| PDF parsing | ✅ | ❌ | P2 | pdfjs-dist |
|
||||
@@ -195,6 +259,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Vision model integration | ✅ | ❌ | P2 | Image understanding |
|
||||
| TTS (Edge TTS) | ✅ | ❌ | P3 | Text-to-speech |
|
||||
| TTS (OpenAI) | ✅ | ❌ | P3 | |
|
||||
| Incremental TTS playback | ✅ | ❌ | P3 | iOS progressive playback |
|
||||
| Sticker-to-image | ✅ | ❌ | P3 | Telegram stickers |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
@@ -213,10 +278,13 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Auth plugins | ✅ | ❌ | |
|
||||
| Memory plugins | ✅ | ❌ | Custom backends |
|
||||
| Tool plugins | ✅ | ✅ | WASM tools |
|
||||
| Hook plugins | ✅ | ❌ | |
|
||||
| Hook plugins | ✅ | ✅ | Declarative hooks from extension capabilities |
|
||||
| Provider plugins | ✅ | ❌ | |
|
||||
| Plugin CLI (`install`, `list`) | ✅ | ✅ | `tool` subcommand |
|
||||
| ClawHub registry | ✅ | ❌ | Discovery |
|
||||
| `before_agent_start` hook | ✅ | ❌ | modelOverride/providerOverride support |
|
||||
| `before_message_write` hook | ✅ | ❌ | Pre-write message interception |
|
||||
| `llm_input`/`llm_output` hooks | ✅ | ❌ | LLM payload inspection |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -235,6 +303,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Legacy migration | ✅ | ➖ | |
|
||||
| State directory | ✅ `~/.openclaw-state/` | ✅ `~/.ironclaw/` | |
|
||||
| Credentials directory | ✅ | ✅ | Session files |
|
||||
| Full model compat fields in schema | ✅ | ❌ | pi-ai model compat exposed in config |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -247,16 +316,19 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Vector memory | ✅ | ✅ | pgvector |
|
||||
| Session-based memory | ✅ | ✅ | |
|
||||
| Hybrid search (BM25 + vector) | ✅ | ✅ | RRF algorithm |
|
||||
| Temporal decay (hybrid search) | ✅ | ❌ | Opt-in time-based scoring factor |
|
||||
| MMR re-ranking | ✅ | ❌ | Maximal marginal relevance for result diversity |
|
||||
| LLM-based query expansion | ✅ | ❌ | Expand FTS queries via LLM |
|
||||
| OpenAI embeddings | ✅ | ✅ | |
|
||||
| Gemini embeddings | ✅ | ❌ | |
|
||||
| Local embeddings | ✅ | ❌ | |
|
||||
| SQLite-vec backend | ✅ | ❌ | IronClaw uses PostgreSQL |
|
||||
| LanceDB backend | ✅ | ❌ | |
|
||||
| LanceDB backend | ✅ | ❌ | Configurable auto-capture max length |
|
||||
| QMD backend | ✅ | ❌ | |
|
||||
| Atomic reindexing | ✅ | ✅ | |
|
||||
| Embeddings batching | ✅ | ❌ | |
|
||||
| Embeddings batching | ✅ | ✅ | `embed_batch` on EmbeddingProvider trait |
|
||||
| Citation support | ✅ | ❌ | |
|
||||
| Memory CLI commands | ✅ | ❌ | `memory search/index/status` |
|
||||
| Memory CLI commands | ✅ | ✅ | `memory search/read/write/tree/status` CLI subcommands |
|
||||
| Flexible path structure | ✅ | ✅ | Filesystem-like API |
|
||||
| Identity files (AGENTS.md, etc.) | ✅ | ✅ | |
|
||||
| Daily logs | ✅ | ✅ | |
|
||||
@@ -272,12 +344,16 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|---------|----------|----------|----------|-------|
|
||||
| iOS app (SwiftUI) | ✅ | 🚫 | - | Out of scope initially |
|
||||
| Android app (Kotlin) | ✅ | 🚫 | - | Out of scope initially |
|
||||
| Apple Watch companion | ✅ | 🚫 | - | Send/receive messages MVP |
|
||||
| Gateway WebSocket client | ✅ | 🚫 | - | |
|
||||
| Camera/photo access | ✅ | 🚫 | - | |
|
||||
| Voice input | ✅ | 🚫 | - | |
|
||||
| Push-to-talk | ✅ | 🚫 | - | |
|
||||
| Location sharing | ✅ | 🚫 | - | |
|
||||
| Node pairing | ✅ | 🚫 | - | |
|
||||
| APNs push notifications | ✅ | 🚫 | - | Wake disconnected nodes before invoke |
|
||||
| Share to OpenClaw (iOS) | ✅ | 🚫 | - | iOS share sheet integration |
|
||||
| Background listening toggle | ✅ | 🚫 | - | iOS background audio |
|
||||
|
||||
### Owner: _Unassigned_ (if ever prioritized)
|
||||
|
||||
@@ -288,12 +364,17 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Feature | OpenClaw | IronClaw | Priority | Notes |
|
||||
|---------|----------|----------|----------|-------|
|
||||
| SwiftUI native app | ✅ | 🚫 | - | Out of scope |
|
||||
| Menu bar presence | ✅ | 🚫 | - | |
|
||||
| Menu bar presence | ✅ | 🚫 | - | Animated menubar icon |
|
||||
| Bundled gateway | ✅ | 🚫 | - | |
|
||||
| Canvas hosting | ✅ | 🚫 | - | |
|
||||
| Voice wake | ✅ | 🚫 | - | |
|
||||
| Canvas hosting | ✅ | 🚫 | - | Agent-controlled panel with placement/resizing |
|
||||
| Voice wake | ✅ | 🚫 | - | Overlay, mic picker, language selection, live meter |
|
||||
| Voice wake overlay | ✅ | 🚫 | - | Partial transcripts, adaptive delays, dismiss animations |
|
||||
| Push-to-talk hotkey | ✅ | 🚫 | - | System-wide hotkey |
|
||||
| Exec approval dialogs | ✅ | ✅ | - | TUI overlay |
|
||||
| iMessage integration | ✅ | 🚫 | - | |
|
||||
| Instances tab | ✅ | 🚫 | - | Presence beacons across instances |
|
||||
| Agent events debug window | ✅ | 🚫 | - | Real-time event inspector |
|
||||
| Sparkle auto-updates | ✅ | 🚫 | - | Appcast distribution |
|
||||
|
||||
### Owner: _Unassigned_ (if ever prioritized)
|
||||
|
||||
@@ -303,14 +384,17 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Priority | Notes |
|
||||
|---------|----------|----------|----------|-------|
|
||||
| Control UI Dashboard | ✅ | ❌ | P2 | Web status/config |
|
||||
| Channel status view | ✅ | ❌ | P2 | |
|
||||
| Control UI Dashboard | ✅ | ✅ | - | Web gateway with chat, memory, jobs, logs, extensions |
|
||||
| Channel status view | ✅ | 🚧 | P2 | Gateway status widget, full channel view pending |
|
||||
| Agent management | ✅ | ❌ | P3 | |
|
||||
| Model selection | ✅ | ✅ | - | TUI only |
|
||||
| Config editing | ✅ | ❌ | P3 | |
|
||||
| Debug/logs viewer | ✅ | ❌ | P3 | |
|
||||
| WebChat interface | ✅ | ❌ | P2 | Browser chat |
|
||||
| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI |
|
||||
| Debug/logs viewer | ✅ | ✅ | - | Real-time log streaming with level/target filters |
|
||||
| WebChat interface | ✅ | ✅ | - | Web gateway chat with SSE/WebSocket |
|
||||
| Canvas system (A2UI) | ✅ | ❌ | P3 | Agent-driven UI, improved asset resolution |
|
||||
| Control UI i18n | ✅ | ❌ | P3 | English, Chinese, Portuguese |
|
||||
| WebChat theme sync | ✅ | ❌ | P3 | Sync with system dark/light mode |
|
||||
| Partial output on abort | ✅ | ❌ | P2 | Preserve partial output when aborting |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -320,21 +404,27 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Priority | Notes |
|
||||
|---------|----------|----------|----------|-------|
|
||||
| Cron jobs | ✅ | ❌ | P2 | Schedule-based tasks |
|
||||
| Timezone support | ✅ | ❌ | P2 | |
|
||||
| One-shot/recurring jobs | ✅ | ❌ | P2 | |
|
||||
| `beforeInbound` hook | ✅ | ❌ | P2 | |
|
||||
| `beforeOutbound` hook | ✅ | ❌ | P2 | |
|
||||
| `beforeToolCall` hook | ✅ | ❌ | P2 | |
|
||||
| `onMessage` hook | ✅ | ❌ | P2 | |
|
||||
| `onSessionStart` hook | ✅ | ❌ | P2 | |
|
||||
| `onSessionEnd` hook | ✅ | ❌ | P2 | |
|
||||
| Cron jobs | ✅ | ✅ | - | Routines with cron trigger |
|
||||
| Cron stagger controls | ✅ | ❌ | P3 | Default stagger for scheduled jobs |
|
||||
| Cron finished-run webhook | ✅ | ❌ | P3 | Webhook on job completion |
|
||||
| Timezone support | ✅ | ✅ | - | Via cron expressions |
|
||||
| One-shot/recurring jobs | ✅ | ✅ | - | Manual + cron triggers |
|
||||
| Channel health monitor | ✅ | ❌ | P2 | Auto-restart with configurable interval |
|
||||
| `beforeInbound` hook | ✅ | ✅ | P2 | |
|
||||
| `beforeOutbound` hook | ✅ | ✅ | P2 | |
|
||||
| `beforeToolCall` hook | ✅ | ✅ | P2 | |
|
||||
| `before_agent_start` hook | ✅ | ❌ | P2 | Model/provider override |
|
||||
| `before_message_write` hook | ✅ | ❌ | P2 | Pre-write interception |
|
||||
| `onMessage` hook | ✅ | ✅ | - | Routines with event trigger |
|
||||
| `onSessionStart` hook | ✅ | ✅ | P2 | |
|
||||
| `onSessionEnd` hook | ✅ | ✅ | P2 | |
|
||||
| `transcribeAudio` hook | ✅ | ❌ | P3 | |
|
||||
| `transformResponse` hook | ✅ | ❌ | P2 | |
|
||||
| Bundled hooks | ✅ | ❌ | P2 | |
|
||||
| Plugin hooks | ✅ | ❌ | P3 | |
|
||||
| Workspace hooks | ✅ | ❌ | P2 | Inline code |
|
||||
| Outbound webhooks | ✅ | ❌ | P2 | |
|
||||
| `transformResponse` hook | ✅ | ✅ | P2 | |
|
||||
| `llm_input`/`llm_output` hooks | ✅ | ❌ | P3 | LLM payload inspection |
|
||||
| Bundled hooks | ✅ | ✅ | P2 | Audit + declarative rule/webhook hooks |
|
||||
| Plugin hooks | ✅ | ✅ | P3 | Registered from WASM `capabilities.json` |
|
||||
| Workspace hooks | ✅ | ✅ | P2 | `hooks/hooks.json` and `hooks/*.hook.json` |
|
||||
| Outbound webhooks | ✅ | ✅ | P2 | Fire-and-forget lifecycle event delivery |
|
||||
| Heartbeat system | ✅ | ✅ | - | Periodic execution |
|
||||
| Gmail pub/sub | ✅ | ❌ | P3 | |
|
||||
|
||||
@@ -346,28 +436,37 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
|
||||
| Feature | OpenClaw | IronClaw | Notes |
|
||||
|---------|----------|----------|-------|
|
||||
| Gateway token auth | ✅ | 🚧 | HTTP webhook secret |
|
||||
| Gateway token auth | ✅ | ✅ | Bearer token auth on web gateway |
|
||||
| Device pairing | ✅ | ❌ | |
|
||||
| Tailscale identity | ✅ | ❌ | |
|
||||
| Trusted-proxy auth | ✅ | ❌ | Header-based reverse proxy auth |
|
||||
| OAuth flows | ✅ | 🚧 | NEAR AI OAuth |
|
||||
| DM pairing verification | ✅ | ❌ | |
|
||||
| Allowlist/blocklist | ✅ | ❌ | |
|
||||
| DM pairing verification | ✅ | ✅ | ironclaw pairing approve, host APIs |
|
||||
| Allowlist/blocklist | ✅ | 🚧 | allow_from + pairing store |
|
||||
| Per-group tool policies | ✅ | ❌ | |
|
||||
| Exec approvals | ✅ | ✅ | TUI overlay |
|
||||
| TLS 1.3 minimum | ✅ | ✅ | reqwest rustls |
|
||||
| SSRF protection | ✅ | ✅ | WASM allowlist |
|
||||
| SSRF IPv6 transition bypass block | ✅ | ❌ | Block IPv4-mapped IPv6 bypasses |
|
||||
| Cron webhook SSRF guard | ✅ | ❌ | SSRF checks on webhook delivery |
|
||||
| Loopback-first | ✅ | 🚧 | HTTP binds 0.0.0.0 |
|
||||
| Docker sandbox | ✅ | ❌ | Uses WASM sandbox |
|
||||
| Docker sandbox | ✅ | ✅ | Orchestrator/worker containers |
|
||||
| Podman support | ✅ | ❌ | Alternative to Docker |
|
||||
| WASM sandbox | ❌ | ✅ | IronClaw innovation |
|
||||
| Sandbox env sanitization | ✅ | 🚧 | Shell tool scrubs env vars (secret detection); docker container env sanitization partial |
|
||||
| Tool policies | ✅ | ✅ | |
|
||||
| Elevated mode | ✅ | ❌ | |
|
||||
| Safe bins allowlist | ✅ | ❌ | |
|
||||
| Safe bins allowlist | ✅ | ❌ | Hardened path trust |
|
||||
| LD*/DYLD* validation | ✅ | ❌ | |
|
||||
| Path traversal prevention | ✅ | ✅ | |
|
||||
| Path traversal prevention | ✅ | ✅ | Including config includes (OC-06) |
|
||||
| Credential theft via env injection | ✅ | 🚧 | Shell env scrubbing + command injection detection; no full OC-09 defense |
|
||||
| Session file permissions (0o600) | ✅ | ✅ | Session token file set to 0o600 in llm/session.rs |
|
||||
| Skill download path restriction | ✅ | ❌ | Prevent arbitrary write targets |
|
||||
| Webhook signature verification | ✅ | ✅ | |
|
||||
| Media URL validation | ✅ | ❌ | |
|
||||
| Prompt injection defense | ✅ | ✅ | Pattern detection, sanitization |
|
||||
| Leak detection | ✅ | ✅ | Secret exfiltration |
|
||||
| Dangerous tool re-enable warning | ✅ | ❌ | Warn when gateway.tools.allow re-enables HTTP tools |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -387,6 +486,9 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Coverage | V8 | tarpaulin/llvm-cov | |
|
||||
| CI/CD | GitHub Actions | GitHub Actions | |
|
||||
| Pre-commit hooks | prek | - | Consider adding |
|
||||
| Docker: Chromium + Xvfb | ✅ | ❌ | Optional browser in container |
|
||||
| Docker: init scripts | ✅ | ❌ | /openclaw-init.d/ support |
|
||||
| Browser: extraArgs config | ✅ | ❌ | Custom Chrome launch arguments |
|
||||
|
||||
### Owner: _Unassigned_
|
||||
|
||||
@@ -397,30 +499,44 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
### P0 - Core (Already Done)
|
||||
- ✅ TUI channel with approval overlays
|
||||
- ✅ HTTP webhook channel
|
||||
- ✅ DM pairing (ironclaw pairing list/approve, host APIs)
|
||||
- ✅ WASM tool sandbox
|
||||
- ✅ Workspace/memory with hybrid search
|
||||
- ✅ Workspace/memory with hybrid search + embeddings batching
|
||||
- ✅ Prompt injection defense
|
||||
- ✅ Heartbeat system
|
||||
- ✅ Session management
|
||||
- ✅ Context compaction
|
||||
- ✅ Model selection
|
||||
- ✅ Gateway control plane + WebSocket
|
||||
- ✅ Web Control UI (chat, memory, jobs, logs, extensions, routines)
|
||||
- ✅ WebChat channel (web gateway)
|
||||
- ✅ Slack channel (WASM tool)
|
||||
- ✅ Telegram channel (WASM tool, MTProto)
|
||||
- ✅ Docker sandbox (orchestrator/worker)
|
||||
- ✅ Cron job scheduling (routines)
|
||||
- ✅ CLI subcommands (onboard, config, status, memory)
|
||||
- ✅ Gateway token auth
|
||||
- ✅ Skills system (prompt-based with trust gating, attenuation, activation criteria)
|
||||
- ✅ Session file permissions (0o600)
|
||||
- ✅ Memory CLI commands (search, read, write, tree, status)
|
||||
- ✅ Shell env scrubbing + command injection detection
|
||||
- ✅ Tinfoil private inference provider
|
||||
- ✅ OpenAI-compatible / OpenRouter provider support
|
||||
|
||||
### P1 - High Priority
|
||||
- ❌ Slack channel (real implementation)
|
||||
- ❌ Telegram channel
|
||||
- ✅ Telegram channel (WASM, DM pairing, caption, /start)
|
||||
- ❌ WhatsApp channel
|
||||
- ❌ Multi-provider failover
|
||||
- ❌ Gateway control plane + WebSocket
|
||||
- ❌ Hooks system (beforeInbound, beforeToolCall, etc.)
|
||||
- ✅ Multi-provider failover (`FailoverProvider` with retryable error classification)
|
||||
- ✅ Hooks system (core lifecycle hooks + bundled/plugin/workspace hooks + outbound webhooks)
|
||||
|
||||
### P2 - Medium Priority
|
||||
- ❌ Cron job scheduling
|
||||
- ❌ Web Control UI
|
||||
- ❌ WebChat channel
|
||||
- ❌ Media handling (images, PDFs)
|
||||
- ❌ CLI subcommands (config, status, memory, doctor)
|
||||
- ❌ Ollama/local model support
|
||||
- ✅ Ollama/local model support (via rig::providers::ollama)
|
||||
- ❌ Configuration hot-reload
|
||||
- ❌ Webhook trigger endpoint in web gateway
|
||||
- ❌ Channel health monitor with auto-restart
|
||||
- ❌ Partial output preservation on abort
|
||||
|
||||
### P3 - Lower Priority
|
||||
- ❌ Discord channel
|
||||
@@ -429,8 +545,12 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
- ❌ Other messaging platforms
|
||||
- ❌ TTS/audio features
|
||||
- ❌ Video support
|
||||
- ❌ Skills system
|
||||
- 🚧 Skills routing blocks (activation criteria exist, but no "Use when / Don't use when")
|
||||
- ❌ Plugin registry
|
||||
- ❌ Streaming (block/tool/Z.AI tool_stream)
|
||||
- ❌ Memory: temporal decay, MMR re-ranking, query expansion
|
||||
- ❌ Control UI i18n
|
||||
- ❌ Stuck loop detection
|
||||
|
||||
---
|
||||
|
||||
@@ -455,9 +575,12 @@ IronClaw intentionally differs from OpenClaw in these ways:
|
||||
|
||||
1. **Rust vs TypeScript**: Native performance, memory safety, single binary distribution
|
||||
2. **WASM sandbox vs Docker**: Lighter weight, faster startup, capability-based security
|
||||
3. **PostgreSQL vs SQLite**: Better suited for production deployments
|
||||
3. **PostgreSQL + libSQL vs SQLite**: Dual-backend (production PG + embedded libSQL for zero-dep local mode)
|
||||
4. **NEAR AI focus**: Primary provider with session-based auth
|
||||
5. **No mobile/desktop apps**: Focus on server-side and CLI initially
|
||||
6. **WASM channels**: Novel extension mechanism not in OpenClaw
|
||||
7. **Tinfoil private inference**: IronClaw-only provider for private/encrypted inference
|
||||
8. **GitHub WASM tool**: Native GitHub integration as WASM tool
|
||||
9. **Prompt-based skills**: Different approach than OpenClaw capability bundles (trust gating, attenuation)
|
||||
|
||||
These are intentional architectural choices, not gaps to be filled.
|
||||
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to the Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by the Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding any notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2026 NEAR AI
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -43,7 +43,10 @@ IronClaw is the AI assistant you can actually trust with your personal and profe
|
||||
|
||||
### Always Available
|
||||
|
||||
- **Multi-channel** - Reach your assistant via CLI, Telegram, WhatsApp, Slack, or HTTP webhooks
|
||||
- **Multi-channel** - REPL, HTTP webhooks, WASM channels (Telegram, Slack), and web gateway
|
||||
- **Docker Sandbox** - Isolated container execution with per-job tokens and orchestrator/worker pattern
|
||||
- **Web Gateway** - Browser UI with real-time SSE/WebSocket streaming
|
||||
- **Routines** - Cron schedules, event triggers, webhook handlers for background automation
|
||||
- **Heartbeat System** - Proactive background execution for monitoring and maintenance tasks
|
||||
- **Parallel Jobs** - Handle multiple requests concurrently with isolated contexts
|
||||
- **Self-repair** - Automatic detection and recovery of stuck operations
|
||||
@@ -65,10 +68,41 @@ IronClaw is the AI assistant you can actually trust with your personal and profe
|
||||
### Prerequisites
|
||||
|
||||
- Rust 1.85+
|
||||
- PostgreSQL 15+ with pgvector extension
|
||||
- NEAR AI session token (or other LLM provider)
|
||||
- PostgreSQL 15+ with [pgvector](https://github.com/pgvector/pgvector) extension
|
||||
- NEAR AI account (authentication handled via setup wizard)
|
||||
|
||||
### Build
|
||||
## Download or Build
|
||||
|
||||
Visit [Releases page](https://github.com/nearai/ironclaw/releases/) to see the latest updates.
|
||||
|
||||
<details>
|
||||
<summary>Install via Windows Installer (Windows)</summary>
|
||||
|
||||
Download the [Windows Installer](https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-x86_64-pc-windows-msvc.msi) and run it.
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Install via powershell script (Windows)</summary>
|
||||
|
||||
```sh
|
||||
irm https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.ps1 | iex
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Install via shell script (macOS, Linux, Windows/WSL)</summary>
|
||||
|
||||
```sh
|
||||
curl --proto '=https' --tlsv1.2 -LsSf https://github.com/nearai/ironclaw/releases/latest/download/ironclaw-installer.sh | sh
|
||||
```
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Compile the source code (Cargo on Windows, Linux, macOS)</summary>
|
||||
|
||||
Install it with `cargo`, just make sure you have [Rust](https://rustup.rs) installed on your computer.
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
@@ -82,6 +116,10 @@ cargo build --release
|
||||
cargo test
|
||||
```
|
||||
|
||||
For **full release** (after modifying channel sources), run `./scripts/build-all.sh` to rebuild channels first.
|
||||
|
||||
</details>
|
||||
|
||||
### Database Setup
|
||||
|
||||
```bash
|
||||
@@ -90,36 +128,20 @@ createdb ironclaw
|
||||
|
||||
# Enable pgvector
|
||||
psql ironclaw -c "CREATE EXTENSION IF NOT EXISTS vector;"
|
||||
|
||||
# Run migrations
|
||||
refinery migrate -c refinery.toml
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy `.env.example` to `.env` and configure:
|
||||
Run the setup wizard to configure IronClaw:
|
||||
|
||||
```bash
|
||||
# Required
|
||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||
NEARAI_SESSION_TOKEN=sess_...
|
||||
|
||||
# Optional: Enable channels
|
||||
TELEGRAM_BOT_TOKEN=...
|
||||
WHATSAPP_ACCESS_TOKEN=...
|
||||
SLACK_BOT_TOKEN=xoxb-...
|
||||
HTTP_PORT=8080
|
||||
ironclaw onboard
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Description | Required |
|
||||
|----------|-------------|----------|
|
||||
| `DATABASE_URL` | PostgreSQL connection string | Yes |
|
||||
| `NEARAI_SESSION_TOKEN` | NEAR AI authentication token | Yes |
|
||||
| `NEARAI_MODEL` | Model to use (default: claude-3-5-sonnet) | No |
|
||||
| `AGENT_MAX_PARALLEL_JOBS` | Max concurrent jobs (default: 5) | No |
|
||||
| `SECRETS_MASTER_KEY` | 32+ byte key for secret encryption | For secrets |
|
||||
The wizard handles database connection, NEAR AI authentication (via browser OAuth),
|
||||
and secrets encryption (using your system keychain). Settings are persisted in the
|
||||
connected database; bootstrap variables (e.g. `DATABASE_URL`, `LLM_BACKEND`) are
|
||||
written to `~/.ironclaw/.env` so they are available before the database connects.
|
||||
|
||||
## Security
|
||||
|
||||
@@ -160,37 +182,42 @@ External content passes through multiple security layers:
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Channels │
|
||||
│ ┌─────┐ ┌──────────┐ ┌──────────┐ ┌───────┐ │
|
||||
│ │ CLI │ │ Telegram │ │ WhatsApp │ │ Slack │ │
|
||||
│ └──┬──┘ └────┬─────┘ └────┬─────┘ └───┬───┘ │
|
||||
│ └──────────┴─────────────┴────────────┘ │
|
||||
│ │ │
|
||||
│ ┌────▼────┐ │
|
||||
│ │ Router │ Intent classification │
|
||||
│ └────┬────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────▼──────────┐ │
|
||||
│ │ Scheduler │ Parallel job management │
|
||||
│ └──────────┬──────────┘ │
|
||||
│ │ │
|
||||
│ ┌───────────────┼───────────────┐ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌─────────┐ ┌─────────┐ ┌─────────┐ │
|
||||
│ │ Worker │ │ Worker │ │ Worker │ LLM reasoning │
|
||||
│ └────┬────┘ └────┬────┘ └────┬────┘ │
|
||||
│ └───────────────┼───────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────▼──────────┐ │
|
||||
│ │ Tool Registry │ │
|
||||
│ │ ┌───────────────┐ │ │
|
||||
│ │ │ Built-in │ │ │
|
||||
│ │ │ MCP │ │ │
|
||||
│ │ │ WASM Sandbox │ │ │
|
||||
│ │ └───────────────┘ │ │
|
||||
│ └─────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ Channels │
|
||||
│ ┌──────┐ ┌──────┐ ┌─────────────┐ ┌─────────────┐ │
|
||||
│ │ REPL │ │ HTTP │ │WASM Channels│ │ Web Gateway │ │
|
||||
│ └──┬───┘ └──┬───┘ └──────┬──────┘ │ (SSE + WS) │ │
|
||||
│ │ │ │ └──────┬──────┘ │
|
||||
│ └─────────┴──────────────┴────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌─────────▼─────────┐ │
|
||||
│ │ Agent Loop │ Intent routing │
|
||||
│ └────┬──────────┬───┘ │
|
||||
│ │ │ │
|
||||
│ ┌──────────▼────┐ ┌──▼───────────────┐ │
|
||||
│ │ Scheduler │ │ Routines Engine │ │
|
||||
│ │(parallel jobs)│ │(cron, event, wh) │ │
|
||||
│ └──────┬────────┘ └────────┬─────────┘ │
|
||||
│ │ │ │
|
||||
│ ┌─────────────┼────────────────────┘ │
|
||||
│ │ │ │
|
||||
│ ┌───▼─────┐ ┌────▼────────────────┐ │
|
||||
│ │ Local │ │ Orchestrator │ │
|
||||
│ │Workers │ │ ┌───────────────┐ │ │
|
||||
│ │(in-proc)│ │ │ Docker Sandbox│ │ │
|
||||
│ └───┬─────┘ │ │ Containers │ │ │
|
||||
│ │ │ │ ┌───────────┐ │ │ │
|
||||
│ │ │ │ │Worker / CC│ │ │ │
|
||||
│ │ │ │ └───────────┘ │ │ │
|
||||
│ │ │ └───────────────┘ │ │
|
||||
│ │ └─────────┬───────────┘ │
|
||||
│ └──────────────────┤ │
|
||||
│ │ │
|
||||
│ ┌───────────▼──────────┐ │
|
||||
│ │ Tool Registry │ │
|
||||
│ │ Built-in, MCP, WASM │ │
|
||||
│ └──────────────────────┘ │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Core Components
|
||||
@@ -201,33 +228,25 @@ External content passes through multiple security layers:
|
||||
| **Router** | Classifies user intent (command, query, task) |
|
||||
| **Scheduler** | Manages parallel job execution with priorities |
|
||||
| **Worker** | Executes jobs with LLM reasoning and tool calls |
|
||||
| **Orchestrator** | Container lifecycle, LLM proxying, per-job auth |
|
||||
| **Web Gateway** | Browser UI with chat, memory, jobs, logs, extensions, routines |
|
||||
| **Routines Engine** | Scheduled (cron) and reactive (event, webhook) background tasks |
|
||||
| **Workspace** | Persistent memory with hybrid search |
|
||||
| **Safety Layer** | Prompt injection defense and content sanitization |
|
||||
|
||||
## Usage
|
||||
|
||||
### CLI Mode
|
||||
|
||||
```bash
|
||||
# Start interactive CLI
|
||||
# First-time setup (configures database, auth, etc.)
|
||||
ironclaw onboard
|
||||
|
||||
# Start interactive REPL
|
||||
cargo run
|
||||
|
||||
# With debug logging
|
||||
RUST_LOG=ironclaw=debug cargo run
|
||||
```
|
||||
|
||||
### HTTP Server
|
||||
|
||||
```bash
|
||||
# Start with HTTP webhook server
|
||||
HTTP_PORT=8080 cargo run
|
||||
|
||||
# Send a request
|
||||
curl -X POST http://localhost:8080/webhook \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"message": "Hello, IronClaw!"}'
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
@@ -238,12 +257,16 @@ cargo fmt
|
||||
cargo clippy --all --benches --tests --examples --all-features
|
||||
|
||||
# Run tests
|
||||
createdb ironclaw_test
|
||||
cargo test
|
||||
|
||||
# Run specific test
|
||||
cargo test test_name
|
||||
```
|
||||
|
||||
- **Telegram channel**: See [docs/TELEGRAM_SETUP.md](docs/TELEGRAM_SETUP.md) for setup and DM pairing.
|
||||
- **Changing channel sources**: Run `./channels-src/telegram/build.sh` before `cargo build` so the updated WASM is bundled.
|
||||
|
||||
## OpenClaw Heritage
|
||||
|
||||
IronClaw is a Rust reimplementation inspired by [OpenClaw](https://github.com/openclaw/openclaw). See [FEATURE_PARITY.md](FEATURE_PARITY.md) for the complete tracking matrix.
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
[package]
|
||||
name = "ironclaw-bench"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
description = "Benchmarking harness for IronClaw agent"
|
||||
license = "MIT OR Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "ironclaw-bench"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ironclaw = { path = ".." }
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Core types
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
|
||||
# Async traits
|
||||
async-trait = "0.1"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
# Scoring
|
||||
regex = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"run_id": "8a48de1f-09ce-4c35-9ad1-dad98fb83a1a",
|
||||
"suite_id": "spot",
|
||||
"config_label": "default",
|
||||
"model": "openai/gpt-5.2",
|
||||
"commit_hash": "2c43b83",
|
||||
"pass_rate": 1.0,
|
||||
"avg_score": 1.0,
|
||||
"total_tasks": 21,
|
||||
"completed_tasks": 21,
|
||||
"total_cost_usd": 0.307053,
|
||||
"total_wall_time_ms": 111009,
|
||||
"started_at": "2026-02-17T22:02:08.206112Z",
|
||||
"finished_at": "2026-02-17T22:03:59.270325Z"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{"task_id":"smoke-greeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2037,"llm_calls":1,"input_tokens":1359,"output_tokens":65,"estimated_cost_usd":0.005052,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"I’m NEAR AI Agent—an autonomous assistant that helps you plan, write, debug, and execute tasks using available tools (like reading/writing files, running shell commands, and making HTTP requests). Tell me what you’re working on and what outcome you want, and I’ll drive it to completion.","started_at":"2026-02-17T22:02:08.206165Z","finished_at":"2026-02-17T22:02:10.243552Z","config_label":"default","error":null}
|
||||
{"task_id":"smoke-math","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2038,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"1081","started_at":"2026-02-17T22:02:10.243933Z","finished_at":"2026-02-17T22:02:12.282133Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2646,"llm_calls":2,"input_tokens":2786,"output_tokens":24,"estimated_cost_usd":0.008718,"tool_calls":[{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Spot check passed","started_at":"2026-02-17T22:02:12.282294Z","finished_at":"2026-02-17T22:02:14.928540Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-time","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2839,"output_tokens":78,"estimated_cost_usd":0.009687,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Current date/time (UTC): **2026-02-17 22:02:16** \nISO 8601: **2026-02-17T22:02:16.045933+00:00**","started_at":"2026-02-17T22:02:14.928810Z","finished_at":"2026-02-17T22:02:18.389188Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-json-query","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2751,"llm_calls":2,"input_tokens":2839,"output_tokens":43,"estimated_cost_usd":0.009162,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Bob","started_at":"2026-02-17T22:02:18.389812Z","finished_at":"2026-02-17T22:02:21.141146Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-shell-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2811,"output_tokens":25,"estimated_cost_usd":0.008808,"tool_calls":[{"name":"shell","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"benchmark test","started_at":"2026-02-17T22:02:21.141384Z","finished_at":"2026-02-17T22:02:24.601812Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-list-dir","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":19146,"llm_calls":2,"input_tokens":4370,"output_tokens":1337,"estimated_cost_usd":0.033165,"tool_calls":[{"name":"list_dir","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```\nclaude-501/\nclaude-505/\nclaude/\ncodex/\ncom.apple.launchd.HqYyNM7hoK/\ncom.apple.launchd.Z0bTtlsPEN/\ncom.apple.launchd.oY8y5gkzN4/\ncom.apple.launchd.sdo3WGu0S7/\ndata-gym-cache/\ndocker-desktop-privileged2203342862/\ngrammers-test/\nnear-agent-target/\nnear-market-bot-push/\nnear-market-bot/\nopenclaw/\npowerlog/\ntest_grpo/\ntest_grpo2/\ntest_grpo_entry/\ntest_grpo_mix/\ntest_grpo_singleton/\ntmpybtjiuc0/\n.d8bd37babaf2f1f7-00000000.node (5.9MB)\n.d8bd37fbfbb0f7ff-00000000.node (5.9MB)\n.d8bd3d9bdf6dd3f7-00000000.node (5.9MB)\n.d8bd3fbfbf21ddf7-00000000.node (5.9MB)\n.d8bd7dfebb38d5ff-00000000.node (5.9MB)\n.d8bdf5deba3bf1f7-00000000.node (5.9MB)\n.d8bdf7cfff76f3f7-00000000.node (5.9MB)\n.d8bdfd8ffe26d5ff-00000000.node (5.9MB)\n.d8bdfdeb9ba8dbff-00000000.node (5.9MB)\n.d8bdfffe9ab2ddff-00000000.node (5.9MB)\n.s.PGSQL.5432 (0B)\n.s.PGSQL.5432.lock (56B)\n__KMP_REGISTERED_LIB_75079 (1.0KB)\nagent_loop_new.rs (28.5KB)\nagent_mod.rs (1.6KB)\nauth_trace.md (10.8KB)\nbench-daily.md (72B)\nbench-log.md (109B)\nbench-meeting.md (160B)\nbench-monday.md (44B)\nbench-prefs.md (61B)\nbench-project.md (213B)\nbench-reminder.md (68B)\nbench-todo.md (153B)\nbench-tuesday.md (40B)\ncac-deck.html (151.5KB)\ncircuit_breaker.rs (22.1KB)\ncli_config.rs (9.1KB)\ncli_service.rs (1.1KB)\ncommands.rs (17.5KB)\nconfig.rs (58.4KB)\nconflicts_summary.md (21.4KB)\ncost_guard.rs (11.3KB)\ndebug_forc2.py (2.8KB)\ndebug_forc3.py (3.0KB)\ndebug_forc4.py (3.0KB)\ndebug_forc5.py (4.0KB)\ndebug_forc6.py (3.9KB)\ndebug_forc7.py (2.7KB)\ndebug_forc8.py (2.9KB)\ndebug_forc_prove.py (3.0KB)\ndispatcher.rs (26.2KB)\ndoctor.rs (8.4KB)\nhygiene.rs (7.3KB)\nironclaw_blog_test.png (21.4KB)\nironclaw_browser_test_viewport.png (156.1KB)\nironclaw_linkedin_debug.png (6.5KB)\nironclaw_spot_test.txt (19B)\nkeys_chain_signatures.rs (6.1KB)\nkeys_error.rs (1.6KB)\nkeys_intents.rs (5.8KB)\nkeys_mod.rs (31.6KB)\nkeys_policy.rs (31.4KB)\nkeys_rpc.rs (8.5KB)\nkeys_signer.rs (8.0KB)\nkeys_spending.rs (5.9KB)\nkeys_transaction.rs (13.7KB)\nkeys_types.rs (17.4KB)\nleak_detection_research_summary.md (13.0KB)\nleak_detector.rs (25.3KB)\nlib_new.rs (5.0KB)\nllm_mod.rs (10.0KB)\nmain.rs (56.8KB)\nmain_bootstrap.rs (11.0KB)\nmain_rs.txt (34.1KB)\nnear_resp.json (39B)\nobs_log.rs (5.9KB)\nobs_mod.rs (2.7KB)\nobs_multi.rs (4.0KB)\nobs_noop.rs (1.5KB)\nobs_traits.rs (4.0KB)\nour_bootstrap.rs (12.6KB)\nour_config.rs (45.2KB)\npost_final_review.py (6.4KB)\npost_review.py (4.6KB)\npost_review2.py (4.8KB)\npost_review3.py (2.2KB)\npr-35.diff (137.4KB)\npr115.diff (1.6MB)\npr126.diff (336.9KB)\npr14_full_diff.txt (193.4KB)\npr39_files.txt (243B)\npr51_files.txt (548B)\npr93_diff.txt (160.4KB)\nprs.json (4.6KB)\nresponse_cache.rs (15.1KB)\nrun_hard_proof_search.sh (545B)\nrun_hard_proof_search2.sh (515B)\nrun_proof_search_v3.sh (523B)\nrun_proof_search_v4.sh (524B)\nservice.rs (11.9KB)\nsettings.rs (34.7KB)\ntest_line.py (789B)\ntest_search.txt (1.8KB)\nthread_ops.rs (38.5KB)\ntool_wit.txt (6.0KB)\ntunnel_cf.rs (3.9KB)\ntunnel_custom.rs (6.2KB)\ntunnel_mod.rs (10.4KB)\ntunnel_ngrok.rs (4.2KB)\ntunnel_none.rs (1.3KB)\ntunnel_ts.rs (3.5KB)\nwasm_capabilities.rs (15.2KB)\nwasm_capabilities_schema.rs (26.2KB)\nwasm_host.rs (25.3KB)\nwasm_wrapper.rs (14.9KB)\n```","started_at":"2026-02-17T22:02:24.602248Z","finished_at":"2026-02-17T22:02:43.748463Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-write-read","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5712,"llm_calls":3,"input_tokens":4489,"output_tokens":73,"estimated_cost_usd":0.014562,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It says: `ironclaw spot check`.","started_at":"2026-02-17T22:02:43.749933Z","finished_at":"2026-02-17T22:02:49.462007Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-shell-json","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4582,"llm_calls":3,"input_tokens":4433,"output_tokens":94,"estimated_cost_usd":0.014709,"tool_calls":[{"name":"shell","duration_ms":0,"success":true},{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"[Called tool `json` with arguments: {\"data\":{\"code\":200,\"status\":\"ok\"},\"operation\":\"query\",\"path\":\"status\"}]","started_at":"2026-02-17T22:02:49.463794Z","finished_at":"2026-02-17T22:02:54.046092Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-time-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4583,"llm_calls":3,"input_tokens":4404,"output_tokens":76,"estimated_cost_usd":0.014352,"tool_calls":[{"name":"time","duration_ms":0,"success":true},{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"2026-02-17T22:02:55.786456+00:00","started_at":"2026-02-17T22:02:54.046306Z","finished_at":"2026-02-17T22:02:58.629389Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-no-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":1020,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Paris.","started_at":"2026-02-17T22:02:58.630502Z","finished_at":"2026-02-17T22:02:59.650655Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-correct-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3980,"llm_calls":2,"input_tokens":2825,"output_tokens":50,"estimated_cost_usd":0.009225,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It’s **22:03:01 UTC** right now (**2026-02-17**).","started_at":"2026-02-17T22:02:59.650901Z","finished_at":"2026-02-17T22:03:03.631122Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-json-validate","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2954,"llm_calls":2,"input_tokens":2822,"output_tokens":43,"estimated_cost_usd":0.009111,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The JSON is valid (`valid: true`).","started_at":"2026-02-17T22:03:03.631527Z","finished_at":"2026-02-17T22:03:06.585971Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-daily","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4890,"llm_calls":3,"input_tokens":4547,"output_tokens":114,"estimated_cost_usd":0.015351,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```md\n# Daily Tasks\n\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\n```","started_at":"2026-02-17T22:03:06.586556Z","finished_at":"2026-02-17T22:03:11.476648Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-reminder","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5607,"llm_calls":3,"input_tokens":4531,"output_tokens":95,"estimated_cost_usd":0.015018,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The appointment is on **March 5th at 2pm**, with **Dr. Smith**.","started_at":"2026-02-17T22:03:11.476873Z","finished_at":"2026-02-17T22:03:17.084343Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-meeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6113,"llm_calls":3,"input_tokens":4750,"output_tokens":122,"estimated_cost_usd":0.01608,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Frontend owner: **Bob** \nLaunch date: **April 15th**","started_at":"2026-02-17T22:03:17.085432Z","finished_at":"2026-02-17T22:03:23.198905Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-append-log","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9880,"llm_calls":6,"input_tokens":9941,"output_tokens":208,"estimated_cost_usd":0.032943,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"shell","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Contents of `/tmp/bench-log.md`:\n1. Morning standup: discussed blockers on auth service \n2. Afternoon: resolved API timeout issue in payment gateway \n\nSummary: You discussed blockers affecting the auth service during the morning standup, and later in the afternoon you resolved an API timeout issue in the payment gateway.","started_at":"2026-02-17T22:03:23.199155Z","finished_at":"2026-02-17T22:03:33.079396Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-recall-detail","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4989,"llm_calls":3,"input_tokens":4721,"output_tokens":121,"estimated_cost_usd":0.015978,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The project uses **PostgreSQL** as the database, and the PM is **James**.","started_at":"2026-02-17T22:03:33.080611Z","finished_at":"2026-02-17T22:03:38.069896Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-todo-priority","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5916,"llm_calls":3,"input_tokens":4676,"output_tokens":110,"estimated_cost_usd":0.015678,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"High priority tasks:\n- Fix login bug\n- Security audit","started_at":"2026-02-17T22:03:38.070162Z","finished_at":"2026-02-17T22:03:43.986831Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-multifile","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6081,"llm_calls":4,"input_tokens":6442,"output_tokens":196,"estimated_cost_usd":0.022266,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"On Tuesday you have **Sprint retro at 11am**, then **team lunch at noon**.","started_at":"2026-02-17T22:03:43.988013Z","finished_at":"2026-02-17T22:03:50.069452Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-update-context","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9164,"llm_calls":5,"input_tokens":8354,"output_tokens":252,"estimated_cost_usd":0.028842,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"apply_patch","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Final read confirms the file now says `timezone: EST`.","started_at":"2026-02-17T22:03:50.070277Z","finished_at":"2026-02-17T22:03:59.235196Z","config_label":"default","error":null}
|
||||
@@ -0,0 +1,21 @@
|
||||
{"id": "smoke-greeting", "prompt": "Hello! Introduce yourself briefly.", "tags": ["smoke"], "assertions": {"response_matches": "(?i)(hello|hi|hey|assistant|agent|help)", "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "smoke-math", "prompt": "What is 47 * 23? Reply with just the number.", "tags": ["smoke"], "assertions": {"response_contains": ["1081"], "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "tool-echo", "prompt": "Use the echo tool to repeat the message: 'Spot check passed'", "tags": ["tool"], "assertions": {"tools_used": ["echo"], "response_contains": ["Spot check passed"], "no_error": true}}
|
||||
{"id": "tool-time", "prompt": "What is the current date and time? Use the time tool.", "tags": ["tool"], "assertions": {"tools_used": ["time"], "response_matches": "20\\d{2}", "no_error": true}}
|
||||
{"id": "tool-json-query", "prompt": "Given this JSON: {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}, use the json tool to extract the second user's name.", "tags": ["tool"], "assertions": {"tools_used": ["json"], "response_contains": ["Bob"], "no_error": true}}
|
||||
{"id": "tool-shell-echo", "prompt": "Use the shell tool to run: echo 'benchmark test'", "tags": ["tool"], "assertions": {"tools_used": ["shell"], "response_contains": ["benchmark test"], "no_error": true}}
|
||||
{"id": "tool-list-dir", "prompt": "Use the list_dir tool to list the contents of the /tmp directory.", "tags": ["tool"], "assertions": {"tools_used": ["list_dir"], "no_error": true}}
|
||||
{"id": "chain-write-read", "prompt": "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt using the write_file tool, then read it back using the read_file tool and tell me what it says.", "tags": ["chain"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["ironclaw spot check"], "no_error": true}}
|
||||
{"id": "chain-shell-json", "prompt": "Run a shell command to output the JSON string '{\"status\": \"ok\", \"code\": 200}', then use the json tool to extract the status field.", "tags": ["chain"], "assertions": {"response_contains": ["ok"], "min_tool_calls": 1, "no_error": true}}
|
||||
{"id": "chain-time-echo", "prompt": "First get the current time using the time tool, then use the echo tool to repeat it back.", "tags": ["chain"], "assertions": {"tools_used": ["time", "echo"], "no_error": true}}
|
||||
{"id": "robust-no-tool", "prompt": "What is the capital of France? Answer directly without using any tools.", "tags": ["robust"], "assertions": {"response_contains": ["Paris"], "max_tool_calls": 0, "no_error": true}}
|
||||
{"id": "robust-correct-tool", "prompt": "What time is it right now?", "tags": ["robust"], "assertions": {"tools_used": ["time"], "tools_not_used": ["shell", "echo"], "no_error": true}}
|
||||
{"id": "robust-json-validate", "prompt": "Use the json tool to validate whether this is valid JSON: {\"key\": \"value\", \"num\": 42}", "tags": ["robust"], "assertions": {"tools_used": ["json"], "tools_not_used": ["shell"], "no_error": true}}
|
||||
{"id": "memory-save-daily", "prompt": "Save these daily tasks to /tmp/bench-daily.md:\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\nThen read the file back and confirm what was saved.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PR #42", "docs", "staging"], "no_error": true}}
|
||||
{"id": "memory-save-reminder", "prompt": "Write a reminder to /tmp/bench-reminder.md: Dentist appointment on March 5th at 2pm with Dr. Smith. Then read the file back and tell me when the appointment is and with whom.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["March 5", "Smith"], "response_matches": "2(:00)?\\s*[Pp][Mm]", "no_error": true}}
|
||||
{"id": "memory-save-meeting", "prompt": "Save these meeting notes to /tmp/bench-meeting.md:\nMeeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\nDecisions:\n- Launch date: April 15th\n- Budget: $50k approved\n- Bob owns frontend, Carol owns backend\nThen read the file back and tell me who owns the frontend and what the launch date is.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["Bob", "frontend", "April 15"], "no_error": true}}
|
||||
{"id": "memory-append-log", "prompt": "Write 'Morning standup: discussed blockers on auth service' to /tmp/bench-log.md. Then append a new line 'Afternoon: resolved API timeout issue in payment gateway' to the same file. Finally read the full file and summarize what happened.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["auth", "timeout"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-recall-detail", "prompt": "Save the following project context to /tmp/bench-project.md:\nProject Ironclad uses Rust for the backend, React for the frontend, and PostgreSQL for the database. The API is deployed on AWS ECS. The lead developer is Sarah and the PM is James. The sprint ends on March 20th.\nThen read it back and answer: What database does the project use, and who is the PM?", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PostgreSQL", "James"], "no_error": true}}
|
||||
{"id": "memory-todo-priority", "prompt": "Write the following to /tmp/bench-todo.md:\n- [ ] Fix login bug (priority: HIGH)\n- [ ] Write unit tests (priority: medium)\n- [ ] Update README (priority: low)\n- [ ] Security audit (priority: HIGH)\nThen read it back and tell me which tasks are high priority.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["login bug", "security audit"], "no_error": true}}
|
||||
{"id": "memory-multifile", "prompt": "Save 'Team standup at 9am, then client demo at 2pm' to /tmp/bench-monday.md and 'Sprint retro at 11am, team lunch at noon' to /tmp/bench-tuesday.md. Then read both files and tell me what's happening on Tuesday.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["retro", "lunch"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-update-context", "prompt": "Write 'User preference: dark mode, timezone: PST, language: English' to /tmp/bench-prefs.md. Then read it back, and rewrite the file changing the timezone to EST. Finally read it one more time and confirm the timezone is now EST.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["EST"], "min_tool_calls": 4, "no_error": true}}
|
||||
@@ -0,0 +1,8 @@
|
||||
task_timeout = "120s"
|
||||
parallelism = 1
|
||||
|
||||
[[matrix]]
|
||||
label = "default"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "benchmarks/data/spot.jsonl"
|
||||
@@ -0,0 +1,243 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// A single entry in the custom JSONL format.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CustomEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
expected: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_contains: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_regex: Option<String>,
|
||||
/// "exact", "contains", "regex", or "llm" (default: "exact")
|
||||
#[serde(default = "default_scorer")]
|
||||
scorer: String,
|
||||
}
|
||||
|
||||
fn default_scorer() -> String {
|
||||
"exact".to_string()
|
||||
}
|
||||
|
||||
/// Custom JSONL benchmark suite.
|
||||
///
|
||||
/// Each line of the JSONL file is a task with `id`, `prompt`, and scoring
|
||||
/// criteria (`expected`, `expected_contains`, `expected_regex`).
|
||||
pub struct CustomSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl CustomSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for CustomSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Custom JSONL"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"custom"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: CustomEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut metadata = serde_json::json!({
|
||||
"scorer": entry.scorer,
|
||||
});
|
||||
if let Some(ref expected) = entry.expected {
|
||||
metadata["expected"] = serde_json::Value::String(expected.clone());
|
||||
}
|
||||
if let Some(ref expected_contains) = entry.expected_contains {
|
||||
metadata["expected_contains"] =
|
||||
serde_json::Value::String(expected_contains.clone());
|
||||
}
|
||||
if let Some(ref expected_regex) = entry.expected_regex {
|
||||
metadata["expected_regex"] = serde_json::Value::String(expected_regex.clone());
|
||||
}
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let scorer = task
|
||||
.metadata
|
||||
.get("scorer")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("exact");
|
||||
|
||||
match scorer {
|
||||
"exact" => {
|
||||
if let Some(expected) = task.metadata.get("expected").and_then(|v| v.as_str()) {
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected' field for exact scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"contains" => {
|
||||
if let Some(expected) = task
|
||||
.metadata
|
||||
.get("expected_contains")
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::contains_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_contains' field for contains scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"regex" => {
|
||||
if let Some(pattern) = task.metadata.get("expected_regex").and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::regex_match(pattern, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_regex' field for regex scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"llm" => {
|
||||
// TODO: LLM-as-judge scoring
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"LLM-as-judge scoring not implemented, returning placeholder 0.5"
|
||||
);
|
||||
Ok(BenchScore::partial(0.5, "LLM scoring not yet implemented"))
|
||||
}
|
||||
other => Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("unknown scorer: {other}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t2", "prompt": "Say hello", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "t1");
|
||||
assert_eq!(tasks[1].id, "t2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_exact_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "4".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_contains_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "Greet me", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "Hello there!".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskResource, TaskSubmission};
|
||||
|
||||
/// GAIA dataset entry (Hugging Face JSONL format).
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GaiaEntry {
|
||||
task_id: String,
|
||||
#[serde(alias = "Question")]
|
||||
question: String,
|
||||
#[serde(alias = "Final answer", alias = "final_answer")]
|
||||
final_answer: String,
|
||||
#[serde(alias = "Level", default)]
|
||||
level: Option<u32>,
|
||||
#[serde(alias = "file_name", default)]
|
||||
file_name: Option<String>,
|
||||
}
|
||||
|
||||
/// GAIA benchmark suite.
|
||||
///
|
||||
/// Tasks are loaded from HuggingFace JSONL exports. Scoring uses normalized
|
||||
/// exact match against the `final_answer` field.
|
||||
pub struct GaiaSuite {
|
||||
dataset_path: PathBuf,
|
||||
attachments_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl GaiaSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
attachments_dir: Option<impl Into<PathBuf>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
attachments_dir: attachments_dir.map(|d| d.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for GaiaSuite {
|
||||
fn name(&self) -> &str {
|
||||
"GAIA"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"gaia"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: GaiaEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("GAIA line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut resources = Vec::new();
|
||||
if let Some(ref file_name) = entry.file_name {
|
||||
if !file_name.is_empty() {
|
||||
if let Some(ref dir) = self.attachments_dir {
|
||||
resources.push(TaskResource {
|
||||
name: file_name.clone(),
|
||||
path: dir.join(file_name).to_string_lossy().to_string(),
|
||||
resource_type: crate::suite::ResourceType::File,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut tags = Vec::new();
|
||||
if let Some(level) = entry.level {
|
||||
tags.push(format!("level-{level}"));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"expected": entry.final_answer,
|
||||
"level": entry.level,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.task_id,
|
||||
prompt: entry.question,
|
||||
context: None,
|
||||
resources,
|
||||
tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let expected = task
|
||||
.metadata
|
||||
.get("expected")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing expected answer in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "What is the capital of France?", "final_answer": "Paris", "Level": 1}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "g1");
|
||||
assert!(tasks[0].tags.contains(&"level-1".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "Capital of France?", "final_answer": "Paris"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Exact match (case insensitive)
|
||||
let submission = TaskSubmission {
|
||||
response: "paris".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
|
||||
// Wrong answer
|
||||
let submission = TaskSubmission {
|
||||
response: "London".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
pub mod custom;
|
||||
pub mod gaia;
|
||||
pub mod spot;
|
||||
pub mod swe_bench;
|
||||
pub mod tau_bench;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchSuite;
|
||||
|
||||
/// List of all known suite IDs.
|
||||
pub const KNOWN_SUITES: &[(&str, &str)] = &[
|
||||
("custom", "Custom JSONL tasks"),
|
||||
("gaia", "GAIA benchmark (knowledge & reasoning)"),
|
||||
("spot", "Spot checks (end-to-end user workflows)"),
|
||||
("tau_bench", "Tau-bench (multi-turn tool use)"),
|
||||
("swe_bench", "SWE-bench Pro (software engineering)"),
|
||||
];
|
||||
|
||||
/// Create a suite adapter by name.
|
||||
pub fn create_suite(name: &str, config: &BenchConfig) -> Result<Box<dyn BenchSuite>, BenchError> {
|
||||
let suite_map = config.suite_config_map();
|
||||
match name {
|
||||
"custom" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'custom' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(custom::CustomSuite::new(dataset_path)))
|
||||
}
|
||||
"gaia" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'gaia' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let attachments_dir = suite_map
|
||||
.get("attachments_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
Ok(Box::new(gaia::GaiaSuite::new(
|
||||
dataset_path,
|
||||
attachments_dir,
|
||||
)))
|
||||
}
|
||||
"spot" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'spot' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(spot::SpotSuite::new(dataset_path)))
|
||||
}
|
||||
"tau_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'tau_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let domain = suite_map
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("retail")
|
||||
.to_string();
|
||||
Ok(Box::new(tau_bench::TauBenchSuite::new(
|
||||
dataset_path,
|
||||
domain,
|
||||
)))
|
||||
}
|
||||
"swe_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'swe_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let workspace_dir = suite_map
|
||||
.get("workspace_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("/tmp/swe-bench")
|
||||
.to_string();
|
||||
let use_docker = suite_map
|
||||
.get("use_docker")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
Ok(Box::new(swe_bench::SweBenchSuite::new(
|
||||
dataset_path,
|
||||
workspace_dir,
|
||||
use_docker,
|
||||
)))
|
||||
}
|
||||
_ => {
|
||||
let available = KNOWN_SUITES
|
||||
.iter()
|
||||
.map(|(id, _)| *id)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
Err(BenchError::SuiteNotFound {
|
||||
name: name.to_string(),
|
||||
available,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,504 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Multi-criterion assertions for a spot check scenario.
|
||||
///
|
||||
/// Each field generates one or more individual checks. The final score is
|
||||
/// `passed_checks / total_checks`, giving a value between 0.0 and 1.0.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct SpotAssertions {
|
||||
/// All must appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_contains: Vec<String>,
|
||||
|
||||
/// None may appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_not_contains: Vec<String>,
|
||||
|
||||
/// Each tool name must appear in the tool_calls list (checked by name,
|
||||
/// not by count; duplicates in tool_calls are collapsed).
|
||||
#[serde(default)]
|
||||
pub tools_used: Vec<String>,
|
||||
|
||||
/// None of these tool names may appear in the tool_calls list.
|
||||
#[serde(default)]
|
||||
pub tools_not_used: Vec<String>,
|
||||
|
||||
/// Regex pattern the response must match.
|
||||
#[serde(default)]
|
||||
pub response_matches: Option<String>,
|
||||
|
||||
/// Hard fail if the task produced an error.
|
||||
#[serde(default)]
|
||||
pub no_error: bool,
|
||||
|
||||
/// Minimum number of tool calls expected (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub min_tool_calls: Option<usize>,
|
||||
|
||||
/// Maximum number of tool calls allowed (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub max_tool_calls: Option<usize>,
|
||||
}
|
||||
|
||||
impl SpotAssertions {
|
||||
/// Evaluate all assertions against a submission, returning (score, failure_details).
|
||||
pub fn evaluate(&self, submission: &TaskSubmission) -> (f64, Vec<String>) {
|
||||
let mut passed: usize = 0;
|
||||
let mut total: usize = 0;
|
||||
let mut failures: Vec<String> = Vec::new();
|
||||
|
||||
// Hard fail: error check
|
||||
if self.no_error {
|
||||
total += 1;
|
||||
if let Some(ref err) = submission.error {
|
||||
failures.push(format!("no_error: task errored with: {err}"));
|
||||
// Hard fail: return 0.0 immediately
|
||||
return (0.0, failures);
|
||||
}
|
||||
passed += 1;
|
||||
}
|
||||
|
||||
let response_lower = submission.response.to_lowercase();
|
||||
|
||||
// response_contains: all must appear
|
||||
for needle in &self.response_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_contains: missing \"{needle}\""));
|
||||
}
|
||||
}
|
||||
|
||||
// response_not_contains: none may appear
|
||||
for needle in &self.response_not_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
failures.push(format!("response_not_contains: found \"{needle}\""));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let tool_set: HashSet<&str> = submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
// tools_used: each must appear
|
||||
for tool in &self.tools_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("tools_used: \"{tool}\" not called"));
|
||||
}
|
||||
}
|
||||
|
||||
// tools_not_used: none may appear
|
||||
for tool in &self.tools_not_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
failures.push(format!("tools_not_used: \"{tool}\" was called"));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// response_matches: regex pattern
|
||||
if let Some(ref pattern) = self.response_matches {
|
||||
total += 1;
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(&submission.response) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_matches: /{pattern}/ did not match"));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
failures.push(format!("response_matches: bad regex: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let call_count = submission.tool_calls.len();
|
||||
|
||||
// min_tool_calls
|
||||
if let Some(min) = self.min_tool_calls {
|
||||
total += 1;
|
||||
if call_count >= min {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"min_tool_calls: expected >= {min}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// max_tool_calls
|
||||
if let Some(max) = self.max_tool_calls {
|
||||
total += 1;
|
||||
if call_count <= max {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"max_tool_calls: expected <= {max}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
return (1.0, failures);
|
||||
}
|
||||
|
||||
let score = passed as f64 / total as f64;
|
||||
(score, failures)
|
||||
}
|
||||
}
|
||||
|
||||
/// JSONL entry for a spot check scenario.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SpotEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
assertions: SpotAssertions,
|
||||
}
|
||||
|
||||
/// Spot benchmark suite: end-to-end checks for real user workflows.
|
||||
///
|
||||
/// Tests conversation, individual tool use, multi-tool chaining, and robustness.
|
||||
/// Each task declares multi-criterion assertions scored as passed/total.
|
||||
pub struct SpotSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl SpotSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SpotSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Spot Checks"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"spot"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SpotEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("spot line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"assertions": serde_json::to_value(&entry.assertions)
|
||||
.map_err(|e| BenchError::Config(format!("spot {}: {}", entry.id, e)))?,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let assertions: SpotAssertions = task
|
||||
.metadata
|
||||
.get("assertions")
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing assertions in metadata".to_string(),
|
||||
})
|
||||
.and_then(|v| {
|
||||
serde_json::from_value(v.clone()).map_err(|e| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("bad assertions: {e}"),
|
||||
})
|
||||
})?;
|
||||
|
||||
let (score, failures) = assertions.evaluate(submission);
|
||||
|
||||
if score >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if score <= 0.0 {
|
||||
Ok(BenchScore::fail(failures.join("; ")))
|
||||
} else {
|
||||
Ok(BenchScore::partial(score, failures.join("; ")))
|
||||
}
|
||||
}
|
||||
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![
|
||||
Arc::new(ironclaw::tools::builtin::ShellTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ReadFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::WriteFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ListDirTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ApplyPatchTool::new()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
fn make_submission(
|
||||
response: &str,
|
||||
tool_calls: Vec<&str>,
|
||||
error: Option<&str>,
|
||||
) -> TaskSubmission {
|
||||
TaskSubmission {
|
||||
response: response.to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: tool_calls.into_iter().map(|s| s.to_string()).collect(),
|
||||
error: error.map(|s| s.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_all_pass() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["hello".to_string()],
|
||||
tools_used: vec!["echo".to_string()],
|
||||
no_error: true,
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello, world!", vec!["echo"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hard_fail_on_error() {
|
||||
let assertions = SpotAssertions {
|
||||
no_error: true,
|
||||
response_contains: vec!["hello".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello!", vec![], Some("timeout after 60s"));
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
assert!(failures[0].contains("no_error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_score() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["alpha".to_string(), "beta".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("alpha is here but not the other", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("beta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_not_contains() {
|
||||
let assertions = SpotAssertions {
|
||||
response_not_contains: vec!["error".to_string(), "fail".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("This is an error message", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_used_and_not_used() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_used: vec!["time".to_string()],
|
||||
tools_not_used: vec!["shell".to_string(), "echo".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The time is now", vec!["time"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_not_used_fails() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_not_used: vec!["shell".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("result", vec!["shell", "time"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"\d{4}".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The year is 2026", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex_fail() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"^\d+$".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("not a number", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_min_max_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
min_tool_calls: Some(2),
|
||||
max_tool_calls: Some(4),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Within range
|
||||
let sub = make_submission("ok", vec!["a", "b", "c"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
// Too few
|
||||
let sub = make_submission("ok", vec!["a"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("min_tool_calls"));
|
||||
|
||||
// Too many
|
||||
let sub = make_submission("ok", vec!["a", "b", "c", "d", "e"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("max_tool_calls"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_zero_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
max_tool_calls: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("just talking", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
let sub = make_submission("oops", vec!["echo"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_assertions() {
|
||||
let assertions = SpotAssertions::default();
|
||||
let sub = make_submission("anything", vec!["whatever"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "tags": ["smoke"], "assertions": {{"response_contains": ["hello"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s2", "prompt": "Echo test", "assertions": {{"tools_used": ["echo"]}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "s1");
|
||||
assert_eq!(tasks[1].id, "s2");
|
||||
assert!(tasks[0].tags.contains(&"smoke".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "assertions": {{"response_contains": ["hello", "world"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Full pass
|
||||
let sub = make_submission("Hello World!", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
|
||||
// Partial
|
||||
let sub = make_submission("Hello there", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert!(score.value > 0.0 && score.value < 1.0);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Error hard fail
|
||||
let sub = make_submission("Hello World!", vec![], Some("boom"));
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Validate that a string is safe for use as a filesystem path component.
|
||||
/// Allows alphanumerics, hyphens, underscores, dots, and forward slashes (for nested paths).
|
||||
/// Rejects absolute paths, `..` traversal, and shell metacharacters.
|
||||
fn is_safe_path_component(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& !s.starts_with('/')
|
||||
&& !s.contains("..")
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
}
|
||||
|
||||
/// Validate that a repo string matches the expected `owner/repo` GitHub format.
|
||||
fn is_valid_github_repo(repo: &str) -> bool {
|
||||
// Match "owner/repo" where both parts are alphanumeric with hyphens/underscores/dots
|
||||
static REPO_PATTERN: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$").unwrap());
|
||||
REPO_PATTERN.is_match(repo)
|
||||
}
|
||||
|
||||
/// Validate that a string looks like a git ref (hex SHA or valid ref name).
|
||||
fn is_valid_git_ref(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
&& !s.contains("..")
|
||||
}
|
||||
|
||||
/// SWE-bench dataset entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SweBenchEntry {
|
||||
instance_id: String,
|
||||
repo: String,
|
||||
base_commit: String,
|
||||
#[serde(default)]
|
||||
problem_statement: String,
|
||||
#[serde(default)]
|
||||
hints_text: Option<String>,
|
||||
#[serde(default)]
|
||||
test_patch: Option<String>,
|
||||
#[serde(default)]
|
||||
patch: Option<String>,
|
||||
}
|
||||
|
||||
/// SWE-bench Pro: real-world software engineering tasks.
|
||||
///
|
||||
/// Each task clones a repo at a specific commit, presents the problem statement,
|
||||
/// and expects the agent to produce a patch. Scoring runs the test suite.
|
||||
pub struct SweBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
workspace_dir: PathBuf,
|
||||
use_docker: bool,
|
||||
}
|
||||
|
||||
impl SweBenchSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
workspace_dir: impl Into<PathBuf>,
|
||||
use_docker: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
workspace_dir: workspace_dir.into(),
|
||||
use_docker,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SweBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"SWE-bench Pro"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"swe_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SweBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("swe_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
if !is_safe_path_component(&entry.instance_id) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: unsafe instance_id \"{}\"",
|
||||
line_num + 1,
|
||||
entry.instance_id,
|
||||
)));
|
||||
}
|
||||
if !is_valid_github_repo(&entry.repo) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid repo format \"{}\"",
|
||||
line_num + 1,
|
||||
entry.repo,
|
||||
)));
|
||||
}
|
||||
if !is_valid_git_ref(&entry.base_commit) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid base_commit \"{}\"",
|
||||
line_num + 1,
|
||||
entry.base_commit,
|
||||
)));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"repo": entry.repo,
|
||||
"base_commit": entry.base_commit,
|
||||
"test_patch": entry.test_patch,
|
||||
"gold_patch": entry.patch,
|
||||
"use_docker": self.use_docker,
|
||||
"workspace_dir": self.workspace_dir.to_string_lossy(),
|
||||
});
|
||||
|
||||
let prompt = if let Some(ref hints) = entry.hints_text {
|
||||
format!("{}\n\nHints:\n{}", entry.problem_statement, hints)
|
||||
} else {
|
||||
entry.problem_statement
|
||||
};
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.instance_id,
|
||||
prompt,
|
||||
context: Some(format!(
|
||||
"Repository: {}, Commit: {}",
|
||||
entry.repo, entry.base_commit
|
||||
)),
|
||||
resources: vec![],
|
||||
tags: vec![format!("repo-{}", entry.repo.replace('/', "-"))],
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn setup_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let repo = task
|
||||
.metadata
|
||||
.get("repo")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing repo in metadata".to_string(),
|
||||
})?;
|
||||
let base_commit = task
|
||||
.metadata
|
||||
.get("base_commit")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing base_commit in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
|
||||
// Clone repo if not already present
|
||||
if !task_dir.exists() {
|
||||
let repo_url = format!("https://github.com/{}.git", repo);
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
&repo_url,
|
||||
&task_dir.to_string_lossy(),
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Checkout the base commit
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
// Shallow clone might not have the commit; fetch more history
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["fetch", "--unshallow"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout retry failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn teardown_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
if task_dir.exists() {
|
||||
// Reset any changes
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["checkout", "."])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["clean", "-fdx"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// For SWE-bench, scoring requires running the test patch against the agent's changes.
|
||||
// This is a simplified version that checks if the agent produced any code changes.
|
||||
|
||||
let test_patch = task.metadata.get("test_patch").and_then(|v| v.as_str());
|
||||
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response from agent"));
|
||||
}
|
||||
|
||||
// If we have a test patch, try to verify the submission
|
||||
if let Some(_test_patch) = test_patch {
|
||||
// TODO: Apply agent's patch, then apply test patch, then run tests.
|
||||
// For now, give partial credit if the agent produced some output.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"SWE-bench test execution not implemented, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"test execution not yet implemented; partial credit for response",
|
||||
))
|
||||
} else {
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"no test_patch available, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"no test_patch available for automated scoring",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "django__django-12345", "repo": "django/django", "base_commit": "abc123", "problem_statement": "Fix the ORM bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "django__django-12345");
|
||||
assert!(tasks[0].tags.contains(&"repo-django-django".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_scoring_no_response() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "s1", "repo": "org/repo", "base_commit": "abc", "problem_statement": "Fix bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: String::new(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_safe_path_component() {
|
||||
assert!(is_safe_path_component("django__django-12345"));
|
||||
assert!(is_safe_path_component("org/repo"));
|
||||
assert!(is_safe_path_component("abc123"));
|
||||
assert!(!is_safe_path_component(""));
|
||||
assert!(!is_safe_path_component("../../etc/passwd"));
|
||||
assert!(!is_safe_path_component("/etc/passwd"));
|
||||
assert!(!is_safe_path_component("foo;rm -rf /"));
|
||||
assert!(!is_safe_path_component("foo bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_github_repo() {
|
||||
assert!(is_valid_github_repo("django/django"));
|
||||
assert!(is_valid_github_repo("org/repo-name"));
|
||||
assert!(is_valid_github_repo("Org.Name/Repo_v2"));
|
||||
assert!(!is_valid_github_repo(""));
|
||||
assert!(!is_valid_github_repo("no-slash"));
|
||||
assert!(!is_valid_github_repo("too/many/slashes"));
|
||||
assert!(!is_valid_github_repo("spa ce/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_git_ref() {
|
||||
assert!(is_valid_git_ref("abc123"));
|
||||
assert!(is_valid_git_ref("deadbeef0123456789abcdef0123456789abcdef"));
|
||||
assert!(is_valid_git_ref("v1.2.3"));
|
||||
assert!(is_valid_git_ref("main"));
|
||||
assert!(!is_valid_git_ref(""));
|
||||
assert!(!is_valid_git_ref("bad..ref"));
|
||||
assert!(!is_valid_git_ref("has space"));
|
||||
assert!(!is_valid_git_ref("semi;colon"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_path_traversal() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "../../etc/passwd", "repo": "org/repo", "base_commit": "abc", "problem_statement": "evil"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("unsafe instance_id"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_bad_repo() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "task1", "repo": "not-a-repo-format", "base_commit": "abc", "problem_statement": "bad"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("invalid repo format"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, ConversationTurn, TaskSubmission};
|
||||
|
||||
/// Tau-bench task entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TauBenchEntry {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
domain: String,
|
||||
instruction: String,
|
||||
#[serde(default)]
|
||||
user_persona: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_state: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
expected_actions: Vec<String>,
|
||||
#[serde(default)]
|
||||
max_turns: Option<usize>,
|
||||
}
|
||||
|
||||
/// Tau-bench: multi-turn tool-calling dialog benchmark.
|
||||
///
|
||||
/// Tests agent ability to handle customer service scenarios with simulated
|
||||
/// domain APIs (retail, airline). Scoring compares final state against expected.
|
||||
pub struct TauBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
domain: String,
|
||||
}
|
||||
|
||||
impl TauBenchSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>, domain: impl Into<String>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
domain: domain.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for TauBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Tau-bench"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"tau_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: TauBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("tau_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
let domain = if entry.domain.is_empty() {
|
||||
self.domain.clone()
|
||||
} else {
|
||||
entry.domain.clone()
|
||||
};
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"domain": domain,
|
||||
"user_persona": entry.user_persona,
|
||||
"expected_state": entry.expected_state,
|
||||
"expected_actions": entry.expected_actions,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.instruction,
|
||||
context: entry.user_persona.clone(),
|
||||
resources: vec![],
|
||||
tags: vec![format!("domain-{domain}")],
|
||||
expected_turns: entry.max_turns,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// Score based on expected actions completion
|
||||
let expected_actions: Vec<String> = task
|
||||
.metadata
|
||||
.get("expected_actions")
|
||||
.and_then(|v| serde_json::from_value(v.clone()).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if expected_actions.is_empty() {
|
||||
// No expected actions defined; score based on whether agent responded
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response"));
|
||||
}
|
||||
return Ok(BenchScore::partial(
|
||||
0.5,
|
||||
"no expected_actions to evaluate against",
|
||||
));
|
||||
}
|
||||
|
||||
// Check which expected actions were actually called
|
||||
let called: std::collections::HashSet<&str> =
|
||||
submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
let matched = expected_actions
|
||||
.iter()
|
||||
.filter(|a| called.contains(a.as_str()))
|
||||
.count();
|
||||
|
||||
let ratio = matched as f64 / expected_actions.len() as f64;
|
||||
if ratio >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if ratio > 0.0 {
|
||||
Ok(BenchScore::partial(
|
||||
ratio,
|
||||
format!(
|
||||
"{}/{} expected actions completed",
|
||||
matched,
|
||||
expected_actions.len()
|
||||
),
|
||||
))
|
||||
} else {
|
||||
Ok(BenchScore::fail(format!(
|
||||
"0/{} expected actions completed",
|
||||
expected_actions.len()
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
// Check if we've exceeded max turns
|
||||
if let Some(max) = task.expected_turns {
|
||||
let user_turns = conversation
|
||||
.iter()
|
||||
.filter(|t| matches!(t.role, crate::suite::TurnRole::User))
|
||||
.count();
|
||||
if user_turns >= max {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-turn simulation requires an LLM to play the customer role.
|
||||
// Until that's implemented, every scenario is single-turn only.
|
||||
// TODO: Use LLM to simulate customer based on user_persona.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"multi-turn simulation not implemented, ending after first turn"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return my order", "expected_actions": ["lookup_order", "process_return"], "max_turns": 3}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].expected_turns, Some(3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return order", "expected_actions": ["lookup_order", "process_return"]}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Partial completion
|
||||
let submission = TaskSubmission {
|
||||
response: "I found your order.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.5);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Full completion
|
||||
let submission = TaskSubmission {
|
||||
response: "Return processed.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string(), "process_return".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use ironclaw::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
use ironclaw::error::ChannelError;
|
||||
|
||||
use crate::results::TraceToolCall;
|
||||
use crate::suite::ConversationTurn;
|
||||
|
||||
/// Truncate a string to at most `max_bytes` without splitting a UTF-8 character.
|
||||
fn truncate_str(s: &str, max_bytes: usize) -> &str {
|
||||
if s.len() <= max_bytes {
|
||||
return s;
|
||||
}
|
||||
let mut end = max_bytes;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
}
|
||||
|
||||
/// Captured state from a benchmark channel run.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ChannelCapture {
|
||||
/// All responses the agent sent back.
|
||||
pub responses: Vec<String>,
|
||||
/// Tool calls observed (name, success, duration_ms).
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
/// Full conversation turns for multi-turn scoring.
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
/// Status messages (for debugging).
|
||||
pub status_log: Vec<String>,
|
||||
}
|
||||
|
||||
/// A headless Channel implementation for benchmarking.
|
||||
///
|
||||
/// Modeled after `ReplChannel`: uses mpsc to inject messages and captures
|
||||
/// all responses and tool status events. Auto-approves tool execution
|
||||
/// so benchmarks run without user interaction.
|
||||
pub struct BenchChannel {
|
||||
/// Sender to inject messages into the agent loop.
|
||||
msg_tx: mpsc::Sender<IncomingMessage>,
|
||||
/// Receiver the agent loop reads from (taken once by `start()`).
|
||||
msg_rx: Mutex<Option<mpsc::Receiver<IncomingMessage>>>,
|
||||
/// Accumulated capture data.
|
||||
capture: Arc<Mutex<ChannelCapture>>,
|
||||
}
|
||||
|
||||
impl BenchChannel {
|
||||
pub fn new() -> (Self, mpsc::Sender<IncomingMessage>) {
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
let channel = Self {
|
||||
msg_tx: tx.clone(),
|
||||
msg_rx: Mutex::new(Some(rx)),
|
||||
capture: Arc::new(Mutex::new(ChannelCapture::default())),
|
||||
};
|
||||
(channel, tx)
|
||||
}
|
||||
|
||||
/// Get a handle to the capture data.
|
||||
pub fn capture(&self) -> Arc<Mutex<ChannelCapture>> {
|
||||
Arc::clone(&self.capture)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Channel for BenchChannel {
|
||||
fn name(&self) -> &str {
|
||||
"bench"
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<MessageStream, ChannelError> {
|
||||
let rx = self
|
||||
.msg_rx
|
||||
.lock()
|
||||
.await
|
||||
.take()
|
||||
.ok_or_else(|| ChannelError::StartupFailed {
|
||||
name: "bench".to_string(),
|
||||
reason: "start() already called".to_string(),
|
||||
})?;
|
||||
Ok(Box::pin(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
async fn respond(
|
||||
&self,
|
||||
_msg: &IncomingMessage,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.responses.push(response.content.clone());
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: crate::suite::TurnRole::Assistant,
|
||||
content: response.content,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_status(
|
||||
&self,
|
||||
status: StatusUpdate,
|
||||
_metadata: &serde_json::Value,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
|
||||
match status {
|
||||
StatusUpdate::ToolCompleted { ref name, success } => {
|
||||
cap.tool_calls.push(TraceToolCall {
|
||||
name: name.clone(),
|
||||
duration_ms: 0, // We don't have precise per-tool timing here
|
||||
success,
|
||||
});
|
||||
cap.status_log
|
||||
.push(format!("tool_completed: {name} success={success}"));
|
||||
}
|
||||
StatusUpdate::ApprovalNeeded { ref request_id, .. } => {
|
||||
// Auto-approve all tools during benchmarks
|
||||
cap.status_log.push(format!("auto_approved: {request_id}"));
|
||||
drop(cap); // Release lock before sending
|
||||
let approval = IncomingMessage::new("bench", "bench-user", "always");
|
||||
let _ = self.msg_tx.send(approval).await;
|
||||
return Ok(());
|
||||
}
|
||||
StatusUpdate::Thinking(ref msg) => {
|
||||
cap.status_log.push(format!("thinking: {msg}"));
|
||||
}
|
||||
StatusUpdate::ToolStarted { ref name } => {
|
||||
cap.status_log.push(format!("tool_started: {name}"));
|
||||
}
|
||||
StatusUpdate::ToolResult {
|
||||
ref name,
|
||||
ref preview,
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"tool_result: {name} -> {}",
|
||||
truncate_str(preview, 100)
|
||||
));
|
||||
}
|
||||
StatusUpdate::StreamChunk(_) => {}
|
||||
StatusUpdate::Status(ref msg) => {
|
||||
cap.status_log.push(format!("status: {msg}"));
|
||||
}
|
||||
StatusUpdate::JobStarted {
|
||||
ref job_id,
|
||||
ref title,
|
||||
..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("job_started: {job_id} ({title})"));
|
||||
}
|
||||
StatusUpdate::AuthRequired {
|
||||
ref extension_name, ..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("auth_required: {extension_name} (auto-skipped)"));
|
||||
}
|
||||
StatusUpdate::AuthCompleted {
|
||||
ref extension_name,
|
||||
success,
|
||||
..
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"auth_completed: {extension_name} success={success}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn broadcast(
|
||||
&self,
|
||||
_user_id: &str,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.status_log.push(format!(
|
||||
"broadcast: {}",
|
||||
truncate_str(&response.content, 100)
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn shutdown(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_responses() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
let capture = channel.capture();
|
||||
|
||||
let msg = IncomingMessage::new("bench", "user", "hello");
|
||||
let response = OutgoingResponse::text("world");
|
||||
channel.respond(&msg, response).await.unwrap();
|
||||
|
||||
let cap = capture.lock().await;
|
||||
assert_eq!(cap.responses.len(), 1);
|
||||
assert_eq!(cap.responses[0], "world");
|
||||
assert_eq!(cap.conversation.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_auto_approves() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
// start() to consume the receiver
|
||||
let _stream = channel.start().await.unwrap();
|
||||
|
||||
let status = StatusUpdate::ApprovalNeeded {
|
||||
request_id: "req-1".to_string(),
|
||||
tool_name: "shell".to_string(),
|
||||
description: "run ls".to_string(),
|
||||
parameters: serde_json::json!({}),
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The approval message was sent through msg_tx,
|
||||
// which means the stream would receive it.
|
||||
// We can't easily read from the stream in this test without
|
||||
// consuming it, but we can verify the status log.
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert!(cap.status_log.iter().any(|s| s.contains("auto_approved")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_tool_events() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
|
||||
let status = StatusUpdate::ToolCompleted {
|
||||
name: "echo".to_string(),
|
||||
success: true,
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert_eq!(cap.tool_calls.len(), 1);
|
||||
assert_eq!(cap.tool_calls[0].name, "echo");
|
||||
assert!(cap.tool_calls[0].success);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// Top-level bench configuration, loaded from TOML.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BenchConfig {
|
||||
/// Where to write results. Default: "./bench-results".
|
||||
#[serde(default = "default_results_dir")]
|
||||
pub results_dir: PathBuf,
|
||||
|
||||
/// Per-task timeout. Default: "300s".
|
||||
#[serde(
|
||||
default = "default_task_timeout",
|
||||
deserialize_with = "deserialize_duration"
|
||||
)]
|
||||
pub task_timeout: Duration,
|
||||
|
||||
/// How many tasks to run in parallel. Default: 1.
|
||||
#[serde(default = "default_parallelism")]
|
||||
pub parallelism: usize,
|
||||
|
||||
/// Model/config matrix entries. At least one required.
|
||||
#[serde(default)]
|
||||
pub matrix: Vec<MatrixEntry>,
|
||||
|
||||
/// Suite-specific configuration (passed through to adapter).
|
||||
#[serde(default = "default_suite_config")]
|
||||
pub suite_config: toml::Value,
|
||||
}
|
||||
|
||||
/// A single model/config combination to benchmark.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MatrixEntry {
|
||||
/// Label for this configuration (used in results).
|
||||
pub label: String,
|
||||
|
||||
/// Model identifier.
|
||||
#[serde(default)]
|
||||
pub model: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchConfig {
|
||||
/// Load from a TOML file.
|
||||
pub fn from_file(path: &Path) -> Result<Self, BenchError> {
|
||||
if !path.exists() {
|
||||
return Err(BenchError::ConfigNotFound {
|
||||
path: path.to_path_buf(),
|
||||
});
|
||||
}
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let config: BenchConfig = toml::from_str(&content)?;
|
||||
if config.matrix.is_empty() {
|
||||
return Err(BenchError::Config(
|
||||
"config must have at least one [[matrix]] entry".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// Create a minimal config for when no config file is provided.
|
||||
/// Uses defaults and optional CLI overrides.
|
||||
pub fn minimal(model: Option<String>) -> Self {
|
||||
let label = model.as_deref().unwrap_or("default").to_string();
|
||||
Self {
|
||||
results_dir: default_results_dir(),
|
||||
task_timeout: default_task_timeout(),
|
||||
parallelism: default_parallelism(),
|
||||
matrix: vec![MatrixEntry { label, model }],
|
||||
suite_config: toml::Value::Table(toml::map::Map::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the suite_config as a generic map for adapter use.
|
||||
pub fn suite_config_map(&self) -> toml::map::Map<String, toml::Value> {
|
||||
match &self.suite_config {
|
||||
toml::Value::Table(map) => map.clone(),
|
||||
_ => toml::map::Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a string value from suite_config.
|
||||
pub fn suite_config_str(&self, key: &str) -> Option<String> {
|
||||
self.suite_config_map()
|
||||
.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_suite_config() -> toml::Value {
|
||||
toml::Value::Table(toml::map::Map::new())
|
||||
}
|
||||
|
||||
fn default_results_dir() -> PathBuf {
|
||||
PathBuf::from("./bench-results")
|
||||
}
|
||||
|
||||
fn default_task_timeout() -> Duration {
|
||||
Duration::from_secs(300)
|
||||
}
|
||||
|
||||
fn default_parallelism() -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
/// Deserialize a duration from a string like "300s", "5m", etc.
|
||||
fn deserialize_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
parse_duration(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn parse_duration(s: &str) -> Result<Duration, String> {
|
||||
let s = s.trim();
|
||||
if let Some(secs) = s.strip_suffix('s') {
|
||||
secs.trim()
|
||||
.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid seconds: {e}"))
|
||||
} else if let Some(mins) = s.strip_suffix('m') {
|
||||
mins.trim()
|
||||
.parse::<u64>()
|
||||
.map(|m| Duration::from_secs(m * 60))
|
||||
.map_err(|e| format!("invalid minutes: {e}"))
|
||||
} else {
|
||||
// Assume seconds if no suffix
|
||||
s.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid duration '{s}': {e}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_duration() {
|
||||
assert_eq!(parse_duration("300s").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("60").unwrap(), Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_config() {
|
||||
let config = BenchConfig::minimal(Some("test-model".to_string()));
|
||||
assert_eq!(config.matrix.len(), 1);
|
||||
assert_eq!(config.matrix[0].label, "test-model");
|
||||
assert_eq!(config.parallelism, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_rejects_empty_matrix() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("empty.toml");
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"
|
||||
results_dir = "./results"
|
||||
task_timeout = "60s"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let err = BenchConfig::from_file(&path).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("at least one [[matrix]]"),
|
||||
"got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_from_toml() {
|
||||
let toml_str = r#"
|
||||
results_dir = "./my-results"
|
||||
task_timeout = "60s"
|
||||
parallelism = 2
|
||||
|
||||
[[matrix]]
|
||||
label = "fast"
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
[[matrix]]
|
||||
label = "full"
|
||||
model = "claude-3-5-sonnet"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "./data/test.jsonl"
|
||||
"#;
|
||||
let config: BenchConfig = toml::from_str(toml_str).unwrap();
|
||||
assert_eq!(config.results_dir, PathBuf::from("./my-results"));
|
||||
assert_eq!(config.task_timeout, Duration::from_secs(60));
|
||||
assert_eq!(config.parallelism, 2);
|
||||
assert_eq!(config.matrix.len(), 2);
|
||||
assert_eq!(
|
||||
config.suite_config_str("dataset_path").unwrap(),
|
||||
"./data/test.jsonl"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BenchError {
|
||||
#[error("Config error: {0}")]
|
||||
Config(String),
|
||||
|
||||
#[error("Config file not found: {path}")]
|
||||
ConfigNotFound { path: PathBuf },
|
||||
|
||||
#[error("Suite {name} not found. Available: {available}")]
|
||||
SuiteNotFound { name: String, available: String },
|
||||
|
||||
#[error("Task {task_id} failed: {reason}")]
|
||||
TaskFailed { task_id: String, reason: String },
|
||||
|
||||
#[error("Scoring error for task {task_id}: {reason}")]
|
||||
Scoring { task_id: String, reason: String },
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("TOML parse error: {0}")]
|
||||
Toml(#[from] toml::de::Error),
|
||||
|
||||
#[error("Agent error: {0}")]
|
||||
Agent(#[from] ironclaw::Error),
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
use rust_decimal::prelude::ToPrimitive;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use ironclaw::error::LlmError;
|
||||
use ironclaw::llm::{
|
||||
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
|
||||
/// Recorded metrics from a single LLM call.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct LlmCallRecord {
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub duration_ms: u64,
|
||||
pub had_tool_calls: bool,
|
||||
}
|
||||
|
||||
/// Wraps an `LlmProvider` to record per-call metrics.
|
||||
///
|
||||
/// The wrapper is transparent to the agent: it delegates every call
|
||||
/// to the inner provider and captures token counts and timings.
|
||||
pub struct InstrumentedLlm {
|
||||
inner: Arc<dyn LlmProvider>,
|
||||
records: Mutex<Vec<LlmCallRecord>>,
|
||||
total_input_tokens: AtomicU32,
|
||||
total_output_tokens: AtomicU32,
|
||||
call_count: AtomicU32,
|
||||
}
|
||||
|
||||
impl InstrumentedLlm {
|
||||
pub fn new(inner: Arc<dyn LlmProvider>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
records: Mutex::new(Vec::new()),
|
||||
total_input_tokens: AtomicU32::new(0),
|
||||
total_output_tokens: AtomicU32::new(0),
|
||||
call_count: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take all recorded call metrics, clearing the internal buffer.
|
||||
pub async fn take_records(&self) -> Vec<LlmCallRecord> {
|
||||
let mut records = self.records.lock().await;
|
||||
std::mem::take(&mut *records)
|
||||
}
|
||||
|
||||
/// Snapshot of total tokens without clearing.
|
||||
pub fn total_input_tokens(&self) -> u32 {
|
||||
self.total_input_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn total_output_tokens(&self) -> u32 {
|
||||
self.total_output_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> u32 {
|
||||
self.call_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Estimated cost using the inner provider's cost-per-token rates.
|
||||
pub fn estimated_cost(&self) -> f64 {
|
||||
let (input_rate, output_rate) = self.inner.cost_per_token();
|
||||
let input_cost =
|
||||
input_rate * Decimal::from(self.total_input_tokens.load(Ordering::Relaxed));
|
||||
let output_cost =
|
||||
output_rate * Decimal::from(self.total_output_tokens.load(Ordering::Relaxed));
|
||||
let total = input_cost + output_cost;
|
||||
total.to_f64().unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Reset all counters and records.
|
||||
pub async fn reset(&self) {
|
||||
self.records.lock().await.clear();
|
||||
self.total_input_tokens.store(0, Ordering::Relaxed);
|
||||
self.total_output_tokens.store(0, Ordering::Relaxed);
|
||||
self.call_count.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn record(
|
||||
&self,
|
||||
input_tokens: u32,
|
||||
output_tokens: u32,
|
||||
duration_ms: u64,
|
||||
had_tool_calls: bool,
|
||||
) {
|
||||
self.total_input_tokens
|
||||
.fetch_add(input_tokens, Ordering::Relaxed);
|
||||
self.total_output_tokens
|
||||
.fetch_add(output_tokens, Ordering::Relaxed);
|
||||
self.call_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.records.lock().await.push(LlmCallRecord {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
had_tool_calls,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for InstrumentedLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
self.inner.cost_per_token()
|
||||
}
|
||||
|
||||
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete_with_tools(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
let had_tool_calls = !response.tool_calls.is_empty();
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
had_tool_calls,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
self.inner.list_models().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ironclaw::llm::{ChatMessage, CompletionRequest, CompletionResponse, FinishReason};
|
||||
|
||||
/// Fake LLM that returns a canned response with known token counts.
|
||||
struct FakeLlm;
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for FakeLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
(
|
||||
Decimal::new(3, 6), // $0.000003 per input token
|
||||
Decimal::new(15, 6), // $0.000015 per output token
|
||||
)
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: CompletionRequest,
|
||||
) -> Result<CompletionResponse, LlmError> {
|
||||
Ok(CompletionResponse {
|
||||
content: "test response".to_string(),
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
finish_reason: FinishReason::Stop,
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
_request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
Ok(ToolCompletionResponse {
|
||||
content: Some("tool response".to_string()),
|
||||
tool_calls: vec![],
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
finish_reason: FinishReason::Stop,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_records_metrics() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
assert_eq!(instrumented.total_input_tokens(), 100);
|
||||
assert_eq!(instrumented.total_output_tokens(), 50);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].input_tokens, 100);
|
||||
assert!(!records[0].had_tool_calls);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_cost_calculation() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
// 100 * 0.000003 + 50 * 0.000015 = 0.0003 + 0.00075 = 0.00105
|
||||
let cost = instrumented.estimated_cost();
|
||||
assert!((cost - 0.00105).abs() < 0.0001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_reset() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
|
||||
instrumented.reset().await;
|
||||
assert_eq!(instrumented.call_count(), 0);
|
||||
assert_eq!(instrumented.total_input_tokens(), 0);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert!(records.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
mod adapters;
|
||||
mod channel;
|
||||
mod config;
|
||||
mod error;
|
||||
mod instrumented_llm;
|
||||
mod results;
|
||||
mod runner;
|
||||
mod scoring;
|
||||
mod suite;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "ironclaw-bench", about = "IronClaw benchmarking harness")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Run a benchmark suite.
|
||||
Run {
|
||||
/// Suite to run (custom, gaia, spot, tau_bench, swe_bench).
|
||||
#[arg(long)]
|
||||
suite: String,
|
||||
|
||||
/// Path to bench config TOML.
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
|
||||
/// Override model for all matrix entries.
|
||||
#[arg(long)]
|
||||
model: Option<String>,
|
||||
|
||||
/// Max tasks to run in parallel.
|
||||
#[arg(long)]
|
||||
parallelism: Option<usize>,
|
||||
|
||||
/// Sample N tasks from the suite (for quick testing).
|
||||
#[arg(long)]
|
||||
sample: Option<usize>,
|
||||
|
||||
/// Only run these task IDs (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
task_ids: Option<Vec<String>>,
|
||||
|
||||
/// Only run tasks with these tags (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
tags: Option<Vec<String>>,
|
||||
|
||||
/// Per-task timeout in seconds.
|
||||
#[arg(long)]
|
||||
timeout_secs: Option<u64>,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
|
||||
/// Resume a previous run by ID.
|
||||
#[arg(long)]
|
||||
resume: Option<Uuid>,
|
||||
},
|
||||
|
||||
/// Show results for a run.
|
||||
Results {
|
||||
/// Run ID or "latest".
|
||||
#[arg(default_value = "latest")]
|
||||
run_id: String,
|
||||
|
||||
/// Output format.
|
||||
#[arg(long, default_value = "table")]
|
||||
format: ResultsFormat,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// Compare two runs.
|
||||
Compare {
|
||||
/// Baseline run ID.
|
||||
baseline: Uuid,
|
||||
|
||||
/// Comparison run ID.
|
||||
comparison: Uuid,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// List available benchmark suites.
|
||||
List,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, clap::ValueEnum)]
|
||||
enum ResultsFormat {
|
||||
Table,
|
||||
Json,
|
||||
Csv,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("ironclaw_bench=info,ironclaw=warn")),
|
||||
)
|
||||
.with(tracing_subscriber::fmt::layer().with_target(false))
|
||||
.init();
|
||||
|
||||
match cli.command {
|
||||
Commands::List => {
|
||||
println!("Available benchmark suites:\n");
|
||||
for (id, desc) in adapters::KNOWN_SUITES {
|
||||
println!(" {:<15} {}", id, desc);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
Commands::Run {
|
||||
suite,
|
||||
config: config_path,
|
||||
model,
|
||||
parallelism,
|
||||
sample,
|
||||
task_ids,
|
||||
tags,
|
||||
timeout_secs,
|
||||
results_dir,
|
||||
resume,
|
||||
} => {
|
||||
// Load or create config
|
||||
let mut bench_config = if let Some(ref path) = config_path {
|
||||
BenchConfig::from_file(path)?
|
||||
} else {
|
||||
BenchConfig::minimal(model.clone())
|
||||
};
|
||||
|
||||
// Apply CLI overrides
|
||||
if let Some(p) = parallelism {
|
||||
bench_config.parallelism = p;
|
||||
}
|
||||
if let Some(t) = timeout_secs {
|
||||
bench_config.task_timeout = std::time::Duration::from_secs(t);
|
||||
}
|
||||
if let Some(ref dir) = results_dir {
|
||||
bench_config.results_dir = dir.clone();
|
||||
}
|
||||
|
||||
// If model override specified and we have matrix entries, update them
|
||||
if let Some(ref m) = model {
|
||||
for entry in &mut bench_config.matrix {
|
||||
entry.model = Some(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Create suite
|
||||
let bench_suite = adapters::create_suite(&suite, &bench_config)?;
|
||||
|
||||
// Initialize ironclaw LLM provider
|
||||
let ironclaw_config = ironclaw::Config::from_env().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to load ironclaw config: {}. Make sure .env is configured.",
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
||||
auth_base_url: ironclaw_config.llm.nearai.auth_base_url.clone(),
|
||||
session_path: ironclaw_config.llm.nearai.session_path.clone(),
|
||||
})
|
||||
.await;
|
||||
session.ensure_authenticated().await?;
|
||||
|
||||
let llm = ironclaw::llm::create_llm_provider(&ironclaw_config.llm, session)?;
|
||||
let safety = Arc::new(ironclaw::safety::SafetyLayer::new(&ironclaw_config.safety));
|
||||
|
||||
let runner = runner::BenchRunner::new(bench_suite, bench_config.clone(), llm, safety);
|
||||
|
||||
// Run for each matrix entry
|
||||
for matrix_entry in &bench_config.matrix {
|
||||
let run_id = runner
|
||||
.run(
|
||||
matrix_entry,
|
||||
sample,
|
||||
task_ids.as_deref(),
|
||||
tags.as_deref(),
|
||||
resume,
|
||||
)
|
||||
.await?;
|
||||
println!("Run complete: {}", run_id);
|
||||
}
|
||||
}
|
||||
Commands::Results {
|
||||
run_id,
|
||||
format,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
let uuid = if run_id == "latest" {
|
||||
results::find_latest_run(&base)?
|
||||
.ok_or_else(|| anyhow::anyhow!("No runs found in {}", base.display()))?
|
||||
} else {
|
||||
Uuid::parse_str(&run_id)?
|
||||
};
|
||||
|
||||
let json_path = results::run_json_path(&base, uuid);
|
||||
let jsonl_path = results::tasks_jsonl_path(&base, uuid);
|
||||
|
||||
let run = results::read_run_result(&json_path)?;
|
||||
let tasks = results::read_task_results(&jsonl_path)?;
|
||||
|
||||
match format {
|
||||
ResultsFormat::Table => {
|
||||
results::print_results_table(&tasks, &run);
|
||||
}
|
||||
ResultsFormat::Json => {
|
||||
let output = serde_json::json!({
|
||||
"run": run,
|
||||
"tasks": tasks,
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output)?);
|
||||
}
|
||||
ResultsFormat::Csv => {
|
||||
println!("task_id,score,label,tokens,cost,turns,time_s");
|
||||
for task in &tasks {
|
||||
println!(
|
||||
"{},{:.3},{},{},{:.4},{},{:.1}",
|
||||
task.task_id,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
task.trace.input_tokens + task.trace.output_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Commands::Compare {
|
||||
baseline,
|
||||
comparison,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
|
||||
let baseline_run = results::read_run_result(&results::run_json_path(&base, baseline))?;
|
||||
let comparison_run =
|
||||
results::read_run_result(&results::run_json_path(&base, comparison))?;
|
||||
|
||||
println!("\nComparison: {} vs {}\n", baseline, comparison);
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12} {:>10}",
|
||||
"Metric", "Baseline", "Comparison", "Delta"
|
||||
);
|
||||
println!("{}", "-".repeat(58));
|
||||
|
||||
let pass_delta = comparison_run.pass_rate - baseline_run.pass_rate;
|
||||
println!(
|
||||
"{:<20} {:>11.1}% {:>11.1}% {:>+9.1}%",
|
||||
"Pass rate",
|
||||
baseline_run.pass_rate * 100.0,
|
||||
comparison_run.pass_rate * 100.0,
|
||||
pass_delta * 100.0,
|
||||
);
|
||||
|
||||
let score_delta = comparison_run.avg_score - baseline_run.avg_score;
|
||||
println!(
|
||||
"{:<20} {:>12.3} {:>12.3} {:>+10.3}",
|
||||
"Avg score", baseline_run.avg_score, comparison_run.avg_score, score_delta,
|
||||
);
|
||||
|
||||
let cost_delta = comparison_run.total_cost_usd - baseline_run.total_cost_usd;
|
||||
println!(
|
||||
"{:<20} {:>11.4}$ {:>11.4}$ {:>+9.4}$",
|
||||
"Total cost",
|
||||
baseline_run.total_cost_usd,
|
||||
comparison_run.total_cost_usd,
|
||||
cost_delta,
|
||||
);
|
||||
|
||||
let time_b = baseline_run.total_wall_time_ms as f64 / 1000.0;
|
||||
let time_c = comparison_run.total_wall_time_ms as f64 / 1000.0;
|
||||
println!(
|
||||
"{:<20} {:>11.1}s {:>11.1}s {:>+9.1}s",
|
||||
"Total time",
|
||||
time_b,
|
||||
time_c,
|
||||
time_c - time_b,
|
||||
);
|
||||
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12}",
|
||||
"Model", baseline_run.model, comparison_run.model,
|
||||
);
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,473 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::{BufRead, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Metrics from a single task run: LLM usage, timing, tool calls.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Trace {
|
||||
pub wall_time_ms: u64,
|
||||
pub llm_calls: u32,
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub estimated_cost_usd: f64,
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
pub turns: u32,
|
||||
pub hit_iteration_limit: bool,
|
||||
pub hit_timeout: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TraceToolCall {
|
||||
pub name: String,
|
||||
pub duration_ms: u64,
|
||||
pub success: bool,
|
||||
}
|
||||
|
||||
/// Result of running a single benchmark task.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResult {
|
||||
pub task_id: String,
|
||||
pub suite_id: String,
|
||||
pub score: BenchScore,
|
||||
pub trace: Trace,
|
||||
pub response: String,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
pub config_label: String,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Aggregate results for a full benchmark run.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RunResult {
|
||||
pub run_id: Uuid,
|
||||
pub suite_id: String,
|
||||
pub config_label: String,
|
||||
pub model: String,
|
||||
/// Short git commit hash at the time of the run.
|
||||
#[serde(default)]
|
||||
pub commit_hash: String,
|
||||
pub pass_rate: f64,
|
||||
pub avg_score: f64,
|
||||
pub total_tasks: usize,
|
||||
pub completed_tasks: usize,
|
||||
pub total_cost_usd: f64,
|
||||
pub total_wall_time_ms: u64,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl RunResult {
|
||||
/// Build aggregate from individual task results.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn from_tasks(
|
||||
run_id: Uuid,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
model: &str,
|
||||
commit_hash: &str,
|
||||
total_tasks: usize,
|
||||
tasks: &[TaskResult],
|
||||
started_at: DateTime<Utc>,
|
||||
) -> Self {
|
||||
let pass_count = tasks.iter().filter(|t| t.score.value >= 1.0).count();
|
||||
let pass_rate = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
pass_count as f64 / tasks.len() as f64
|
||||
};
|
||||
let avg_score = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
tasks.iter().map(|t| t.score.value).sum::<f64>() / tasks.len() as f64
|
||||
};
|
||||
let total_cost: f64 = tasks.iter().map(|t| t.trace.estimated_cost_usd).sum();
|
||||
let total_wall: u64 = tasks.iter().map(|t| t.trace.wall_time_ms).sum();
|
||||
|
||||
Self {
|
||||
run_id,
|
||||
suite_id: suite_id.to_string(),
|
||||
config_label: config_label.to_string(),
|
||||
model: model.to_string(),
|
||||
commit_hash: commit_hash.to_string(),
|
||||
pass_rate,
|
||||
avg_score,
|
||||
total_tasks,
|
||||
completed_tasks: tasks.len(),
|
||||
total_cost_usd: total_cost,
|
||||
total_wall_time_ms: total_wall,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a single task result as one JSON line to the JSONL file.
|
||||
pub fn append_task_result(path: &Path, result: &TaskResult) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)?;
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Overwrite the JSONL file with the given results (used after scoring).
|
||||
pub fn write_task_results(path: &Path, results: &[TaskResult]) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::File::create(path)?;
|
||||
for result in results {
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read all task results from a JSONL file.
|
||||
pub fn read_task_results(path: &Path) -> Result<Vec<TaskResult>, BenchError> {
|
||||
if !path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let file = std::fs::File::open(path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut results = Vec::new();
|
||||
for line in reader.lines() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let result: TaskResult = serde_json::from_str(trimmed)?;
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Write the aggregate run result as JSON.
|
||||
pub fn write_run_result(path: &Path, result: &RunResult) -> Result<(), BenchError> {
|
||||
let json = serde_json::to_string_pretty(result)?;
|
||||
std::fs::write(path, json)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the aggregate run result from JSON.
|
||||
pub fn read_run_result(path: &Path) -> Result<RunResult, BenchError> {
|
||||
let json = std::fs::read_to_string(path)?;
|
||||
let result: RunResult = serde_json::from_str(&json)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Get the set of already-completed task IDs from a JSONL file (for resume).
|
||||
///
|
||||
/// Only includes tasks that have been scored (label != "pending"). Tasks that
|
||||
/// were written but not scored (e.g., from an interrupted run) will be re-executed.
|
||||
pub fn completed_task_ids(path: &Path) -> Result<HashSet<String>, BenchError> {
|
||||
let results = read_task_results(path)?;
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.filter(|r| r.score.label != "pending")
|
||||
.map(|r| r.task_id)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Get the results directory for a specific run.
|
||||
pub fn run_dir(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
base.join(run_id.to_string())
|
||||
}
|
||||
|
||||
/// Get the tasks JSONL path for a run.
|
||||
pub fn tasks_jsonl_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("tasks.jsonl")
|
||||
}
|
||||
|
||||
/// Get the run JSON path for a run.
|
||||
pub fn run_json_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("run.json")
|
||||
}
|
||||
|
||||
/// Find the latest run directory by the modification time of its `run.json`.
|
||||
///
|
||||
/// Falls back to `tasks.jsonl` mtime, then directory mtime. This avoids the
|
||||
/// issue where modifying files inside a directory doesn't update the directory's
|
||||
/// mtime on many filesystems.
|
||||
pub fn find_latest_run(base: &Path) -> Result<Option<Uuid>, BenchError> {
|
||||
if !base.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut entries: Vec<_> = std::fs::read_dir(base)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_type().map(|ft| ft.is_dir()).unwrap_or(false))
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_string_lossy().to_string();
|
||||
let uuid = Uuid::parse_str(&name).ok()?;
|
||||
let dir_path = e.path();
|
||||
// Prefer run.json mtime, fall back to tasks.jsonl, then directory
|
||||
let modified = std::fs::metadata(dir_path.join("run.json"))
|
||||
.and_then(|m| m.modified())
|
||||
.or_else(|_| {
|
||||
std::fs::metadata(dir_path.join("tasks.jsonl")).and_then(|m| m.modified())
|
||||
})
|
||||
.or_else(|_| e.metadata().and_then(|m| m.modified()))
|
||||
.ok()?;
|
||||
Some((uuid, modified))
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
Ok(entries.first().map(|(uuid, _)| *uuid))
|
||||
}
|
||||
|
||||
/// Print a summary table of task results.
|
||||
pub fn print_results_table(tasks: &[TaskResult], run: &RunResult) {
|
||||
println!();
|
||||
let commit_suffix = if run.commit_hash.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" | Commit: {}", run.commit_hash)
|
||||
};
|
||||
println!(
|
||||
"Run: {} | Suite: {} | Model: {}{}",
|
||||
run.run_id, run.suite_id, run.model, commit_suffix
|
||||
);
|
||||
println!(
|
||||
"Pass rate: {:.1}% | Avg score: {:.3} | Tasks: {}/{} | Cost: ${:.4} | Time: {:.1}s",
|
||||
run.pass_rate * 100.0,
|
||||
run.avg_score,
|
||||
run.completed_tasks,
|
||||
run.total_tasks,
|
||||
run.total_cost_usd,
|
||||
run.total_wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
println!();
|
||||
|
||||
// Header
|
||||
println!(
|
||||
"{:<30} {:>6} {:>7} {:>8} {:>10} {:>6} {:>8}",
|
||||
"Task ID", "Score", "Label", "Tokens", "Cost", "Turns", "Time"
|
||||
);
|
||||
println!("{}", "-".repeat(80));
|
||||
|
||||
for task in tasks {
|
||||
let total_tokens = task.trace.input_tokens + task.trace.output_tokens;
|
||||
let task_id_display = if task.task_id.len() > 28 {
|
||||
let truncated: String = task.task_id.chars().take(25).collect();
|
||||
format!("{truncated}...")
|
||||
} else {
|
||||
task.task_id.clone()
|
||||
};
|
||||
println!(
|
||||
"{:<30} {:>6.3} {:>7} {:>8} {:>10.4} {:>6} {:>7.1}s",
|
||||
task_id_display,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
total_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_run_result_from_tasks() {
|
||||
let tasks = vec![
|
||||
TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 1000,
|
||||
llm_calls: 2,
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
estimated_cost_usd: 0.01,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
TaskResult {
|
||||
task_id: "t2".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some("wrong".to_string()),
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 2000,
|
||||
llm_calls: 3,
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
estimated_cost_usd: 0.02,
|
||||
tool_calls: vec![],
|
||||
turns: 2,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "wrong answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
];
|
||||
|
||||
let run = RunResult::from_tasks(
|
||||
Uuid::new_v4(),
|
||||
"custom",
|
||||
"default",
|
||||
"test-model",
|
||||
"abc1234",
|
||||
2,
|
||||
&tasks,
|
||||
Utc::now(),
|
||||
);
|
||||
|
||||
assert_eq!(run.pass_rate, 0.5);
|
||||
assert_eq!(run.avg_score, 0.5);
|
||||
assert_eq!(run.total_tasks, 2);
|
||||
assert_eq!(run.completed_tasks, 2);
|
||||
assert!((run.total_cost_usd - 0.03).abs() < f64::EPSILON);
|
||||
assert_eq!(run.total_wall_time_ms, 3000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jsonl_roundtrip() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "round-trip-test".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 500,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "hello".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
|
||||
append_task_result(&path, &result).expect("append");
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let loaded = read_task_results(&path).expect("read");
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].task_id, "round-trip-test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completed_task_ids() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "unique-id-1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "x".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let ids = completed_task_ids(&path).expect("ids");
|
||||
assert!(ids.contains("unique-id-1"));
|
||||
assert!(!ids.contains("unique-id-2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_task_results_overwrites() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
// Write initial "pending" result via append
|
||||
let pending = TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "spot".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "42".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &pending).expect("append");
|
||||
|
||||
// Verify pending score
|
||||
let before = read_task_results(&path).expect("read");
|
||||
assert_eq!(before.len(), 1);
|
||||
assert_eq!(before[0].score.label, "pending");
|
||||
|
||||
// Overwrite with scored result
|
||||
let mut scored = pending;
|
||||
scored.score = BenchScore::pass();
|
||||
write_task_results(&path, &[scored]).expect("write");
|
||||
|
||||
// Verify scored result replaced pending
|
||||
let after = read_task_results(&path).expect("read");
|
||||
assert_eq!(after.len(), 1);
|
||||
assert_eq!(after[0].score.label, "pass");
|
||||
assert_eq!(after[0].score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,550 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use ironclaw::agent::{Agent, AgentDeps};
|
||||
use ironclaw::channels::{ChannelManager, IncomingMessage};
|
||||
use ironclaw::config::AgentConfig;
|
||||
use ironclaw::llm::LlmProvider;
|
||||
use ironclaw::safety::SafetyLayer;
|
||||
use ironclaw::tools::ToolRegistry;
|
||||
|
||||
use crate::channel::BenchChannel;
|
||||
use crate::config::{BenchConfig, MatrixEntry};
|
||||
use crate::error::BenchError;
|
||||
use crate::instrumented_llm::InstrumentedLlm;
|
||||
use crate::results::{
|
||||
RunResult, TaskResult, Trace, append_task_result, completed_task_ids, run_dir, run_json_path,
|
||||
tasks_jsonl_path, write_run_result, write_task_results,
|
||||
};
|
||||
use crate::suite::{BenchSuite, BenchTask, ConversationTurn, TaskSubmission, TurnRole};
|
||||
|
||||
/// Parameters for running a single task in isolation.
|
||||
struct TaskRunParams<'a> {
|
||||
task: &'a BenchTask,
|
||||
suite_id: &'a str,
|
||||
config_label: &'a str,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
timeout: std::time::Duration,
|
||||
additional_tools: &'a [Arc<dyn ironclaw::tools::Tool>],
|
||||
}
|
||||
|
||||
/// Orchestrates benchmark execution: loads tasks, runs agent per task,
|
||||
/// scores results, writes JSONL output.
|
||||
pub struct BenchRunner {
|
||||
suite: Arc<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
}
|
||||
|
||||
impl BenchRunner {
|
||||
pub fn new(
|
||||
suite: Box<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
suite: Arc::from(suite),
|
||||
config,
|
||||
llm,
|
||||
safety,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the benchmark for one matrix entry.
|
||||
///
|
||||
/// Returns the run_id for result retrieval.
|
||||
pub async fn run(
|
||||
&self,
|
||||
matrix: &MatrixEntry,
|
||||
sample: Option<usize>,
|
||||
task_filter: Option<&[String]>,
|
||||
tag_filter: Option<&[String]>,
|
||||
resume_run_id: Option<Uuid>,
|
||||
) -> Result<Uuid, BenchError> {
|
||||
let run_id = resume_run_id.unwrap_or_else(Uuid::new_v4);
|
||||
let results_base = &self.config.results_dir;
|
||||
let dir = run_dir(results_base, run_id);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
|
||||
let jsonl_path = tasks_jsonl_path(results_base, run_id);
|
||||
let json_path = run_json_path(results_base, run_id);
|
||||
|
||||
// Load completed task IDs for resume support
|
||||
let completed: HashSet<String> = if resume_run_id.is_some() {
|
||||
completed_task_ids(&jsonl_path)?
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
|
||||
if !completed.is_empty() {
|
||||
tracing::info!(
|
||||
"Resuming run {}: {} tasks already completed",
|
||||
run_id,
|
||||
completed.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Load all tasks once (used for both execution and scoring)
|
||||
let all_tasks = self.suite.load_tasks().await?;
|
||||
let task_index: HashMap<String, BenchTask> = all_tasks
|
||||
.iter()
|
||||
.map(|t| (t.id.clone(), t.clone()))
|
||||
.collect();
|
||||
|
||||
// Filter tasks for execution
|
||||
let mut tasks = all_tasks;
|
||||
|
||||
if let Some(ids) = task_filter {
|
||||
let id_set: HashSet<&str> = ids.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| id_set.contains(t.id.as_str()));
|
||||
}
|
||||
|
||||
if let Some(tags) = tag_filter {
|
||||
let tag_set: HashSet<&str> = tags.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| t.tags.iter().any(|tag| tag_set.contains(tag.as_str())));
|
||||
}
|
||||
|
||||
// Filter out already-completed tasks
|
||||
tasks.retain(|t| !completed.contains(&t.id));
|
||||
|
||||
// Sample if requested
|
||||
if let Some(n) = sample {
|
||||
tasks.truncate(n);
|
||||
}
|
||||
|
||||
let total_tasks = tasks.len() + completed.len();
|
||||
let model_label = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
let commit_hash = git_short_hash();
|
||||
tracing::info!(
|
||||
"[{} @ {}] Running {} tasks for suite '{}' (run: {})",
|
||||
model_label,
|
||||
commit_hash,
|
||||
tasks.len(),
|
||||
self.suite.id(),
|
||||
run_id
|
||||
);
|
||||
|
||||
let started_at = Utc::now();
|
||||
let all_results: Arc<Mutex<Vec<TaskResult>>> =
|
||||
Arc::new(Mutex::new(Vec::with_capacity(tasks.len())));
|
||||
|
||||
if self.config.parallelism <= 1 {
|
||||
// Sequential execution
|
||||
let additional_tools = self.suite.additional_tools();
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed.len(),
|
||||
total_tasks,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = self.suite.setup_task(task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
task,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
continue;
|
||||
}
|
||||
let params = TaskRunParams {
|
||||
task,
|
||||
suite_id: self.suite.id(),
|
||||
config_label: &matrix.label,
|
||||
llm: Arc::clone(&self.llm),
|
||||
safety: Arc::clone(&self.safety),
|
||||
timeout: task.timeout.unwrap_or(self.config.task_timeout),
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = self.suite.teardown_task(task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
}
|
||||
} else {
|
||||
// Parallel execution with bounded concurrency
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.config.parallelism));
|
||||
let shared_tools: Arc<[Arc<dyn ironclaw::tools::Tool>]> =
|
||||
Arc::from(self.suite.additional_tools());
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for (i, task) in tasks.into_iter().enumerate() {
|
||||
let sem = Arc::clone(&semaphore);
|
||||
let suite = Arc::clone(&self.suite);
|
||||
let config_label = matrix.label.clone();
|
||||
let llm = Arc::clone(&self.llm);
|
||||
let safety = Arc::clone(&self.safety);
|
||||
let timeout = task.timeout.unwrap_or(self.config.task_timeout);
|
||||
let results_ref = Arc::clone(&all_results);
|
||||
let completed_count = completed.len();
|
||||
let total = total_tasks;
|
||||
let additional_tools = Arc::clone(&shared_tools);
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = match sem.acquire().await {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
tracing::error!("Semaphore closed for task {}", task.id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed_count,
|
||||
total,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = suite.setup_task(&task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
&task,
|
||||
suite.id(),
|
||||
&config_label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
results_ref.lock().await.push(result);
|
||||
return;
|
||||
}
|
||||
let suite_id = suite.id().to_string();
|
||||
let params = TaskRunParams {
|
||||
task: &task,
|
||||
suite_id: &suite_id,
|
||||
config_label: &config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = suite.teardown_task(&task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
results_ref.lock().await.push(result);
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
tracing::error!("Task panicked: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Write all results to JSONL after parallel execution completes.
|
||||
// This avoids the race condition of concurrent file appends.
|
||||
let results = all_results.lock().await;
|
||||
for result in results.iter() {
|
||||
append_task_result(&jsonl_path, result)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Score all results using the cached task index
|
||||
let results = all_results.lock().await;
|
||||
let mut scored: Vec<TaskResult> = Vec::with_capacity(results.len());
|
||||
for result in results.iter() {
|
||||
if let Some(task) = task_index.get(&result.task_id) {
|
||||
let submission = TaskSubmission {
|
||||
response: result.response.clone(),
|
||||
conversation: vec![],
|
||||
tool_calls: result
|
||||
.trace
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|tc| tc.name.clone())
|
||||
.collect(),
|
||||
error: result.error.clone(),
|
||||
};
|
||||
match self.suite.score(task, &submission).await {
|
||||
Ok(score) => {
|
||||
let mut scored_result = result.clone();
|
||||
scored_result.score = score;
|
||||
scored.push(scored_result);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Scoring failed for {}: {}", result.task_id, e);
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Combine with any previously completed results for the aggregate
|
||||
let mut all_for_aggregate = crate::results::read_task_results(&jsonl_path)?;
|
||||
// De-duplicate (prefer the newer scored versions)
|
||||
let scored_ids: HashSet<String> = scored.iter().map(|r| r.task_id.clone()).collect();
|
||||
all_for_aggregate.retain(|r| !scored_ids.contains(&r.task_id));
|
||||
all_for_aggregate.extend(scored);
|
||||
|
||||
// Rewrite JSONL with scored results so `results` command shows final scores
|
||||
write_task_results(&jsonl_path, &all_for_aggregate)?;
|
||||
|
||||
let model_name = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
|
||||
let run_result = RunResult::from_tasks(
|
||||
run_id,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
model_name,
|
||||
&commit_hash,
|
||||
total_tasks,
|
||||
&all_for_aggregate,
|
||||
started_at,
|
||||
);
|
||||
|
||||
write_run_result(&json_path, &run_result)?;
|
||||
|
||||
tracing::info!(
|
||||
"[{} @ {}] Run {} complete: {:.1}% pass rate, {:.3} avg score, ${:.4} cost",
|
||||
model_name,
|
||||
commit_hash,
|
||||
run_id,
|
||||
run_result.pass_rate * 100.0,
|
||||
run_result.avg_score,
|
||||
run_result.total_cost_usd,
|
||||
);
|
||||
|
||||
Ok(run_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a single benchmark task in complete isolation.
|
||||
///
|
||||
/// Creates a fresh Agent + BenchChannel + InstrumentedLlm for the task,
|
||||
/// injects the prompt, waits for the response, and returns the result.
|
||||
///
|
||||
/// # Current limitations
|
||||
///
|
||||
/// - **Single-turn only**: After the first assistant response, `/quit` is sent.
|
||||
/// Multi-turn suites (e.g., Tau-bench's `next_user_message()`) are not yet wired.
|
||||
/// - **Resources not injected**: `BenchTask.resources` (e.g., GAIA file attachments)
|
||||
/// are not included in the prompt or made available via the workspace.
|
||||
/// - **Conversation not captured**: `TaskSubmission.conversation` is always empty,
|
||||
/// which prevents multi-turn scoring hooks from working.
|
||||
async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult {
|
||||
let TaskRunParams {
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools,
|
||||
} = params;
|
||||
|
||||
let started_at = Utc::now();
|
||||
let start = Instant::now();
|
||||
|
||||
// Wrap LLM with instrumentation
|
||||
let instrumented = Arc::new(InstrumentedLlm::new(llm));
|
||||
|
||||
// Create bench channel
|
||||
let (bench_channel, msg_tx) = BenchChannel::new();
|
||||
let capture = bench_channel.capture();
|
||||
|
||||
// Build tool registry
|
||||
let tools = Arc::new(ToolRegistry::new());
|
||||
tools.register_builtin_tools();
|
||||
|
||||
// Register additional suite-specific tools
|
||||
for tool in additional_tools {
|
||||
tools.register(Arc::clone(tool)).await;
|
||||
}
|
||||
|
||||
// Build agent config (minimal, headless)
|
||||
let agent_config = AgentConfig {
|
||||
name: format!("bench-{}", task.id),
|
||||
max_parallel_jobs: 1,
|
||||
job_timeout: timeout,
|
||||
stuck_threshold: timeout,
|
||||
repair_check_interval: timeout + std::time::Duration::from_secs(999),
|
||||
max_repair_attempts: 0,
|
||||
use_planning: false,
|
||||
session_idle_timeout: timeout,
|
||||
allow_local_tools: true,
|
||||
max_cost_per_day_cents: None,
|
||||
max_actions_per_hour: None,
|
||||
};
|
||||
|
||||
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
|
||||
ironclaw::agent::cost_guard::CostGuardConfig::default(),
|
||||
));
|
||||
|
||||
let deps = AgentDeps {
|
||||
store: None,
|
||||
llm: instrumented.clone() as Arc<dyn LlmProvider>,
|
||||
cheap_llm: None,
|
||||
safety,
|
||||
tools,
|
||||
workspace: None,
|
||||
extension_manager: None,
|
||||
skill_registry: None,
|
||||
skills_config: ironclaw::config::SkillsConfig::default(),
|
||||
hooks: Arc::new(ironclaw::hooks::HookRegistry::new()),
|
||||
cost_guard,
|
||||
};
|
||||
|
||||
let mut channels = ChannelManager::new();
|
||||
channels.add(Box::new(bench_channel));
|
||||
|
||||
let agent = Agent::new(agent_config, deps, channels, None, None, None, None, None);
|
||||
|
||||
// Build the full prompt with context
|
||||
let full_prompt = if let Some(ref ctx) = task.context {
|
||||
format!("{}\n\nContext:\n{}", task.prompt, ctx)
|
||||
} else {
|
||||
task.prompt.clone()
|
||||
};
|
||||
|
||||
// Inject the task prompt
|
||||
let incoming = IncomingMessage::new("bench", "bench-user", &full_prompt);
|
||||
if msg_tx.send(incoming).await.is_err() {
|
||||
return make_error_result(
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
started_at,
|
||||
"failed to send prompt",
|
||||
);
|
||||
}
|
||||
|
||||
// Record prompt in conversation
|
||||
{
|
||||
let mut cap = capture.lock().await;
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: TurnRole::User,
|
||||
content: full_prompt,
|
||||
});
|
||||
}
|
||||
|
||||
// Run agent with timeout.
|
||||
// After the first response, send /quit to end the session.
|
||||
let quit_tx = msg_tx.clone();
|
||||
let capture_for_quit = Arc::clone(&capture);
|
||||
let quit_handle = tokio::spawn(async move {
|
||||
// Poll for first response
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
let cap = capture_for_quit.lock().await;
|
||||
if !cap.responses.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Give a small grace period for any final status events
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
let quit = IncomingMessage::new("bench", "bench-user", "/quit");
|
||||
let _ = quit_tx.send(quit).await;
|
||||
});
|
||||
|
||||
let agent_result = tokio::time::timeout(timeout, agent.run()).await;
|
||||
|
||||
quit_handle.abort();
|
||||
|
||||
let wall_time = start.elapsed();
|
||||
let hit_timeout = agent_result.is_err();
|
||||
|
||||
if let Ok(Err(e)) = &agent_result {
|
||||
tracing::warn!("Agent error for task {}: {}", task.id, e);
|
||||
}
|
||||
|
||||
// Extract results from capture
|
||||
let cap = capture.lock().await;
|
||||
let response = cap.responses.last().cloned().unwrap_or_default();
|
||||
|
||||
let trace = Trace {
|
||||
wall_time_ms: wall_time.as_millis() as u64,
|
||||
llm_calls: instrumented.call_count(),
|
||||
input_tokens: instrumented.total_input_tokens(),
|
||||
output_tokens: instrumented.total_output_tokens(),
|
||||
estimated_cost_usd: instrumented.estimated_cost(),
|
||||
tool_calls: cap.tool_calls.clone(),
|
||||
turns: cap.responses.len() as u32,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout,
|
||||
};
|
||||
|
||||
let error = if hit_timeout {
|
||||
Some(format!("timeout after {}s", timeout.as_secs()))
|
||||
} else if let Ok(Err(e)) = &agent_result {
|
||||
Some(e.to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace,
|
||||
response,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_error_result(
|
||||
task: &BenchTask,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
reason: &str,
|
||||
) -> TaskResult {
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore::fail(reason),
|
||||
trace: Trace {
|
||||
wall_time_ms: 0,
|
||||
llm_calls: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 0,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: String::new(),
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error: Some(reason.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the short git commit hash of HEAD, or "unknown" if not in a repo.
|
||||
fn git_short_hash() -> String {
|
||||
std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| {
|
||||
if o.status.success() {
|
||||
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
use regex::Regex;
|
||||
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Normalize an answer string for comparison: lowercase, trim whitespace,
|
||||
/// strip trailing punctuation, collapse internal whitespace.
|
||||
pub fn normalize_answer(s: &str) -> String {
|
||||
let trimmed = s.trim().to_lowercase();
|
||||
let collapsed: String = trimmed.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
collapsed.trim_end_matches(['.', ',', ';', '!']).to_string()
|
||||
}
|
||||
|
||||
/// Exact match after normalization.
|
||||
pub fn exact_match(expected: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_expected == norm_actual {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!(
|
||||
"expected \"{norm_expected}\", got \"{norm_actual}\""
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer contains the expected substring (normalized).
|
||||
pub fn contains_match(expected_substring: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected_substring);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_actual.contains(&norm_expected) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not contain \"{norm_expected}\""))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer matches a regex pattern.
|
||||
pub fn regex_match(pattern: &str, actual: &str) -> BenchScore {
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(actual) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not match pattern /{pattern}/"))
|
||||
}
|
||||
}
|
||||
Err(e) => BenchScore::fail(format!("invalid regex pattern: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_normalize_answer() {
|
||||
assert_eq!(normalize_answer(" Hello World. "), "hello world");
|
||||
assert_eq!(normalize_answer("Yes!"), "yes");
|
||||
assert_eq!(normalize_answer("42"), "42");
|
||||
assert_eq!(normalize_answer(" "), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_pass() {
|
||||
let score = exact_match("Hello World", " hello world. ");
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_fail() {
|
||||
let score = exact_match("hello", "world");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_pass() {
|
||||
let score = contains_match("world", "Hello World!");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_fail() {
|
||||
let score = contains_match("xyz", "Hello World!");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_pass() {
|
||||
let score = regex_match(r"\d{4}", "The year is 2024.");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_fail() {
|
||||
let score = regex_match(r"\d{4}", "No numbers here.");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_invalid_pattern() {
|
||||
let score = regex_match(r"[invalid", "anything");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert!(
|
||||
score
|
||||
.details
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("invalid regex")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// A single task in a benchmark suite.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchTask {
|
||||
pub id: String,
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub context: Option<String>,
|
||||
#[serde(default)]
|
||||
pub resources: Vec<TaskResource>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub expected_turns: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub timeout: Option<Duration>,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A resource attached to a benchmark task (file, URL, etc.).
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResource {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
#[serde(default)]
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ResourceType {
|
||||
#[default]
|
||||
File,
|
||||
Url,
|
||||
Directory,
|
||||
}
|
||||
|
||||
/// What the agent produced for scoring.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TaskSubmission {
|
||||
pub response: String,
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
pub tool_calls: Vec<String>,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A single turn in a multi-turn conversation.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ConversationTurn {
|
||||
pub role: TurnRole,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnRole {
|
||||
User,
|
||||
Assistant,
|
||||
System,
|
||||
}
|
||||
|
||||
/// Score for a single task.
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchScore {
|
||||
/// 0.0 to 1.0 (1.0 = perfect).
|
||||
pub value: f64,
|
||||
/// "pass" / "fail" / "partial".
|
||||
pub label: String,
|
||||
#[serde(default)]
|
||||
pub details: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchScore {
|
||||
pub fn pass() -> Self {
|
||||
Self {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fail(details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn partial(value: f64, details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: value.clamp(0.0, 1.0),
|
||||
label: "partial".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for benchmark suite adapters.
|
||||
///
|
||||
/// Each suite (GAIA, Tau-bench, custom, etc.) implements this trait
|
||||
/// to provide task loading, scoring, and optional lifecycle hooks.
|
||||
#[async_trait]
|
||||
#[allow(dead_code)]
|
||||
pub trait BenchSuite: Send + Sync {
|
||||
/// Human-readable name (e.g., "GAIA Validation").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Machine ID (e.g., "gaia").
|
||||
fn id(&self) -> &str;
|
||||
|
||||
/// Load all tasks from the suite's data source.
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError>;
|
||||
|
||||
/// Score the agent's submission against the expected answer.
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError>;
|
||||
|
||||
/// Optional: set up environment before running a task (clone repo, init DB, etc.).
|
||||
async fn setup_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: tear down environment after a task completes.
|
||||
async fn teardown_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: additional tools to register for this suite's tasks.
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// Multi-turn: generate next simulated user message based on conversation so far.
|
||||
/// Return `None` to end the conversation.
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
_task: &BenchTask,
|
||||
_conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
//! Build script: compile Telegram channel WASM from source.
|
||||
//!
|
||||
//! Do not commit compiled WASM binaries — they are a supply chain risk.
|
||||
//! This script builds telegram.wasm from channels-src/telegram before the main crate compiles.
|
||||
//!
|
||||
//! Reproducible build:
|
||||
//! cargo build --release
|
||||
//! (build.rs invokes the channel build automatically)
|
||||
//!
|
||||
//! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let root = PathBuf::from(&manifest_dir);
|
||||
let channel_dir = root.join("channels-src/telegram");
|
||||
let wasm_out = channel_dir.join("telegram.wasm");
|
||||
|
||||
// Rerun when channel source or build script changes
|
||||
println!("cargo:rerun-if-changed=channels-src/telegram/src");
|
||||
println!("cargo:rerun-if-changed=channels-src/telegram/Cargo.toml");
|
||||
println!("cargo:rerun-if-changed=wit/channel.wit");
|
||||
|
||||
if !channel_dir.is_dir() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build WASM module
|
||||
let status = match Command::new("cargo")
|
||||
.args([
|
||||
"build",
|
||||
"--release",
|
||||
"--target",
|
||||
"wasm32-wasip2",
|
||||
"--manifest-path",
|
||||
channel_dir.join("Cargo.toml").to_str().unwrap(),
|
||||
])
|
||||
.current_dir(&root)
|
||||
.status()
|
||||
{
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
eprintln!(
|
||||
"cargo:warning=Telegram channel build failed. Run: ./channels-src/telegram/build.sh"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
if !status.success() {
|
||||
eprintln!(
|
||||
"cargo:warning=Telegram channel build failed. Run: ./channels-src/telegram/build.sh"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let raw_wasm = channel_dir.join("target/wasm32-wasip2/release/telegram_channel.wasm");
|
||||
if !raw_wasm.exists() {
|
||||
eprintln!(
|
||||
"cargo:warning=Telegram WASM output not found at {:?}",
|
||||
raw_wasm
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert to component and strip (wasm-tools)
|
||||
let component_ok = Command::new("wasm-tools")
|
||||
.args([
|
||||
"component",
|
||||
"new",
|
||||
raw_wasm.to_str().unwrap(),
|
||||
"-o",
|
||||
wasm_out.to_str().unwrap(),
|
||||
])
|
||||
.current_dir(&root)
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false);
|
||||
|
||||
if !component_ok {
|
||||
// Fallback: copy raw module if wasm-tools unavailable
|
||||
if std::fs::copy(&raw_wasm, &wasm_out).is_err() {
|
||||
eprintln!("cargo:warning=wasm-tools not found. Run: cargo install wasm-tools");
|
||||
}
|
||||
} else {
|
||||
// Strip debug info (use temp file to avoid clobbering)
|
||||
let stripped = wasm_out.with_extension("wasm.stripped");
|
||||
let strip_ok = Command::new("wasm-tools")
|
||||
.args([
|
||||
"strip",
|
||||
wasm_out.to_str().unwrap(),
|
||||
"-o",
|
||||
stripped.to_str().unwrap(),
|
||||
])
|
||||
.current_dir(&root)
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false);
|
||||
if strip_ok {
|
||||
let _ = std::fs::rename(&stripped, &wasm_out);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "discord-channel"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Discord channel for IronClaw"
|
||||
license = "MIT OR Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
wit-bindgen = "0.41.0"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[profile.release]
|
||||
strip = true
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
|
||||
|
||||
|
||||
[workspace]
|
||||
@@ -0,0 +1,121 @@
|
||||
# Discord Channel for IronClaw
|
||||
|
||||
WASM channel for Discord integration - handle slash commands and button interactions via webhooks.
|
||||
|
||||
## Features
|
||||
|
||||
- **Slash Commands** - Process Discord slash commands
|
||||
- **Button Interactions** - Handle button clicks
|
||||
- **Thread Support** - Respond in threads
|
||||
- **DM Support** - Handle direct messages
|
||||
|
||||
## Setup
|
||||
|
||||
1. Create a Discord Application at <https://discord.com/developers/applications>
|
||||
2. Create a Bot and get the token
|
||||
3. Set up Interactions URL to point to your IronClaw instance
|
||||
4. Copy the Application ID and Public Key
|
||||
5. Store in IronClaw secrets:
|
||||
|
||||
```bash
|
||||
ironclaw secret set discord_bot_token YOUR_BOT_TOKEN
|
||||
```
|
||||
|
||||
**Note:** The `discord_bot_token` secret is the only value read directly by this
|
||||
Discord channel WASM component. The `discord_app_id` and `discord_public_key`
|
||||
secrets are used by the IronClaw host (for example, to verify Discord
|
||||
interaction signatures and manage slash command registration) and are not
|
||||
accessed from the WASM module itself.
|
||||
|
||||
## Discord Configuration
|
||||
|
||||
### Register Slash Commands
|
||||
|
||||
```bash
|
||||
curl -X POST \
|
||||
-H "Authorization: Bot YOUR_BOT_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
https://discord.com/api/v10/applications/YOUR_APP_ID/commands \
|
||||
-d '{
|
||||
"name": "ask",
|
||||
"description": "Ask the AI agent",
|
||||
"options": [{
|
||||
"name": "question",
|
||||
"description": "Your question",
|
||||
"type": 3,
|
||||
"required": true
|
||||
}]
|
||||
}'
|
||||
```
|
||||
|
||||
### Set Interactions Endpoint
|
||||
|
||||
In your Discord app settings, set:
|
||||
|
||||
- Interactions Endpoint URL: `https://your-ironclaw.com/webhook/discord`
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Slash Command
|
||||
|
||||
User types: `/ask question: What is the weather?`
|
||||
|
||||
The agent receives:
|
||||
|
||||
```text
|
||||
User: @username
|
||||
Content: /ask question: What is the weather?
|
||||
```
|
||||
|
||||
### Button Click
|
||||
|
||||
When a user clicks a button in a message, the agent receives:
|
||||
|
||||
```text
|
||||
User: @username
|
||||
Content: [Button clicked] Original message content
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
If an internal error occurs (e.g., metadata serialization failure), the tool attempts to send an ephemeral message to the user:
|
||||
|
||||
```text
|
||||
❌ Internal Error: Failed to process command metadata.
|
||||
```
|
||||
|
||||
Check the host logs for detailed error information.
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Embeds
|
||||
|
||||
To send embeds, include an `embeds` array in the `metadata_json` field of the agent's response. The structure should match the Discord API `embed` object.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Invalid Signature"
|
||||
|
||||
- Check that `discord_public_key` is set correctly in IronClaw secrets.
|
||||
- This validation happens on the host before reaching the WASM.
|
||||
|
||||
### "401 Unauthorized"
|
||||
|
||||
- Check that `discord_bot_token` is set correctly in IronClaw secrets.
|
||||
- Ensure the bot is added to the server.
|
||||
|
||||
### "Interaction Failed"
|
||||
|
||||
- The interaction might have timed out (Discord requires a response within 3 seconds).
|
||||
- The `interactions_endpoint_url` might be unreachable.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
cd channels-src/discord
|
||||
cargo build --target wasm32-wasi --release
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT/Apache-2.0
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"type": "channel",
|
||||
"name": "discord",
|
||||
"description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages",
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
{ "host": "discord.com", "path_prefix": "/api/v10" }
|
||||
],
|
||||
"credentials": {
|
||||
"discord_bot_token": {
|
||||
"secret_name": "discord_bot_token",
|
||||
"location": { "type": "header", "header_name": "Authorization", "prefix": "Bot " },
|
||||
"host_patterns": ["discord.com"]
|
||||
}
|
||||
},
|
||||
"rate_limit": {
|
||||
"requests_per_minute": 60,
|
||||
"requests_per_hour": 3600
|
||||
}
|
||||
},
|
||||
"secrets": {
|
||||
"allowed_names": ["discord_bot_token", "discord_*"]
|
||||
},
|
||||
"channel": {
|
||||
"allowed_paths": ["/webhook/discord"],
|
||||
"allow_polling": false,
|
||||
"callback_timeout_secs": 45,
|
||||
"workspace_prefix": "channels/discord/",
|
||||
"emit_rate_limit": {
|
||||
"messages_per_minute": 100,
|
||||
"messages_per_hour": 5000
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"require_signature_verification": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,476 @@
|
||||
//! Discord Gateway/Webhook channel for IronClaw.
|
||||
//!
|
||||
//! This WASM component implements the channel interface for handling Discord
|
||||
//! interactions via webhooks and sending messages back to Discord.
|
||||
//!
|
||||
//! # Features
|
||||
//!
|
||||
//! - URL verification for Discord interactions
|
||||
//! - Slash command handling
|
||||
//! - Message event parsing (@mentions, DMs)
|
||||
//! - Thread support for conversations
|
||||
//! - Response posting via Discord Web API
|
||||
//! - Automatic message truncation (> 2000 chars)
|
||||
//!
|
||||
//! # Security
|
||||
//!
|
||||
//! - Signature validation is handled by the host (webhook secrets)
|
||||
//! - Bot token is injected by host during HTTP requests
|
||||
//! - WASM never sees raw credentials
|
||||
|
||||
wit_bindgen::generate!({
|
||||
world: "sandboxed-channel",
|
||||
path: "../../wit/channel.wit",
|
||||
});
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use exports::near::agent::channel::{
|
||||
AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest,
|
||||
OutgoingHttpResponse, StatusUpdate,
|
||||
};
|
||||
use near::agent::channel_host::{self, EmittedMessage};
|
||||
|
||||
/// Discord interaction wrapper.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiscordInteraction {
|
||||
/// Interaction type (1=Ping, 2=ApplicationCommand, 3=MessageComponent)
|
||||
#[serde(rename = "type")]
|
||||
interaction_type: u8,
|
||||
|
||||
/// Interaction ID
|
||||
id: String,
|
||||
|
||||
/// Application ID
|
||||
application_id: String,
|
||||
|
||||
/// Guild ID (if in server)
|
||||
#[allow(dead_code)] // Part of API payload, currently unused
|
||||
guild_id: Option<String>,
|
||||
|
||||
/// Channel ID
|
||||
channel_id: Option<String>,
|
||||
|
||||
/// Member info (if in server)
|
||||
member: Option<DiscordMember>,
|
||||
|
||||
/// User info (if DM)
|
||||
user: Option<DiscordUser>,
|
||||
|
||||
/// Command data (for slash commands)
|
||||
data: Option<DiscordCommandData>,
|
||||
|
||||
/// Message (for component interactions)
|
||||
message: Option<DiscordMessage>,
|
||||
|
||||
/// Token for responding
|
||||
token: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct DiscordMember {
|
||||
user: DiscordUser,
|
||||
#[allow(dead_code)] // Part of API payload, currently unused
|
||||
nick: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct DiscordUser {
|
||||
id: String,
|
||||
username: String,
|
||||
global_name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct DiscordCommandData {
|
||||
#[allow(dead_code)] // Part of API payload, currently unused
|
||||
id: String,
|
||||
name: String,
|
||||
options: Option<Vec<DiscordCommandOption>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct DiscordCommandOption {
|
||||
name: String,
|
||||
value: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
struct DiscordMessage {
|
||||
#[allow(dead_code)] // Part of API payload, currently unused
|
||||
id: String,
|
||||
content: String,
|
||||
channel_id: String,
|
||||
#[allow(dead_code)] // Part of API payload, currently unused
|
||||
author: DiscordUser,
|
||||
}
|
||||
|
||||
/// Metadata stored with emitted messages for response routing.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct DiscordMessageMetadata {
|
||||
/// Discord channel ID
|
||||
channel_id: String,
|
||||
|
||||
/// Interaction ID for followups
|
||||
interaction_id: String,
|
||||
|
||||
/// Interaction token for responding
|
||||
token: String,
|
||||
|
||||
/// Application ID
|
||||
application_id: String,
|
||||
|
||||
/// Thread ID (for forum threads)
|
||||
thread_id: Option<String>,
|
||||
}
|
||||
|
||||
struct DiscordChannel;
|
||||
|
||||
impl Guest for DiscordChannel {
|
||||
fn on_start(_config_json: String) -> Result<ChannelConfig, String> {
|
||||
channel_host::log(channel_host::LogLevel::Info, "Discord channel starting");
|
||||
|
||||
Ok(ChannelConfig {
|
||||
display_name: "Discord".to_string(),
|
||||
http_endpoints: vec![HttpEndpointConfig {
|
||||
path: "/webhook/discord".to_string(),
|
||||
methods: vec!["POST".to_string()],
|
||||
require_secret: true,
|
||||
}],
|
||||
poll: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn on_http_request(req: IncomingHttpRequest) -> OutgoingHttpResponse {
|
||||
let body_str = match std::str::from_utf8(&req.body) {
|
||||
Ok(s) => s,
|
||||
Err(_) => {
|
||||
return json_response(400, serde_json::json!({"error": "Invalid UTF-8 body"}));
|
||||
}
|
||||
};
|
||||
|
||||
let interaction: DiscordInteraction = match serde_json::from_str(body_str) {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to parse Discord interaction: {}", e),
|
||||
);
|
||||
return json_response(400, serde_json::json!({"error": "Invalid interaction"}));
|
||||
}
|
||||
};
|
||||
|
||||
match interaction.interaction_type {
|
||||
// Ping - Discord verification
|
||||
1 => {
|
||||
channel_host::log(channel_host::LogLevel::Info, "Responding to Discord ping");
|
||||
json_response(200, serde_json::json!({"type": 1}))
|
||||
}
|
||||
|
||||
// Application Command (slash command)
|
||||
2 => {
|
||||
handle_slash_command(&interaction);
|
||||
json_response(
|
||||
200,
|
||||
serde_json::json!({
|
||||
"type": 5,
|
||||
"data": {
|
||||
"content": "🤔 Thinking..."
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// Message Component (buttons, selects)
|
||||
3 => {
|
||||
if let Some(ref message) = interaction.message {
|
||||
handle_message_component(&interaction, message);
|
||||
}
|
||||
json_response(200, serde_json::json!({"type": 6}))
|
||||
}
|
||||
|
||||
_ => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Warn,
|
||||
&format!(
|
||||
"Unknown Discord interaction type: {}",
|
||||
interaction.interaction_type
|
||||
),
|
||||
);
|
||||
json_response(200, serde_json::json!({"type": 6}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn on_poll() {}
|
||||
|
||||
fn on_respond(response: AgentResponse) -> Result<(), String> {
|
||||
let metadata: DiscordMessageMetadata = serde_json::from_str(&response.metadata_json)
|
||||
.map_err(|e| format!("Failed to parse metadata: {}", e))?;
|
||||
|
||||
// Use webhook endpoint for followup
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
metadata.application_id, metadata.token
|
||||
);
|
||||
|
||||
// Truncate content to 2000 characters to comply with Discord limits
|
||||
let content = truncate_message(&response.content);
|
||||
|
||||
let mut payload = serde_json::json!({
|
||||
"content": content,
|
||||
});
|
||||
|
||||
// Check for embeds in metadata
|
||||
if let Ok(meta_json) = serde_json::from_str::<serde_json::Value>(&response.metadata_json) {
|
||||
if let Some(embeds) = meta_json.get("embeds") {
|
||||
payload["embeds"] = embeds.clone();
|
||||
}
|
||||
}
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({
|
||||
"Content-Type": "application/json"
|
||||
});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(http_response) => {
|
||||
if http_response.status >= 200 && http_response.status < 300 {
|
||||
channel_host::log(channel_host::LogLevel::Debug, "Posted followup to Discord");
|
||||
Ok(())
|
||||
} else {
|
||||
let body_str = String::from_utf8_lossy(&http_response.body);
|
||||
Err(format!(
|
||||
"Discord API error: {} - {}",
|
||||
http_response.status, body_str
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn on_status(_update: StatusUpdate) {}
|
||||
|
||||
fn on_shutdown() {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
"Discord channel shutting down",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
.map(|m| &m.user)
|
||||
.or(interaction.user.as_ref());
|
||||
let user_id = user.map(|u| u.id.clone()).unwrap_or_default();
|
||||
let user_name = user
|
||||
.map(|u| {
|
||||
u.global_name
|
||||
.as_ref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(&u.username)
|
||||
.clone()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let channel_id = interaction.channel_id.clone().unwrap_or_default();
|
||||
|
||||
let command_name = interaction
|
||||
.data
|
||||
.as_ref()
|
||||
.map(|d| d.name.clone())
|
||||
.unwrap_or_default();
|
||||
let options = interaction.data.as_ref().and_then(|d| d.options.clone());
|
||||
|
||||
let content = if let Some(opts) = options {
|
||||
let opt_str = opts
|
||||
.iter()
|
||||
.map(|o| format!("{}: {}", o.name, o.value))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
format!("/{} {}", command_name, opt_str)
|
||||
} else {
|
||||
format!("/{}", command_name)
|
||||
};
|
||||
|
||||
let metadata = DiscordMessageMetadata {
|
||||
channel_id: channel_id.clone(),
|
||||
interaction_id: interaction.id.clone(),
|
||||
token: interaction.token.clone(),
|
||||
application_id: interaction.application_id.clone(),
|
||||
thread_id: None,
|
||||
};
|
||||
|
||||
let metadata_json = match serde_json::to_string(&metadata) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize metadata: {}", e),
|
||||
);
|
||||
// Attempt to notify user of internal error
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
interaction.application_id, interaction.token
|
||||
);
|
||||
let payload = serde_json::json!({
|
||||
"content": "❌ Internal Error: Failed to process command metadata.",
|
||||
"flags": 64 // Ephemeral
|
||||
});
|
||||
let _ = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&serde_json::json!({"Content-Type": "application/json"}).to_string(),
|
||||
Some(&serde_json::to_vec(&payload).unwrap_or_default()),
|
||||
None,
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
channel_host::emit_message(&EmittedMessage {
|
||||
user_id,
|
||||
user_name: Some(user_name),
|
||||
content,
|
||||
thread_id: None,
|
||||
metadata_json,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordMessage) {
|
||||
// Check member first (for server contexts), then user (for DMs)
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
.map(|m| &m.user)
|
||||
.or(interaction.user.as_ref());
|
||||
let user_id = user.map(|u| u.id.clone()).unwrap_or_default();
|
||||
let user_name = user
|
||||
.map(|u| {
|
||||
u.global_name
|
||||
.as_ref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or(&u.username)
|
||||
.clone()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let channel_id = message.channel_id.clone();
|
||||
|
||||
let metadata = DiscordMessageMetadata {
|
||||
channel_id: channel_id.clone(),
|
||||
interaction_id: interaction.id.clone(),
|
||||
token: interaction.token.clone(),
|
||||
application_id: interaction.application_id.clone(),
|
||||
thread_id: None,
|
||||
};
|
||||
|
||||
let metadata_json = match serde_json::to_string(&metadata) {
|
||||
Ok(json) => json,
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize metadata: {}", e),
|
||||
);
|
||||
return; // Don't emit message if metadata can't be serialized
|
||||
}
|
||||
};
|
||||
|
||||
channel_host::emit_message(&EmittedMessage {
|
||||
user_id,
|
||||
user_name: Some(user_name),
|
||||
content: format!("[Button clicked] {}", message.content),
|
||||
thread_id: None,
|
||||
metadata_json,
|
||||
});
|
||||
}
|
||||
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
OutgoingHttpResponse {
|
||||
status,
|
||||
headers_json: headers.to_string(),
|
||||
body,
|
||||
}
|
||||
}
|
||||
|
||||
export!(DiscordChannel);
|
||||
|
||||
fn truncate_message(content: &str) -> String {
|
||||
if content.len() <= 2000 {
|
||||
content.to_string()
|
||||
} else {
|
||||
let max_bytes = 1990;
|
||||
let cutoff = content
|
||||
.char_indices()
|
||||
.map(|(i, c)| i + c.len_utf8())
|
||||
.take_while(|&end| end <= max_bytes)
|
||||
.last()
|
||||
.unwrap_or(0);
|
||||
let mut truncated = content[..cutoff].to_string();
|
||||
truncated.push_str("\n... (truncated)");
|
||||
truncated
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_truncate_message() {
|
||||
let short = "Hello world";
|
||||
assert_eq!(truncate_message(short), short);
|
||||
|
||||
let long = "a".repeat(2005);
|
||||
let truncated = truncate_message(&long);
|
||||
assert_eq!(truncated.len(), 2006); // 1990 + 16 chars suffix
|
||||
assert!(truncated.ends_with("\n... (truncated)"));
|
||||
|
||||
// Test with multibyte characters (Euro sign is 3 bytes)
|
||||
// 1000 chars * 3 bytes = 3000 bytes
|
||||
let multi = "€".repeat(1000);
|
||||
let truncated_multi = truncate_message(&multi);
|
||||
|
||||
// 1990 bytes limit. 1990 / 3 = 663 with remainder 1.
|
||||
// Should truncate at 663 chars (1989 bytes).
|
||||
// Suffix is 16 bytes. Total: 1989 + 16 = 2005 bytes.
|
||||
assert!(truncated_multi.len() <= 2006);
|
||||
assert!(truncated_multi.len() >= 2006 - 4); // Allow for max utf8 char width variance
|
||||
assert!(truncated_multi.ends_with("\n... (truncated)"));
|
||||
|
||||
let content_part = &truncated_multi[..truncated_multi.len() - 16];
|
||||
assert!(content_part.chars().all(|c| c == '€'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_metadata_serialization() {
|
||||
let metadata = DiscordMessageMetadata {
|
||||
channel_id: "123".into(),
|
||||
interaction_id: "456".into(),
|
||||
token: "abc".into(),
|
||||
application_id: "789".into(),
|
||||
thread_id: None,
|
||||
};
|
||||
let json = serde_json::to_string(&metadata).unwrap();
|
||||
let parsed: DiscordMessageMetadata = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed.channel_id, "123");
|
||||
assert_eq!(parsed.interaction_id, "456");
|
||||
}
|
||||
}
|
||||
Generated
+497
@@ -0,0 +1,497 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.101"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hmac"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
|
||||
dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "id-arena"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||
|
||||
[[package]]
|
||||
name = "leb128"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.180"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "slack-channel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"hmac",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
|
||||
[[package]]
|
||||
name = "spdx"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.114"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-encoder"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1"
|
||||
dependencies = [
|
||||
"leb128",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-metadata"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"indexmap",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"spdx",
|
||||
"wasm-encoder",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bitflags",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
"wit-bindgen-rust-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-core"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"indexmap",
|
||||
"prettyplease",
|
||||
"syn",
|
||||
"wasm-metadata",
|
||||
"wit-bindgen-core",
|
||||
"wit-component",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust-macro"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wit-bindgen-core",
|
||||
"wit-bindgen-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-component"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags",
|
||||
"indexmap",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"wasm-encoder",
|
||||
"wasm-metadata",
|
||||
"wasmparser",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-parser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"id-arena",
|
||||
"indexmap",
|
||||
"log",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"unicode-xid",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445"
|
||||
@@ -27,3 +27,5 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -30,7 +30,7 @@ if [ -f "$WASM_PATH" ]; then
|
||||
wasm-tools strip slack.wasm -o slack.wasm
|
||||
|
||||
echo "Built: slack.wasm ($(du -h slack.wasm | cut -f1))"
|
||||
echo "Copy slack.wasm and slack.capabilities.json to ~/.near-agent/channels/"
|
||||
echo "Copy slack.wasm and slack.capabilities.json to ~/.ironclaw/channels/"
|
||||
else
|
||||
echo "Error: WASM output not found at $WASM_PATH"
|
||||
exit 1
|
||||
|
||||
@@ -27,7 +27,7 @@ use serde::{Deserialize, Serialize};
|
||||
// Re-export generated types
|
||||
use exports::near::agent::channel::{
|
||||
AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest,
|
||||
OutgoingHttpResponse, PollConfig,
|
||||
OutgoingHttpResponse, StatusUpdate,
|
||||
};
|
||||
use near::agent::channel_host::{self, EmittedMessage};
|
||||
|
||||
@@ -108,7 +108,10 @@ struct SlackPostMessageResponse {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SlackConfig {
|
||||
/// Name of secret containing signing secret (for verification by host).
|
||||
/// Parsed from config for forward compatibility; not yet used in WASM
|
||||
/// (host handles signature verification).
|
||||
#[serde(default = "default_signing_secret_name")]
|
||||
#[allow(dead_code)]
|
||||
signing_secret_name: String,
|
||||
}
|
||||
|
||||
@@ -175,11 +178,7 @@ impl Guest for SlackChannel {
|
||||
// Actual event callback
|
||||
"event_callback" => {
|
||||
if let Some(event) = event_wrapper.event {
|
||||
handle_slack_event(
|
||||
event,
|
||||
event_wrapper.team_id,
|
||||
event_wrapper.event_id,
|
||||
);
|
||||
handle_slack_event(event, event_wrapper.team_id, event_wrapper.event_id);
|
||||
}
|
||||
// Always respond 200 quickly to Slack (they have a 3s timeout)
|
||||
json_response(200, serde_json::json!({"ok": true}))
|
||||
@@ -230,6 +229,7 @@ impl Guest for SlackChannel {
|
||||
"https://slack.com/api/chat.postMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
@@ -243,14 +243,15 @@ impl Guest for SlackChannel {
|
||||
|
||||
// Parse Slack response
|
||||
let slack_response: SlackPostMessageResponse =
|
||||
serde_json::from_slice(&http_response.body).map_err(|e| {
|
||||
format!("Failed to parse Slack response: {}", e)
|
||||
})?;
|
||||
serde_json::from_slice(&http_response.body)
|
||||
.map_err(|e| format!("Failed to parse Slack response: {}", e))?;
|
||||
|
||||
if !slack_response.ok {
|
||||
return Err(format!(
|
||||
"Slack API error: {}",
|
||||
slack_response.error.unwrap_or_else(|| "unknown".to_string())
|
||||
slack_response
|
||||
.error
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
@@ -269,23 +270,24 @@ impl Guest for SlackChannel {
|
||||
}
|
||||
}
|
||||
|
||||
fn on_status(_update: StatusUpdate) {}
|
||||
|
||||
fn on_shutdown() {
|
||||
channel_host::log(channel_host::LogLevel::Info, "Slack channel shutting down");
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle a Slack event and emit message if applicable.
|
||||
fn handle_slack_event(
|
||||
event: SlackEvent,
|
||||
team_id: Option<String>,
|
||||
_event_id: Option<String>,
|
||||
) {
|
||||
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
|
||||
match event.event_type.as_str() {
|
||||
// Direct mention of the bot
|
||||
"app_mention" => {
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) =
|
||||
(event.user, event.channel.clone(), event.text, event.ts.clone())
|
||||
{
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) = (
|
||||
event.user,
|
||||
event.channel.clone(),
|
||||
event.text,
|
||||
event.ts.clone(),
|
||||
) {
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
}
|
||||
}
|
||||
@@ -297,9 +299,12 @@ fn handle_slack_event(
|
||||
return;
|
||||
}
|
||||
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) =
|
||||
(event.user, event.channel.clone(), event.text, event.ts.clone())
|
||||
{
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) = (
|
||||
event.user,
|
||||
event.channel.clone(),
|
||||
event.text,
|
||||
event.ts.clone(),
|
||||
) {
|
||||
// Only process DMs (channel IDs starting with D)
|
||||
if channel.starts_with('D') {
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
@@ -333,13 +338,18 @@ fn emit_message(
|
||||
team_id,
|
||||
};
|
||||
|
||||
let metadata_json =
|
||||
serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string());
|
||||
let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|e| {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize Slack metadata: {}", e),
|
||||
);
|
||||
"{}".to_string()
|
||||
});
|
||||
|
||||
// Strip @ mentions of the bot from the text for cleaner messages
|
||||
let cleaned_text = strip_bot_mention(&text);
|
||||
|
||||
channel_host::emit_message(EmittedMessage {
|
||||
channel_host::emit_message(&EmittedMessage {
|
||||
user_id,
|
||||
user_name: None, // Could fetch from Slack API if needed
|
||||
content: cleaned_text,
|
||||
@@ -362,7 +372,13 @@ fn strip_bot_mention(text: &str) -> String {
|
||||
|
||||
/// Create a JSON HTTP response.
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
let body = serde_json::to_vec(&value).unwrap_or_else(|e| {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize JSON response: {}", e),
|
||||
);
|
||||
Vec::new()
|
||||
});
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
OutgoingHttpResponse {
|
||||
|
||||
@@ -16,9 +16,14 @@ wit-bindgen = "0.36"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
|
||||
# Exclude from parent workspace (this is a standalone WASM component)
|
||||
[workspace]
|
||||
|
||||
[profile.release]
|
||||
# Optimize for size
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -32,8 +32,8 @@ if [ -f "$WASM_PATH" ]; then
|
||||
echo "Built: telegram.wasm ($(du -h telegram.wasm | cut -f1))"
|
||||
echo ""
|
||||
echo "To install:"
|
||||
echo " mkdir -p ~/.near-agent/channels"
|
||||
echo " cp telegram.wasm telegram.capabilities.json ~/.near-agent/channels/"
|
||||
echo " mkdir -p ~/.ironclaw/channels"
|
||||
echo " cp telegram.wasm telegram.capabilities.json ~/.ironclaw/channels/"
|
||||
echo ""
|
||||
echo "Then add your bot token to secrets:"
|
||||
echo " # Set TELEGRAM_BOT_TOKEN in your environment or secrets store"
|
||||
|
||||
+473
-101
@@ -72,6 +72,10 @@ struct TelegramMessage {
|
||||
/// Message text.
|
||||
text: Option<String>,
|
||||
|
||||
/// Caption for media (photo, video, document, etc.).
|
||||
#[serde(default)]
|
||||
caption: Option<String>,
|
||||
|
||||
/// Original message if this is a reply.
|
||||
reply_to_message: Option<Box<TelegramMessage>>,
|
||||
|
||||
@@ -160,6 +164,21 @@ const POLLING_STATE_PATH: &str = "state/last_update_id";
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "telegram";
|
||||
|
||||
/// Workspace path for persisting bot_username for mention detection in groups.
|
||||
const BOT_USERNAME_PATH: &str = "state/bot_username";
|
||||
|
||||
/// Workspace path for persisting respond_to_all_group_messages flag.
|
||||
const RESPOND_TO_ALL_GROUP_PATH: &str = "state/respond_to_all_group_messages";
|
||||
|
||||
// ============================================================================
|
||||
// Channel Metadata
|
||||
// ============================================================================
|
||||
@@ -196,6 +215,14 @@ struct TelegramConfig {
|
||||
#[serde(default)]
|
||||
owner_id: Option<i64>,
|
||||
|
||||
/// DM policy: "pairing" (default), "allowlist", or "open".
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
|
||||
/// Allowed sender IDs/usernames from config (merged with pairing-approved store).
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
|
||||
/// Whether to respond to all group messages (not just mentions).
|
||||
#[serde(default)]
|
||||
respond_to_all_group_messages: bool,
|
||||
@@ -257,6 +284,24 @@ impl Guest for TelegramChannel {
|
||||
);
|
||||
}
|
||||
|
||||
// Persist dm_policy and allow_from for DM pairing in handle_message
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing").to_string();
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, &dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
// Persist bot_username and respond_to_all_group_messages for group handling
|
||||
let _ = channel_host::workspace_write(
|
||||
BOT_USERNAME_PATH,
|
||||
&config.bot_username.unwrap_or_default(),
|
||||
);
|
||||
let _ = channel_host::workspace_write(
|
||||
RESPOND_TO_ALL_GROUP_PATH,
|
||||
&config.respond_to_all_group_messages.to_string(),
|
||||
);
|
||||
|
||||
// Mode is determined by whether the host injected a tunnel_url
|
||||
// If tunnel is configured, use webhooks. Otherwise, use polling.
|
||||
let webhook_mode = config.tunnel_url.is_some();
|
||||
@@ -388,7 +433,9 @@ impl Guest for TelegramChannel {
|
||||
|
||||
let headers = serde_json::json!({});
|
||||
|
||||
let result = channel_host::http_request("GET", &url, &headers.to_string(), None);
|
||||
// 35s HTTP timeout outlives Telegram's 30s server-side long-poll
|
||||
let result =
|
||||
channel_host::http_request("GET", &url, &headers.to_string(), None, Some(35_000));
|
||||
|
||||
match result {
|
||||
Ok(response) => {
|
||||
@@ -461,72 +508,52 @@ impl Guest for TelegramChannel {
|
||||
}
|
||||
|
||||
fn on_respond(response: AgentResponse) -> Result<(), String> {
|
||||
// Parse metadata to get chat info
|
||||
let metadata: TelegramMessageMetadata = serde_json::from_str(&response.metadata_json)
|
||||
.map_err(|e| format!("Failed to parse metadata: {}", e))?;
|
||||
|
||||
// Build sendMessage payload
|
||||
let mut payload = serde_json::json!({
|
||||
"chat_id": metadata.chat_id,
|
||||
"text": response.content,
|
||||
"parse_mode": "Markdown",
|
||||
});
|
||||
|
||||
// Reply to the original message for context
|
||||
payload["reply_to_message_id"] = serde_json::Value::Number(metadata.message_id.into());
|
||||
|
||||
let payload_bytes = serde_json::to_vec(&payload)
|
||||
.map_err(|e| format!("Failed to serialize payload: {}", e))?;
|
||||
|
||||
// Make HTTP request to Telegram API
|
||||
// The bot token is injected into the URL by the host
|
||||
let headers = serde_json::json!({
|
||||
"Content-Type": "application/json"
|
||||
});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
// Try sending with Markdown first; fall back to plain text if Telegram
|
||||
// can't parse the entities (e.g. model leaked <tool_call> with underscores).
|
||||
let result = send_message(
|
||||
metadata.chat_id,
|
||||
&response.content,
|
||||
metadata.message_id,
|
||||
Some("Markdown"),
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(http_response) => {
|
||||
if http_response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&http_response.body);
|
||||
return Err(format!(
|
||||
"Telegram API returned status {}: {}",
|
||||
http_response.status, body_str
|
||||
));
|
||||
}
|
||||
|
||||
// Parse Telegram response
|
||||
let api_response: TelegramApiResponse<SentMessage> =
|
||||
serde_json::from_slice(&http_response.body)
|
||||
.map_err(|e| format!("Failed to parse Telegram response: {}", e))?;
|
||||
|
||||
if !api_response.ok {
|
||||
return Err(format!(
|
||||
"Telegram API error: {}",
|
||||
api_response
|
||||
.description
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
Ok(msg_id) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Sent message to chat {}: message_id={}",
|
||||
metadata.chat_id,
|
||||
api_response.result.map(|r| r.message_id).unwrap_or(0)
|
||||
metadata.chat_id, msg_id
|
||||
),
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
Err(SendError::ParseEntities(detail)) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Warn,
|
||||
&format!("Markdown parse failed ({}), retrying as plain text", detail),
|
||||
);
|
||||
let msg_id = send_message(
|
||||
metadata.chat_id,
|
||||
&response.content,
|
||||
metadata.message_id,
|
||||
None,
|
||||
)
|
||||
.map_err(|e| format!("Plain-text retry also failed: {}", e))?;
|
||||
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Sent plain-text message to chat {}: message_id={}",
|
||||
metadata.chat_id, msg_id
|
||||
),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,6 +595,7 @@ impl Guest for TelegramChannel {
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendChatAction",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
if let Err(e) = result {
|
||||
@@ -586,6 +614,101 @@ impl Guest for TelegramChannel {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Send Message Helper
|
||||
// ============================================================================
|
||||
|
||||
/// Errors from send_message, split so callers can match on parse-entity failures.
|
||||
enum SendError {
|
||||
/// Telegram returned 400 with "can't parse entities" (Markdown issue).
|
||||
ParseEntities(String),
|
||||
/// Any other failure.
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SendError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
SendError::ParseEntities(detail) => write!(f, "parse entities error: {}", detail),
|
||||
SendError::Other(msg) => write!(f, "{}", msg),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a message via the Telegram Bot API.
|
||||
///
|
||||
/// Returns the sent message_id on success. When `parse_mode` is set and
|
||||
/// Telegram returns a 400 "can't parse entities" error, returns
|
||||
/// `SendError::ParseEntities` so the caller can retry without formatting.
|
||||
fn send_message(
|
||||
chat_id: i64,
|
||||
text: &str,
|
||||
reply_to_message_id: i64,
|
||||
parse_mode: Option<&str>,
|
||||
) -> Result<i64, SendError> {
|
||||
let mut payload = serde_json::json!({
|
||||
"chat_id": chat_id,
|
||||
"text": text,
|
||||
"reply_to_message_id": reply_to_message_id,
|
||||
});
|
||||
|
||||
if let Some(mode) = parse_mode {
|
||||
payload["parse_mode"] = serde_json::Value::String(mode.to_string());
|
||||
}
|
||||
|
||||
let payload_bytes = serde_json::to_vec(&payload)
|
||||
.map_err(|e| SendError::Other(format!("Failed to serialize payload: {}", e)))?;
|
||||
|
||||
let headers = serde_json::json!({ "Content-Type": "application/json" });
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(http_response) => {
|
||||
if http_response.status == 400 {
|
||||
let body_str = String::from_utf8_lossy(&http_response.body);
|
||||
if body_str.contains("can't parse entities") {
|
||||
return Err(SendError::ParseEntities(body_str.to_string()));
|
||||
}
|
||||
return Err(SendError::Other(format!(
|
||||
"Telegram API returned 400: {}",
|
||||
body_str
|
||||
)));
|
||||
}
|
||||
|
||||
if http_response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&http_response.body);
|
||||
return Err(SendError::Other(format!(
|
||||
"Telegram API returned status {}: {}",
|
||||
http_response.status, body_str
|
||||
)));
|
||||
}
|
||||
|
||||
let api_response: TelegramApiResponse<SentMessage> =
|
||||
serde_json::from_slice(&http_response.body)
|
||||
.map_err(|e| SendError::Other(format!("Failed to parse response: {}", e)))?;
|
||||
|
||||
if !api_response.ok {
|
||||
return Err(SendError::Other(format!(
|
||||
"Telegram API error: {}",
|
||||
api_response
|
||||
.description
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(api_response.result.map(|r| r.message_id).unwrap_or(0))
|
||||
}
|
||||
Err(e) => Err(SendError::Other(format!("HTTP request failed: {}", e))),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Webhook Management
|
||||
// ============================================================================
|
||||
@@ -604,6 +727,7 @@ fn delete_webhook() -> Result<(), String> {
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/deleteWebhook",
|
||||
&headers.to_string(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
@@ -666,6 +790,7 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<()
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/setWebhook",
|
||||
&headers.to_string(),
|
||||
Some(&body_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
@@ -700,6 +825,48 @@ fn register_webhook(tunnel_url: &str, webhook_secret: Option<&str>) -> Result<()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Pairing Reply
|
||||
// ============================================================================
|
||||
|
||||
/// Send a pairing code message to a chat. Used when an unknown user DMs the bot.
|
||||
fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
|
||||
let payload = serde_json::json!({
|
||||
"chat_id": chat_id,
|
||||
"text": format!(
|
||||
"To pair with this bot, run: `ironclaw pairing approve telegram {}`",
|
||||
code
|
||||
),
|
||||
"parse_mode": "Markdown",
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize payload: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({
|
||||
"Content-Type": "application/json"
|
||||
});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) => {
|
||||
if response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
return Err(format!("HTTP {}: {}", response.status, body_str));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Update Handling
|
||||
// ============================================================================
|
||||
@@ -719,11 +886,16 @@ fn handle_update(update: TelegramUpdate) {
|
||||
|
||||
/// Process a single message.
|
||||
fn handle_message(message: TelegramMessage) {
|
||||
// Skip messages without text
|
||||
let text = match message.text {
|
||||
Some(t) if !t.is_empty() => t,
|
||||
_ => return,
|
||||
};
|
||||
// Use text or caption (for media messages)
|
||||
let content = message
|
||||
.text
|
||||
.filter(|t| !t.is_empty())
|
||||
.or_else(|| message.caption.filter(|c| !c.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if content.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Skip messages without a sender (channel posts)
|
||||
let from = match message.from {
|
||||
@@ -736,41 +908,105 @@ fn handle_message(message: TelegramMessage) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Owner validation: silently drop messages from non-owner users
|
||||
if let Some(owner_id_str) = channel_host::workspace_read(OWNER_ID_PATH) {
|
||||
if !owner_id_str.is_empty() {
|
||||
if let Ok(owner_id) = owner_id_str.parse::<i64>() {
|
||||
if from.id != owner_id {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner user {} (owner: {})",
|
||||
from.id, owner_id
|
||||
),
|
||||
);
|
||||
return;
|
||||
let is_private = message.chat.chat_type == "private";
|
||||
|
||||
// Owner validation: when owner_id is set, only that user can message
|
||||
let owner_id_str = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
|
||||
if let Some(ref id_str) = owner_id_str {
|
||||
if let Ok(owner_id) = id_str.parse::<i64>() {
|
||||
if from.id != owner_id {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner user {} (owner: {})",
|
||||
from.id, owner_id
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if is_private {
|
||||
// No owner_id: apply dm_policy for private chats
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy != "open" {
|
||||
// Build effective allow list: config allow_from + pairing store
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
let id_str = from.id.to_string();
|
||||
let username_opt = from.username.as_deref();
|
||||
let is_allowed = allowed.contains(&"*".to_string())
|
||||
|| allowed.contains(&id_str)
|
||||
|| username_opt.map_or(false, |u| allowed.contains(&u.to_string()));
|
||||
|
||||
if !is_allowed {
|
||||
if dm_policy == "pairing" {
|
||||
// Upsert pairing request and send reply
|
||||
let meta = serde_json::json!({
|
||||
"chat_id": message.chat.id,
|
||||
"user_id": from.id,
|
||||
"username": username_opt,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, &id_str, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for user {} (chat {}): code {}",
|
||||
from.id, message.chat.id, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
let _ = send_pairing_reply(message.chat.id, &result.code);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let is_private = message.chat.chat_type == "private";
|
||||
|
||||
// For group chats, check if the bot was mentioned
|
||||
// TODO: Read bot_username from config and check mentions
|
||||
// For now, process all messages in private chats and groups
|
||||
// For group chats, only respond if bot was mentioned or respond_to_all is enabled
|
||||
if !is_private {
|
||||
// In groups, only respond if there's a bot mention or command
|
||||
// This is a simplified check - proper implementation would use entities
|
||||
let has_command = text.starts_with('/');
|
||||
let has_mention = text.contains('@');
|
||||
let respond_to_all = channel_host::workspace_read(RESPOND_TO_ALL_GROUP_PATH)
|
||||
.as_deref()
|
||||
.unwrap_or("false")
|
||||
== "true";
|
||||
|
||||
if !has_command && !has_mention {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!("Ignoring group message without mention: {}", text),
|
||||
);
|
||||
return;
|
||||
if !respond_to_all {
|
||||
let has_command = content.starts_with('/');
|
||||
let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH).unwrap_or_default();
|
||||
let has_bot_mention = if bot_username.is_empty() {
|
||||
content.contains('@')
|
||||
} else {
|
||||
let mention = format!("@{}", bot_username);
|
||||
content.to_lowercase().contains(&mention.to_lowercase())
|
||||
};
|
||||
|
||||
if !has_command && !has_bot_mention {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!("Ignoring group message without mention: {}", content),
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -792,17 +1028,39 @@ fn handle_message(message: TelegramMessage) {
|
||||
let metadata_json = serde_json::to_string(&metadata).unwrap_or_else(|_| "{}".to_string());
|
||||
|
||||
// Clean the message text (strip bot mentions and commands)
|
||||
let cleaned_text = clean_message_text(&text);
|
||||
let bot_username = channel_host::workspace_read(BOT_USERNAME_PATH).unwrap_or_default();
|
||||
let cleaned_text = clean_message_text(
|
||||
&content,
|
||||
if bot_username.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(bot_username.as_str())
|
||||
},
|
||||
);
|
||||
|
||||
if cleaned_text.is_empty() {
|
||||
// Determine what to emit to the agent.
|
||||
// - `/start` (no args): emit a welcome placeholder so the agent greets the user
|
||||
// - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through
|
||||
// so Submission::parse() can handle it
|
||||
// - Commands with args (e.g. `/start hello`): cleaned_text already has the args
|
||||
// - Plain text: pass through as-is
|
||||
let trimmed_content = content.trim();
|
||||
let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
"[User started the bot]".to_string()
|
||||
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
|
||||
// Bare control command like /interrupt, /stop, /help — pass through raw
|
||||
trimmed_content.to_string()
|
||||
} else if cleaned_text.is_empty() {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
cleaned_text
|
||||
};
|
||||
|
||||
// Emit the message to the agent
|
||||
channel_host::emit_message(&EmittedMessage {
|
||||
user_id: from.id.to_string(),
|
||||
user_name: Some(user_name),
|
||||
content: cleaned_text,
|
||||
content: content_to_emit,
|
||||
thread_id: None, // Telegram doesn't have threads in the same way
|
||||
metadata_json,
|
||||
});
|
||||
@@ -817,7 +1075,8 @@ fn handle_message(message: TelegramMessage) {
|
||||
}
|
||||
|
||||
/// Clean message text by removing bot commands and @mentions at the start.
|
||||
fn clean_message_text(text: &str) -> String {
|
||||
/// When bot_username is set, only strips that specific mention; otherwise strips any leading @mention.
|
||||
fn clean_message_text(text: &str, bot_username: Option<&str>) -> String {
|
||||
let mut result = text.trim().to_string();
|
||||
|
||||
// Remove leading /command
|
||||
@@ -832,11 +1091,30 @@ fn clean_message_text(text: &str) -> String {
|
||||
|
||||
// Remove leading @mention
|
||||
if result.starts_with('@') {
|
||||
if let Some(space_idx) = result.find(' ') {
|
||||
result = result[space_idx..].trim_start().to_string();
|
||||
if let Some(bot) = bot_username {
|
||||
let mention = format!("@{}", bot);
|
||||
let mention_lower = mention.to_lowercase();
|
||||
let result_lower = result.to_lowercase();
|
||||
if result_lower.starts_with(&mention_lower) {
|
||||
let rest = result[mention.len()..].trim_start();
|
||||
if rest.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
result = rest.to_string();
|
||||
} else if let Some(space_idx) = result.find(' ') {
|
||||
// Different leading @mention - only strip if it's the bot
|
||||
let first_word = &result[..space_idx];
|
||||
if first_word.eq_ignore_ascii_case(&mention) {
|
||||
result = result[space_idx..].trim_start().to_string();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Just a mention with no text
|
||||
return String::new();
|
||||
// No bot_username: strip any leading @mention
|
||||
if let Some(space_idx) = result.find(' ') {
|
||||
result = result[space_idx..].trim_start().to_string();
|
||||
} else {
|
||||
return String::new();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -872,12 +1150,93 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_clean_message_text() {
|
||||
assert_eq!(clean_message_text("/start hello"), "hello");
|
||||
assert_eq!(clean_message_text("@bot hello world"), "hello world");
|
||||
assert_eq!(clean_message_text("/start"), "");
|
||||
assert_eq!(clean_message_text("@botname"), "");
|
||||
assert_eq!(clean_message_text("just text"), "just text");
|
||||
assert_eq!(clean_message_text(" spaced "), "spaced");
|
||||
// Without bot_username: strips any leading @mention
|
||||
assert_eq!(clean_message_text("/start hello", None), "hello");
|
||||
assert_eq!(clean_message_text("@bot hello world", None), "hello world");
|
||||
assert_eq!(clean_message_text("/start", None), "");
|
||||
assert_eq!(clean_message_text("@botname", None), "");
|
||||
assert_eq!(clean_message_text("just text", None), "just text");
|
||||
assert_eq!(clean_message_text(" spaced ", None), "spaced");
|
||||
|
||||
// With bot_username: only strips @MyBot, not @alice
|
||||
assert_eq!(clean_message_text("@MyBot hello", Some("MyBot")), "hello");
|
||||
assert_eq!(clean_message_text("@mybot hi", Some("MyBot")), "hi");
|
||||
assert_eq!(
|
||||
clean_message_text("@alice hello", Some("MyBot")),
|
||||
"@alice hello"
|
||||
);
|
||||
assert_eq!(clean_message_text("@MyBot", Some("MyBot")), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean_message_text_bare_commands() {
|
||||
// Bare commands return empty (the caller decides what to emit)
|
||||
assert_eq!(clean_message_text("/start", None), "");
|
||||
assert_eq!(clean_message_text("/interrupt", None), "");
|
||||
assert_eq!(clean_message_text("/stop", None), "");
|
||||
assert_eq!(clean_message_text("/help", None), "");
|
||||
assert_eq!(clean_message_text("/undo", None), "");
|
||||
assert_eq!(clean_message_text("/ping", None), "");
|
||||
|
||||
// Commands with args: command prefix stripped, args returned
|
||||
assert_eq!(clean_message_text("/start hello", None), "hello");
|
||||
assert_eq!(clean_message_text("/help me please", None), "me please");
|
||||
assert_eq!(clean_message_text("/model claude-opus-4-6", None), "claude-opus-4-6");
|
||||
}
|
||||
|
||||
/// Tests for the content_to_emit logic in handle_message.
|
||||
/// Since handle_message uses WASM host calls, we test the decision logic inline.
|
||||
#[test]
|
||||
fn test_content_to_emit_logic() {
|
||||
// Simulates the content_to_emit decision for various inputs.
|
||||
// This mirrors the logic in handle_message after clean_message_text.
|
||||
fn resolve_content(content: &str) -> Option<String> {
|
||||
let cleaned_text = clean_message_text(content, None);
|
||||
let trimmed_content = content.trim();
|
||||
if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
Some("[User started the bot]".to_string())
|
||||
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
|
||||
Some(trimmed_content.to_string())
|
||||
} else if cleaned_text.is_empty() {
|
||||
None // would return/skip in handle_message
|
||||
} else {
|
||||
Some(cleaned_text)
|
||||
}
|
||||
}
|
||||
|
||||
// /start → welcome placeholder
|
||||
assert_eq!(resolve_content("/start"), Some("[User started the bot]".to_string()));
|
||||
assert_eq!(resolve_content("/Start"), Some("[User started the bot]".to_string()));
|
||||
assert_eq!(resolve_content(" /start "), Some("[User started the bot]".to_string()));
|
||||
|
||||
// /start with args → pass args through
|
||||
assert_eq!(resolve_content("/start hello"), Some("hello".to_string()));
|
||||
|
||||
// Control commands → pass through raw so Submission::parse() can match
|
||||
assert_eq!(resolve_content("/interrupt"), Some("/interrupt".to_string()));
|
||||
assert_eq!(resolve_content("/stop"), Some("/stop".to_string()));
|
||||
assert_eq!(resolve_content("/help"), Some("/help".to_string()));
|
||||
assert_eq!(resolve_content("/undo"), Some("/undo".to_string()));
|
||||
assert_eq!(resolve_content("/redo"), Some("/redo".to_string()));
|
||||
assert_eq!(resolve_content("/ping"), Some("/ping".to_string()));
|
||||
assert_eq!(resolve_content("/tools"), Some("/tools".to_string()));
|
||||
assert_eq!(resolve_content("/compact"), Some("/compact".to_string()));
|
||||
assert_eq!(resolve_content("/clear"), Some("/clear".to_string()));
|
||||
assert_eq!(resolve_content("/version"), Some("/version".to_string()));
|
||||
|
||||
// Commands with args → cleaned text (command stripped)
|
||||
assert_eq!(resolve_content("/help me please"), Some("me please".to_string()));
|
||||
|
||||
// Plain text → pass through
|
||||
assert_eq!(resolve_content("hello world"), Some("hello world".to_string()));
|
||||
assert_eq!(resolve_content("just text"), Some("just text".to_string()));
|
||||
|
||||
// Empty / whitespace → skip (None)
|
||||
assert_eq!(resolve_content(""), None);
|
||||
assert_eq!(resolve_content(" "), None);
|
||||
|
||||
// Bare @mention without bot → skip
|
||||
assert_eq!(resolve_content("@botname"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -945,4 +1304,17 @@ mod tests {
|
||||
assert_eq!(from.id, 789);
|
||||
assert_eq!(from.first_name, "John");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_message_with_caption() {
|
||||
let json = r#"{
|
||||
"message_id": 1,
|
||||
"from": {"id": 1, "is_bot": false, "first_name": "A"},
|
||||
"chat": {"id": 1, "type": "private"},
|
||||
"caption": "What's in this image?"
|
||||
}"#;
|
||||
let msg: TelegramMessage = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(msg.text, None);
|
||||
assert_eq!(msg.caption.as_deref(), Some("What's in this image?"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,43 +1 @@
|
||||
{
|
||||
"type": "channel",
|
||||
"name": "telegram",
|
||||
"description": "Telegram Bot API channel for receiving and responding to Telegram messages",
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
{ "host": "api.telegram.org", "path_prefix": "/bot" }
|
||||
],
|
||||
"credentials": {
|
||||
"telegram_bot": {
|
||||
"secret_name": "telegram_bot_token",
|
||||
"location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" },
|
||||
"host_patterns": ["api.telegram.org"]
|
||||
}
|
||||
},
|
||||
"rate_limit": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 1000
|
||||
}
|
||||
},
|
||||
"secrets": {
|
||||
"allowed_names": ["telegram_*"]
|
||||
},
|
||||
"channel": {
|
||||
"allowed_paths": ["/webhook/telegram"],
|
||||
"allow_polling": true,
|
||||
"min_poll_interval_ms": 30000,
|
||||
"workspace_prefix": "channels/telegram/",
|
||||
"emit_rate_limit": {
|
||||
"messages_per_minute": 100,
|
||||
"messages_per_hour": 5000
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"bot_username": null,
|
||||
"owner_id": null,
|
||||
"respond_to_all_group_messages": false,
|
||||
"polling_enabled": false,
|
||||
"poll_interval_ms": 30000
|
||||
}
|
||||
}
|
||||
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
|
||||
|
||||
Binary file not shown.
@@ -16,3 +16,5 @@ serde_json = "1"
|
||||
opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
|
||||
[workspace]
|
||||
@@ -30,7 +30,7 @@ use serde::{Deserialize, Serialize};
|
||||
// Re-export generated types
|
||||
use exports::near::agent::channel::{
|
||||
AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest,
|
||||
OutgoingHttpResponse,
|
||||
OutgoingHttpResponse, StatusUpdate,
|
||||
};
|
||||
use near::agent::channel_host::{self, EmittedMessage};
|
||||
|
||||
@@ -254,10 +254,19 @@ struct WhatsAppChannel;
|
||||
|
||||
impl Guest for WhatsAppChannel {
|
||||
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
|
||||
let config: WhatsAppConfig = serde_json::from_str(&config_json).unwrap_or(WhatsAppConfig {
|
||||
api_version: default_api_version(),
|
||||
reply_to_message: default_reply_to_message(),
|
||||
});
|
||||
let config: WhatsAppConfig = match serde_json::from_str(&config_json) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Warn,
|
||||
&format!("Failed to parse WhatsApp config, using defaults: {}", e),
|
||||
);
|
||||
WhatsAppConfig {
|
||||
api_version: default_api_version(),
|
||||
reply_to_message: default_reply_to_message(),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
@@ -267,6 +276,9 @@ impl Guest for WhatsAppChannel {
|
||||
),
|
||||
);
|
||||
|
||||
// Persist api_version in workspace so on_respond() can read it
|
||||
let _ = channel_host::workspace_write("channels/whatsapp/api_version", &config.api_version);
|
||||
|
||||
// WhatsApp Cloud API is webhook-only, no polling available
|
||||
Ok(ChannelConfig {
|
||||
display_name: "WhatsApp".to_string(),
|
||||
@@ -327,11 +339,16 @@ impl Guest for WhatsAppChannel {
|
||||
let metadata: WhatsAppMessageMetadata = serde_json::from_str(&response.metadata_json)
|
||||
.map_err(|e| format!("Failed to parse metadata: {}", e))?;
|
||||
|
||||
// Read api_version from workspace (set during on_start), fallback to default
|
||||
let api_version = channel_host::workspace_read("channels/whatsapp/api_version")
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "v18.0".to_string());
|
||||
|
||||
// Build WhatsApp API URL with token placeholder
|
||||
// Host will replace {WHATSAPP_ACCESS_TOKEN} with actual token in Authorization header
|
||||
let api_url = format!(
|
||||
"https://graph.facebook.com/v18.0/{}/messages",
|
||||
metadata.phone_number_id
|
||||
"https://graph.facebook.com/{}/{}/messages",
|
||||
api_version, metadata.phone_number_id
|
||||
);
|
||||
|
||||
// Build sendMessage payload
|
||||
@@ -361,6 +378,7 @@ impl Guest for WhatsAppChannel {
|
||||
&api_url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
@@ -417,6 +435,8 @@ impl Guest for WhatsAppChannel {
|
||||
}
|
||||
}
|
||||
|
||||
fn on_status(_update: StatusUpdate) {}
|
||||
|
||||
fn on_shutdown() {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Cloud SQL Auth Proxy
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
DynamicUser=yes
|
||||
ExecStart=/usr/local/bin/cloud-sql-proxy ironclaw-prod:us-central1:ironclaw-db --port=5432
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,30 @@
|
||||
# WARNING: Replace all CHANGE_ME values before deploying.
|
||||
# Do not use placeholder passwords in production.
|
||||
DATABASE_URL=postgres://ironclaw:CHANGE_ME@localhost:5432/ironclaw
|
||||
|
||||
# NEAR AI Cloud (API key auth, Chat Completions API)
|
||||
# Get an API key from https://cloud.near.ai
|
||||
NEARAI_API_KEY=CHANGE_ME
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||
|
||||
# Or use NEAR AI Chat (session token auth, Responses API):
|
||||
# NEARAI_SESSION_TOKEN=sess_...
|
||||
# NEARAI_BASE_URL=https://private.near.ai
|
||||
|
||||
# Agent
|
||||
AGENT_NAME=ironclaw
|
||||
CLI_ENABLED=false
|
||||
|
||||
# Web Gateway
|
||||
GATEWAY_ENABLED=true
|
||||
# 0.0.0.0 binds to all interfaces (required for Docker --network=host).
|
||||
# Use 127.0.0.1 if running outside Docker or for local-only access.
|
||||
GATEWAY_HOST=0.0.0.0
|
||||
GATEWAY_PORT=3000
|
||||
GATEWAY_AUTH_TOKEN=CHANGE_ME
|
||||
|
||||
# Disabled for initial deploy
|
||||
SANDBOX_ENABLED=false
|
||||
HEARTBEAT_ENABLED=false
|
||||
EMBEDDING_ENABLED=false
|
||||
@@ -0,0 +1,20 @@
|
||||
[Unit]
|
||||
Description=IronClaw AI Assistant
|
||||
After=cloud-sql-proxy.service docker.service
|
||||
Requires=cloud-sql-proxy.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStartPre=/usr/bin/docker pull us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest
|
||||
ExecStart=/usr/bin/docker run --rm \
|
||||
--name ironclaw \
|
||||
--env-file /opt/ironclaw/.env \
|
||||
--network=host \
|
||||
us-central1-docker.pkg.dev/ironclaw-prod/ironclaw/agent:latest \
|
||||
--no-onboard
|
||||
ExecStop=/usr/bin/docker stop ironclaw
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# VM bootstrap script for IronClaw on GCP Compute Engine.
|
||||
#
|
||||
# Run on a fresh Debian 12 VM after SSH:
|
||||
# sudo bash setup.sh
|
||||
#
|
||||
# Prerequisites:
|
||||
# - VM has the ironclaw-vm service account attached
|
||||
# - Cloud SQL Auth Proxy accessible via IAM
|
||||
# - Artifact Registry image pushed
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Must run as root
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "ERROR: This script must be run as root (sudo bash setup.sh)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Installing Docker"
|
||||
apt-get update
|
||||
apt-get install -y docker.io
|
||||
systemctl enable docker
|
||||
systemctl start docker
|
||||
|
||||
echo "==> Installing Cloud SQL Auth Proxy"
|
||||
curl -fsSL -o /usr/local/bin/cloud-sql-proxy \
|
||||
https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.14.3/cloud-sql-proxy.linux.amd64
|
||||
chmod +x /usr/local/bin/cloud-sql-proxy
|
||||
|
||||
echo "==> Installing systemd services"
|
||||
cp /tmp/deploy/cloud-sql-proxy.service /etc/systemd/system/
|
||||
cp /tmp/deploy/ironclaw.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
|
||||
echo "==> Starting Cloud SQL Auth Proxy"
|
||||
systemctl enable cloud-sql-proxy
|
||||
systemctl start cloud-sql-proxy
|
||||
|
||||
echo "==> Configuring Docker registry auth"
|
||||
# The VM service account provides Artifact Registry access
|
||||
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
||||
|
||||
echo "==> Creating config directory"
|
||||
# Owned by root, readable only by root. Docker reads --env-file as root
|
||||
# before dropping to uid 1000 (ironclaw) inside the container.
|
||||
mkdir -p /opt/ironclaw
|
||||
chmod 700 /opt/ironclaw
|
||||
|
||||
if [ ! -f /opt/ironclaw/.env ]; then
|
||||
echo "WARNING: /opt/ironclaw/.env does not exist."
|
||||
echo "Create it with your configuration before starting IronClaw."
|
||||
echo "See deploy/env.example for the required variables."
|
||||
echo ""
|
||||
echo "Then run: systemctl enable ironclaw && systemctl start ironclaw"
|
||||
else
|
||||
chmod 600 /opt/ironclaw/.env
|
||||
echo "==> Starting IronClaw"
|
||||
systemctl enable ironclaw
|
||||
systemctl start ironclaw
|
||||
fi
|
||||
|
||||
echo "==> Setup complete"
|
||||
echo ""
|
||||
echo "Verify with:"
|
||||
echo " systemctl status cloud-sql-proxy"
|
||||
echo " systemctl status ironclaw"
|
||||
echo " docker logs ironclaw"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Local development only — do NOT use these credentials in production.
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: ironclaw
|
||||
POSTGRES_USER: ironclaw
|
||||
POSTGRES_PASSWORD: ironclaw # dev-only, change for any non-local deployment
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ironclaw"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
@@ -246,13 +246,46 @@ Create `my-channel.capabilities.json`:
|
||||
|
||||
## Building and Deploying
|
||||
|
||||
### Supply Chain Security: No Committed Binaries
|
||||
|
||||
**Do not commit compiled WASM binaries.** They are a supply chain risk — the binary in a PR may not match the source. IronClaw builds channels from source:
|
||||
|
||||
- `cargo build` automatically builds `telegram.wasm` via `build.rs`
|
||||
- The built binary is in `.gitignore` and is not committed
|
||||
- CI should run `cargo build` (or `./scripts/build-all.sh`) to produce releases
|
||||
|
||||
**Reproducible build:**
|
||||
```bash
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
Prerequisites: `rustup target add wasm32-wasip2`, `cargo install wasm-tools` (optional; fallback copies raw WASM if unavailable).
|
||||
|
||||
### Telegram Channel (Manual Build)
|
||||
|
||||
```bash
|
||||
# Add WASM target if needed
|
||||
rustup target add wasm32-wasip2
|
||||
|
||||
# Build Telegram channel
|
||||
./channels-src/telegram/build.sh
|
||||
|
||||
# Install (or use ironclaw onboard to install bundled channel)
|
||||
mkdir -p ~/.ironclaw/channels
|
||||
cp channels-src/telegram/telegram.wasm channels-src/telegram/telegram.capabilities.json ~/.ironclaw/channels/
|
||||
```
|
||||
|
||||
**Note**: The main IronClaw binary bundles `telegram.wasm` via `include_bytes!`. When modifying the Telegram channel source, run `./channels-src/telegram/build.sh` **before** building the main crate, so the updated WASM is included.
|
||||
|
||||
### Other Channels
|
||||
|
||||
```bash
|
||||
# Build the WASM component
|
||||
cd channels/my-channel
|
||||
cargo component build --release
|
||||
cd channels-src/my-channel
|
||||
cargo build --release --target wasm32-wasip2
|
||||
|
||||
# Deploy to ~/.ironclaw/channels/
|
||||
cp target/wasm32-wasip1/release/my_channel.wasm ~/.ironclaw/channels/my-channel.wasm
|
||||
cp target/wasm32-wasip2/release/my_channel.wasm ~/.ironclaw/channels/my-channel.wasm
|
||||
cp my-channel.capabilities.json ~/.ironclaw/channels/
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
# Telegram Channel Setup
|
||||
|
||||
This guide covers configuring the Telegram channel for IronClaw, including DM pairing for access control.
|
||||
|
||||
## Overview
|
||||
|
||||
The Telegram channel lets you interact with IronClaw via Telegram DMs and groups. It supports:
|
||||
|
||||
- **Webhook mode** (recommended): Instant delivery via tunnel
|
||||
- **Polling mode**: No tunnel required; ~30s delay
|
||||
- **DM pairing**: Approve unknown users before they can message the agent
|
||||
- **Group mentions**: `@YourBot` or `/command` to trigger in groups
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- IronClaw installed and configured (`ironclaw onboard`)
|
||||
- A Telegram bot token from [@BotFather](https://t.me/BotFather)
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Create a Bot
|
||||
|
||||
1. Message [@BotFather](https://t.me/BotFather) on Telegram
|
||||
2. Send `/newbot` and follow the prompts
|
||||
3. Copy the bot token (e.g., `123456789:ABCdefGHIjklMNOpqrsTUVwxyz`)
|
||||
|
||||
### 2. Configure via Setup Wizard
|
||||
|
||||
```bash
|
||||
ironclaw onboard
|
||||
```
|
||||
|
||||
When prompted, enable the Telegram channel and paste your bot token. The wizard will:
|
||||
|
||||
- Validate the token
|
||||
- Optionally configure a webhook secret
|
||||
- Set up tunnel (if you want webhook mode)
|
||||
|
||||
### 3. (Optional) Configure Tunnel for Webhooks
|
||||
|
||||
For instant message delivery, expose your agent via a tunnel:
|
||||
|
||||
```bash
|
||||
# ngrok
|
||||
ngrok http 8080
|
||||
|
||||
# Cloudflare
|
||||
cloudflared tunnel --url http://localhost:8080
|
||||
```
|
||||
|
||||
Set the tunnel URL in settings or via `TUNNEL_URL` env var. Without a tunnel, the channel uses polling (~30s delay).
|
||||
|
||||
## DM Pairing
|
||||
|
||||
When an unknown user DMs your bot, they receive a pairing code. You must approve them before they can message the agent.
|
||||
|
||||
### Flow
|
||||
|
||||
1. Unknown user sends a message to your bot
|
||||
2. Bot replies: `To pair with this bot, run: ironclaw pairing approve telegram ABC12345`
|
||||
3. You run: `ironclaw pairing approve telegram ABC12345`
|
||||
4. User is added to the allow list; future messages are delivered
|
||||
|
||||
### Commands
|
||||
|
||||
```bash
|
||||
# List pending pairing requests
|
||||
ironclaw pairing list telegram
|
||||
|
||||
# List as JSON
|
||||
ironclaw pairing list telegram --json
|
||||
|
||||
# Approve a user by code
|
||||
ironclaw pairing approve telegram ABC12345
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
Edit `~/.ironclaw/channels/telegram.capabilities.json` (or the config injected by the host):
|
||||
|
||||
| Option | Values | Default | Description |
|
||||
|--------|--------|---------|-------------|
|
||||
| `dm_policy` | `open`, `allowlist`, `pairing` | `pairing` | `open` = allow all; `allowlist` = config + approved only; `pairing` = allowlist + send pairing reply to unknown |
|
||||
| `allow_from` | `["user_id", "username", "*"]` | `[]` | Pre-approved IDs/usernames. `*` allows everyone. |
|
||||
| `owner_id` | Telegram user ID | `null` | When set, only this user can message (overrides dm_policy) |
|
||||
| `bot_username` | Bot username (no @) | `null` | Used for mention detection in groups; when set, only strips this mention from messages |
|
||||
| `respond_to_all_group_messages` | `true`/`false` | `false` | When true, respond to all group messages; when false, only @mentions and /commands |
|
||||
|
||||
## Manual Installation
|
||||
|
||||
If the channel isn't installed via the wizard:
|
||||
|
||||
```bash
|
||||
# Build the Telegram channel (requires wasm32-wasip2 target)
|
||||
rustup target add wasm32-wasip2
|
||||
./channels-src/telegram/build.sh
|
||||
|
||||
# Install
|
||||
mkdir -p ~/.ironclaw/channels
|
||||
cp channels-src/telegram/telegram.wasm channels-src/telegram/telegram.capabilities.json ~/.ironclaw/channels/
|
||||
```
|
||||
|
||||
## Secrets
|
||||
|
||||
The channel expects a secret named `telegram_bot_token`. Configure via:
|
||||
|
||||
- **Setup wizard**: Saves to encrypted secrets store
|
||||
- **Environment**: `TELEGRAM_BOT_TOKEN=your_token`
|
||||
- **Secrets store**: `ironclaw` CLI (if available)
|
||||
|
||||
## Webhook Secret (Optional)
|
||||
|
||||
For webhook validation, set `telegram_webhook_secret` in secrets. Telegram will send `X-Telegram-Bot-Api-Secret-Token` with each request; the host validates it before forwarding.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Messages not delivered
|
||||
|
||||
- **Polling mode**: Check logs for `getUpdates` errors. Ensure the bot token is valid.
|
||||
- **Webhook mode**: Verify tunnel is running and `TUNNEL_URL` is correct. Telegram requires HTTPS.
|
||||
|
||||
### Pairing code not received
|
||||
|
||||
- Verify the channel can send messages (HTTP allowlist includes `api.telegram.org`)
|
||||
- Check `dm_policy` is `pairing` (not `allowlist` which blocks without reply)
|
||||
|
||||
### Group mentions not working
|
||||
|
||||
- Set `bot_username` in config to your bot's username (e.g., `MyIronClawBot`)
|
||||
- Ensure the message contains `@YourBot` or starts with `/`
|
||||
|
||||
### "Connection refused" when starting
|
||||
|
||||
- For webhook mode: Start your tunnel before `ironclaw run`
|
||||
- For polling only: No tunnel needed; ignore tunnel-related warnings
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"http": {
|
||||
"allowlist": [
|
||||
{
|
||||
"host": "slack.com",
|
||||
"path_prefix": "/api/",
|
||||
"methods": ["GET", "POST"]
|
||||
}
|
||||
],
|
||||
"credentials": {
|
||||
"slack_bot_token": {
|
||||
"secret_name": "slack_bot_token",
|
||||
"location": { "type": "bearer" },
|
||||
"host_patterns": ["slack.com"]
|
||||
}
|
||||
},
|
||||
"rate_limit": {
|
||||
"requests_per_minute": 50,
|
||||
"requests_per_hour": 1000
|
||||
},
|
||||
"timeout_secs": 30
|
||||
},
|
||||
"secrets": {
|
||||
"allowed_names": ["slack_bot_token"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
-- Add project_dir and user_id columns for sandbox job tracking.
|
||||
-- user_id was previously hardcoded to "default" in the Rust layer;
|
||||
-- now it's persisted so we can filter per-user.
|
||||
|
||||
ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS project_dir TEXT;
|
||||
ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS user_id TEXT NOT NULL DEFAULT 'default';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_source ON agent_jobs(source);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_user ON agent_jobs(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_created ON agent_jobs(created_at DESC);
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Track which mode a sandbox job uses (worker vs claude_code).
|
||||
ALTER TABLE agent_jobs ADD COLUMN IF NOT EXISTS job_mode TEXT NOT NULL DEFAULT 'worker';
|
||||
|
||||
-- Persist Claude Code streaming events so they survive restarts and can be
|
||||
-- loaded when the frontend opens a job detail view after the fact.
|
||||
CREATE TABLE IF NOT EXISTS claude_code_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL REFERENCES agent_jobs(id),
|
||||
event_type TEXT NOT NULL,
|
||||
data JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cc_events_job ON claude_code_events(job_id, id);
|
||||
@@ -0,0 +1,73 @@
|
||||
-- Routines: scheduled and reactive job system.
|
||||
--
|
||||
-- A routine is a named, persistent, user-owned task with a trigger and an action.
|
||||
-- Triggers fire independently (cron, event, webhook, manual) so only the
|
||||
-- relevant routine's prompt hits the LLM, not the whole checklist.
|
||||
|
||||
CREATE TABLE routines (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
user_id TEXT NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
|
||||
-- Trigger definition
|
||||
trigger_type TEXT NOT NULL, -- 'cron', 'event', 'webhook', 'manual'
|
||||
trigger_config JSONB NOT NULL, -- type-specific config (schedule, pattern, etc.)
|
||||
|
||||
-- Action definition
|
||||
action_type TEXT NOT NULL, -- 'lightweight', 'full_job'
|
||||
action_config JSONB NOT NULL, -- prompt, context_paths, max_tokens / title, max_iterations
|
||||
|
||||
-- Guardrails
|
||||
cooldown_secs INTEGER NOT NULL DEFAULT 300,
|
||||
max_concurrent INTEGER NOT NULL DEFAULT 1,
|
||||
dedup_window_secs INTEGER, -- NULL = no dedup
|
||||
|
||||
-- Notification preferences
|
||||
notify_channel TEXT, -- NULL = use default
|
||||
notify_user TEXT NOT NULL DEFAULT 'default',
|
||||
notify_on_success BOOLEAN NOT NULL DEFAULT false,
|
||||
notify_on_failure BOOLEAN NOT NULL DEFAULT true,
|
||||
notify_on_attention BOOLEAN NOT NULL DEFAULT true,
|
||||
|
||||
-- Runtime state (updated by engine)
|
||||
state JSONB NOT NULL DEFAULT '{}',
|
||||
last_run_at TIMESTAMPTZ,
|
||||
next_fire_at TIMESTAMPTZ, -- pre-computed for cron triggers
|
||||
run_count BIGINT NOT NULL DEFAULT 0,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
UNIQUE (user_id, name)
|
||||
);
|
||||
|
||||
-- Fast lookup: "which cron routines need to fire right now?"
|
||||
CREATE INDEX idx_routines_next_fire
|
||||
ON routines (next_fire_at)
|
||||
WHERE enabled AND next_fire_at IS NOT NULL;
|
||||
|
||||
-- Fast lookup: event triggers for a user
|
||||
CREATE INDEX idx_routines_event_triggers
|
||||
ON routines (user_id)
|
||||
WHERE enabled AND trigger_type = 'event';
|
||||
|
||||
-- Audit log of individual routine executions.
|
||||
CREATE TABLE routine_runs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
routine_id UUID NOT NULL REFERENCES routines(id) ON DELETE CASCADE,
|
||||
trigger_type TEXT NOT NULL,
|
||||
trigger_detail TEXT, -- e.g. matched message preview, cron expression
|
||||
started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
completed_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'running', -- running, ok, attention, failed
|
||||
result_summary TEXT,
|
||||
tokens_used INTEGER,
|
||||
job_id UUID REFERENCES agent_jobs(id), -- non-NULL for full_job runs
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX idx_routine_runs_routine ON routine_runs (routine_id);
|
||||
CREATE INDEX idx_routine_runs_status ON routine_runs (status) WHERE status = 'running';
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Rename claude_code_events to job_events (generic for all sandbox job types).
|
||||
ALTER TABLE claude_code_events RENAME TO job_events;
|
||||
ALTER INDEX idx_cc_events_job RENAME TO idx_job_events_job;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Settings table: key-value store for all user configuration.
|
||||
--
|
||||
-- Replaces ~/.ironclaw/settings.json, session.json, and mcp-servers.json.
|
||||
-- Keys use dotted paths matching the existing Settings.get()/set() convention
|
||||
-- (e.g., "agent.name", "sandbox.enabled", "mcp_servers").
|
||||
-- One row per setting so individual values can be updated atomically.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
user_id TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value JSONB NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (user_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_settings_user ON settings (user_id);
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Allow embedding vectors of any dimension (not just 1536).
|
||||
-- This supports Ollama models (768-dim nomic-embed-text, 1024-dim mxbai-embed-large)
|
||||
-- alongside OpenAI models (1536-dim text-embedding-3-small, 3072-dim text-embedding-3-large).
|
||||
--
|
||||
-- NOTE: HNSW indexes require a fixed dimension, so we drop the index.
|
||||
-- Exact (sequential) cosine distance search still works without the index.
|
||||
-- For a personal assistant workspace the dataset is small enough that this
|
||||
-- has negligible impact on query latency.
|
||||
|
||||
-- Drop dependent views first
|
||||
DROP VIEW IF EXISTS chunks_pending_embedding;
|
||||
DROP VIEW IF EXISTS memory_documents_summary;
|
||||
|
||||
DROP INDEX IF EXISTS idx_memory_chunks_embedding;
|
||||
|
||||
ALTER TABLE memory_chunks
|
||||
ALTER COLUMN embedding TYPE vector
|
||||
USING embedding::vector;
|
||||
|
||||
-- Recreate the views
|
||||
CREATE VIEW memory_documents_summary AS
|
||||
SELECT
|
||||
d.id,
|
||||
d.user_id,
|
||||
d.path,
|
||||
d.created_at,
|
||||
d.updated_at,
|
||||
COUNT(c.id) as chunk_count,
|
||||
COUNT(c.embedding) as embedded_chunk_count
|
||||
FROM memory_documents d
|
||||
LEFT JOIN memory_chunks c ON c.document_id = d.id
|
||||
GROUP BY d.id;
|
||||
|
||||
CREATE VIEW chunks_pending_embedding AS
|
||||
SELECT
|
||||
c.id as chunk_id,
|
||||
c.document_id,
|
||||
d.user_id,
|
||||
d.path,
|
||||
LENGTH(c.content) as content_length
|
||||
FROM memory_chunks c
|
||||
JOIN memory_documents d ON d.id = c.document_id
|
||||
WHERE c.embedding IS NULL;
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"bundles": {
|
||||
"google": {
|
||||
"display_name": "Google Suite",
|
||||
"description": "Gmail, Calendar, Drive, Docs, Sheets, Slides",
|
||||
"extensions": [
|
||||
"tools/gmail",
|
||||
"tools/google-calendar",
|
||||
"tools/google-docs",
|
||||
"tools/google-drive",
|
||||
"tools/google-sheets",
|
||||
"tools/google-slides"
|
||||
],
|
||||
"shared_auth": "google_oauth_token"
|
||||
},
|
||||
"messaging": {
|
||||
"display_name": "Messaging Channels",
|
||||
"description": "Discord, Telegram, Slack, and WhatsApp channels",
|
||||
"extensions": [
|
||||
"channels/discord",
|
||||
"channels/telegram",
|
||||
"channels/slack",
|
||||
"channels/whatsapp"
|
||||
],
|
||||
"shared_auth": null
|
||||
},
|
||||
"default": {
|
||||
"display_name": "Recommended Set",
|
||||
"description": "Core tools and channels for a productive setup",
|
||||
"extensions": [
|
||||
"tools/github",
|
||||
"tools/gmail",
|
||||
"tools/google-calendar",
|
||||
"tools/google-drive",
|
||||
"tools/slack",
|
||||
"channels/telegram",
|
||||
"channels/slack"
|
||||
],
|
||||
"shared_auth": null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "discord",
|
||||
"display_name": "Discord",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Discord Gateway/Webhook channel for slash commands, buttons, and messages",
|
||||
"keywords": ["messaging", "chat", "discord", "bot"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/discord",
|
||||
"capabilities": "discord.capabilities.json",
|
||||
"crate_name": "discord-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Discord",
|
||||
"secrets": ["discord_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://discord.com/developers/applications"
|
||||
},
|
||||
|
||||
"tags": ["messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Slack Events API channel for receiving and responding to Slack messages",
|
||||
"keywords": ["messaging", "chat", "workspace", "slack"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/slack",
|
||||
"capabilities": "slack.capabilities.json",
|
||||
"crate_name": "slack-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token", "slack_signing_secret"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://api.slack.com/apps"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "telegram",
|
||||
"display_name": "Telegram",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "Telegram Bot API channel for receiving and responding to messages",
|
||||
"keywords": ["messaging", "bot", "chat", "telegram"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/telegram",
|
||||
"capabilities": "telegram.capabilities.json",
|
||||
"crate_name": "telegram-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Telegram",
|
||||
"secrets": ["telegram_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://t.me/BotFather"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "whatsapp",
|
||||
"display_name": "WhatsApp",
|
||||
"kind": "channel",
|
||||
"version": "0.1.0",
|
||||
"description": "WhatsApp Cloud API channel for receiving and responding to messages",
|
||||
"keywords": ["messaging", "chat", "whatsapp", "meta"],
|
||||
|
||||
"source": {
|
||||
"dir": "channels-src/whatsapp",
|
||||
"capabilities": "whatsapp.capabilities.json",
|
||||
"crate_name": "whatsapp-channel"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "Meta",
|
||||
"secrets": ["whatsapp_access_token", "whatsapp_verify_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://developers.facebook.com/apps/"
|
||||
},
|
||||
|
||||
"tags": ["messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "github",
|
||||
"display_name": "GitHub",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "GitHub integration for issues, PRs, repos, and code search",
|
||||
"keywords": ["git", "code", "issues", "pull-requests", "repositories"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/github",
|
||||
"capabilities": "github-tool.capabilities.json",
|
||||
"crate_name": "github-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "manual",
|
||||
"provider": "GitHub",
|
||||
"secrets": ["github_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://github.com/settings/tokens"
|
||||
},
|
||||
|
||||
"tags": ["default", "development"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "gmail",
|
||||
"display_name": "Gmail",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Read, send, and manage Gmail messages and threads",
|
||||
"keywords": ["email", "google", "mail", "messaging"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/gmail",
|
||||
"capabilities": "gmail-tool.capabilities.json",
|
||||
"crate_name": "gmail-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "messaging"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-calendar",
|
||||
"display_name": "Google Calendar",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create, read, update, and delete Google Calendar events",
|
||||
"keywords": ["calendar", "google", "scheduling", "events"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-calendar",
|
||||
"capabilities": "google-calendar-tool.capabilities.json",
|
||||
"crate_name": "google-calendar-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-docs",
|
||||
"display_name": "Google Docs",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create and edit Google Docs documents",
|
||||
"keywords": ["documents", "google", "writing", "docs"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-docs",
|
||||
"capabilities": "google-docs-tool.capabilities.json",
|
||||
"crate_name": "google-docs-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-drive",
|
||||
"display_name": "Google Drive",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Upload, download, search, and manage Google Drive files and folders",
|
||||
"keywords": ["storage", "google", "files", "drive"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-drive",
|
||||
"capabilities": "google-drive-tool.capabilities.json",
|
||||
"crate_name": "google-drive-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["default", "google", "storage"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-sheets",
|
||||
"display_name": "Google Sheets",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Read and write Google Sheets spreadsheet data",
|
||||
"keywords": ["spreadsheets", "google", "data", "sheets"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-sheets",
|
||||
"capabilities": "google-sheets-tool.capabilities.json",
|
||||
"crate_name": "google-sheets-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "google-slides",
|
||||
"display_name": "Google Slides",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Create and edit Google Slides presentations",
|
||||
"keywords": ["presentations", "google", "slides"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/google-slides",
|
||||
"capabilities": "google-slides-tool.capabilities.json",
|
||||
"crate_name": "google-slides-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Google",
|
||||
"secrets": ["google_oauth_token"],
|
||||
"shared_auth": "google_oauth_token",
|
||||
"setup_url": "https://console.cloud.google.com/apis/credentials"
|
||||
},
|
||||
|
||||
"tags": ["google", "productivity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "okta",
|
||||
"display_name": "Okta",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Okta SSO for user profile, app catalog, and SSO launch links",
|
||||
"keywords": ["sso", "identity", "authentication", "okta"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/okta",
|
||||
"capabilities": "okta-tool.capabilities.json",
|
||||
"crate_name": "okta-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Okta",
|
||||
"secrets": ["okta_oauth_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://developer.okta.com/docs/guides/implement-oauth-for-okta/main/"
|
||||
},
|
||||
|
||||
"tags": ["identity"]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "slack",
|
||||
"display_name": "Slack",
|
||||
"kind": "tool",
|
||||
"version": "0.1.0",
|
||||
"description": "Post messages, read channels, and manage conversations via Slack API",
|
||||
"keywords": ["messaging", "chat", "workspace"],
|
||||
|
||||
"source": {
|
||||
"dir": "tools-src/slack",
|
||||
"capabilities": "slack-tool.capabilities.json",
|
||||
"crate_name": "slack-tool"
|
||||
},
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
"auth_summary": {
|
||||
"method": "oauth",
|
||||
"provider": "Slack",
|
||||
"secrets": ["slack_bot_token"],
|
||||
"shared_auth": null,
|
||||
"setup_url": "https://api.slack.com/apps"
|
||||
},
|
||||
|
||||
"tags": ["default", "messaging"]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user