mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-26 15:40:18 +00:00
Compare commits
24
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
497b93cebb | ||
|
|
37c0158765 | ||
|
|
5a70e3e1ef | ||
|
|
bbb2d5c4dd | ||
|
|
0a30c95ee1 | ||
|
|
b3bf50f10e | ||
|
|
48b5323ec9 | ||
|
|
3124ab2b7f | ||
|
|
dbd3e0807f | ||
|
|
436066415b | ||
|
|
3d4c647216 | ||
|
|
b68d67bd35 | ||
|
|
493e4578d0 | ||
|
|
250551799b | ||
|
|
c038c7705b | ||
|
|
98ee648fcb | ||
|
|
2cdd1acb1e | ||
|
|
542268fde5 | ||
|
|
3b6105d5ea | ||
|
|
df8616b604 | ||
|
|
e8dcb52fda | ||
|
|
1f18422b88 | ||
|
|
448383cfb0 | ||
|
|
7df356c109 |
@@ -286,8 +286,8 @@ impl Tool for <Name>Tool {
|
||||
false // Set true if tool processes external data
|
||||
}
|
||||
|
||||
fn requires_approval(&self) -> bool {
|
||||
false // Set true if tool is destructive or contacts external services
|
||||
fn requires_approval(&self, _params: &serde_json::Value) -> crate::tools::tool::ApprovalRequirement {
|
||||
crate::tools::tool::ApprovalRequirement::Never // Set to UnlessAutoApproved or Always as needed
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
+25
-12
@@ -6,21 +6,19 @@ DATABASE_POOL_SIZE=10
|
||||
# LLM_BACKEND=nearai # default
|
||||
# Possible values: nearai, ollama, openai_compatible, openai, anthropic, tinfoil
|
||||
|
||||
# === NEAR AI Chat (Responses API, session token auth) ===
|
||||
# Default mode. Uses browser OAuth (GitHub/Google) on first run.
|
||||
# Session token stored in ~/.ironclaw/session.json automatically.
|
||||
# For hosting providers: set NEARAI_SESSION_TOKEN env var directly.
|
||||
# === NEAR AI (Chat Completions API) ===
|
||||
# Two auth modes:
|
||||
# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run.
|
||||
# Session token stored in ~/.ironclaw/session.json automatically.
|
||||
# Base URL defaults to https://private.near.ai
|
||||
# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai.
|
||||
# Base URL defaults to https://cloud-api.near.ai
|
||||
NEARAI_MODEL=zai-org/GLM-5-FP8
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
NEARAI_AUTH_URL=https://private.near.ai
|
||||
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
|
||||
|
||||
# === NEAR AI Cloud (Chat Completions API, API key auth) ===
|
||||
# Auto-selected when NEARAI_API_KEY is set. Get a key from cloud.near.ai.
|
||||
# NEARAI_API_KEY=...
|
||||
# NEARAI_BASE_URL=https://cloud-api.near.ai # default for cloud mode
|
||||
# NEARAI_API_MODE=chat_completions # auto-detected from API key
|
||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||
|
||||
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
|
||||
|
||||
@@ -35,11 +33,26 @@ NEARAI_AUTH_URL=https://private.near.ai
|
||||
# LLM_BASE_URL=http://localhost:1234/v1
|
||||
# LLM_API_KEY=sk-... # optional for local servers
|
||||
|
||||
# === OpenRouter (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=anthropic/claude-sonnet-4
|
||||
# === OpenRouter (300+ models via OpenAI-compatible) ===
|
||||
# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
# LLM_API_KEY=sk-or-...
|
||||
# LLM_EXTRA_HEADERS=HTTP-Referer:https://myapp.com,X-Title:MyApp
|
||||
|
||||
# === Together AI (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://api.together.xyz/v1
|
||||
# LLM_API_KEY=...
|
||||
|
||||
# === Fireworks AI (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://api.fireworks.ai/inference/v1
|
||||
# LLM_API_KEY=fw_...
|
||||
|
||||
# For full provider setup guide see docs/LLM_PROVIDERS.md
|
||||
|
||||
|
||||
# Channel Configuration
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
tests/test-pages/**/*.html linguist-generated=true
|
||||
@@ -214,14 +214,113 @@ jobs:
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
# Build WASM extension bundles (tar.gz with .wasm + .capabilities.json)
|
||||
build-wasm-extensions:
|
||||
needs:
|
||||
- plan
|
||||
if: ${{ needs.plan.outputs.publishing == 'true' }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install Rust toolchain + wasm target
|
||||
run: |
|
||||
rustup target add wasm32-wasip2
|
||||
cargo install cargo-component --locked || true
|
||||
- uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
key: wasm-extensions
|
||||
- name: Build and package WASM extensions
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p target/wasm-bundles
|
||||
|
||||
# Process each manifest in registry/tools/ and registry/channels/
|
||||
for manifest in registry/tools/*.json registry/channels/*.json; do
|
||||
[ -f "$manifest" ] || continue
|
||||
|
||||
name=$(jq -r '.name' "$manifest")
|
||||
source_dir=$(jq -r '.source.dir' "$manifest")
|
||||
caps_file=$(jq -r '.source.capabilities' "$manifest")
|
||||
crate_name=$(jq -r '.source.crate_name' "$manifest")
|
||||
|
||||
if [ ! -d "$source_dir" ]; then
|
||||
echo "::warning::Source dir '$source_dir' not found for '$name', skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "=== Building $name from $source_dir ==="
|
||||
|
||||
# Build WASM component
|
||||
cargo component build --release --manifest-path "$source_dir/Cargo.toml" || {
|
||||
echo "::warning::Build failed for '$name', skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
# Find the built WASM file (Cargo uses underscores in artifact names)
|
||||
wasm_artifact="${crate_name//-/_}"
|
||||
wasm_path=""
|
||||
for target_dir in wasm32-wasip2 wasm32-wasip1 wasm32-wasi; do
|
||||
candidate="$source_dir/target/$target_dir/release/${wasm_artifact}.wasm"
|
||||
if [ -f "$candidate" ]; then
|
||||
wasm_path="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$wasm_path" ]; then
|
||||
echo "::warning::No WASM output found for '$name', skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Copy files with standardized names for the archive
|
||||
cp "$wasm_path" "target/wasm-bundles/${name}.wasm"
|
||||
|
||||
caps_path="$source_dir/$caps_file"
|
||||
if [ -f "$caps_path" ]; then
|
||||
cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json"
|
||||
else
|
||||
echo "::warning::No capabilities file at '$caps_path' for '$name'"
|
||||
fi
|
||||
|
||||
# Create tar.gz bundle
|
||||
bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz"
|
||||
(cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi)
|
||||
|
||||
# Compute SHA256
|
||||
sha256=$(sha256sum "$bundle" | cut -d' ' -f1)
|
||||
echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt
|
||||
|
||||
# Clean up intermediate files
|
||||
rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json"
|
||||
|
||||
echo " -> $bundle ($sha256)"
|
||||
done
|
||||
|
||||
echo "=== WASM bundles built ==="
|
||||
ls -la target/wasm-bundles/
|
||||
- name: "Upload WASM bundles"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-wasm-extensions
|
||||
path: |
|
||||
target/wasm-bundles/*.tar.gz
|
||||
target/wasm-bundles/checksums.txt
|
||||
|
||||
# Determines if we should publish/announce
|
||||
host:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
- build-wasm-extensions
|
||||
# Only run if we're "publishing", and only if plan, local, global, and wasm didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
|
||||
@@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.9.0](https://github.com/nearai/ironclaw/compare/v0.8.0...v0.9.0) - 2026-02-21
|
||||
|
||||
### Added
|
||||
|
||||
- add TEE attestation shield to web gateway UI ([#275](https://github.com/nearai/ironclaw/pull/275))
|
||||
- configurable tool iterations, auto-approve, and policy fix ([#251](https://github.com/nearai/ironclaw/pull/251))
|
||||
|
||||
### Fixed
|
||||
|
||||
- add X-Accel-Buffering header to SSE endpoints ([#277](https://github.com/nearai/ironclaw/pull/277))
|
||||
|
||||
## [0.8.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.7.0...ironclaw-v0.8.0) - 2026-02-20
|
||||
|
||||
### Added
|
||||
|
||||
- extension registry with metadata catalog and onboarding integration ([#238](https://github.com/nearai/ironclaw/pull/238))
|
||||
- *(models)* add GPT-5.3 Codex, full GPT-5.x family, Claude 4.x series, o4-mini ([#197](https://github.com/nearai/ironclaw/pull/197))
|
||||
- wire memory hygiene into the heartbeat loop ([#195](https://github.com/nearai/ironclaw/pull/195))
|
||||
|
||||
### Fixed
|
||||
|
||||
- persist WASM channel workspace writes across callbacks ([#264](https://github.com/nearai/ironclaw/pull/264))
|
||||
- consolidate per-module ENV_MUTEX into crate-wide test lock ([#246](https://github.com/nearai/ironclaw/pull/246))
|
||||
- remove auto-proceed fake user message injection from agent loop ([#255](https://github.com/nearai/ironclaw/pull/255))
|
||||
- onboarding errors reset flow and remote server auth (#185, #186) ([#248](https://github.com/nearai/ironclaw/pull/248))
|
||||
- parallelize tool call execution via JoinSet ([#219](https://github.com/nearai/ironclaw/pull/219)) ([#252](https://github.com/nearai/ironclaw/pull/252))
|
||||
- prevent pipe deadlock in shell command execution ([#140](https://github.com/nearai/ironclaw/pull/140))
|
||||
- persist turns after approval and add agent-level tests ([#250](https://github.com/nearai/ironclaw/pull/250))
|
||||
|
||||
### Other
|
||||
|
||||
- add automated PR labeling system ([#253](https://github.com/nearai/ironclaw/pull/253))
|
||||
- update CLAUDE.md for recently merged features ([#183](https://github.com/nearai/ironclaw/pull/183))
|
||||
|
||||
## [0.7.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.6.0...ironclaw-v0.7.0) - 2026-02-19
|
||||
|
||||
### Added
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
# Format code
|
||||
cargo fmt
|
||||
|
||||
# Lint (address warnings before committing)
|
||||
# Lint (fix ALL warnings before committing, including pre-existing ones)
|
||||
cargo clippy --all --benches --tests --examples --all-features
|
||||
|
||||
# Run all tests
|
||||
@@ -120,8 +120,7 @@ src/
|
||||
├── llm/ # LLM integration (multi-provider)
|
||||
│ ├── mod.rs # Provider factory, LlmBackend enum
|
||||
│ ├── provider.rs # LlmProvider trait, message types
|
||||
│ ├── nearai.rs # NEAR AI Responses API provider
|
||||
│ ├── nearai_chat.rs # NEAR AI Chat Completions fallback
|
||||
│ ├── nearai_chat.rs # NEAR AI Chat Completions provider (session token + API key auth)
|
||||
│ ├── reasoning.rs # Planning, tool selection, evaluation
|
||||
│ ├── session.rs # Session token management with auto-renewal
|
||||
│ ├── circuit_breaker.rs # Circuit breaker for provider failures
|
||||
@@ -322,7 +321,10 @@ cargo check --all-features # all features
|
||||
```
|
||||
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||
|
||||
**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate.
|
||||
|
||||
**Mechanical verification before committing:** Run these checks on changed files before committing:
|
||||
- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings
|
||||
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||
@@ -339,13 +341,12 @@ LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
|
||||
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
|
||||
|
||||
# NEAR AI (when LLM_BACKEND=nearai, the default)
|
||||
# Two modes: "NEAR AI Chat" (session token) or "NEAR AI Cloud" (API key)
|
||||
# NEAR AI Chat (Responses API, default):
|
||||
NEARAI_SESSION_TOKEN=sess_... # session token for chat-api
|
||||
# Two auth modes: session token (default) or API key
|
||||
# Session token auth (default): uses browser OAuth on first run
|
||||
NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
# NEAR AI Cloud (Chat Completions API, auto-selected when API key is set):
|
||||
# API key auth: set NEARAI_API_KEY, base URL defaults to cloud-api.near.ai
|
||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||
# NEARAI_BASE_URL=https://cloud-api.near.ai
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
|
||||
# Agent settings
|
||||
@@ -408,11 +409,13 @@ TINFOIL_MODEL=kimi-k2-5 # Default model
|
||||
|
||||
IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai` (default), `openai`, `anthropic`, `ollama`, `openai_compatible`, and `tinfoil`.
|
||||
|
||||
**NEAR AI Chat** -- Uses the NEAR AI Responses API (`https://private.near.ai/v1/responses`). Authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google). Supports response chaining (delta-only follow-up messages) for efficient multi-turn conversations. This is the default mode when no `NEARAI_API_KEY` is set. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment. Configure with `NEARAI_BASE_URL` (default: `https://private.near.ai`).
|
||||
**NEAR AI** -- Uses the Chat Completions API with dual auth support. Session token auth (default): authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google), base URL defaults to `https://private.near.ai`. API key auth: set `NEARAI_API_KEY` (from `cloud.near.ai`), base URL defaults to `https://cloud-api.near.ai`. Both modes use the same Chat Completions endpoint. Tool messages are flattened to plain text for compatibility. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment.
|
||||
|
||||
**NEAR AI Cloud** -- Uses the OpenAI-compatible Chat Completions API (`https://cloud-api.near.ai/v1/chat/completions`). Authenticates with API keys from `cloud.near.ai`. Auto-selected when `NEARAI_API_KEY` is set (or explicitly via `NEARAI_API_MODE=chat_completions`). Tool messages are flattened to plain text for compatibility. Configure with `NEARAI_API_KEY` and `NEARAI_BASE_URL` (default: `https://cloud-api.near.ai`).
|
||||
|
||||
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API only (not the Responses API, so tool calls are adapted to chat format). Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
|
||||
**OpenAI-compatible** -- Any endpoint that speaks the OpenAI API (vLLM, LiteLLM, OpenRouter, etc.). Configure with `LLM_BASE_URL`, `LLM_API_KEY` (optional), `LLM_MODEL`. Set `LLM_EXTRA_HEADERS` to inject custom HTTP headers into every request (format: `Key:Value,Key2:Value2`), useful for OpenRouter attribution headers like `HTTP-Referer` and `X-Title`.
|
||||
|
||||
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API. Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
|
||||
|
||||
## Database
|
||||
|
||||
|
||||
Generated
+480
-28
@@ -11,6 +11,12 @@ dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
@@ -64,6 +70,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"const-random",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy 0.8.37",
|
||||
@@ -188,6 +195,15 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063"
|
||||
|
||||
[[package]]
|
||||
name = "astral-tl"
|
||||
version = "0.7.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-broadcast"
|
||||
version = "0.7.2"
|
||||
@@ -924,6 +940,26 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
|
||||
dependencies = [
|
||||
"const-random-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random-macro"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"once_cell",
|
||||
"tiny-keccak",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
@@ -1099,6 +1135,21 @@ dependencies = [
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
@@ -1244,6 +1295,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
@@ -1255,6 +1312,29 @@ dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa",
|
||||
"phf 0.13.1",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser-macros"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctr"
|
||||
version = "0.9.2"
|
||||
@@ -1497,12 +1577,33 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa-short"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
|
||||
dependencies = [
|
||||
"dtoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ego-tree"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
@@ -1680,6 +1781,16 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
@@ -1692,6 +1803,12 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
@@ -1743,6 +1860,16 @@ version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||
dependencies = [
|
||||
"mac",
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.31"
|
||||
@@ -1883,6 +2010,15 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getopts"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
|
||||
dependencies = [
|
||||
"unicode-width 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -2001,7 +2137,7 @@ version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"foldhash 0.1.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -2010,6 +2146,11 @@ name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
@@ -2065,6 +2206,54 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-escape"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
|
||||
dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-to-markdown-rs"
|
||||
version = "2.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb31d75f2fdbc8d889d78a912e10c22c30451afb44ee3310f5bfcabf79a31a17"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"astral-tl",
|
||||
"base64 0.22.1",
|
||||
"html-escape",
|
||||
"html5ever 0.38.0",
|
||||
"lru",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.38.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.12"
|
||||
@@ -2490,7 +2679,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ironclaw"
|
||||
version = "0.7.0"
|
||||
version = "0.9.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"aho-corasick",
|
||||
@@ -2508,9 +2697,11 @@ dependencies = [
|
||||
"deadpool-postgres",
|
||||
"dirs 6.0.0",
|
||||
"dotenvy",
|
||||
"flate2",
|
||||
"fs4",
|
||||
"futures",
|
||||
"hkdf",
|
||||
"html-to-markdown-rs",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper-util",
|
||||
@@ -2521,6 +2712,7 @@ dependencies = [
|
||||
"postgres-types",
|
||||
"pretty_assertions",
|
||||
"rand 0.8.5",
|
||||
"readabilityrs",
|
||||
"refinery",
|
||||
"regex",
|
||||
"reqwest",
|
||||
@@ -2536,6 +2728,7 @@ dependencies = [
|
||||
"serde_yml",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"termimad",
|
||||
"testcontainers-modules",
|
||||
@@ -2559,30 +2752,6 @@ dependencies = [
|
||||
"zbus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ironclaw-bench"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"clap",
|
||||
"futures",
|
||||
"ironclaw",
|
||||
"regex",
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"toml",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-docker"
|
||||
version = "0.2.0"
|
||||
@@ -2663,6 +2832,21 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuchikikiki"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b73885c6a3cefdf7a1db0327cefbe4b9b72cac94cae4b19ede4fa492d8af02a0"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"crc",
|
||||
"cssparser",
|
||||
"html5ever 0.38.0",
|
||||
"indexmap 2.13.0",
|
||||
"precomputed-hash",
|
||||
"selectors 0.35.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex"
|
||||
version = "3.5.1"
|
||||
@@ -2813,7 +2997,7 @@ dependencies = [
|
||||
"log",
|
||||
"memchr",
|
||||
"phf 0.11.3",
|
||||
"phf_codegen",
|
||||
"phf_codegen 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
"uncased",
|
||||
]
|
||||
@@ -2907,12 +3091,27 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.4.3"
|
||||
@@ -2922,6 +3121,28 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.4.3",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.5.0",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -3014,6 +3235,16 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.1"
|
||||
@@ -3052,6 +3283,12 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
@@ -3422,6 +3659,7 @@ version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.13.1",
|
||||
"serde",
|
||||
]
|
||||
@@ -3432,10 +3670,20 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
@@ -3446,6 +3694,29 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
@@ -3609,6 +3880,12 @@ dependencies = [
|
||||
"zerocopy 0.8.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "pretty_assertions"
|
||||
version = "1.4.1"
|
||||
@@ -3876,6 +4153,24 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "readabilityrs"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eb174b0af6c181a87d68b42800806657bfbdf88b566f819aaadb9d2a7b7699d"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"kuchikikiki",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"url",
|
||||
"v_htmlescape",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
@@ -4418,6 +4713,21 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
"getopts",
|
||||
"html5ever 0.36.1",
|
||||
"precomputed-hash",
|
||||
"selectors 0.33.0",
|
||||
"tendril 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seahash"
|
||||
version = "4.1.0"
|
||||
@@ -4489,6 +4799,44 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
@@ -4627,6 +4975,15 @@ dependencies = [
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yml"
|
||||
version = "0.0.12"
|
||||
@@ -4719,6 +5076,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
@@ -4790,6 +5153,30 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared 0.13.1",
|
||||
"precomputed-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.5"
|
||||
@@ -4927,6 +5314,17 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.16"
|
||||
@@ -4946,6 +5344,27 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
@@ -5089,6 +5508,15 @@ dependencies = [
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiny-keccak"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
|
||||
dependencies = [
|
||||
"crunchy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.2"
|
||||
@@ -5730,6 +6158,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -5754,6 +6188,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "v_htmlescape"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
@@ -6289,6 +6729,18 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
|
||||
dependencies = [
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
|
||||
+18
-5
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
members = [".", "benchmarks"]
|
||||
members = ["."]
|
||||
exclude = [
|
||||
"channels-src/discord",
|
||||
"channels-src/telegram",
|
||||
@@ -19,7 +19,7 @@ exclude = [
|
||||
|
||||
[package]
|
||||
name = "ironclaw"
|
||||
version = "0.7.0"
|
||||
version = "0.9.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.92"
|
||||
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
||||
@@ -41,7 +41,7 @@ tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -82,7 +82,7 @@ clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
# Terminal
|
||||
crossterm = "0.28"
|
||||
rustyline = { version = "17", features = ["derive", "with-file-history"] }
|
||||
rustyline = { version = "17", features = ["custom-bindings", "derive", "with-file-history"] }
|
||||
termimad = "0.34"
|
||||
|
||||
# Channel integrations
|
||||
@@ -137,6 +137,10 @@ rig-core = "0.30"
|
||||
# Docker sandbox
|
||||
bollard = "0.18"
|
||||
|
||||
# Archive extraction for WASM extension bundles
|
||||
flate2 = "1"
|
||||
tar = "0.4"
|
||||
|
||||
# HTTP proxy for sandboxed network access
|
||||
hyper = { version = "1.5", features = ["server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] }
|
||||
@@ -145,6 +149,10 @@ bytes = "1"
|
||||
base64 = "0.22.1"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
# HTML to Markdown conversion (feature gated)
|
||||
html-to-markdown-rs = { version = "2.3", optional = true }
|
||||
readabilityrs = { version = "0.1.2", optional = true }
|
||||
|
||||
# macOS keychain
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
security-framework = "3"
|
||||
@@ -162,7 +170,7 @@ pretty_assertions = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = ["postgres", "libsql"]
|
||||
default = ["postgres", "libsql", "html-to-markdown"]
|
||||
postgres = [
|
||||
"dep:deadpool-postgres",
|
||||
"dep:tokio-postgres",
|
||||
@@ -173,6 +181,11 @@ postgres = [
|
||||
]
|
||||
libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
|
||||
|
||||
[[test]]
|
||||
name = "html_to_markdown"
|
||||
required-features = ["html-to-markdown"]
|
||||
|
||||
# The profile that 'cargo dist' will build with
|
||||
[profile.dist]
|
||||
|
||||
@@ -143,6 +143,23 @@ and secrets encryption (using your system keychain). Settings are persisted in t
|
||||
connected database; bootstrap variables (e.g. `DATABASE_URL`, `LLM_BACKEND`) are
|
||||
written to `~/.ironclaw/.env` so they are available before the database connects.
|
||||
|
||||
### Alternative LLM Providers
|
||||
|
||||
IronClaw defaults to NEAR AI but works with any OpenAI-compatible endpoint.
|
||||
Popular options include **OpenRouter** (300+ models), **Together AI**, **Fireworks AI**,
|
||||
**Ollama** (local), and self-hosted servers like **vLLM** or **LiteLLM**.
|
||||
|
||||
Select *"OpenAI-compatible"* in the wizard, or set environment variables directly:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
LLM_API_KEY=sk-or-...
|
||||
LLM_MODEL=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
See [docs/LLM_PROVIDERS.md](docs/LLM_PROVIDERS.md) for a full provider guide.
|
||||
|
||||
## Security
|
||||
|
||||
IronClaw implements defense in depth to protect your data and prevent misuse.
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
[package]
|
||||
name = "ironclaw-bench"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
description = "Benchmarking harness for IronClaw agent"
|
||||
license = "MIT OR Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "ironclaw-bench"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ironclaw = { path = ".." }
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Core types
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
|
||||
# Async traits
|
||||
async-trait = "0.1"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
# Scoring
|
||||
regex = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"run_id": "8a48de1f-09ce-4c35-9ad1-dad98fb83a1a",
|
||||
"suite_id": "spot",
|
||||
"config_label": "default",
|
||||
"model": "openai/gpt-5.2",
|
||||
"commit_hash": "2c43b83",
|
||||
"pass_rate": 1.0,
|
||||
"avg_score": 1.0,
|
||||
"total_tasks": 21,
|
||||
"completed_tasks": 21,
|
||||
"total_cost_usd": 0.307053,
|
||||
"total_wall_time_ms": 111009,
|
||||
"started_at": "2026-02-17T22:02:08.206112Z",
|
||||
"finished_at": "2026-02-17T22:03:59.270325Z"
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{"task_id":"smoke-greeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2037,"llm_calls":1,"input_tokens":1359,"output_tokens":65,"estimated_cost_usd":0.005052,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"I’m NEAR AI Agent—an autonomous assistant that helps you plan, write, debug, and execute tasks using available tools (like reading/writing files, running shell commands, and making HTTP requests). Tell me what you’re working on and what outcome you want, and I’ll drive it to completion.","started_at":"2026-02-17T22:02:08.206165Z","finished_at":"2026-02-17T22:02:10.243552Z","config_label":"default","error":null}
|
||||
{"task_id":"smoke-math","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2038,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"1081","started_at":"2026-02-17T22:02:10.243933Z","finished_at":"2026-02-17T22:02:12.282133Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2646,"llm_calls":2,"input_tokens":2786,"output_tokens":24,"estimated_cost_usd":0.008718,"tool_calls":[{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Spot check passed","started_at":"2026-02-17T22:02:12.282294Z","finished_at":"2026-02-17T22:02:14.928540Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-time","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2839,"output_tokens":78,"estimated_cost_usd":0.009687,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Current date/time (UTC): **2026-02-17 22:02:16** \nISO 8601: **2026-02-17T22:02:16.045933+00:00**","started_at":"2026-02-17T22:02:14.928810Z","finished_at":"2026-02-17T22:02:18.389188Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-json-query","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2751,"llm_calls":2,"input_tokens":2839,"output_tokens":43,"estimated_cost_usd":0.009162,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Bob","started_at":"2026-02-17T22:02:18.389812Z","finished_at":"2026-02-17T22:02:21.141146Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-shell-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2811,"output_tokens":25,"estimated_cost_usd":0.008808,"tool_calls":[{"name":"shell","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"benchmark test","started_at":"2026-02-17T22:02:21.141384Z","finished_at":"2026-02-17T22:02:24.601812Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-list-dir","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":19146,"llm_calls":2,"input_tokens":4370,"output_tokens":1337,"estimated_cost_usd":0.033165,"tool_calls":[{"name":"list_dir","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```\nclaude-501/\nclaude-505/\nclaude/\ncodex/\ncom.apple.launchd.HqYyNM7hoK/\ncom.apple.launchd.Z0bTtlsPEN/\ncom.apple.launchd.oY8y5gkzN4/\ncom.apple.launchd.sdo3WGu0S7/\ndata-gym-cache/\ndocker-desktop-privileged2203342862/\ngrammers-test/\nnear-agent-target/\nnear-market-bot-push/\nnear-market-bot/\nopenclaw/\npowerlog/\ntest_grpo/\ntest_grpo2/\ntest_grpo_entry/\ntest_grpo_mix/\ntest_grpo_singleton/\ntmpybtjiuc0/\n.d8bd37babaf2f1f7-00000000.node (5.9MB)\n.d8bd37fbfbb0f7ff-00000000.node (5.9MB)\n.d8bd3d9bdf6dd3f7-00000000.node (5.9MB)\n.d8bd3fbfbf21ddf7-00000000.node (5.9MB)\n.d8bd7dfebb38d5ff-00000000.node (5.9MB)\n.d8bdf5deba3bf1f7-00000000.node (5.9MB)\n.d8bdf7cfff76f3f7-00000000.node (5.9MB)\n.d8bdfd8ffe26d5ff-00000000.node (5.9MB)\n.d8bdfdeb9ba8dbff-00000000.node (5.9MB)\n.d8bdfffe9ab2ddff-00000000.node (5.9MB)\n.s.PGSQL.5432 (0B)\n.s.PGSQL.5432.lock (56B)\n__KMP_REGISTERED_LIB_75079 (1.0KB)\nagent_loop_new.rs (28.5KB)\nagent_mod.rs (1.6KB)\nauth_trace.md (10.8KB)\nbench-daily.md (72B)\nbench-log.md (109B)\nbench-meeting.md (160B)\nbench-monday.md (44B)\nbench-prefs.md (61B)\nbench-project.md (213B)\nbench-reminder.md (68B)\nbench-todo.md (153B)\nbench-tuesday.md (40B)\ncac-deck.html (151.5KB)\ncircuit_breaker.rs (22.1KB)\ncli_config.rs (9.1KB)\ncli_service.rs (1.1KB)\ncommands.rs (17.5KB)\nconfig.rs (58.4KB)\nconflicts_summary.md (21.4KB)\ncost_guard.rs (11.3KB)\ndebug_forc2.py (2.8KB)\ndebug_forc3.py (3.0KB)\ndebug_forc4.py (3.0KB)\ndebug_forc5.py (4.0KB)\ndebug_forc6.py (3.9KB)\ndebug_forc7.py (2.7KB)\ndebug_forc8.py (2.9KB)\ndebug_forc_prove.py (3.0KB)\ndispatcher.rs (26.2KB)\ndoctor.rs (8.4KB)\nhygiene.rs (7.3KB)\nironclaw_blog_test.png (21.4KB)\nironclaw_browser_test_viewport.png (156.1KB)\nironclaw_linkedin_debug.png (6.5KB)\nironclaw_spot_test.txt (19B)\nkeys_chain_signatures.rs (6.1KB)\nkeys_error.rs (1.6KB)\nkeys_intents.rs (5.8KB)\nkeys_mod.rs (31.6KB)\nkeys_policy.rs (31.4KB)\nkeys_rpc.rs (8.5KB)\nkeys_signer.rs (8.0KB)\nkeys_spending.rs (5.9KB)\nkeys_transaction.rs (13.7KB)\nkeys_types.rs (17.4KB)\nleak_detection_research_summary.md (13.0KB)\nleak_detector.rs (25.3KB)\nlib_new.rs (5.0KB)\nllm_mod.rs (10.0KB)\nmain.rs (56.8KB)\nmain_bootstrap.rs (11.0KB)\nmain_rs.txt (34.1KB)\nnear_resp.json (39B)\nobs_log.rs (5.9KB)\nobs_mod.rs (2.7KB)\nobs_multi.rs (4.0KB)\nobs_noop.rs (1.5KB)\nobs_traits.rs (4.0KB)\nour_bootstrap.rs (12.6KB)\nour_config.rs (45.2KB)\npost_final_review.py (6.4KB)\npost_review.py (4.6KB)\npost_review2.py (4.8KB)\npost_review3.py (2.2KB)\npr-35.diff (137.4KB)\npr115.diff (1.6MB)\npr126.diff (336.9KB)\npr14_full_diff.txt (193.4KB)\npr39_files.txt (243B)\npr51_files.txt (548B)\npr93_diff.txt (160.4KB)\nprs.json (4.6KB)\nresponse_cache.rs (15.1KB)\nrun_hard_proof_search.sh (545B)\nrun_hard_proof_search2.sh (515B)\nrun_proof_search_v3.sh (523B)\nrun_proof_search_v4.sh (524B)\nservice.rs (11.9KB)\nsettings.rs (34.7KB)\ntest_line.py (789B)\ntest_search.txt (1.8KB)\nthread_ops.rs (38.5KB)\ntool_wit.txt (6.0KB)\ntunnel_cf.rs (3.9KB)\ntunnel_custom.rs (6.2KB)\ntunnel_mod.rs (10.4KB)\ntunnel_ngrok.rs (4.2KB)\ntunnel_none.rs (1.3KB)\ntunnel_ts.rs (3.5KB)\nwasm_capabilities.rs (15.2KB)\nwasm_capabilities_schema.rs (26.2KB)\nwasm_host.rs (25.3KB)\nwasm_wrapper.rs (14.9KB)\n```","started_at":"2026-02-17T22:02:24.602248Z","finished_at":"2026-02-17T22:02:43.748463Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-write-read","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5712,"llm_calls":3,"input_tokens":4489,"output_tokens":73,"estimated_cost_usd":0.014562,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It says: `ironclaw spot check`.","started_at":"2026-02-17T22:02:43.749933Z","finished_at":"2026-02-17T22:02:49.462007Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-shell-json","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4582,"llm_calls":3,"input_tokens":4433,"output_tokens":94,"estimated_cost_usd":0.014709,"tool_calls":[{"name":"shell","duration_ms":0,"success":true},{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"[Called tool `json` with arguments: {\"data\":{\"code\":200,\"status\":\"ok\"},\"operation\":\"query\",\"path\":\"status\"}]","started_at":"2026-02-17T22:02:49.463794Z","finished_at":"2026-02-17T22:02:54.046092Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-time-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4583,"llm_calls":3,"input_tokens":4404,"output_tokens":76,"estimated_cost_usd":0.014352,"tool_calls":[{"name":"time","duration_ms":0,"success":true},{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"2026-02-17T22:02:55.786456+00:00","started_at":"2026-02-17T22:02:54.046306Z","finished_at":"2026-02-17T22:02:58.629389Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-no-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":1020,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Paris.","started_at":"2026-02-17T22:02:58.630502Z","finished_at":"2026-02-17T22:02:59.650655Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-correct-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3980,"llm_calls":2,"input_tokens":2825,"output_tokens":50,"estimated_cost_usd":0.009225,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It’s **22:03:01 UTC** right now (**2026-02-17**).","started_at":"2026-02-17T22:02:59.650901Z","finished_at":"2026-02-17T22:03:03.631122Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-json-validate","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2954,"llm_calls":2,"input_tokens":2822,"output_tokens":43,"estimated_cost_usd":0.009111,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The JSON is valid (`valid: true`).","started_at":"2026-02-17T22:03:03.631527Z","finished_at":"2026-02-17T22:03:06.585971Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-daily","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4890,"llm_calls":3,"input_tokens":4547,"output_tokens":114,"estimated_cost_usd":0.015351,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```md\n# Daily Tasks\n\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\n```","started_at":"2026-02-17T22:03:06.586556Z","finished_at":"2026-02-17T22:03:11.476648Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-reminder","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5607,"llm_calls":3,"input_tokens":4531,"output_tokens":95,"estimated_cost_usd":0.015018,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The appointment is on **March 5th at 2pm**, with **Dr. Smith**.","started_at":"2026-02-17T22:03:11.476873Z","finished_at":"2026-02-17T22:03:17.084343Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-meeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6113,"llm_calls":3,"input_tokens":4750,"output_tokens":122,"estimated_cost_usd":0.01608,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Frontend owner: **Bob** \nLaunch date: **April 15th**","started_at":"2026-02-17T22:03:17.085432Z","finished_at":"2026-02-17T22:03:23.198905Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-append-log","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9880,"llm_calls":6,"input_tokens":9941,"output_tokens":208,"estimated_cost_usd":0.032943,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"shell","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Contents of `/tmp/bench-log.md`:\n1. Morning standup: discussed blockers on auth service \n2. Afternoon: resolved API timeout issue in payment gateway \n\nSummary: You discussed blockers affecting the auth service during the morning standup, and later in the afternoon you resolved an API timeout issue in the payment gateway.","started_at":"2026-02-17T22:03:23.199155Z","finished_at":"2026-02-17T22:03:33.079396Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-recall-detail","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4989,"llm_calls":3,"input_tokens":4721,"output_tokens":121,"estimated_cost_usd":0.015978,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The project uses **PostgreSQL** as the database, and the PM is **James**.","started_at":"2026-02-17T22:03:33.080611Z","finished_at":"2026-02-17T22:03:38.069896Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-todo-priority","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5916,"llm_calls":3,"input_tokens":4676,"output_tokens":110,"estimated_cost_usd":0.015678,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"High priority tasks:\n- Fix login bug\n- Security audit","started_at":"2026-02-17T22:03:38.070162Z","finished_at":"2026-02-17T22:03:43.986831Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-multifile","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6081,"llm_calls":4,"input_tokens":6442,"output_tokens":196,"estimated_cost_usd":0.022266,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"On Tuesday you have **Sprint retro at 11am**, then **team lunch at noon**.","started_at":"2026-02-17T22:03:43.988013Z","finished_at":"2026-02-17T22:03:50.069452Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-update-context","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9164,"llm_calls":5,"input_tokens":8354,"output_tokens":252,"estimated_cost_usd":0.028842,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"apply_patch","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Final read confirms the file now says `timezone: EST`.","started_at":"2026-02-17T22:03:50.070277Z","finished_at":"2026-02-17T22:03:59.235196Z","config_label":"default","error":null}
|
||||
@@ -1,21 +0,0 @@
|
||||
{"id": "smoke-greeting", "prompt": "Hello! Introduce yourself briefly.", "tags": ["smoke"], "assertions": {"response_matches": "(?i)(hello|hi|hey|assistant|agent|help)", "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "smoke-math", "prompt": "What is 47 * 23? Reply with just the number.", "tags": ["smoke"], "assertions": {"response_contains": ["1081"], "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "tool-echo", "prompt": "Use the echo tool to repeat the message: 'Spot check passed'", "tags": ["tool"], "assertions": {"tools_used": ["echo"], "response_contains": ["Spot check passed"], "no_error": true}}
|
||||
{"id": "tool-time", "prompt": "What is the current date and time? Use the time tool.", "tags": ["tool"], "assertions": {"tools_used": ["time"], "response_matches": "20\\d{2}", "no_error": true}}
|
||||
{"id": "tool-json-query", "prompt": "Given this JSON: {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}, use the json tool to extract the second user's name.", "tags": ["tool"], "assertions": {"tools_used": ["json"], "response_contains": ["Bob"], "no_error": true}}
|
||||
{"id": "tool-shell-echo", "prompt": "Use the shell tool to run: echo 'benchmark test'", "tags": ["tool"], "assertions": {"tools_used": ["shell"], "response_contains": ["benchmark test"], "no_error": true}}
|
||||
{"id": "tool-list-dir", "prompt": "Use the list_dir tool to list the contents of the /tmp directory.", "tags": ["tool"], "assertions": {"tools_used": ["list_dir"], "no_error": true}}
|
||||
{"id": "chain-write-read", "prompt": "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt using the write_file tool, then read it back using the read_file tool and tell me what it says.", "tags": ["chain"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["ironclaw spot check"], "no_error": true}}
|
||||
{"id": "chain-shell-json", "prompt": "Run a shell command to output the JSON string '{\"status\": \"ok\", \"code\": 200}', then use the json tool to extract the status field.", "tags": ["chain"], "assertions": {"response_contains": ["ok"], "min_tool_calls": 1, "no_error": true}}
|
||||
{"id": "chain-time-echo", "prompt": "First get the current time using the time tool, then use the echo tool to repeat it back.", "tags": ["chain"], "assertions": {"tools_used": ["time", "echo"], "no_error": true}}
|
||||
{"id": "robust-no-tool", "prompt": "What is the capital of France? Answer directly without using any tools.", "tags": ["robust"], "assertions": {"response_contains": ["Paris"], "max_tool_calls": 0, "no_error": true}}
|
||||
{"id": "robust-correct-tool", "prompt": "What time is it right now?", "tags": ["robust"], "assertions": {"tools_used": ["time"], "tools_not_used": ["shell", "echo"], "no_error": true}}
|
||||
{"id": "robust-json-validate", "prompt": "Use the json tool to validate whether this is valid JSON: {\"key\": \"value\", \"num\": 42}", "tags": ["robust"], "assertions": {"tools_used": ["json"], "tools_not_used": ["shell"], "no_error": true}}
|
||||
{"id": "memory-save-daily", "prompt": "Save these daily tasks to /tmp/bench-daily.md:\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\nThen read the file back and confirm what was saved.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PR #42", "docs", "staging"], "no_error": true}}
|
||||
{"id": "memory-save-reminder", "prompt": "Write a reminder to /tmp/bench-reminder.md: Dentist appointment on March 5th at 2pm with Dr. Smith. Then read the file back and tell me when the appointment is and with whom.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["March 5", "Smith"], "response_matches": "2(:00)?\\s*[Pp][Mm]", "no_error": true}}
|
||||
{"id": "memory-save-meeting", "prompt": "Save these meeting notes to /tmp/bench-meeting.md:\nMeeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\nDecisions:\n- Launch date: April 15th\n- Budget: $50k approved\n- Bob owns frontend, Carol owns backend\nThen read the file back and tell me who owns the frontend and what the launch date is.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["Bob", "frontend", "April 15"], "no_error": true}}
|
||||
{"id": "memory-append-log", "prompt": "Write 'Morning standup: discussed blockers on auth service' to /tmp/bench-log.md. Then append a new line 'Afternoon: resolved API timeout issue in payment gateway' to the same file. Finally read the full file and summarize what happened.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["auth", "timeout"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-recall-detail", "prompt": "Save the following project context to /tmp/bench-project.md:\nProject Ironclad uses Rust for the backend, React for the frontend, and PostgreSQL for the database. The API is deployed on AWS ECS. The lead developer is Sarah and the PM is James. The sprint ends on March 20th.\nThen read it back and answer: What database does the project use, and who is the PM?", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PostgreSQL", "James"], "no_error": true}}
|
||||
{"id": "memory-todo-priority", "prompt": "Write the following to /tmp/bench-todo.md:\n- [ ] Fix login bug (priority: HIGH)\n- [ ] Write unit tests (priority: medium)\n- [ ] Update README (priority: low)\n- [ ] Security audit (priority: HIGH)\nThen read it back and tell me which tasks are high priority.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["login bug", "security audit"], "no_error": true}}
|
||||
{"id": "memory-multifile", "prompt": "Save 'Team standup at 9am, then client demo at 2pm' to /tmp/bench-monday.md and 'Sprint retro at 11am, team lunch at noon' to /tmp/bench-tuesday.md. Then read both files and tell me what's happening on Tuesday.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["retro", "lunch"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-update-context", "prompt": "Write 'User preference: dark mode, timezone: PST, language: English' to /tmp/bench-prefs.md. Then read it back, and rewrite the file changing the timezone to EST. Finally read it one more time and confirm the timezone is now EST.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["EST"], "min_tool_calls": 4, "no_error": true}}
|
||||
@@ -1,8 +0,0 @@
|
||||
task_timeout = "120s"
|
||||
parallelism = 1
|
||||
|
||||
[[matrix]]
|
||||
label = "default"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "benchmarks/data/spot.jsonl"
|
||||
@@ -1,243 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// A single entry in the custom JSONL format.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CustomEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
expected: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_contains: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_regex: Option<String>,
|
||||
/// "exact", "contains", "regex", or "llm" (default: "exact")
|
||||
#[serde(default = "default_scorer")]
|
||||
scorer: String,
|
||||
}
|
||||
|
||||
fn default_scorer() -> String {
|
||||
"exact".to_string()
|
||||
}
|
||||
|
||||
/// Custom JSONL benchmark suite.
|
||||
///
|
||||
/// Each line of the JSONL file is a task with `id`, `prompt`, and scoring
|
||||
/// criteria (`expected`, `expected_contains`, `expected_regex`).
|
||||
pub struct CustomSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl CustomSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for CustomSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Custom JSONL"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"custom"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: CustomEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut metadata = serde_json::json!({
|
||||
"scorer": entry.scorer,
|
||||
});
|
||||
if let Some(ref expected) = entry.expected {
|
||||
metadata["expected"] = serde_json::Value::String(expected.clone());
|
||||
}
|
||||
if let Some(ref expected_contains) = entry.expected_contains {
|
||||
metadata["expected_contains"] =
|
||||
serde_json::Value::String(expected_contains.clone());
|
||||
}
|
||||
if let Some(ref expected_regex) = entry.expected_regex {
|
||||
metadata["expected_regex"] = serde_json::Value::String(expected_regex.clone());
|
||||
}
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let scorer = task
|
||||
.metadata
|
||||
.get("scorer")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("exact");
|
||||
|
||||
match scorer {
|
||||
"exact" => {
|
||||
if let Some(expected) = task.metadata.get("expected").and_then(|v| v.as_str()) {
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected' field for exact scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"contains" => {
|
||||
if let Some(expected) = task
|
||||
.metadata
|
||||
.get("expected_contains")
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::contains_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_contains' field for contains scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"regex" => {
|
||||
if let Some(pattern) = task.metadata.get("expected_regex").and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::regex_match(pattern, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_regex' field for regex scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"llm" => {
|
||||
// TODO: LLM-as-judge scoring
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"LLM-as-judge scoring not implemented, returning placeholder 0.5"
|
||||
);
|
||||
Ok(BenchScore::partial(0.5, "LLM scoring not yet implemented"))
|
||||
}
|
||||
other => Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("unknown scorer: {other}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t2", "prompt": "Say hello", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "t1");
|
||||
assert_eq!(tasks[1].id, "t2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_exact_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "4".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_contains_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "Greet me", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "Hello there!".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskResource, TaskSubmission};
|
||||
|
||||
/// GAIA dataset entry (Hugging Face JSONL format).
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GaiaEntry {
|
||||
task_id: String,
|
||||
#[serde(alias = "Question")]
|
||||
question: String,
|
||||
#[serde(alias = "Final answer", alias = "final_answer")]
|
||||
final_answer: String,
|
||||
#[serde(alias = "Level", default)]
|
||||
level: Option<u32>,
|
||||
#[serde(alias = "file_name", default)]
|
||||
file_name: Option<String>,
|
||||
}
|
||||
|
||||
/// GAIA benchmark suite.
|
||||
///
|
||||
/// Tasks are loaded from HuggingFace JSONL exports. Scoring uses normalized
|
||||
/// exact match against the `final_answer` field.
|
||||
pub struct GaiaSuite {
|
||||
dataset_path: PathBuf,
|
||||
attachments_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl GaiaSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
attachments_dir: Option<impl Into<PathBuf>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
attachments_dir: attachments_dir.map(|d| d.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for GaiaSuite {
|
||||
fn name(&self) -> &str {
|
||||
"GAIA"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"gaia"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: GaiaEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("GAIA line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut resources = Vec::new();
|
||||
if let Some(ref file_name) = entry.file_name {
|
||||
if !file_name.is_empty() {
|
||||
if let Some(ref dir) = self.attachments_dir {
|
||||
resources.push(TaskResource {
|
||||
name: file_name.clone(),
|
||||
path: dir.join(file_name).to_string_lossy().to_string(),
|
||||
resource_type: crate::suite::ResourceType::File,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut tags = Vec::new();
|
||||
if let Some(level) = entry.level {
|
||||
tags.push(format!("level-{level}"));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"expected": entry.final_answer,
|
||||
"level": entry.level,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.task_id,
|
||||
prompt: entry.question,
|
||||
context: None,
|
||||
resources,
|
||||
tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let expected = task
|
||||
.metadata
|
||||
.get("expected")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing expected answer in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "What is the capital of France?", "final_answer": "Paris", "Level": 1}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "g1");
|
||||
assert!(tasks[0].tags.contains(&"level-1".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "Capital of France?", "final_answer": "Paris"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Exact match (case insensitive)
|
||||
let submission = TaskSubmission {
|
||||
response: "paris".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
|
||||
// Wrong answer
|
||||
let submission = TaskSubmission {
|
||||
response: "London".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
pub mod custom;
|
||||
pub mod gaia;
|
||||
pub mod spot;
|
||||
pub mod swe_bench;
|
||||
pub mod tau_bench;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchSuite;
|
||||
|
||||
/// List of all known suite IDs.
|
||||
pub const KNOWN_SUITES: &[(&str, &str)] = &[
|
||||
("custom", "Custom JSONL tasks"),
|
||||
("gaia", "GAIA benchmark (knowledge & reasoning)"),
|
||||
("spot", "Spot checks (end-to-end user workflows)"),
|
||||
("tau_bench", "Tau-bench (multi-turn tool use)"),
|
||||
("swe_bench", "SWE-bench Pro (software engineering)"),
|
||||
];
|
||||
|
||||
/// Create a suite adapter by name.
|
||||
pub fn create_suite(name: &str, config: &BenchConfig) -> Result<Box<dyn BenchSuite>, BenchError> {
|
||||
let suite_map = config.suite_config_map();
|
||||
match name {
|
||||
"custom" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'custom' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(custom::CustomSuite::new(dataset_path)))
|
||||
}
|
||||
"gaia" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'gaia' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let attachments_dir = suite_map
|
||||
.get("attachments_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
Ok(Box::new(gaia::GaiaSuite::new(
|
||||
dataset_path,
|
||||
attachments_dir,
|
||||
)))
|
||||
}
|
||||
"spot" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'spot' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(spot::SpotSuite::new(dataset_path)))
|
||||
}
|
||||
"tau_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'tau_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let domain = suite_map
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("retail")
|
||||
.to_string();
|
||||
Ok(Box::new(tau_bench::TauBenchSuite::new(
|
||||
dataset_path,
|
||||
domain,
|
||||
)))
|
||||
}
|
||||
"swe_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'swe_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let workspace_dir = suite_map
|
||||
.get("workspace_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("/tmp/swe-bench")
|
||||
.to_string();
|
||||
let use_docker = suite_map
|
||||
.get("use_docker")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
Ok(Box::new(swe_bench::SweBenchSuite::new(
|
||||
dataset_path,
|
||||
workspace_dir,
|
||||
use_docker,
|
||||
)))
|
||||
}
|
||||
_ => {
|
||||
let available = KNOWN_SUITES
|
||||
.iter()
|
||||
.map(|(id, _)| *id)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
Err(BenchError::SuiteNotFound {
|
||||
name: name.to_string(),
|
||||
available,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,504 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Multi-criterion assertions for a spot check scenario.
|
||||
///
|
||||
/// Each field generates one or more individual checks. The final score is
|
||||
/// `passed_checks / total_checks`, giving a value between 0.0 and 1.0.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct SpotAssertions {
|
||||
/// All must appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_contains: Vec<String>,
|
||||
|
||||
/// None may appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_not_contains: Vec<String>,
|
||||
|
||||
/// Each tool name must appear in the tool_calls list (checked by name,
|
||||
/// not by count; duplicates in tool_calls are collapsed).
|
||||
#[serde(default)]
|
||||
pub tools_used: Vec<String>,
|
||||
|
||||
/// None of these tool names may appear in the tool_calls list.
|
||||
#[serde(default)]
|
||||
pub tools_not_used: Vec<String>,
|
||||
|
||||
/// Regex pattern the response must match.
|
||||
#[serde(default)]
|
||||
pub response_matches: Option<String>,
|
||||
|
||||
/// Hard fail if the task produced an error.
|
||||
#[serde(default)]
|
||||
pub no_error: bool,
|
||||
|
||||
/// Minimum number of tool calls expected (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub min_tool_calls: Option<usize>,
|
||||
|
||||
/// Maximum number of tool calls allowed (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub max_tool_calls: Option<usize>,
|
||||
}
|
||||
|
||||
impl SpotAssertions {
|
||||
/// Evaluate all assertions against a submission, returning (score, failure_details).
|
||||
pub fn evaluate(&self, submission: &TaskSubmission) -> (f64, Vec<String>) {
|
||||
let mut passed: usize = 0;
|
||||
let mut total: usize = 0;
|
||||
let mut failures: Vec<String> = Vec::new();
|
||||
|
||||
// Hard fail: error check
|
||||
if self.no_error {
|
||||
total += 1;
|
||||
if let Some(ref err) = submission.error {
|
||||
failures.push(format!("no_error: task errored with: {err}"));
|
||||
// Hard fail: return 0.0 immediately
|
||||
return (0.0, failures);
|
||||
}
|
||||
passed += 1;
|
||||
}
|
||||
|
||||
let response_lower = submission.response.to_lowercase();
|
||||
|
||||
// response_contains: all must appear
|
||||
for needle in &self.response_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_contains: missing \"{needle}\""));
|
||||
}
|
||||
}
|
||||
|
||||
// response_not_contains: none may appear
|
||||
for needle in &self.response_not_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
failures.push(format!("response_not_contains: found \"{needle}\""));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let tool_set: HashSet<&str> = submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
// tools_used: each must appear
|
||||
for tool in &self.tools_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("tools_used: \"{tool}\" not called"));
|
||||
}
|
||||
}
|
||||
|
||||
// tools_not_used: none may appear
|
||||
for tool in &self.tools_not_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
failures.push(format!("tools_not_used: \"{tool}\" was called"));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// response_matches: regex pattern
|
||||
if let Some(ref pattern) = self.response_matches {
|
||||
total += 1;
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(&submission.response) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_matches: /{pattern}/ did not match"));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
failures.push(format!("response_matches: bad regex: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let call_count = submission.tool_calls.len();
|
||||
|
||||
// min_tool_calls
|
||||
if let Some(min) = self.min_tool_calls {
|
||||
total += 1;
|
||||
if call_count >= min {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"min_tool_calls: expected >= {min}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// max_tool_calls
|
||||
if let Some(max) = self.max_tool_calls {
|
||||
total += 1;
|
||||
if call_count <= max {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"max_tool_calls: expected <= {max}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
return (1.0, failures);
|
||||
}
|
||||
|
||||
let score = passed as f64 / total as f64;
|
||||
(score, failures)
|
||||
}
|
||||
}
|
||||
|
||||
/// JSONL entry for a spot check scenario.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SpotEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
assertions: SpotAssertions,
|
||||
}
|
||||
|
||||
/// Spot benchmark suite: end-to-end checks for real user workflows.
|
||||
///
|
||||
/// Tests conversation, individual tool use, multi-tool chaining, and robustness.
|
||||
/// Each task declares multi-criterion assertions scored as passed/total.
|
||||
pub struct SpotSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl SpotSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SpotSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Spot Checks"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"spot"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SpotEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("spot line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"assertions": serde_json::to_value(&entry.assertions)
|
||||
.map_err(|e| BenchError::Config(format!("spot {}: {}", entry.id, e)))?,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let assertions: SpotAssertions = task
|
||||
.metadata
|
||||
.get("assertions")
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing assertions in metadata".to_string(),
|
||||
})
|
||||
.and_then(|v| {
|
||||
serde_json::from_value(v.clone()).map_err(|e| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("bad assertions: {e}"),
|
||||
})
|
||||
})?;
|
||||
|
||||
let (score, failures) = assertions.evaluate(submission);
|
||||
|
||||
if score >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if score <= 0.0 {
|
||||
Ok(BenchScore::fail(failures.join("; ")))
|
||||
} else {
|
||||
Ok(BenchScore::partial(score, failures.join("; ")))
|
||||
}
|
||||
}
|
||||
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![
|
||||
Arc::new(ironclaw::tools::builtin::ShellTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ReadFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::WriteFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ListDirTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ApplyPatchTool::new()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
fn make_submission(
|
||||
response: &str,
|
||||
tool_calls: Vec<&str>,
|
||||
error: Option<&str>,
|
||||
) -> TaskSubmission {
|
||||
TaskSubmission {
|
||||
response: response.to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: tool_calls.into_iter().map(|s| s.to_string()).collect(),
|
||||
error: error.map(|s| s.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_all_pass() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["hello".to_string()],
|
||||
tools_used: vec!["echo".to_string()],
|
||||
no_error: true,
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello, world!", vec!["echo"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hard_fail_on_error() {
|
||||
let assertions = SpotAssertions {
|
||||
no_error: true,
|
||||
response_contains: vec!["hello".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello!", vec![], Some("timeout after 60s"));
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
assert!(failures[0].contains("no_error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_score() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["alpha".to_string(), "beta".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("alpha is here but not the other", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("beta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_not_contains() {
|
||||
let assertions = SpotAssertions {
|
||||
response_not_contains: vec!["error".to_string(), "fail".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("This is an error message", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_used_and_not_used() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_used: vec!["time".to_string()],
|
||||
tools_not_used: vec!["shell".to_string(), "echo".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The time is now", vec!["time"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_not_used_fails() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_not_used: vec!["shell".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("result", vec!["shell", "time"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"\d{4}".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The year is 2026", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex_fail() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"^\d+$".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("not a number", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_min_max_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
min_tool_calls: Some(2),
|
||||
max_tool_calls: Some(4),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Within range
|
||||
let sub = make_submission("ok", vec!["a", "b", "c"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
// Too few
|
||||
let sub = make_submission("ok", vec!["a"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("min_tool_calls"));
|
||||
|
||||
// Too many
|
||||
let sub = make_submission("ok", vec!["a", "b", "c", "d", "e"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("max_tool_calls"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_zero_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
max_tool_calls: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("just talking", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
let sub = make_submission("oops", vec!["echo"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_assertions() {
|
||||
let assertions = SpotAssertions::default();
|
||||
let sub = make_submission("anything", vec!["whatever"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "tags": ["smoke"], "assertions": {{"response_contains": ["hello"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s2", "prompt": "Echo test", "assertions": {{"tools_used": ["echo"]}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "s1");
|
||||
assert_eq!(tasks[1].id, "s2");
|
||||
assert!(tasks[0].tags.contains(&"smoke".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "assertions": {{"response_contains": ["hello", "world"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Full pass
|
||||
let sub = make_submission("Hello World!", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
|
||||
// Partial
|
||||
let sub = make_submission("Hello there", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert!(score.value > 0.0 && score.value < 1.0);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Error hard fail
|
||||
let sub = make_submission("Hello World!", vec![], Some("boom"));
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
}
|
||||
@@ -1,416 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Validate that a string is safe for use as a filesystem path component.
|
||||
/// Allows alphanumerics, hyphens, underscores, dots, and forward slashes (for nested paths).
|
||||
/// Rejects absolute paths, `..` traversal, and shell metacharacters.
|
||||
fn is_safe_path_component(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& !s.starts_with('/')
|
||||
&& !s.contains("..")
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
}
|
||||
|
||||
/// Validate that a repo string matches the expected `owner/repo` GitHub format.
|
||||
fn is_valid_github_repo(repo: &str) -> bool {
|
||||
// Match "owner/repo" where both parts are alphanumeric with hyphens/underscores/dots
|
||||
static REPO_PATTERN: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$").unwrap());
|
||||
REPO_PATTERN.is_match(repo)
|
||||
}
|
||||
|
||||
/// Validate that a string looks like a git ref (hex SHA or valid ref name).
|
||||
fn is_valid_git_ref(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
&& !s.contains("..")
|
||||
}
|
||||
|
||||
/// SWE-bench dataset entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SweBenchEntry {
|
||||
instance_id: String,
|
||||
repo: String,
|
||||
base_commit: String,
|
||||
#[serde(default)]
|
||||
problem_statement: String,
|
||||
#[serde(default)]
|
||||
hints_text: Option<String>,
|
||||
#[serde(default)]
|
||||
test_patch: Option<String>,
|
||||
#[serde(default)]
|
||||
patch: Option<String>,
|
||||
}
|
||||
|
||||
/// SWE-bench Pro: real-world software engineering tasks.
|
||||
///
|
||||
/// Each task clones a repo at a specific commit, presents the problem statement,
|
||||
/// and expects the agent to produce a patch. Scoring runs the test suite.
|
||||
pub struct SweBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
workspace_dir: PathBuf,
|
||||
use_docker: bool,
|
||||
}
|
||||
|
||||
impl SweBenchSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
workspace_dir: impl Into<PathBuf>,
|
||||
use_docker: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
workspace_dir: workspace_dir.into(),
|
||||
use_docker,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SweBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"SWE-bench Pro"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"swe_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SweBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("swe_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
if !is_safe_path_component(&entry.instance_id) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: unsafe instance_id \"{}\"",
|
||||
line_num + 1,
|
||||
entry.instance_id,
|
||||
)));
|
||||
}
|
||||
if !is_valid_github_repo(&entry.repo) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid repo format \"{}\"",
|
||||
line_num + 1,
|
||||
entry.repo,
|
||||
)));
|
||||
}
|
||||
if !is_valid_git_ref(&entry.base_commit) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid base_commit \"{}\"",
|
||||
line_num + 1,
|
||||
entry.base_commit,
|
||||
)));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"repo": entry.repo,
|
||||
"base_commit": entry.base_commit,
|
||||
"test_patch": entry.test_patch,
|
||||
"gold_patch": entry.patch,
|
||||
"use_docker": self.use_docker,
|
||||
"workspace_dir": self.workspace_dir.to_string_lossy(),
|
||||
});
|
||||
|
||||
let prompt = if let Some(ref hints) = entry.hints_text {
|
||||
format!("{}\n\nHints:\n{}", entry.problem_statement, hints)
|
||||
} else {
|
||||
entry.problem_statement
|
||||
};
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.instance_id,
|
||||
prompt,
|
||||
context: Some(format!(
|
||||
"Repository: {}, Commit: {}",
|
||||
entry.repo, entry.base_commit
|
||||
)),
|
||||
resources: vec![],
|
||||
tags: vec![format!("repo-{}", entry.repo.replace('/', "-"))],
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn setup_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let repo = task
|
||||
.metadata
|
||||
.get("repo")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing repo in metadata".to_string(),
|
||||
})?;
|
||||
let base_commit = task
|
||||
.metadata
|
||||
.get("base_commit")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing base_commit in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
|
||||
// Clone repo if not already present
|
||||
if !task_dir.exists() {
|
||||
let repo_url = format!("https://github.com/{}.git", repo);
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
&repo_url,
|
||||
&task_dir.to_string_lossy(),
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Checkout the base commit
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
// Shallow clone might not have the commit; fetch more history
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["fetch", "--unshallow"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout retry failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn teardown_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
if task_dir.exists() {
|
||||
// Reset any changes
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["checkout", "."])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["clean", "-fdx"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// For SWE-bench, scoring requires running the test patch against the agent's changes.
|
||||
// This is a simplified version that checks if the agent produced any code changes.
|
||||
|
||||
let test_patch = task.metadata.get("test_patch").and_then(|v| v.as_str());
|
||||
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response from agent"));
|
||||
}
|
||||
|
||||
// If we have a test patch, try to verify the submission
|
||||
if let Some(_test_patch) = test_patch {
|
||||
// TODO: Apply agent's patch, then apply test patch, then run tests.
|
||||
// For now, give partial credit if the agent produced some output.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"SWE-bench test execution not implemented, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"test execution not yet implemented; partial credit for response",
|
||||
))
|
||||
} else {
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"no test_patch available, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"no test_patch available for automated scoring",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "django__django-12345", "repo": "django/django", "base_commit": "abc123", "problem_statement": "Fix the ORM bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "django__django-12345");
|
||||
assert!(tasks[0].tags.contains(&"repo-django-django".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_scoring_no_response() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "s1", "repo": "org/repo", "base_commit": "abc", "problem_statement": "Fix bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: String::new(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_safe_path_component() {
|
||||
assert!(is_safe_path_component("django__django-12345"));
|
||||
assert!(is_safe_path_component("org/repo"));
|
||||
assert!(is_safe_path_component("abc123"));
|
||||
assert!(!is_safe_path_component(""));
|
||||
assert!(!is_safe_path_component("../../etc/passwd"));
|
||||
assert!(!is_safe_path_component("/etc/passwd"));
|
||||
assert!(!is_safe_path_component("foo;rm -rf /"));
|
||||
assert!(!is_safe_path_component("foo bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_github_repo() {
|
||||
assert!(is_valid_github_repo("django/django"));
|
||||
assert!(is_valid_github_repo("org/repo-name"));
|
||||
assert!(is_valid_github_repo("Org.Name/Repo_v2"));
|
||||
assert!(!is_valid_github_repo(""));
|
||||
assert!(!is_valid_github_repo("no-slash"));
|
||||
assert!(!is_valid_github_repo("too/many/slashes"));
|
||||
assert!(!is_valid_github_repo("spa ce/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_git_ref() {
|
||||
assert!(is_valid_git_ref("abc123"));
|
||||
assert!(is_valid_git_ref("deadbeef0123456789abcdef0123456789abcdef"));
|
||||
assert!(is_valid_git_ref("v1.2.3"));
|
||||
assert!(is_valid_git_ref("main"));
|
||||
assert!(!is_valid_git_ref(""));
|
||||
assert!(!is_valid_git_ref("bad..ref"));
|
||||
assert!(!is_valid_git_ref("has space"));
|
||||
assert!(!is_valid_git_ref("semi;colon"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_path_traversal() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "../../etc/passwd", "repo": "org/repo", "base_commit": "abc", "problem_statement": "evil"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("unsafe instance_id"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_bad_repo() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "task1", "repo": "not-a-repo-format", "base_commit": "abc", "problem_statement": "bad"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("invalid repo format"));
|
||||
}
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, ConversationTurn, TaskSubmission};
|
||||
|
||||
/// Tau-bench task entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TauBenchEntry {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
domain: String,
|
||||
instruction: String,
|
||||
#[serde(default)]
|
||||
user_persona: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_state: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
expected_actions: Vec<String>,
|
||||
#[serde(default)]
|
||||
max_turns: Option<usize>,
|
||||
}
|
||||
|
||||
/// Tau-bench: multi-turn tool-calling dialog benchmark.
|
||||
///
|
||||
/// Tests agent ability to handle customer service scenarios with simulated
|
||||
/// domain APIs (retail, airline). Scoring compares final state against expected.
|
||||
pub struct TauBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
domain: String,
|
||||
}
|
||||
|
||||
impl TauBenchSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>, domain: impl Into<String>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
domain: domain.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for TauBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Tau-bench"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"tau_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: TauBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("tau_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
let domain = if entry.domain.is_empty() {
|
||||
self.domain.clone()
|
||||
} else {
|
||||
entry.domain.clone()
|
||||
};
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"domain": domain,
|
||||
"user_persona": entry.user_persona,
|
||||
"expected_state": entry.expected_state,
|
||||
"expected_actions": entry.expected_actions,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.instruction,
|
||||
context: entry.user_persona.clone(),
|
||||
resources: vec![],
|
||||
tags: vec![format!("domain-{domain}")],
|
||||
expected_turns: entry.max_turns,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// Score based on expected actions completion
|
||||
let expected_actions: Vec<String> = task
|
||||
.metadata
|
||||
.get("expected_actions")
|
||||
.and_then(|v| serde_json::from_value(v.clone()).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if expected_actions.is_empty() {
|
||||
// No expected actions defined; score based on whether agent responded
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response"));
|
||||
}
|
||||
return Ok(BenchScore::partial(
|
||||
0.5,
|
||||
"no expected_actions to evaluate against",
|
||||
));
|
||||
}
|
||||
|
||||
// Check which expected actions were actually called
|
||||
let called: std::collections::HashSet<&str> =
|
||||
submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
let matched = expected_actions
|
||||
.iter()
|
||||
.filter(|a| called.contains(a.as_str()))
|
||||
.count();
|
||||
|
||||
let ratio = matched as f64 / expected_actions.len() as f64;
|
||||
if ratio >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if ratio > 0.0 {
|
||||
Ok(BenchScore::partial(
|
||||
ratio,
|
||||
format!(
|
||||
"{}/{} expected actions completed",
|
||||
matched,
|
||||
expected_actions.len()
|
||||
),
|
||||
))
|
||||
} else {
|
||||
Ok(BenchScore::fail(format!(
|
||||
"0/{} expected actions completed",
|
||||
expected_actions.len()
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
// Check if we've exceeded max turns
|
||||
if let Some(max) = task.expected_turns {
|
||||
let user_turns = conversation
|
||||
.iter()
|
||||
.filter(|t| matches!(t.role, crate::suite::TurnRole::User))
|
||||
.count();
|
||||
if user_turns >= max {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-turn simulation requires an LLM to play the customer role.
|
||||
// Until that's implemented, every scenario is single-turn only.
|
||||
// TODO: Use LLM to simulate customer based on user_persona.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"multi-turn simulation not implemented, ending after first turn"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return my order", "expected_actions": ["lookup_order", "process_return"], "max_turns": 3}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].expected_turns, Some(3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return order", "expected_actions": ["lookup_order", "process_return"]}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Partial completion
|
||||
let submission = TaskSubmission {
|
||||
response: "I found your order.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.5);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Full completion
|
||||
let submission = TaskSubmission {
|
||||
response: "Return processed.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string(), "process_return".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use ironclaw::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
use ironclaw::error::ChannelError;
|
||||
|
||||
use crate::results::TraceToolCall;
|
||||
use crate::suite::ConversationTurn;
|
||||
|
||||
/// Truncate a string to at most `max_bytes` without splitting a UTF-8 character.
|
||||
fn truncate_str(s: &str, max_bytes: usize) -> &str {
|
||||
if s.len() <= max_bytes {
|
||||
return s;
|
||||
}
|
||||
let mut end = max_bytes;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
}
|
||||
|
||||
/// Captured state from a benchmark channel run.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ChannelCapture {
|
||||
/// All responses the agent sent back.
|
||||
pub responses: Vec<String>,
|
||||
/// Tool calls observed (name, success, duration_ms).
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
/// Full conversation turns for multi-turn scoring.
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
/// Status messages (for debugging).
|
||||
pub status_log: Vec<String>,
|
||||
}
|
||||
|
||||
/// A headless Channel implementation for benchmarking.
|
||||
///
|
||||
/// Modeled after `ReplChannel`: uses mpsc to inject messages and captures
|
||||
/// all responses and tool status events. Auto-approves tool execution
|
||||
/// so benchmarks run without user interaction.
|
||||
pub struct BenchChannel {
|
||||
/// Sender to inject messages into the agent loop.
|
||||
msg_tx: mpsc::Sender<IncomingMessage>,
|
||||
/// Receiver the agent loop reads from (taken once by `start()`).
|
||||
msg_rx: Mutex<Option<mpsc::Receiver<IncomingMessage>>>,
|
||||
/// Accumulated capture data.
|
||||
capture: Arc<Mutex<ChannelCapture>>,
|
||||
}
|
||||
|
||||
impl BenchChannel {
|
||||
pub fn new() -> (Self, mpsc::Sender<IncomingMessage>) {
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
let channel = Self {
|
||||
msg_tx: tx.clone(),
|
||||
msg_rx: Mutex::new(Some(rx)),
|
||||
capture: Arc::new(Mutex::new(ChannelCapture::default())),
|
||||
};
|
||||
(channel, tx)
|
||||
}
|
||||
|
||||
/// Get a handle to the capture data.
|
||||
pub fn capture(&self) -> Arc<Mutex<ChannelCapture>> {
|
||||
Arc::clone(&self.capture)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Channel for BenchChannel {
|
||||
fn name(&self) -> &str {
|
||||
"bench"
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<MessageStream, ChannelError> {
|
||||
let rx = self
|
||||
.msg_rx
|
||||
.lock()
|
||||
.await
|
||||
.take()
|
||||
.ok_or_else(|| ChannelError::StartupFailed {
|
||||
name: "bench".to_string(),
|
||||
reason: "start() already called".to_string(),
|
||||
})?;
|
||||
Ok(Box::pin(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
async fn respond(
|
||||
&self,
|
||||
_msg: &IncomingMessage,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.responses.push(response.content.clone());
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: crate::suite::TurnRole::Assistant,
|
||||
content: response.content,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_status(
|
||||
&self,
|
||||
status: StatusUpdate,
|
||||
_metadata: &serde_json::Value,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
|
||||
match status {
|
||||
StatusUpdate::ToolCompleted { ref name, success } => {
|
||||
cap.tool_calls.push(TraceToolCall {
|
||||
name: name.clone(),
|
||||
duration_ms: 0, // We don't have precise per-tool timing here
|
||||
success,
|
||||
});
|
||||
cap.status_log
|
||||
.push(format!("tool_completed: {name} success={success}"));
|
||||
}
|
||||
StatusUpdate::ApprovalNeeded { ref request_id, .. } => {
|
||||
// Auto-approve all tools during benchmarks
|
||||
cap.status_log.push(format!("auto_approved: {request_id}"));
|
||||
drop(cap); // Release lock before sending
|
||||
let approval = IncomingMessage::new("bench", "bench-user", "always");
|
||||
let _ = self.msg_tx.send(approval).await;
|
||||
return Ok(());
|
||||
}
|
||||
StatusUpdate::Thinking(ref msg) => {
|
||||
cap.status_log.push(format!("thinking: {msg}"));
|
||||
}
|
||||
StatusUpdate::ToolStarted { ref name } => {
|
||||
cap.status_log.push(format!("tool_started: {name}"));
|
||||
}
|
||||
StatusUpdate::ToolResult {
|
||||
ref name,
|
||||
ref preview,
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"tool_result: {name} -> {}",
|
||||
truncate_str(preview, 100)
|
||||
));
|
||||
}
|
||||
StatusUpdate::StreamChunk(_) => {}
|
||||
StatusUpdate::Status(ref msg) => {
|
||||
cap.status_log.push(format!("status: {msg}"));
|
||||
}
|
||||
StatusUpdate::JobStarted {
|
||||
ref job_id,
|
||||
ref title,
|
||||
..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("job_started: {job_id} ({title})"));
|
||||
}
|
||||
StatusUpdate::AuthRequired {
|
||||
ref extension_name, ..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("auth_required: {extension_name} (auto-skipped)"));
|
||||
}
|
||||
StatusUpdate::AuthCompleted {
|
||||
ref extension_name,
|
||||
success,
|
||||
..
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"auth_completed: {extension_name} success={success}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn broadcast(
|
||||
&self,
|
||||
_user_id: &str,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.status_log.push(format!(
|
||||
"broadcast: {}",
|
||||
truncate_str(&response.content, 100)
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn shutdown(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_responses() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
let capture = channel.capture();
|
||||
|
||||
let msg = IncomingMessage::new("bench", "user", "hello");
|
||||
let response = OutgoingResponse::text("world");
|
||||
channel.respond(&msg, response).await.unwrap();
|
||||
|
||||
let cap = capture.lock().await;
|
||||
assert_eq!(cap.responses.len(), 1);
|
||||
assert_eq!(cap.responses[0], "world");
|
||||
assert_eq!(cap.conversation.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_auto_approves() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
// start() to consume the receiver
|
||||
let _stream = channel.start().await.unwrap();
|
||||
|
||||
let status = StatusUpdate::ApprovalNeeded {
|
||||
request_id: "req-1".to_string(),
|
||||
tool_name: "shell".to_string(),
|
||||
description: "run ls".to_string(),
|
||||
parameters: serde_json::json!({}),
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The approval message was sent through msg_tx,
|
||||
// which means the stream would receive it.
|
||||
// We can't easily read from the stream in this test without
|
||||
// consuming it, but we can verify the status log.
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert!(cap.status_log.iter().any(|s| s.contains("auto_approved")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_tool_events() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
|
||||
let status = StatusUpdate::ToolCompleted {
|
||||
name: "echo".to_string(),
|
||||
success: true,
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert_eq!(cap.tool_calls.len(), 1);
|
||||
assert_eq!(cap.tool_calls[0].name, "echo");
|
||||
assert!(cap.tool_calls[0].success);
|
||||
}
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// Top-level bench configuration, loaded from TOML.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BenchConfig {
|
||||
/// Where to write results. Default: "./bench-results".
|
||||
#[serde(default = "default_results_dir")]
|
||||
pub results_dir: PathBuf,
|
||||
|
||||
/// Per-task timeout. Default: "300s".
|
||||
#[serde(
|
||||
default = "default_task_timeout",
|
||||
deserialize_with = "deserialize_duration"
|
||||
)]
|
||||
pub task_timeout: Duration,
|
||||
|
||||
/// How many tasks to run in parallel. Default: 1.
|
||||
#[serde(default = "default_parallelism")]
|
||||
pub parallelism: usize,
|
||||
|
||||
/// Model/config matrix entries. At least one required.
|
||||
#[serde(default)]
|
||||
pub matrix: Vec<MatrixEntry>,
|
||||
|
||||
/// Suite-specific configuration (passed through to adapter).
|
||||
#[serde(default = "default_suite_config")]
|
||||
pub suite_config: toml::Value,
|
||||
}
|
||||
|
||||
/// A single model/config combination to benchmark.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MatrixEntry {
|
||||
/// Label for this configuration (used in results).
|
||||
pub label: String,
|
||||
|
||||
/// Model identifier.
|
||||
#[serde(default)]
|
||||
pub model: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchConfig {
|
||||
/// Load from a TOML file.
|
||||
pub fn from_file(path: &Path) -> Result<Self, BenchError> {
|
||||
if !path.exists() {
|
||||
return Err(BenchError::ConfigNotFound {
|
||||
path: path.to_path_buf(),
|
||||
});
|
||||
}
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let config: BenchConfig = toml::from_str(&content)?;
|
||||
if config.matrix.is_empty() {
|
||||
return Err(BenchError::Config(
|
||||
"config must have at least one [[matrix]] entry".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// Create a minimal config for when no config file is provided.
|
||||
/// Uses defaults and optional CLI overrides.
|
||||
pub fn minimal(model: Option<String>) -> Self {
|
||||
let label = model.as_deref().unwrap_or("default").to_string();
|
||||
Self {
|
||||
results_dir: default_results_dir(),
|
||||
task_timeout: default_task_timeout(),
|
||||
parallelism: default_parallelism(),
|
||||
matrix: vec![MatrixEntry { label, model }],
|
||||
suite_config: toml::Value::Table(toml::map::Map::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the suite_config as a generic map for adapter use.
|
||||
pub fn suite_config_map(&self) -> toml::map::Map<String, toml::Value> {
|
||||
match &self.suite_config {
|
||||
toml::Value::Table(map) => map.clone(),
|
||||
_ => toml::map::Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a string value from suite_config.
|
||||
pub fn suite_config_str(&self, key: &str) -> Option<String> {
|
||||
self.suite_config_map()
|
||||
.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_suite_config() -> toml::Value {
|
||||
toml::Value::Table(toml::map::Map::new())
|
||||
}
|
||||
|
||||
fn default_results_dir() -> PathBuf {
|
||||
PathBuf::from("./bench-results")
|
||||
}
|
||||
|
||||
fn default_task_timeout() -> Duration {
|
||||
Duration::from_secs(300)
|
||||
}
|
||||
|
||||
fn default_parallelism() -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
/// Deserialize a duration from a string like "300s", "5m", etc.
|
||||
fn deserialize_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
parse_duration(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn parse_duration(s: &str) -> Result<Duration, String> {
|
||||
let s = s.trim();
|
||||
if let Some(secs) = s.strip_suffix('s') {
|
||||
secs.trim()
|
||||
.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid seconds: {e}"))
|
||||
} else if let Some(mins) = s.strip_suffix('m') {
|
||||
mins.trim()
|
||||
.parse::<u64>()
|
||||
.map(|m| Duration::from_secs(m * 60))
|
||||
.map_err(|e| format!("invalid minutes: {e}"))
|
||||
} else {
|
||||
// Assume seconds if no suffix
|
||||
s.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid duration '{s}': {e}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_duration() {
|
||||
assert_eq!(parse_duration("300s").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("60").unwrap(), Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_config() {
|
||||
let config = BenchConfig::minimal(Some("test-model".to_string()));
|
||||
assert_eq!(config.matrix.len(), 1);
|
||||
assert_eq!(config.matrix[0].label, "test-model");
|
||||
assert_eq!(config.parallelism, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_rejects_empty_matrix() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("empty.toml");
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"
|
||||
results_dir = "./results"
|
||||
task_timeout = "60s"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let err = BenchConfig::from_file(&path).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("at least one [[matrix]]"),
|
||||
"got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_from_toml() {
|
||||
let toml_str = r#"
|
||||
results_dir = "./my-results"
|
||||
task_timeout = "60s"
|
||||
parallelism = 2
|
||||
|
||||
[[matrix]]
|
||||
label = "fast"
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
[[matrix]]
|
||||
label = "full"
|
||||
model = "claude-3-5-sonnet"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "./data/test.jsonl"
|
||||
"#;
|
||||
let config: BenchConfig = toml::from_str(toml_str).unwrap();
|
||||
assert_eq!(config.results_dir, PathBuf::from("./my-results"));
|
||||
assert_eq!(config.task_timeout, Duration::from_secs(60));
|
||||
assert_eq!(config.parallelism, 2);
|
||||
assert_eq!(config.matrix.len(), 2);
|
||||
assert_eq!(
|
||||
config.suite_config_str("dataset_path").unwrap(),
|
||||
"./data/test.jsonl"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BenchError {
|
||||
#[error("Config error: {0}")]
|
||||
Config(String),
|
||||
|
||||
#[error("Config file not found: {path}")]
|
||||
ConfigNotFound { path: PathBuf },
|
||||
|
||||
#[error("Suite {name} not found. Available: {available}")]
|
||||
SuiteNotFound { name: String, available: String },
|
||||
|
||||
#[error("Task {task_id} failed: {reason}")]
|
||||
TaskFailed { task_id: String, reason: String },
|
||||
|
||||
#[error("Scoring error for task {task_id}: {reason}")]
|
||||
Scoring { task_id: String, reason: String },
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("TOML parse error: {0}")]
|
||||
Toml(#[from] toml::de::Error),
|
||||
|
||||
#[error("Agent error: {0}")]
|
||||
Agent(#[from] ironclaw::Error),
|
||||
}
|
||||
@@ -1,251 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
use rust_decimal::prelude::ToPrimitive;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use ironclaw::error::LlmError;
|
||||
use ironclaw::llm::{
|
||||
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
|
||||
/// Recorded metrics from a single LLM call.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct LlmCallRecord {
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub duration_ms: u64,
|
||||
pub had_tool_calls: bool,
|
||||
}
|
||||
|
||||
/// Wraps an `LlmProvider` to record per-call metrics.
|
||||
///
|
||||
/// The wrapper is transparent to the agent: it delegates every call
|
||||
/// to the inner provider and captures token counts and timings.
|
||||
pub struct InstrumentedLlm {
|
||||
inner: Arc<dyn LlmProvider>,
|
||||
records: Mutex<Vec<LlmCallRecord>>,
|
||||
total_input_tokens: AtomicU32,
|
||||
total_output_tokens: AtomicU32,
|
||||
call_count: AtomicU32,
|
||||
}
|
||||
|
||||
impl InstrumentedLlm {
|
||||
pub fn new(inner: Arc<dyn LlmProvider>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
records: Mutex::new(Vec::new()),
|
||||
total_input_tokens: AtomicU32::new(0),
|
||||
total_output_tokens: AtomicU32::new(0),
|
||||
call_count: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take all recorded call metrics, clearing the internal buffer.
|
||||
pub async fn take_records(&self) -> Vec<LlmCallRecord> {
|
||||
let mut records = self.records.lock().await;
|
||||
std::mem::take(&mut *records)
|
||||
}
|
||||
|
||||
/// Snapshot of total tokens without clearing.
|
||||
pub fn total_input_tokens(&self) -> u32 {
|
||||
self.total_input_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn total_output_tokens(&self) -> u32 {
|
||||
self.total_output_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> u32 {
|
||||
self.call_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Estimated cost using the inner provider's cost-per-token rates.
|
||||
pub fn estimated_cost(&self) -> f64 {
|
||||
let (input_rate, output_rate) = self.inner.cost_per_token();
|
||||
let input_cost =
|
||||
input_rate * Decimal::from(self.total_input_tokens.load(Ordering::Relaxed));
|
||||
let output_cost =
|
||||
output_rate * Decimal::from(self.total_output_tokens.load(Ordering::Relaxed));
|
||||
let total = input_cost + output_cost;
|
||||
total.to_f64().unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Reset all counters and records.
|
||||
pub async fn reset(&self) {
|
||||
self.records.lock().await.clear();
|
||||
self.total_input_tokens.store(0, Ordering::Relaxed);
|
||||
self.total_output_tokens.store(0, Ordering::Relaxed);
|
||||
self.call_count.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn record(
|
||||
&self,
|
||||
input_tokens: u32,
|
||||
output_tokens: u32,
|
||||
duration_ms: u64,
|
||||
had_tool_calls: bool,
|
||||
) {
|
||||
self.total_input_tokens
|
||||
.fetch_add(input_tokens, Ordering::Relaxed);
|
||||
self.total_output_tokens
|
||||
.fetch_add(output_tokens, Ordering::Relaxed);
|
||||
self.call_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.records.lock().await.push(LlmCallRecord {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
had_tool_calls,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for InstrumentedLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
self.inner.cost_per_token()
|
||||
}
|
||||
|
||||
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete_with_tools(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
let had_tool_calls = !response.tool_calls.is_empty();
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
had_tool_calls,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
self.inner.list_models().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ironclaw::llm::{ChatMessage, CompletionRequest, CompletionResponse, FinishReason};
|
||||
|
||||
/// Fake LLM that returns a canned response with known token counts.
|
||||
struct FakeLlm;
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for FakeLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
(
|
||||
Decimal::new(3, 6), // $0.000003 per input token
|
||||
Decimal::new(15, 6), // $0.000015 per output token
|
||||
)
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: CompletionRequest,
|
||||
) -> Result<CompletionResponse, LlmError> {
|
||||
Ok(CompletionResponse {
|
||||
content: "test response".to_string(),
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
_request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
Ok(ToolCompletionResponse {
|
||||
content: Some("tool response".to_string()),
|
||||
tool_calls: vec![],
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_records_metrics() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
assert_eq!(instrumented.total_input_tokens(), 100);
|
||||
assert_eq!(instrumented.total_output_tokens(), 50);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].input_tokens, 100);
|
||||
assert!(!records[0].had_tool_calls);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_cost_calculation() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
// 100 * 0.000003 + 50 * 0.000015 = 0.0003 + 0.00075 = 0.00105
|
||||
let cost = instrumented.estimated_cost();
|
||||
assert!((cost - 0.00105).abs() < 0.0001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_reset() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
|
||||
instrumented.reset().await;
|
||||
assert_eq!(instrumented.call_count(), 0);
|
||||
assert_eq!(instrumented.total_input_tokens(), 0);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert!(records.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,313 +0,0 @@
|
||||
mod adapters;
|
||||
mod channel;
|
||||
mod config;
|
||||
mod error;
|
||||
mod instrumented_llm;
|
||||
mod results;
|
||||
mod runner;
|
||||
mod scoring;
|
||||
mod suite;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "ironclaw-bench", about = "IronClaw benchmarking harness")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Run a benchmark suite.
|
||||
Run {
|
||||
/// Suite to run (custom, gaia, spot, tau_bench, swe_bench).
|
||||
#[arg(long)]
|
||||
suite: String,
|
||||
|
||||
/// Path to bench config TOML.
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
|
||||
/// Override model for all matrix entries.
|
||||
#[arg(long)]
|
||||
model: Option<String>,
|
||||
|
||||
/// Max tasks to run in parallel.
|
||||
#[arg(long)]
|
||||
parallelism: Option<usize>,
|
||||
|
||||
/// Sample N tasks from the suite (for quick testing).
|
||||
#[arg(long)]
|
||||
sample: Option<usize>,
|
||||
|
||||
/// Only run these task IDs (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
task_ids: Option<Vec<String>>,
|
||||
|
||||
/// Only run tasks with these tags (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
tags: Option<Vec<String>>,
|
||||
|
||||
/// Per-task timeout in seconds.
|
||||
#[arg(long)]
|
||||
timeout_secs: Option<u64>,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
|
||||
/// Resume a previous run by ID.
|
||||
#[arg(long)]
|
||||
resume: Option<Uuid>,
|
||||
},
|
||||
|
||||
/// Show results for a run.
|
||||
Results {
|
||||
/// Run ID or "latest".
|
||||
#[arg(default_value = "latest")]
|
||||
run_id: String,
|
||||
|
||||
/// Output format.
|
||||
#[arg(long, default_value = "table")]
|
||||
format: ResultsFormat,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// Compare two runs.
|
||||
Compare {
|
||||
/// Baseline run ID.
|
||||
baseline: Uuid,
|
||||
|
||||
/// Comparison run ID.
|
||||
comparison: Uuid,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// List available benchmark suites.
|
||||
List,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, clap::ValueEnum)]
|
||||
enum ResultsFormat {
|
||||
Table,
|
||||
Json,
|
||||
Csv,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("ironclaw_bench=info,ironclaw=warn")),
|
||||
)
|
||||
.with(tracing_subscriber::fmt::layer().with_target(false))
|
||||
.init();
|
||||
|
||||
match cli.command {
|
||||
Commands::List => {
|
||||
println!("Available benchmark suites:\n");
|
||||
for (id, desc) in adapters::KNOWN_SUITES {
|
||||
println!(" {:<15} {}", id, desc);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
Commands::Run {
|
||||
suite,
|
||||
config: config_path,
|
||||
model,
|
||||
parallelism,
|
||||
sample,
|
||||
task_ids,
|
||||
tags,
|
||||
timeout_secs,
|
||||
results_dir,
|
||||
resume,
|
||||
} => {
|
||||
// Load or create config
|
||||
let mut bench_config = if let Some(ref path) = config_path {
|
||||
BenchConfig::from_file(path)?
|
||||
} else {
|
||||
BenchConfig::minimal(model.clone())
|
||||
};
|
||||
|
||||
// Apply CLI overrides
|
||||
if let Some(p) = parallelism {
|
||||
bench_config.parallelism = p;
|
||||
}
|
||||
if let Some(t) = timeout_secs {
|
||||
bench_config.task_timeout = std::time::Duration::from_secs(t);
|
||||
}
|
||||
if let Some(ref dir) = results_dir {
|
||||
bench_config.results_dir = dir.clone();
|
||||
}
|
||||
|
||||
// If model override specified and we have matrix entries, update them
|
||||
if let Some(ref m) = model {
|
||||
for entry in &mut bench_config.matrix {
|
||||
entry.model = Some(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Create suite
|
||||
let bench_suite = adapters::create_suite(&suite, &bench_config)?;
|
||||
|
||||
// Initialize ironclaw LLM provider
|
||||
let ironclaw_config = ironclaw::Config::from_env().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to load ironclaw config: {}. Make sure .env is configured.",
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
||||
auth_base_url: ironclaw_config.llm.nearai.auth_base_url.clone(),
|
||||
session_path: ironclaw_config.llm.nearai.session_path.clone(),
|
||||
})
|
||||
.await;
|
||||
session.ensure_authenticated().await?;
|
||||
|
||||
let llm = ironclaw::llm::create_llm_provider(&ironclaw_config.llm, session)?;
|
||||
let safety = Arc::new(ironclaw::safety::SafetyLayer::new(&ironclaw_config.safety));
|
||||
|
||||
let runner = runner::BenchRunner::new(bench_suite, bench_config.clone(), llm, safety);
|
||||
|
||||
// Run for each matrix entry
|
||||
for matrix_entry in &bench_config.matrix {
|
||||
let run_id = runner
|
||||
.run(
|
||||
matrix_entry,
|
||||
sample,
|
||||
task_ids.as_deref(),
|
||||
tags.as_deref(),
|
||||
resume,
|
||||
)
|
||||
.await?;
|
||||
println!("Run complete: {}", run_id);
|
||||
}
|
||||
}
|
||||
Commands::Results {
|
||||
run_id,
|
||||
format,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
let uuid = if run_id == "latest" {
|
||||
results::find_latest_run(&base)?
|
||||
.ok_or_else(|| anyhow::anyhow!("No runs found in {}", base.display()))?
|
||||
} else {
|
||||
Uuid::parse_str(&run_id)?
|
||||
};
|
||||
|
||||
let json_path = results::run_json_path(&base, uuid);
|
||||
let jsonl_path = results::tasks_jsonl_path(&base, uuid);
|
||||
|
||||
let run = results::read_run_result(&json_path)?;
|
||||
let tasks = results::read_task_results(&jsonl_path)?;
|
||||
|
||||
match format {
|
||||
ResultsFormat::Table => {
|
||||
results::print_results_table(&tasks, &run);
|
||||
}
|
||||
ResultsFormat::Json => {
|
||||
let output = serde_json::json!({
|
||||
"run": run,
|
||||
"tasks": tasks,
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output)?);
|
||||
}
|
||||
ResultsFormat::Csv => {
|
||||
println!("task_id,score,label,tokens,cost,turns,time_s");
|
||||
for task in &tasks {
|
||||
println!(
|
||||
"{},{:.3},{},{},{:.4},{},{:.1}",
|
||||
task.task_id,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
task.trace.input_tokens + task.trace.output_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Commands::Compare {
|
||||
baseline,
|
||||
comparison,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
|
||||
let baseline_run = results::read_run_result(&results::run_json_path(&base, baseline))?;
|
||||
let comparison_run =
|
||||
results::read_run_result(&results::run_json_path(&base, comparison))?;
|
||||
|
||||
println!("\nComparison: {} vs {}\n", baseline, comparison);
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12} {:>10}",
|
||||
"Metric", "Baseline", "Comparison", "Delta"
|
||||
);
|
||||
println!("{}", "-".repeat(58));
|
||||
|
||||
let pass_delta = comparison_run.pass_rate - baseline_run.pass_rate;
|
||||
println!(
|
||||
"{:<20} {:>11.1}% {:>11.1}% {:>+9.1}%",
|
||||
"Pass rate",
|
||||
baseline_run.pass_rate * 100.0,
|
||||
comparison_run.pass_rate * 100.0,
|
||||
pass_delta * 100.0,
|
||||
);
|
||||
|
||||
let score_delta = comparison_run.avg_score - baseline_run.avg_score;
|
||||
println!(
|
||||
"{:<20} {:>12.3} {:>12.3} {:>+10.3}",
|
||||
"Avg score", baseline_run.avg_score, comparison_run.avg_score, score_delta,
|
||||
);
|
||||
|
||||
let cost_delta = comparison_run.total_cost_usd - baseline_run.total_cost_usd;
|
||||
println!(
|
||||
"{:<20} {:>11.4}$ {:>11.4}$ {:>+9.4}$",
|
||||
"Total cost",
|
||||
baseline_run.total_cost_usd,
|
||||
comparison_run.total_cost_usd,
|
||||
cost_delta,
|
||||
);
|
||||
|
||||
let time_b = baseline_run.total_wall_time_ms as f64 / 1000.0;
|
||||
let time_c = comparison_run.total_wall_time_ms as f64 / 1000.0;
|
||||
println!(
|
||||
"{:<20} {:>11.1}s {:>11.1}s {:>+9.1}s",
|
||||
"Total time",
|
||||
time_b,
|
||||
time_c,
|
||||
time_c - time_b,
|
||||
);
|
||||
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12}",
|
||||
"Model", baseline_run.model, comparison_run.model,
|
||||
);
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,473 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::{BufRead, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Metrics from a single task run: LLM usage, timing, tool calls.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Trace {
|
||||
pub wall_time_ms: u64,
|
||||
pub llm_calls: u32,
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub estimated_cost_usd: f64,
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
pub turns: u32,
|
||||
pub hit_iteration_limit: bool,
|
||||
pub hit_timeout: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TraceToolCall {
|
||||
pub name: String,
|
||||
pub duration_ms: u64,
|
||||
pub success: bool,
|
||||
}
|
||||
|
||||
/// Result of running a single benchmark task.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResult {
|
||||
pub task_id: String,
|
||||
pub suite_id: String,
|
||||
pub score: BenchScore,
|
||||
pub trace: Trace,
|
||||
pub response: String,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
pub config_label: String,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Aggregate results for a full benchmark run.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RunResult {
|
||||
pub run_id: Uuid,
|
||||
pub suite_id: String,
|
||||
pub config_label: String,
|
||||
pub model: String,
|
||||
/// Short git commit hash at the time of the run.
|
||||
#[serde(default)]
|
||||
pub commit_hash: String,
|
||||
pub pass_rate: f64,
|
||||
pub avg_score: f64,
|
||||
pub total_tasks: usize,
|
||||
pub completed_tasks: usize,
|
||||
pub total_cost_usd: f64,
|
||||
pub total_wall_time_ms: u64,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl RunResult {
|
||||
/// Build aggregate from individual task results.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn from_tasks(
|
||||
run_id: Uuid,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
model: &str,
|
||||
commit_hash: &str,
|
||||
total_tasks: usize,
|
||||
tasks: &[TaskResult],
|
||||
started_at: DateTime<Utc>,
|
||||
) -> Self {
|
||||
let pass_count = tasks.iter().filter(|t| t.score.value >= 1.0).count();
|
||||
let pass_rate = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
pass_count as f64 / tasks.len() as f64
|
||||
};
|
||||
let avg_score = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
tasks.iter().map(|t| t.score.value).sum::<f64>() / tasks.len() as f64
|
||||
};
|
||||
let total_cost: f64 = tasks.iter().map(|t| t.trace.estimated_cost_usd).sum();
|
||||
let total_wall: u64 = tasks.iter().map(|t| t.trace.wall_time_ms).sum();
|
||||
|
||||
Self {
|
||||
run_id,
|
||||
suite_id: suite_id.to_string(),
|
||||
config_label: config_label.to_string(),
|
||||
model: model.to_string(),
|
||||
commit_hash: commit_hash.to_string(),
|
||||
pass_rate,
|
||||
avg_score,
|
||||
total_tasks,
|
||||
completed_tasks: tasks.len(),
|
||||
total_cost_usd: total_cost,
|
||||
total_wall_time_ms: total_wall,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a single task result as one JSON line to the JSONL file.
|
||||
pub fn append_task_result(path: &Path, result: &TaskResult) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)?;
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Overwrite the JSONL file with the given results (used after scoring).
|
||||
pub fn write_task_results(path: &Path, results: &[TaskResult]) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::File::create(path)?;
|
||||
for result in results {
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read all task results from a JSONL file.
|
||||
pub fn read_task_results(path: &Path) -> Result<Vec<TaskResult>, BenchError> {
|
||||
if !path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let file = std::fs::File::open(path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut results = Vec::new();
|
||||
for line in reader.lines() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let result: TaskResult = serde_json::from_str(trimmed)?;
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Write the aggregate run result as JSON.
|
||||
pub fn write_run_result(path: &Path, result: &RunResult) -> Result<(), BenchError> {
|
||||
let json = serde_json::to_string_pretty(result)?;
|
||||
std::fs::write(path, json)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the aggregate run result from JSON.
|
||||
pub fn read_run_result(path: &Path) -> Result<RunResult, BenchError> {
|
||||
let json = std::fs::read_to_string(path)?;
|
||||
let result: RunResult = serde_json::from_str(&json)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Get the set of already-completed task IDs from a JSONL file (for resume).
|
||||
///
|
||||
/// Only includes tasks that have been scored (label != "pending"). Tasks that
|
||||
/// were written but not scored (e.g., from an interrupted run) will be re-executed.
|
||||
pub fn completed_task_ids(path: &Path) -> Result<HashSet<String>, BenchError> {
|
||||
let results = read_task_results(path)?;
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.filter(|r| r.score.label != "pending")
|
||||
.map(|r| r.task_id)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Get the results directory for a specific run.
|
||||
pub fn run_dir(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
base.join(run_id.to_string())
|
||||
}
|
||||
|
||||
/// Get the tasks JSONL path for a run.
|
||||
pub fn tasks_jsonl_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("tasks.jsonl")
|
||||
}
|
||||
|
||||
/// Get the run JSON path for a run.
|
||||
pub fn run_json_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("run.json")
|
||||
}
|
||||
|
||||
/// Find the latest run directory by the modification time of its `run.json`.
|
||||
///
|
||||
/// Falls back to `tasks.jsonl` mtime, then directory mtime. This avoids the
|
||||
/// issue where modifying files inside a directory doesn't update the directory's
|
||||
/// mtime on many filesystems.
|
||||
pub fn find_latest_run(base: &Path) -> Result<Option<Uuid>, BenchError> {
|
||||
if !base.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut entries: Vec<_> = std::fs::read_dir(base)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_type().map(|ft| ft.is_dir()).unwrap_or(false))
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_string_lossy().to_string();
|
||||
let uuid = Uuid::parse_str(&name).ok()?;
|
||||
let dir_path = e.path();
|
||||
// Prefer run.json mtime, fall back to tasks.jsonl, then directory
|
||||
let modified = std::fs::metadata(dir_path.join("run.json"))
|
||||
.and_then(|m| m.modified())
|
||||
.or_else(|_| {
|
||||
std::fs::metadata(dir_path.join("tasks.jsonl")).and_then(|m| m.modified())
|
||||
})
|
||||
.or_else(|_| e.metadata().and_then(|m| m.modified()))
|
||||
.ok()?;
|
||||
Some((uuid, modified))
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
Ok(entries.first().map(|(uuid, _)| *uuid))
|
||||
}
|
||||
|
||||
/// Print a summary table of task results.
|
||||
pub fn print_results_table(tasks: &[TaskResult], run: &RunResult) {
|
||||
println!();
|
||||
let commit_suffix = if run.commit_hash.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" | Commit: {}", run.commit_hash)
|
||||
};
|
||||
println!(
|
||||
"Run: {} | Suite: {} | Model: {}{}",
|
||||
run.run_id, run.suite_id, run.model, commit_suffix
|
||||
);
|
||||
println!(
|
||||
"Pass rate: {:.1}% | Avg score: {:.3} | Tasks: {}/{} | Cost: ${:.4} | Time: {:.1}s",
|
||||
run.pass_rate * 100.0,
|
||||
run.avg_score,
|
||||
run.completed_tasks,
|
||||
run.total_tasks,
|
||||
run.total_cost_usd,
|
||||
run.total_wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
println!();
|
||||
|
||||
// Header
|
||||
println!(
|
||||
"{:<30} {:>6} {:>7} {:>8} {:>10} {:>6} {:>8}",
|
||||
"Task ID", "Score", "Label", "Tokens", "Cost", "Turns", "Time"
|
||||
);
|
||||
println!("{}", "-".repeat(80));
|
||||
|
||||
for task in tasks {
|
||||
let total_tokens = task.trace.input_tokens + task.trace.output_tokens;
|
||||
let task_id_display = if task.task_id.len() > 28 {
|
||||
let truncated: String = task.task_id.chars().take(25).collect();
|
||||
format!("{truncated}...")
|
||||
} else {
|
||||
task.task_id.clone()
|
||||
};
|
||||
println!(
|
||||
"{:<30} {:>6.3} {:>7} {:>8} {:>10.4} {:>6} {:>7.1}s",
|
||||
task_id_display,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
total_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_run_result_from_tasks() {
|
||||
let tasks = vec![
|
||||
TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 1000,
|
||||
llm_calls: 2,
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
estimated_cost_usd: 0.01,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
TaskResult {
|
||||
task_id: "t2".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some("wrong".to_string()),
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 2000,
|
||||
llm_calls: 3,
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
estimated_cost_usd: 0.02,
|
||||
tool_calls: vec![],
|
||||
turns: 2,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "wrong answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
];
|
||||
|
||||
let run = RunResult::from_tasks(
|
||||
Uuid::new_v4(),
|
||||
"custom",
|
||||
"default",
|
||||
"test-model",
|
||||
"abc1234",
|
||||
2,
|
||||
&tasks,
|
||||
Utc::now(),
|
||||
);
|
||||
|
||||
assert_eq!(run.pass_rate, 0.5);
|
||||
assert_eq!(run.avg_score, 0.5);
|
||||
assert_eq!(run.total_tasks, 2);
|
||||
assert_eq!(run.completed_tasks, 2);
|
||||
assert!((run.total_cost_usd - 0.03).abs() < f64::EPSILON);
|
||||
assert_eq!(run.total_wall_time_ms, 3000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jsonl_roundtrip() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "round-trip-test".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 500,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "hello".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
|
||||
append_task_result(&path, &result).expect("append");
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let loaded = read_task_results(&path).expect("read");
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].task_id, "round-trip-test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completed_task_ids() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "unique-id-1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "x".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let ids = completed_task_ids(&path).expect("ids");
|
||||
assert!(ids.contains("unique-id-1"));
|
||||
assert!(!ids.contains("unique-id-2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_task_results_overwrites() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
// Write initial "pending" result via append
|
||||
let pending = TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "spot".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "42".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &pending).expect("append");
|
||||
|
||||
// Verify pending score
|
||||
let before = read_task_results(&path).expect("read");
|
||||
assert_eq!(before.len(), 1);
|
||||
assert_eq!(before[0].score.label, "pending");
|
||||
|
||||
// Overwrite with scored result
|
||||
let mut scored = pending;
|
||||
scored.score = BenchScore::pass();
|
||||
write_task_results(&path, &[scored]).expect("write");
|
||||
|
||||
// Verify scored result replaced pending
|
||||
let after = read_task_results(&path).expect("read");
|
||||
assert_eq!(after.len(), 1);
|
||||
assert_eq!(after[0].score.label, "pass");
|
||||
assert_eq!(after[0].score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,550 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use ironclaw::agent::{Agent, AgentDeps};
|
||||
use ironclaw::channels::{ChannelManager, IncomingMessage};
|
||||
use ironclaw::config::AgentConfig;
|
||||
use ironclaw::llm::LlmProvider;
|
||||
use ironclaw::safety::SafetyLayer;
|
||||
use ironclaw::tools::ToolRegistry;
|
||||
|
||||
use crate::channel::BenchChannel;
|
||||
use crate::config::{BenchConfig, MatrixEntry};
|
||||
use crate::error::BenchError;
|
||||
use crate::instrumented_llm::InstrumentedLlm;
|
||||
use crate::results::{
|
||||
RunResult, TaskResult, Trace, append_task_result, completed_task_ids, run_dir, run_json_path,
|
||||
tasks_jsonl_path, write_run_result, write_task_results,
|
||||
};
|
||||
use crate::suite::{BenchSuite, BenchTask, ConversationTurn, TaskSubmission, TurnRole};
|
||||
|
||||
/// Parameters for running a single task in isolation.
|
||||
struct TaskRunParams<'a> {
|
||||
task: &'a BenchTask,
|
||||
suite_id: &'a str,
|
||||
config_label: &'a str,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
timeout: std::time::Duration,
|
||||
additional_tools: &'a [Arc<dyn ironclaw::tools::Tool>],
|
||||
}
|
||||
|
||||
/// Orchestrates benchmark execution: loads tasks, runs agent per task,
|
||||
/// scores results, writes JSONL output.
|
||||
pub struct BenchRunner {
|
||||
suite: Arc<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
}
|
||||
|
||||
impl BenchRunner {
|
||||
pub fn new(
|
||||
suite: Box<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
suite: Arc::from(suite),
|
||||
config,
|
||||
llm,
|
||||
safety,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the benchmark for one matrix entry.
|
||||
///
|
||||
/// Returns the run_id for result retrieval.
|
||||
pub async fn run(
|
||||
&self,
|
||||
matrix: &MatrixEntry,
|
||||
sample: Option<usize>,
|
||||
task_filter: Option<&[String]>,
|
||||
tag_filter: Option<&[String]>,
|
||||
resume_run_id: Option<Uuid>,
|
||||
) -> Result<Uuid, BenchError> {
|
||||
let run_id = resume_run_id.unwrap_or_else(Uuid::new_v4);
|
||||
let results_base = &self.config.results_dir;
|
||||
let dir = run_dir(results_base, run_id);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
|
||||
let jsonl_path = tasks_jsonl_path(results_base, run_id);
|
||||
let json_path = run_json_path(results_base, run_id);
|
||||
|
||||
// Load completed task IDs for resume support
|
||||
let completed: HashSet<String> = if resume_run_id.is_some() {
|
||||
completed_task_ids(&jsonl_path)?
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
|
||||
if !completed.is_empty() {
|
||||
tracing::info!(
|
||||
"Resuming run {}: {} tasks already completed",
|
||||
run_id,
|
||||
completed.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Load all tasks once (used for both execution and scoring)
|
||||
let all_tasks = self.suite.load_tasks().await?;
|
||||
let task_index: HashMap<String, BenchTask> = all_tasks
|
||||
.iter()
|
||||
.map(|t| (t.id.clone(), t.clone()))
|
||||
.collect();
|
||||
|
||||
// Filter tasks for execution
|
||||
let mut tasks = all_tasks;
|
||||
|
||||
if let Some(ids) = task_filter {
|
||||
let id_set: HashSet<&str> = ids.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| id_set.contains(t.id.as_str()));
|
||||
}
|
||||
|
||||
if let Some(tags) = tag_filter {
|
||||
let tag_set: HashSet<&str> = tags.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| t.tags.iter().any(|tag| tag_set.contains(tag.as_str())));
|
||||
}
|
||||
|
||||
// Filter out already-completed tasks
|
||||
tasks.retain(|t| !completed.contains(&t.id));
|
||||
|
||||
// Sample if requested
|
||||
if let Some(n) = sample {
|
||||
tasks.truncate(n);
|
||||
}
|
||||
|
||||
let total_tasks = tasks.len() + completed.len();
|
||||
let model_label = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
let commit_hash = git_short_hash();
|
||||
tracing::info!(
|
||||
"[{} @ {}] Running {} tasks for suite '{}' (run: {})",
|
||||
model_label,
|
||||
commit_hash,
|
||||
tasks.len(),
|
||||
self.suite.id(),
|
||||
run_id
|
||||
);
|
||||
|
||||
let started_at = Utc::now();
|
||||
let all_results: Arc<Mutex<Vec<TaskResult>>> =
|
||||
Arc::new(Mutex::new(Vec::with_capacity(tasks.len())));
|
||||
|
||||
if self.config.parallelism <= 1 {
|
||||
// Sequential execution
|
||||
let additional_tools = self.suite.additional_tools();
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed.len(),
|
||||
total_tasks,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = self.suite.setup_task(task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
task,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
continue;
|
||||
}
|
||||
let params = TaskRunParams {
|
||||
task,
|
||||
suite_id: self.suite.id(),
|
||||
config_label: &matrix.label,
|
||||
llm: Arc::clone(&self.llm),
|
||||
safety: Arc::clone(&self.safety),
|
||||
timeout: task.timeout.unwrap_or(self.config.task_timeout),
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = self.suite.teardown_task(task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
}
|
||||
} else {
|
||||
// Parallel execution with bounded concurrency
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.config.parallelism));
|
||||
let shared_tools: Arc<[Arc<dyn ironclaw::tools::Tool>]> =
|
||||
Arc::from(self.suite.additional_tools());
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for (i, task) in tasks.into_iter().enumerate() {
|
||||
let sem = Arc::clone(&semaphore);
|
||||
let suite = Arc::clone(&self.suite);
|
||||
let config_label = matrix.label.clone();
|
||||
let llm = Arc::clone(&self.llm);
|
||||
let safety = Arc::clone(&self.safety);
|
||||
let timeout = task.timeout.unwrap_or(self.config.task_timeout);
|
||||
let results_ref = Arc::clone(&all_results);
|
||||
let completed_count = completed.len();
|
||||
let total = total_tasks;
|
||||
let additional_tools = Arc::clone(&shared_tools);
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = match sem.acquire().await {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
tracing::error!("Semaphore closed for task {}", task.id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed_count,
|
||||
total,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = suite.setup_task(&task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
&task,
|
||||
suite.id(),
|
||||
&config_label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
results_ref.lock().await.push(result);
|
||||
return;
|
||||
}
|
||||
let suite_id = suite.id().to_string();
|
||||
let params = TaskRunParams {
|
||||
task: &task,
|
||||
suite_id: &suite_id,
|
||||
config_label: &config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = suite.teardown_task(&task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
results_ref.lock().await.push(result);
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
tracing::error!("Task panicked: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Write all results to JSONL after parallel execution completes.
|
||||
// This avoids the race condition of concurrent file appends.
|
||||
let results = all_results.lock().await;
|
||||
for result in results.iter() {
|
||||
append_task_result(&jsonl_path, result)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Score all results using the cached task index
|
||||
let results = all_results.lock().await;
|
||||
let mut scored: Vec<TaskResult> = Vec::with_capacity(results.len());
|
||||
for result in results.iter() {
|
||||
if let Some(task) = task_index.get(&result.task_id) {
|
||||
let submission = TaskSubmission {
|
||||
response: result.response.clone(),
|
||||
conversation: vec![],
|
||||
tool_calls: result
|
||||
.trace
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|tc| tc.name.clone())
|
||||
.collect(),
|
||||
error: result.error.clone(),
|
||||
};
|
||||
match self.suite.score(task, &submission).await {
|
||||
Ok(score) => {
|
||||
let mut scored_result = result.clone();
|
||||
scored_result.score = score;
|
||||
scored.push(scored_result);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Scoring failed for {}: {}", result.task_id, e);
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Combine with any previously completed results for the aggregate
|
||||
let mut all_for_aggregate = crate::results::read_task_results(&jsonl_path)?;
|
||||
// De-duplicate (prefer the newer scored versions)
|
||||
let scored_ids: HashSet<String> = scored.iter().map(|r| r.task_id.clone()).collect();
|
||||
all_for_aggregate.retain(|r| !scored_ids.contains(&r.task_id));
|
||||
all_for_aggregate.extend(scored);
|
||||
|
||||
// Rewrite JSONL with scored results so `results` command shows final scores
|
||||
write_task_results(&jsonl_path, &all_for_aggregate)?;
|
||||
|
||||
let model_name = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
|
||||
let run_result = RunResult::from_tasks(
|
||||
run_id,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
model_name,
|
||||
&commit_hash,
|
||||
total_tasks,
|
||||
&all_for_aggregate,
|
||||
started_at,
|
||||
);
|
||||
|
||||
write_run_result(&json_path, &run_result)?;
|
||||
|
||||
tracing::info!(
|
||||
"[{} @ {}] Run {} complete: {:.1}% pass rate, {:.3} avg score, ${:.4} cost",
|
||||
model_name,
|
||||
commit_hash,
|
||||
run_id,
|
||||
run_result.pass_rate * 100.0,
|
||||
run_result.avg_score,
|
||||
run_result.total_cost_usd,
|
||||
);
|
||||
|
||||
Ok(run_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a single benchmark task in complete isolation.
|
||||
///
|
||||
/// Creates a fresh Agent + BenchChannel + InstrumentedLlm for the task,
|
||||
/// injects the prompt, waits for the response, and returns the result.
|
||||
///
|
||||
/// # Current limitations
|
||||
///
|
||||
/// - **Single-turn only**: After the first assistant response, `/quit` is sent.
|
||||
/// Multi-turn suites (e.g., Tau-bench's `next_user_message()`) are not yet wired.
|
||||
/// - **Resources not injected**: `BenchTask.resources` (e.g., GAIA file attachments)
|
||||
/// are not included in the prompt or made available via the workspace.
|
||||
/// - **Conversation not captured**: `TaskSubmission.conversation` is always empty,
|
||||
/// which prevents multi-turn scoring hooks from working.
|
||||
async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult {
|
||||
let TaskRunParams {
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools,
|
||||
} = params;
|
||||
|
||||
let started_at = Utc::now();
|
||||
let start = Instant::now();
|
||||
|
||||
// Wrap LLM with instrumentation
|
||||
let instrumented = Arc::new(InstrumentedLlm::new(llm));
|
||||
|
||||
// Create bench channel
|
||||
let (bench_channel, msg_tx) = BenchChannel::new();
|
||||
let capture = bench_channel.capture();
|
||||
|
||||
// Build tool registry
|
||||
let tools = Arc::new(ToolRegistry::new());
|
||||
tools.register_builtin_tools();
|
||||
|
||||
// Register additional suite-specific tools
|
||||
for tool in additional_tools {
|
||||
tools.register(Arc::clone(tool)).await;
|
||||
}
|
||||
|
||||
// Build agent config (minimal, headless)
|
||||
let agent_config = AgentConfig {
|
||||
name: format!("bench-{}", task.id),
|
||||
max_parallel_jobs: 1,
|
||||
job_timeout: timeout,
|
||||
stuck_threshold: timeout,
|
||||
repair_check_interval: timeout + std::time::Duration::from_secs(999),
|
||||
max_repair_attempts: 0,
|
||||
use_planning: false,
|
||||
session_idle_timeout: timeout,
|
||||
allow_local_tools: true,
|
||||
max_cost_per_day_cents: None,
|
||||
max_actions_per_hour: None,
|
||||
};
|
||||
|
||||
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
|
||||
ironclaw::agent::cost_guard::CostGuardConfig::default(),
|
||||
));
|
||||
|
||||
let deps = AgentDeps {
|
||||
store: None,
|
||||
llm: instrumented.clone() as Arc<dyn LlmProvider>,
|
||||
cheap_llm: None,
|
||||
safety,
|
||||
tools,
|
||||
workspace: None,
|
||||
extension_manager: None,
|
||||
skill_registry: None,
|
||||
skills_config: ironclaw::config::SkillsConfig::default(),
|
||||
hooks: Arc::new(ironclaw::hooks::HookRegistry::new()),
|
||||
cost_guard,
|
||||
};
|
||||
|
||||
let mut channels = ChannelManager::new();
|
||||
channels.add(Box::new(bench_channel));
|
||||
|
||||
let agent = Agent::new(agent_config, deps, channels, None, None, None, None, None);
|
||||
|
||||
// Build the full prompt with context
|
||||
let full_prompt = if let Some(ref ctx) = task.context {
|
||||
format!("{}\n\nContext:\n{}", task.prompt, ctx)
|
||||
} else {
|
||||
task.prompt.clone()
|
||||
};
|
||||
|
||||
// Inject the task prompt
|
||||
let incoming = IncomingMessage::new("bench", "bench-user", &full_prompt);
|
||||
if msg_tx.send(incoming).await.is_err() {
|
||||
return make_error_result(
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
started_at,
|
||||
"failed to send prompt",
|
||||
);
|
||||
}
|
||||
|
||||
// Record prompt in conversation
|
||||
{
|
||||
let mut cap = capture.lock().await;
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: TurnRole::User,
|
||||
content: full_prompt,
|
||||
});
|
||||
}
|
||||
|
||||
// Run agent with timeout.
|
||||
// After the first response, send /quit to end the session.
|
||||
let quit_tx = msg_tx.clone();
|
||||
let capture_for_quit = Arc::clone(&capture);
|
||||
let quit_handle = tokio::spawn(async move {
|
||||
// Poll for first response
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
let cap = capture_for_quit.lock().await;
|
||||
if !cap.responses.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Give a small grace period for any final status events
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
let quit = IncomingMessage::new("bench", "bench-user", "/quit");
|
||||
let _ = quit_tx.send(quit).await;
|
||||
});
|
||||
|
||||
let agent_result = tokio::time::timeout(timeout, agent.run()).await;
|
||||
|
||||
quit_handle.abort();
|
||||
|
||||
let wall_time = start.elapsed();
|
||||
let hit_timeout = agent_result.is_err();
|
||||
|
||||
if let Ok(Err(e)) = &agent_result {
|
||||
tracing::warn!("Agent error for task {}: {}", task.id, e);
|
||||
}
|
||||
|
||||
// Extract results from capture
|
||||
let cap = capture.lock().await;
|
||||
let response = cap.responses.last().cloned().unwrap_or_default();
|
||||
|
||||
let trace = Trace {
|
||||
wall_time_ms: wall_time.as_millis() as u64,
|
||||
llm_calls: instrumented.call_count(),
|
||||
input_tokens: instrumented.total_input_tokens(),
|
||||
output_tokens: instrumented.total_output_tokens(),
|
||||
estimated_cost_usd: instrumented.estimated_cost(),
|
||||
tool_calls: cap.tool_calls.clone(),
|
||||
turns: cap.responses.len() as u32,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout,
|
||||
};
|
||||
|
||||
let error = if hit_timeout {
|
||||
Some(format!("timeout after {}s", timeout.as_secs()))
|
||||
} else if let Ok(Err(e)) = &agent_result {
|
||||
Some(e.to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace,
|
||||
response,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_error_result(
|
||||
task: &BenchTask,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
reason: &str,
|
||||
) -> TaskResult {
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore::fail(reason),
|
||||
trace: Trace {
|
||||
wall_time_ms: 0,
|
||||
llm_calls: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 0,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: String::new(),
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error: Some(reason.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the short git commit hash of HEAD, or "unknown" if not in a repo.
|
||||
fn git_short_hash() -> String {
|
||||
std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| {
|
||||
if o.status.success() {
|
||||
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
use regex::Regex;
|
||||
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Normalize an answer string for comparison: lowercase, trim whitespace,
|
||||
/// strip trailing punctuation, collapse internal whitespace.
|
||||
pub fn normalize_answer(s: &str) -> String {
|
||||
let trimmed = s.trim().to_lowercase();
|
||||
let collapsed: String = trimmed.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
collapsed.trim_end_matches(['.', ',', ';', '!']).to_string()
|
||||
}
|
||||
|
||||
/// Exact match after normalization.
|
||||
pub fn exact_match(expected: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_expected == norm_actual {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!(
|
||||
"expected \"{norm_expected}\", got \"{norm_actual}\""
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer contains the expected substring (normalized).
|
||||
pub fn contains_match(expected_substring: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected_substring);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_actual.contains(&norm_expected) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not contain \"{norm_expected}\""))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer matches a regex pattern.
|
||||
pub fn regex_match(pattern: &str, actual: &str) -> BenchScore {
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(actual) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not match pattern /{pattern}/"))
|
||||
}
|
||||
}
|
||||
Err(e) => BenchScore::fail(format!("invalid regex pattern: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_normalize_answer() {
|
||||
assert_eq!(normalize_answer(" Hello World. "), "hello world");
|
||||
assert_eq!(normalize_answer("Yes!"), "yes");
|
||||
assert_eq!(normalize_answer("42"), "42");
|
||||
assert_eq!(normalize_answer(" "), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_pass() {
|
||||
let score = exact_match("Hello World", " hello world. ");
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_fail() {
|
||||
let score = exact_match("hello", "world");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_pass() {
|
||||
let score = contains_match("world", "Hello World!");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_fail() {
|
||||
let score = contains_match("xyz", "Hello World!");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_pass() {
|
||||
let score = regex_match(r"\d{4}", "The year is 2024.");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_fail() {
|
||||
let score = regex_match(r"\d{4}", "No numbers here.");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_invalid_pattern() {
|
||||
let score = regex_match(r"[invalid", "anything");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert!(
|
||||
score
|
||||
.details
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("invalid regex")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// A single task in a benchmark suite.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchTask {
|
||||
pub id: String,
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub context: Option<String>,
|
||||
#[serde(default)]
|
||||
pub resources: Vec<TaskResource>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub expected_turns: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub timeout: Option<Duration>,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A resource attached to a benchmark task (file, URL, etc.).
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResource {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
#[serde(default)]
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ResourceType {
|
||||
#[default]
|
||||
File,
|
||||
Url,
|
||||
Directory,
|
||||
}
|
||||
|
||||
/// What the agent produced for scoring.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TaskSubmission {
|
||||
pub response: String,
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
pub tool_calls: Vec<String>,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A single turn in a multi-turn conversation.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ConversationTurn {
|
||||
pub role: TurnRole,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnRole {
|
||||
User,
|
||||
Assistant,
|
||||
System,
|
||||
}
|
||||
|
||||
/// Score for a single task.
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchScore {
|
||||
/// 0.0 to 1.0 (1.0 = perfect).
|
||||
pub value: f64,
|
||||
/// "pass" / "fail" / "partial".
|
||||
pub label: String,
|
||||
#[serde(default)]
|
||||
pub details: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchScore {
|
||||
pub fn pass() -> Self {
|
||||
Self {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fail(details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn partial(value: f64, details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: value.clamp(0.0, 1.0),
|
||||
label: "partial".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for benchmark suite adapters.
|
||||
///
|
||||
/// Each suite (GAIA, Tau-bench, custom, etc.) implements this trait
|
||||
/// to provide task loading, scoring, and optional lifecycle hooks.
|
||||
#[async_trait]
|
||||
#[allow(dead_code)]
|
||||
pub trait BenchSuite: Send + Sync {
|
||||
/// Human-readable name (e.g., "GAIA Validation").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Machine ID (e.g., "gaia").
|
||||
fn id(&self) -> &str;
|
||||
|
||||
/// Load all tasks from the suite's data source.
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError>;
|
||||
|
||||
/// Score the agent's submission against the expected answer.
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError>;
|
||||
|
||||
/// Optional: set up environment before running a task (clone repo, init DB, etc.).
|
||||
async fn setup_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: tear down environment after a task completes.
|
||||
async fn teardown_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: additional tools to register for this suite's tasks.
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// Multi-turn: generate next simulated user message based on conversation so far.
|
||||
/// Return `None` to end the conversation.
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
_task: &BenchTask,
|
||||
_conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -10,12 +10,17 @@
|
||||
//! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let root = PathBuf::from(&manifest_dir);
|
||||
|
||||
// ── Embed registry manifests ────────────────────────────────────────
|
||||
embed_registry_catalog(&root);
|
||||
|
||||
// ── Build Telegram channel WASM ─────────────────────────────────────
|
||||
let channel_dir = root.join("channels-src/telegram");
|
||||
let wasm_out = channel_dir.join("telegram.wasm");
|
||||
|
||||
@@ -104,3 +109,89 @@ fn main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect all registry manifests into a single JSON blob at compile time.
|
||||
///
|
||||
/// Output: `$OUT_DIR/embedded_catalog.json` with structure:
|
||||
/// ```json
|
||||
/// { "tools": [...], "channels": [...], "bundles": {...} }
|
||||
/// ```
|
||||
fn embed_registry_catalog(root: &Path) {
|
||||
use std::fs;
|
||||
|
||||
let registry_dir = root.join("registry");
|
||||
|
||||
// Rerun if the bundles file changes (per-file watches for tools/channels
|
||||
// are emitted inside collect_json_files to track content changes reliably).
|
||||
println!("cargo:rerun-if-changed=registry/_bundles.json");
|
||||
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
|
||||
let out_path = out_dir.join("embedded_catalog.json");
|
||||
|
||||
if !registry_dir.is_dir() {
|
||||
// No registry dir: write empty catalog
|
||||
fs::write(
|
||||
&out_path,
|
||||
r#"{"tools":[],"channels":[],"bundles":{"bundles":{}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
return;
|
||||
}
|
||||
|
||||
let mut tools = Vec::new();
|
||||
let mut channels = Vec::new();
|
||||
|
||||
// Collect tool manifests
|
||||
let tools_dir = registry_dir.join("tools");
|
||||
if tools_dir.is_dir() {
|
||||
collect_json_files(&tools_dir, &mut tools);
|
||||
}
|
||||
|
||||
// Collect channel manifests
|
||||
let channels_dir = registry_dir.join("channels");
|
||||
if channels_dir.is_dir() {
|
||||
collect_json_files(&channels_dir, &mut channels);
|
||||
}
|
||||
|
||||
// Read bundles
|
||||
let bundles_path = registry_dir.join("_bundles.json");
|
||||
let bundles_raw = if bundles_path.is_file() {
|
||||
fs::read_to_string(&bundles_path).unwrap_or_else(|_| r#"{"bundles":{}}"#.to_string())
|
||||
} else {
|
||||
r#"{"bundles":{}}"#.to_string()
|
||||
};
|
||||
|
||||
// Build the combined JSON
|
||||
let catalog = format!(
|
||||
r#"{{"tools":[{}],"channels":[{}],"bundles":{}}}"#,
|
||||
tools.join(","),
|
||||
channels.join(","),
|
||||
bundles_raw,
|
||||
);
|
||||
|
||||
fs::write(&out_path, catalog).unwrap();
|
||||
}
|
||||
|
||||
/// Read all .json files from a directory and push their raw contents into `out`.
|
||||
fn collect_json_files(dir: &Path, out: &mut Vec<String>) {
|
||||
use std::fs;
|
||||
|
||||
let mut entries: Vec<_> = fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("json")
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Sort for deterministic output
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
|
||||
for entry in entries {
|
||||
// Emit per-file watch so Cargo reruns when file contents change
|
||||
println!("cargo:rerun-if-changed={}", entry.path().display());
|
||||
if let Ok(content) = fs::read_to_string(entry.path()) {
|
||||
out.push(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+401
@@ -0,0 +1,401 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "discord-channel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "id-arena"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||
|
||||
[[package]]
|
||||
name = "leb128"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
|
||||
[[package]]
|
||||
name = "spdx"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-encoder"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1"
|
||||
dependencies = [
|
||||
"leb128",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-metadata"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"indexmap",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"spdx",
|
||||
"wasm-encoder",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bitflags",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
"wit-bindgen-rust-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-core"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"indexmap",
|
||||
"prettyplease",
|
||||
"syn",
|
||||
"wasm-metadata",
|
||||
"wit-bindgen-core",
|
||||
"wit-component",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust-macro"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wit-bindgen-core",
|
||||
"wit-bindgen-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-component"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags",
|
||||
"indexmap",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"wasm-encoder",
|
||||
"wasm-metadata",
|
||||
"wasmparser",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-parser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"id-arena",
|
||||
"indexmap",
|
||||
"log",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"unicode-xid",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
@@ -9,7 +9,7 @@ publish = false
|
||||
[dependencies]
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
wit-bindgen = "0.41.0"
|
||||
wit-bindgen = "0.36"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
@@ -2,6 +2,15 @@
|
||||
"type": "channel",
|
||||
"name": "discord",
|
||||
"description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "discord_bot_token",
|
||||
"prompt": "Enter your Discord Bot Token (from Developer Portal)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
@@ -10,7 +19,7 @@
|
||||
"credentials": {
|
||||
"discord_bot_token": {
|
||||
"secret_name": "discord_bot_token",
|
||||
"location": { "type": "header", "header_name": "Authorization", "prefix": "Bot " },
|
||||
"location": { "type": "header", "name": "Authorization", "prefix": "Bot " },
|
||||
"host_patterns": ["discord.com"]
|
||||
}
|
||||
},
|
||||
@@ -34,6 +43,9 @@
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"require_signature_verification": true
|
||||
"require_signature_verification": true,
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
+226
-16
@@ -124,12 +124,57 @@ struct DiscordMessageMetadata {
|
||||
thread_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "discord";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiscordConfig {
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
require_signature_verification: bool,
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
struct DiscordChannel;
|
||||
|
||||
impl Guest for DiscordChannel {
|
||||
fn on_start(_config_json: String) -> Result<ChannelConfig, String> {
|
||||
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
|
||||
let config: DiscordConfig = serde_json::from_str(&config_json)
|
||||
.map_err(|e| format!("Failed to parse config: {}", e))?;
|
||||
|
||||
channel_host::log(channel_host::LogLevel::Info, "Discord channel starting");
|
||||
|
||||
// Persist owner_id so subsequent callbacks can read it
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
// Persist dm_policy and allow_from for DM pairing
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
Ok(ChannelConfig {
|
||||
display_name: "Discord".to_string(),
|
||||
http_endpoints: vec![HttpEndpointConfig {
|
||||
@@ -169,16 +214,21 @@ impl Guest for DiscordChannel {
|
||||
|
||||
// Application Command (slash command)
|
||||
2 => {
|
||||
handle_slash_command(&interaction);
|
||||
json_response(
|
||||
200,
|
||||
serde_json::json!({
|
||||
"type": 5,
|
||||
"data": {
|
||||
"content": "🤔 Thinking..."
|
||||
}
|
||||
}),
|
||||
)
|
||||
if handle_slash_command(&interaction) {
|
||||
json_response(200, serde_json::json!({"type": 5}))
|
||||
} else {
|
||||
// Permission denied — ephemeral response
|
||||
json_response(
|
||||
200,
|
||||
serde_json::json!({
|
||||
"type": 4,
|
||||
"data": {
|
||||
"content": "You are not authorized to use this bot.",
|
||||
"flags": 64
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Message Component (buttons, selects)
|
||||
@@ -270,7 +320,8 @@ impl Guest for DiscordChannel {
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
/// Returns true if the message was emitted, false if permission denied.
|
||||
fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
@@ -287,6 +338,22 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
// DM if no guild member context (only direct user field set)
|
||||
let is_dm = interaction.member.is_none();
|
||||
|
||||
// Permission check
|
||||
if !check_sender_permission(
|
||||
&user_id,
|
||||
Some(&user_name),
|
||||
is_dm,
|
||||
Some(&PairingReplyCtx {
|
||||
application_id: interaction.application_id.clone(),
|
||||
token: interaction.token.clone(),
|
||||
}),
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let channel_id = interaction.channel_id.clone().unwrap_or_default();
|
||||
|
||||
let command_name = interaction
|
||||
@@ -322,14 +389,13 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize metadata: {}", e),
|
||||
);
|
||||
// Attempt to notify user of internal error
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
interaction.application_id, interaction.token
|
||||
);
|
||||
let payload = serde_json::json!({
|
||||
"content": "❌ Internal Error: Failed to process command metadata.",
|
||||
"flags": 64 // Ephemeral
|
||||
"flags": 64
|
||||
});
|
||||
let _ = channel_host::http_request(
|
||||
"POST",
|
||||
@@ -338,7 +404,7 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
Some(&serde_json::to_vec(&payload).unwrap_or_default()),
|
||||
None,
|
||||
);
|
||||
return;
|
||||
return true; // Error, but not a permission denial
|
||||
}
|
||||
};
|
||||
|
||||
@@ -349,10 +415,10 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
thread_id: None,
|
||||
metadata_json,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordMessage) {
|
||||
// Check member first (for server contexts), then user (for DMs)
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
@@ -369,6 +435,11 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let is_dm = interaction.member.is_none();
|
||||
if !check_sender_permission(&user_id, Some(&user_name), is_dm, None) {
|
||||
return;
|
||||
}
|
||||
|
||||
let channel_id = message.channel_id.clone();
|
||||
|
||||
let metadata = DiscordMessageMetadata {
|
||||
@@ -399,6 +470,145 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Context needed to send a pairing reply via Discord webhook followup.
|
||||
struct PairingReplyCtx {
|
||||
application_id: String,
|
||||
token: String,
|
||||
}
|
||||
|
||||
/// Check if a sender is permitted to interact with the bot.
|
||||
/// Returns true if allowed, false if denied (pairing reply sent if applicable).
|
||||
fn check_sender_permission(
|
||||
user_id: &str,
|
||||
username: Option<&str>,
|
||||
is_dm: bool,
|
||||
reply_ctx: Option<&PairingReplyCtx>,
|
||||
) -> bool {
|
||||
// 1. Owner check (highest priority, applies to all contexts)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if user_id != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping interaction from non-owner user {} (owner: {})",
|
||||
user_id, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (only for DMs when no owner_id)
|
||||
if !is_dm {
|
||||
return true; // Guild interactions bypass DM policy
|
||||
}
|
||||
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list: config allow_from + pairing store
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender against allow list
|
||||
let is_allowed = allowed.contains(&"*".to_string())
|
||||
|| allowed.contains(&user_id.to_string())
|
||||
|| username.is_some_and(|u| allowed.contains(&u.to_string()));
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"user_id": user_id,
|
||||
"username": username,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for user {}: code {}",
|
||||
user_id, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
if let Some(ctx) = reply_ctx {
|
||||
let _ = send_pairing_reply(ctx, &result.code);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code as an ephemeral Discord followup message.
|
||||
fn send_pairing_reply(ctx: &PairingReplyCtx, code: &str) -> Result<(), String> {
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
ctx.application_id, ctx.token
|
||||
);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"content": format!(
|
||||
"To pair with this bot, run: `ironclaw pairing approve discord {}`",
|
||||
code
|
||||
),
|
||||
"flags": 64 // Ephemeral — only visible to the sender
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"Discord API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
@@ -2,6 +2,20 @@
|
||||
"type": "channel",
|
||||
"name": "slack",
|
||||
"description": "Slack Events API channel for receiving and responding to Slack messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "slack_bot_token",
|
||||
"prompt": "Enter your Slack Bot OAuth Token (xoxb-...)",
|
||||
"optional": false
|
||||
},
|
||||
{
|
||||
"name": "slack_signing_secret",
|
||||
"prompt": "Enter your Slack Signing Secret (from App Credentials)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
@@ -33,6 +47,9 @@
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"signing_secret_name": "slack_signing_secret"
|
||||
"signing_secret_name": "slack_signing_secret",
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,15 +104,31 @@ struct SlackPostMessageResponse {
|
||||
ts: Option<String>,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "slack";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SlackConfig {
|
||||
/// Name of secret containing signing secret (for verification by host).
|
||||
/// Parsed from config for forward compatibility; not yet used in WASM
|
||||
/// (host handles signature verification).
|
||||
#[serde(default = "default_signing_secret_name")]
|
||||
#[allow(dead_code)]
|
||||
signing_secret_name: String,
|
||||
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_signing_secret_name() -> String {
|
||||
@@ -123,12 +139,30 @@ struct SlackChannel;
|
||||
|
||||
impl Guest for SlackChannel {
|
||||
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
|
||||
// Parse configuration
|
||||
let _config: SlackConfig = serde_json::from_str(&config_json)
|
||||
let config: SlackConfig = serde_json::from_str(&config_json)
|
||||
.map_err(|e| format!("Failed to parse config: {}", e))?;
|
||||
|
||||
channel_host::log(channel_host::LogLevel::Info, "Slack channel starting");
|
||||
|
||||
// Persist owner_id so subsequent callbacks can read it
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
// Persist dm_policy and allow_from for DM pairing
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
Ok(ChannelConfig {
|
||||
display_name: "Slack".to_string(),
|
||||
http_endpoints: vec![HttpEndpointConfig {
|
||||
@@ -136,7 +170,7 @@ impl Guest for SlackChannel {
|
||||
methods: vec!["POST".to_string()],
|
||||
require_secret: true,
|
||||
}],
|
||||
poll: None, // Slack uses push via webhooks, no polling needed
|
||||
poll: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -280,7 +314,7 @@ impl Guest for SlackChannel {
|
||||
/// Handle a Slack event and emit message if applicable.
|
||||
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
|
||||
match event.event_type.as_str() {
|
||||
// Direct mention of the bot
|
||||
// Direct mention of the bot (always in a channel, not a DM)
|
||||
"app_mention" => {
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) = (
|
||||
event.user,
|
||||
@@ -288,6 +322,10 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
|
||||
event.text,
|
||||
event.ts.clone(),
|
||||
) {
|
||||
// app_mention is always in a channel (not DM)
|
||||
if !check_sender_permission(&user, &channel, false) {
|
||||
return;
|
||||
}
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
}
|
||||
}
|
||||
@@ -307,6 +345,9 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
|
||||
) {
|
||||
// Only process DMs (channel IDs starting with D)
|
||||
if channel.starts_with('D') {
|
||||
if !check_sender_permission(&user, &channel, true) {
|
||||
return;
|
||||
}
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
}
|
||||
}
|
||||
@@ -358,6 +399,126 @@ fn emit_message(
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Check if a sender is permitted. Returns true if allowed.
|
||||
/// For pairing mode, sends a pairing code DM if denied.
|
||||
fn check_sender_permission(user_id: &str, channel_id: &str, is_dm: bool) -> bool {
|
||||
// 1. Owner check (highest priority, applies to all contexts)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if user_id != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner user {} (owner: {})",
|
||||
user_id, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (only for DMs when no owner_id)
|
||||
if !is_dm {
|
||||
return true; // Channel messages bypass DM policy
|
||||
}
|
||||
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list: config allow_from + pairing store
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender (Slack events only have user ID, not username)
|
||||
let is_allowed =
|
||||
allowed.contains(&"*".to_string()) || allowed.contains(&user_id.to_string());
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"user_id": user_id,
|
||||
"channel_id": channel_id,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for user {}: code {}",
|
||||
user_id, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
let _ = send_pairing_reply(channel_id, &result.code);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code message via Slack chat.postMessage.
|
||||
fn send_pairing_reply(channel_id: &str, code: &str) -> Result<(), String> {
|
||||
let payload = serde_json::json!({
|
||||
"channel": channel_id,
|
||||
"text": format!(
|
||||
"To pair with this bot, run: `ironclaw pairing approve slack {}`",
|
||||
code
|
||||
),
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
"https://slack.com/api/chat.postMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status == 200 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"Slack API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip leading bot mention from text.
|
||||
fn strip_bot_mention(text: &str) -> String {
|
||||
// Slack mentions look like <@U12345678>
|
||||
|
||||
@@ -25,5 +25,3 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -76,6 +76,9 @@ struct TelegramMessage {
|
||||
#[serde(default)]
|
||||
caption: Option<String>,
|
||||
|
||||
/// Voice message.
|
||||
voice: Option<TelegramVoice>,
|
||||
|
||||
/// Original message if this is a reply.
|
||||
reply_to_message: Option<Box<TelegramMessage>>,
|
||||
|
||||
@@ -139,6 +142,36 @@ struct MessageEntity {
|
||||
user: Option<TelegramUser>,
|
||||
}
|
||||
|
||||
/// Telegram Voice object.
|
||||
/// https://core.telegram.org/bots/api#voice
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramVoice {
|
||||
/// Identifier for this file, which can be used to download the file.
|
||||
file_id: String,
|
||||
|
||||
/// Duration of the audio in seconds.
|
||||
duration: u32,
|
||||
|
||||
/// MIME type of the file.
|
||||
#[serde(default)]
|
||||
mime_type: Option<String>,
|
||||
|
||||
/// File size in bytes.
|
||||
#[serde(default)]
|
||||
file_size: Option<i64>,
|
||||
}
|
||||
|
||||
/// Telegram File object returned by getFile.
|
||||
/// https://core.telegram.org/bots/api#file
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramFile {
|
||||
/// Identifier for this file.
|
||||
file_id: String,
|
||||
|
||||
/// File path for downloading. Use https://api.telegram.org/file/bot<token>/<file_path>.
|
||||
file_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Telegram API response wrapper.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramApiResponse<T> {
|
||||
@@ -867,6 +900,87 @@ fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Voice File Download
|
||||
// ============================================================================
|
||||
|
||||
/// Download a voice file from Telegram by file_id.
|
||||
///
|
||||
/// 1. Call getFile to get the file_path.
|
||||
/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}.
|
||||
fn download_voice_file(file_id: &str) -> Result<Vec<u8>, String> {
|
||||
// Reject file_id containing curly braces to prevent credential placeholder
|
||||
// injection (e.g., a malicious file_id like "{OPENAI_API_KEY}" would be
|
||||
// interpreted by the host-side credential injector).
|
||||
if file_id.contains('{') || file_id.contains('}') {
|
||||
return Err("invalid file_id: contains forbidden characters".to_string());
|
||||
}
|
||||
|
||||
// Step 1: Call getFile to get file_path
|
||||
// Double braces `{{...}}` produce a literal `{TELEGRAM_BOT_TOKEN}` placeholder
|
||||
// in the URL, which the host-side credential injector replaces with the real token.
|
||||
let get_file_url = format!(
|
||||
"https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}",
|
||||
file_id
|
||||
);
|
||||
|
||||
let headers = serde_json::json!({});
|
||||
let result = channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("getFile request failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
return Err(format!("getFile returned {}: {}", response.status, body_str));
|
||||
}
|
||||
|
||||
let api_response: TelegramApiResponse<TelegramFile> =
|
||||
serde_json::from_slice(&response.body)
|
||||
.map_err(|e| format!("Failed to parse getFile response: {}", e))?;
|
||||
|
||||
if !api_response.ok {
|
||||
return Err(format!(
|
||||
"getFile API error: {}",
|
||||
api_response
|
||||
.description
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
let file = api_response
|
||||
.result
|
||||
.ok_or_else(|| "getFile returned no result".to_string())?;
|
||||
|
||||
let file_path = file
|
||||
.file_path
|
||||
.ok_or_else(|| "getFile returned no file_path".to_string())?;
|
||||
|
||||
// Sanitize file_path against credential placeholder injection
|
||||
if file_path.contains('{') || file_path.contains('}') {
|
||||
return Err("invalid file_path: contains forbidden characters".to_string());
|
||||
}
|
||||
|
||||
// Step 2: Download the actual file bytes
|
||||
let download_url = format!(
|
||||
"https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}",
|
||||
file_path
|
||||
);
|
||||
|
||||
let result =
|
||||
channel_host::http_request("GET", &download_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("File download failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
return Err(format!(
|
||||
"File download returned status {}",
|
||||
response.status
|
||||
));
|
||||
}
|
||||
|
||||
Ok(response.body)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Update Handling
|
||||
// ============================================================================
|
||||
@@ -886,6 +1000,9 @@ fn handle_update(update: TelegramUpdate) {
|
||||
|
||||
/// Process a single message.
|
||||
fn handle_message(message: TelegramMessage) {
|
||||
// Check for voice note first (voice-only messages have no text)
|
||||
let is_voice = message.voice.is_some();
|
||||
|
||||
// Use text or caption (for media messages)
|
||||
let content = message
|
||||
.text
|
||||
@@ -893,7 +1010,8 @@ fn handle_message(message: TelegramMessage) {
|
||||
.or_else(|| message.caption.filter(|c| !c.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if content.is_empty() {
|
||||
// Allow voice notes through even when content is empty
|
||||
if content.is_empty() && !is_voice {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1038,19 +1156,65 @@ fn handle_message(message: TelegramMessage) {
|
||||
},
|
||||
);
|
||||
|
||||
// Handle voice notes: download and attach audio bytes.
|
||||
// Note: download is synchronous (two HTTP roundtrips to Telegram API).
|
||||
// This blocks the WASM execution for the current polling tick.
|
||||
let mut attachments = Vec::new();
|
||||
let mut voice_download_failed = false;
|
||||
if let Some(ref voice) = message.voice {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Voice note from user {} (duration: {}s, file_id: {})",
|
||||
from.id, voice.duration, voice.file_id
|
||||
),
|
||||
);
|
||||
|
||||
match download_voice_file(&voice.file_id) {
|
||||
Ok(audio_bytes) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Downloaded voice file: {} bytes", audio_bytes.len()),
|
||||
);
|
||||
attachments.push(channel_host::Attachment {
|
||||
kind: channel_host::AttachmentKind::Audio,
|
||||
mime_type: voice
|
||||
.mime_type
|
||||
.clone()
|
||||
.unwrap_or_else(|| "audio/ogg".to_string()),
|
||||
data: audio_bytes,
|
||||
filename: Some(format!("voice_{}.ogg", voice.file_id)),
|
||||
duration_secs: Some(voice.duration),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to download voice file: {}", e),
|
||||
);
|
||||
voice_download_failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine what to emit to the agent.
|
||||
// - Voice notes: use "[Voice note]" as content (transcription happens host-side)
|
||||
// - `/start` (no args): emit a welcome placeholder so the agent greets the user
|
||||
// - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through
|
||||
// so Submission::parse() can handle it
|
||||
// - Commands with args (e.g. `/start hello`): cleaned_text already has the args
|
||||
// - Plain text: pass through as-is
|
||||
let trimmed_content = content.trim();
|
||||
let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
let content_to_emit = if is_voice && voice_download_failed && content.is_empty() {
|
||||
"[Voice note: download failed]".to_string()
|
||||
} else if is_voice && content.is_empty() {
|
||||
"[Voice note]".to_string()
|
||||
} else if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
"[User started the bot]".to_string()
|
||||
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
|
||||
// Bare control command like /interrupt, /stop, /help — pass through raw
|
||||
trimmed_content.to_string()
|
||||
} else if cleaned_text.is_empty() {
|
||||
} else if cleaned_text.is_empty() && !is_voice {
|
||||
return;
|
||||
} else {
|
||||
cleaned_text
|
||||
@@ -1063,6 +1227,7 @@ fn handle_message(message: TelegramMessage) {
|
||||
content: content_to_emit,
|
||||
thread_id: None, // Telegram doesn't have threads in the same way
|
||||
metadata_json,
|
||||
attachments,
|
||||
});
|
||||
|
||||
channel_host::log(
|
||||
|
||||
@@ -1 +1,55 @@
|
||||
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
|
||||
{
|
||||
"type": "channel",
|
||||
"name": "telegram",
|
||||
"description": "Telegram Bot API channel for receiving and responding to Telegram messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "telegram_bot_token",
|
||||
"prompt": "Enter your Telegram Bot API token (from @BotFather)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
{ "host": "api.telegram.org", "path_prefix": "/bot" },
|
||||
{ "host": "api.telegram.org", "path_prefix": "/file/bot" }
|
||||
],
|
||||
"credentials": {
|
||||
"telegram_bot": {
|
||||
"secret_name": "telegram_bot_token",
|
||||
"location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" },
|
||||
"host_patterns": ["api.telegram.org"]
|
||||
}
|
||||
},
|
||||
"rate_limit": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 1000
|
||||
}
|
||||
},
|
||||
"secrets": {
|
||||
"allowed_names": ["telegram_*"]
|
||||
},
|
||||
"channel": {
|
||||
"allowed_paths": ["/webhook/telegram"],
|
||||
"allow_polling": true,
|
||||
"min_poll_interval_ms": 30000,
|
||||
"workspace_prefix": "channels/telegram/",
|
||||
"emit_rate_limit": {
|
||||
"messages_per_minute": 100,
|
||||
"messages_per_hour": 5000
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"bot_username": null,
|
||||
"owner_id": null,
|
||||
"respond_to_all_group_messages": false,
|
||||
"polling_enabled": false,
|
||||
"poll_interval_ms": 30000,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,15 @@ struct WhatsAppMessageMetadata {
|
||||
timestamp: String,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "whatsapp";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WhatsAppConfig {
|
||||
@@ -236,6 +245,15 @@ struct WhatsAppConfig {
|
||||
/// Whether to reply to the original message (thread context)
|
||||
#[serde(default = "default_reply_to_message")]
|
||||
reply_to_message: bool,
|
||||
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_api_version() -> String {
|
||||
@@ -264,6 +282,9 @@ impl Guest for WhatsAppChannel {
|
||||
WhatsAppConfig {
|
||||
api_version: default_api_version(),
|
||||
reply_to_message: default_reply_to_message(),
|
||||
owner_id: None,
|
||||
dm_policy: None,
|
||||
allow_from: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -279,6 +300,24 @@ impl Guest for WhatsAppChannel {
|
||||
// Persist api_version in workspace so on_respond() can read it
|
||||
let _ = channel_host::workspace_write("channels/whatsapp/api_version", &config.api_version);
|
||||
|
||||
// Persist permission config for handle_message
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
// WhatsApp Cloud API is webhook-only, no polling available
|
||||
Ok(ChannelConfig {
|
||||
display_name: "WhatsApp".to_string(),
|
||||
@@ -604,6 +643,15 @@ fn handle_message(
|
||||
// Look up sender's name from contacts
|
||||
let user_name = contact_names.get(&message.from).cloned();
|
||||
|
||||
// Permission check (WhatsApp is always DM)
|
||||
if !check_sender_permission(
|
||||
&message.from,
|
||||
user_name.as_deref(),
|
||||
phone_number_id,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build metadata for response routing
|
||||
// This is critical - the response handler uses this to know where to send
|
||||
let metadata = WhatsAppMessageMetadata {
|
||||
@@ -637,6 +685,149 @@ fn handle_message(
|
||||
// Utilities
|
||||
// ============================================================================
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Check if a sender is permitted. Returns true if allowed.
|
||||
/// WhatsApp is always 1-to-1 (DM), so dm_policy always applies.
|
||||
fn check_sender_permission(
|
||||
sender_phone: &str,
|
||||
user_name: Option<&str>,
|
||||
phone_number_id: &str,
|
||||
) -> bool {
|
||||
// 1. Owner check (highest priority)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if sender_phone != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner {} (owner: {})",
|
||||
sender_phone, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (WhatsApp is always DM)
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender (phone number or name)
|
||||
let is_allowed = allowed.contains(&"*".to_string())
|
||||
|| allowed.contains(&sender_phone.to_string())
|
||||
|| user_name.is_some_and(|u| allowed.contains(&u.to_string()));
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"phone": sender_phone,
|
||||
"name": user_name,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, sender_phone, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for {}: code {}",
|
||||
sender_phone, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
let _ = send_pairing_reply(sender_phone, phone_number_id, &result.code);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code message via WhatsApp Cloud API.
|
||||
fn send_pairing_reply(
|
||||
recipient_phone: &str,
|
||||
phone_number_id: &str,
|
||||
code: &str,
|
||||
) -> Result<(), String> {
|
||||
let api_version = channel_host::workspace_read("channels/whatsapp/api_version")
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "v18.0".to_string());
|
||||
|
||||
let url = format!(
|
||||
"https://graph.facebook.com/{}/{}/messages",
|
||||
api_version, phone_number_id
|
||||
);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"messaging_product": "whatsapp",
|
||||
"recipient_type": "individual",
|
||||
"to": recipient_phone,
|
||||
"type": "text",
|
||||
"text": {
|
||||
"preview_url": false,
|
||||
"body": format!(
|
||||
"To pair with this bot, run: ironclaw pairing approve whatsapp {}",
|
||||
code
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer {WHATSAPP_ACCESS_TOKEN}"
|
||||
});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"WhatsApp API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a JSON HTTP response.
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
|
||||
@@ -48,6 +48,9 @@
|
||||
},
|
||||
"config": {
|
||||
"api_version": "v18.0",
|
||||
"reply_to_message": true
|
||||
"reply_to_message": true,
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# LLM Provider Configuration
|
||||
|
||||
IronClaw defaults to NEAR AI for model access, but supports any OpenAI-compatible
|
||||
endpoint as well as Anthropic and Ollama directly. This guide covers the most common
|
||||
configurations.
|
||||
|
||||
## Provider Overview
|
||||
|
||||
| Provider | Backend value | Requires API key | Notes |
|
||||
|---|---|---|---|
|
||||
| NEAR AI | `nearai` | OAuth (browser) | Default; multi-model |
|
||||
| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | Claude models |
|
||||
| OpenAI | `openai` | `OPENAI_API_KEY` | GPT models |
|
||||
| Ollama | `ollama` | No | Local inference |
|
||||
| OpenRouter | `openai_compatible` | `LLM_API_KEY` | 300+ models |
|
||||
| Together AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
|
||||
| Fireworks AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
|
||||
| vLLM / LiteLLM | `openai_compatible` | Optional | Self-hosted |
|
||||
| LM Studio | `openai_compatible` | No | Local GUI |
|
||||
|
||||
---
|
||||
|
||||
## NEAR AI (default)
|
||||
|
||||
No additional configuration required. On first run, `ironclaw onboard` opens a browser
|
||||
for OAuth authentication. Credentials are saved to `~/.ironclaw/session.json`.
|
||||
|
||||
```env
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Anthropic (Claude)
|
||||
|
||||
```env
|
||||
LLM_BACKEND=anthropic
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
```
|
||||
|
||||
Popular models: `claude-sonnet-4-20250514`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-20241022`
|
||||
|
||||
---
|
||||
|
||||
## OpenAI (GPT)
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai
|
||||
OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
Popular models: `gpt-4o`, `gpt-4o-mini`, `o3-mini`
|
||||
|
||||
---
|
||||
|
||||
## Ollama (local)
|
||||
|
||||
Install Ollama from [ollama.com](https://ollama.com), pull a model, then:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=ollama
|
||||
OLLAMA_MODEL=llama3.2
|
||||
# OLLAMA_BASE_URL=http://localhost:11434 # default
|
||||
```
|
||||
|
||||
Pull a model first: `ollama pull llama3.2`
|
||||
|
||||
---
|
||||
|
||||
## OpenAI-Compatible Endpoints
|
||||
|
||||
All providers below use `LLM_BACKEND=openai_compatible`. Set `LLM_BASE_URL` to the
|
||||
provider's OpenAI-compatible endpoint and `LLM_API_KEY` to your API key.
|
||||
|
||||
### OpenRouter
|
||||
|
||||
[OpenRouter](https://openrouter.ai) routes to 300+ models from a single API key.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
LLM_API_KEY=sk-or-...
|
||||
LLM_MODEL=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
Popular OpenRouter model IDs:
|
||||
|
||||
| Model | ID |
|
||||
|---|---|
|
||||
| Claude Sonnet 4 | `anthropic/claude-sonnet-4` |
|
||||
| GPT-4o | `openai/gpt-4o` |
|
||||
| Llama 4 Maverick | `meta-llama/llama-4-maverick` |
|
||||
| Gemini 2.0 Flash | `google/gemini-2.0-flash-001` |
|
||||
| Mistral Small | `mistralai/mistral-small-3.1-24b-instruct` |
|
||||
|
||||
Browse all models at [openrouter.ai/models](https://openrouter.ai/models).
|
||||
|
||||
### Together AI
|
||||
|
||||
[Together AI](https://www.together.ai) provides fast inference for open-source models.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://api.together.xyz/v1
|
||||
LLM_API_KEY=...
|
||||
LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
|
||||
```
|
||||
|
||||
Popular Together AI model IDs:
|
||||
|
||||
| Model | ID |
|
||||
|---|---|
|
||||
| Llama 3.3 70B | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
|
||||
| DeepSeek R1 | `deepseek-ai/DeepSeek-R1` |
|
||||
| Qwen 2.5 72B | `Qwen/Qwen2.5-72B-Instruct-Turbo` |
|
||||
|
||||
### Fireworks AI
|
||||
|
||||
[Fireworks AI](https://fireworks.ai) offers fast inference with compound AI system support.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://api.fireworks.ai/inference/v1
|
||||
LLM_API_KEY=fw_...
|
||||
LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
|
||||
```
|
||||
|
||||
### vLLM / LiteLLM (self-hosted)
|
||||
|
||||
For self-hosted inference servers:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:8000/v1
|
||||
LLM_API_KEY=token-abc123 # set to any string if auth is not configured
|
||||
LLM_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
```
|
||||
|
||||
LiteLLM proxy (forwards to any backend, including Bedrock, Vertex, Azure):
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:4000/v1
|
||||
LLM_API_KEY=sk-...
|
||||
LLM_MODEL=gpt-4o # as configured in litellm config.yaml
|
||||
```
|
||||
|
||||
### LM Studio (local GUI)
|
||||
|
||||
Start LM Studio's local server, then:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:1234/v1
|
||||
LLM_MODEL=llama-3.2-3b-instruct-q4_K_M
|
||||
# LLM_API_KEY is not required for LM Studio
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Using the Setup Wizard
|
||||
|
||||
Instead of editing `.env` manually, run the onboarding wizard:
|
||||
|
||||
```bash
|
||||
ironclaw onboard
|
||||
```
|
||||
|
||||
Select **"OpenAI-compatible"** for OpenRouter, Together AI, Fireworks, vLLM, LiteLLM,
|
||||
or LM Studio. You will be prompted for the base URL and (optionally) an API key.
|
||||
The model name is configured in the following step.
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/okta-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
+10
-5
@@ -358,10 +358,6 @@ impl Agent {
|
||||
}
|
||||
});
|
||||
|
||||
tracing::info!(
|
||||
"Heartbeat enabled with {}s interval",
|
||||
hb_config.interval_secs
|
||||
);
|
||||
let hygiene = self
|
||||
.hygiene_config
|
||||
.as_ref()
|
||||
@@ -373,6 +369,7 @@ impl Agent {
|
||||
hygiene,
|
||||
workspace.clone(),
|
||||
self.cheap_llm().clone(),
|
||||
self.safety().clone(),
|
||||
Some(notify_tx),
|
||||
))
|
||||
} else {
|
||||
@@ -685,7 +682,15 @@ impl Agent {
|
||||
|
||||
// Convert SubmissionResult to response string
|
||||
match result? {
|
||||
SubmissionResult::Response { content } => Ok(Some(content)),
|
||||
SubmissionResult::Response { content } => {
|
||||
// Suppress silent replies (e.g. from group chat "nothing to say" responses)
|
||||
if crate::llm::is_silent_reply(&content) {
|
||||
tracing::debug!("Suppressing silent reply token");
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(content))
|
||||
}
|
||||
}
|
||||
SubmissionResult::Ok { message } => Ok(message),
|
||||
SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))),
|
||||
SubmissionResult::Interrupted => Ok(Some("Interrupted.".into())),
|
||||
|
||||
+10
-7
@@ -13,7 +13,7 @@ use crate::agent::submission::SubmissionResult;
|
||||
use crate::agent::{Agent, MessageIntent};
|
||||
use crate::channels::{IncomingMessage, StatusUpdate};
|
||||
use crate::error::Error;
|
||||
use crate::llm::ChatMessage;
|
||||
use crate::llm::{ChatMessage, Reasoning};
|
||||
|
||||
impl Agent {
|
||||
/// Handle job-related intents without turn tracking.
|
||||
@@ -235,6 +235,7 @@ impl Agent {
|
||||
crate::workspace::hygiene::HygieneConfig::default(),
|
||||
workspace.clone(),
|
||||
self.llm().clone(),
|
||||
self.safety().clone(),
|
||||
);
|
||||
|
||||
match runner.check_heartbeat().await {
|
||||
@@ -295,10 +296,11 @@ impl Agent {
|
||||
.with_max_tokens(512)
|
||||
.with_temperature(0.3);
|
||||
|
||||
match self.llm().complete(request).await {
|
||||
Ok(response) => Ok(SubmissionResult::response(format!(
|
||||
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
|
||||
match reasoning.complete(request).await {
|
||||
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
|
||||
"Thread Summary:\n\n{}",
|
||||
response.content.trim()
|
||||
text.trim()
|
||||
))),
|
||||
Err(e) => Ok(SubmissionResult::error(format!("Summarize failed: {}", e))),
|
||||
}
|
||||
@@ -342,10 +344,11 @@ impl Agent {
|
||||
.with_max_tokens(512)
|
||||
.with_temperature(0.5);
|
||||
|
||||
match self.llm().complete(request).await {
|
||||
Ok(response) => Ok(SubmissionResult::response(format!(
|
||||
let reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
|
||||
match reasoning.complete(request).await {
|
||||
Ok((text, _usage)) => Ok(SubmissionResult::response(format!(
|
||||
"Suggested Next Steps:\n\n{}",
|
||||
response.content.trim()
|
||||
text.trim()
|
||||
))),
|
||||
Err(e) => Ok(SubmissionResult::error(format!("Suggest failed: {}", e))),
|
||||
}
|
||||
|
||||
@@ -12,7 +12,8 @@ use chrono::Utc;
|
||||
use crate::agent::context_monitor::{CompactionStrategy, ContextBreakdown};
|
||||
use crate::agent::session::Thread;
|
||||
use crate::error::Error;
|
||||
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider};
|
||||
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::workspace::Workspace;
|
||||
|
||||
/// Result of a compaction operation.
|
||||
@@ -33,12 +34,13 @@ pub struct CompactionResult {
|
||||
/// Compacts conversation context to stay within limits.
|
||||
pub struct ContextCompactor {
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
}
|
||||
|
||||
impl ContextCompactor {
|
||||
/// Create a new context compactor.
|
||||
pub fn new(llm: Arc<dyn LlmProvider>) -> Self {
|
||||
Self { llm }
|
||||
pub fn new(llm: Arc<dyn LlmProvider>, safety: Arc<SafetyLayer>) -> Self {
|
||||
Self { llm, safety }
|
||||
}
|
||||
|
||||
/// Compact a thread's context using the given strategy.
|
||||
@@ -231,8 +233,9 @@ Be brief but capture all important details. Use bullet points."#,
|
||||
.with_max_tokens(1024)
|
||||
.with_temperature(0.3);
|
||||
|
||||
let response = self.llm.complete(request).await?;
|
||||
Ok(response.content)
|
||||
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
|
||||
let (text, _) = reasoning.complete(request).await?;
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
/// Write a summary to the workspace daily log.
|
||||
|
||||
+60
-1
@@ -4,7 +4,7 @@
|
||||
//! to prevent runaway agents from burning through API credits. Especially
|
||||
//! important for daemon/heartbeat modes where the agent acts autonomously.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -53,6 +53,14 @@ impl std::fmt::Display for CostLimitExceeded {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-model token usage counters.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ModelTokens {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cost: Decimal,
|
||||
}
|
||||
|
||||
/// Tracks costs and action rates, enforcing configurable limits.
|
||||
///
|
||||
/// Thread-safe; designed to be shared via `Arc<CostGuard>`.
|
||||
@@ -67,6 +75,9 @@ pub struct CostGuard {
|
||||
|
||||
/// Flag set when daily budget is exceeded to short-circuit checks.
|
||||
budget_exceeded: AtomicBool,
|
||||
|
||||
/// Per-model token usage since startup.
|
||||
model_tokens: Mutex<HashMap<String, ModelTokens>>,
|
||||
}
|
||||
|
||||
struct DailyCost {
|
||||
@@ -85,6 +96,7 @@ impl CostGuard {
|
||||
}),
|
||||
action_window: Mutex::new(VecDeque::new()),
|
||||
budget_exceeded: AtomicBool::new(false),
|
||||
model_tokens: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,6 +204,15 @@ impl CostGuard {
|
||||
window.push_back(Instant::now());
|
||||
}
|
||||
|
||||
// Track per-model token usage
|
||||
{
|
||||
let mut tokens = self.model_tokens.lock().await;
|
||||
let entry = tokens.entry(model.to_string()).or_default();
|
||||
entry.input_tokens += u64::from(input_tokens);
|
||||
entry.output_tokens += u64::from(output_tokens);
|
||||
entry.cost += cost;
|
||||
}
|
||||
|
||||
cost
|
||||
}
|
||||
|
||||
@@ -215,6 +236,11 @@ impl CostGuard {
|
||||
}
|
||||
window.len() as u64
|
||||
}
|
||||
|
||||
/// Per-model token usage since startup.
|
||||
pub async fn model_usage(&self) -> HashMap<String, ModelTokens> {
|
||||
self.model_tokens.lock().await.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a Decimal USD amount to whole cents (truncated).
|
||||
@@ -336,4 +362,37 @@ mod tests {
|
||||
assert!(rate.to_string().contains("101 actions"));
|
||||
assert!(rate.to_string().contains("100 allowed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_usage_per_model_tracking() {
|
||||
let guard = CostGuard::new(CostGuardConfig::default());
|
||||
|
||||
// Initially empty
|
||||
assert!(guard.model_usage().await.is_empty());
|
||||
|
||||
// Record calls for two different models
|
||||
guard.record_llm_call("gpt-4o", 1000, 500).await;
|
||||
guard.record_llm_call("gpt-4o", 2000, 1000).await;
|
||||
guard
|
||||
.record_llm_call("claude-3-5-sonnet-20241022", 500, 200)
|
||||
.await;
|
||||
|
||||
let usage = guard.model_usage().await;
|
||||
assert_eq!(usage.len(), 2);
|
||||
|
||||
let gpt = usage.get("gpt-4o").expect("gpt-4o should be tracked");
|
||||
assert_eq!(gpt.input_tokens, 3000);
|
||||
assert_eq!(gpt.output_tokens, 1500);
|
||||
assert!(gpt.cost > Decimal::ZERO);
|
||||
|
||||
let claude = usage
|
||||
.get("claude-3-5-sonnet-20241022")
|
||||
.expect("claude should be tracked");
|
||||
assert_eq!(claude.input_tokens, 500);
|
||||
assert_eq!(claude.output_tokens, 200);
|
||||
assert!(claude.cost > Decimal::ZERO);
|
||||
|
||||
// Costs should differ since models have different pricing
|
||||
assert_ne!(gpt.cost, claude.cost);
|
||||
}
|
||||
}
|
||||
|
||||
+75
-37
@@ -40,9 +40,17 @@ impl Agent {
|
||||
thread_id: Uuid,
|
||||
initial_messages: Vec<ChatMessage>,
|
||||
) -> Result<AgenticLoopResult, Error> {
|
||||
// Detect group chat from channel metadata (needed before loading system prompt)
|
||||
let is_group_chat = message
|
||||
.metadata
|
||||
.get("chat_type")
|
||||
.and_then(|v| v.as_str())
|
||||
.is_some_and(|t| t == "group" || t == "channel" || t == "supergroup");
|
||||
|
||||
// Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.)
|
||||
// In group chats, MEMORY.md is excluded to prevent leaking personal context.
|
||||
let system_prompt = if let Some(ws) = self.workspace() {
|
||||
match ws.system_prompt().await {
|
||||
match ws.system_prompt_for_context(is_group_chat).await {
|
||||
Ok(prompt) if !prompt.is_empty() => Some(prompt),
|
||||
Ok(_) => None,
|
||||
Err(e) => {
|
||||
@@ -94,7 +102,10 @@ impl Agent {
|
||||
None
|
||||
};
|
||||
|
||||
let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
|
||||
let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone())
|
||||
.with_channel(message.channel.clone())
|
||||
.with_model_name(self.llm().active_model_name())
|
||||
.with_group_chat(is_group_chat);
|
||||
if let Some(prompt) = system_prompt {
|
||||
reasoning = reasoning.with_system_prompt(prompt);
|
||||
}
|
||||
@@ -108,14 +119,21 @@ impl Agent {
|
||||
// Create a JobContext for tool execution (chat doesn't have a real job)
|
||||
let job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session");
|
||||
|
||||
const MAX_TOOL_ITERATIONS: usize = 10;
|
||||
let max_tool_iterations = self.config.max_tool_iterations;
|
||||
// Force a text-only response on the last iteration to guarantee termination
|
||||
// instead of hard-erroring. The penultimate iteration also gets a nudge
|
||||
// message so the LLM knows it should wrap up.
|
||||
let force_text_at = max_tool_iterations;
|
||||
let nudge_at = max_tool_iterations.saturating_sub(1);
|
||||
let mut iteration = 0;
|
||||
loop {
|
||||
iteration += 1;
|
||||
if iteration > MAX_TOOL_ITERATIONS {
|
||||
// Hard ceiling one past the forced-text iteration (should never be reached
|
||||
// since force_text_at guarantees a text response, but kept as a safety net).
|
||||
if iteration > max_tool_iterations + 1 {
|
||||
return Err(crate::error::LlmError::InvalidResponse {
|
||||
provider: "agent".to_string(),
|
||||
reason: format!("Exceeded maximum tool iterations ({})", MAX_TOOL_ITERATIONS),
|
||||
reason: format!("Exceeded maximum tool iterations ({max_tool_iterations})"),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
@@ -143,6 +161,19 @@ impl Agent {
|
||||
.into());
|
||||
}
|
||||
|
||||
// Inject a nudge message when approaching the iteration limit so the
|
||||
// LLM is aware it should produce a final answer on the next turn.
|
||||
if iteration == nudge_at {
|
||||
context_messages.push(ChatMessage::system(
|
||||
"You are approaching the tool call limit. \
|
||||
Provide your best final answer on the next response \
|
||||
using the information you have gathered so far. \
|
||||
Do not call any more tools.",
|
||||
));
|
||||
}
|
||||
|
||||
let force_text = iteration >= force_text_at;
|
||||
|
||||
// Refresh tool definitions each iteration so newly built tools become visible
|
||||
let tool_defs = self.tools().tool_definitions().await;
|
||||
|
||||
@@ -162,8 +193,9 @@ impl Agent {
|
||||
tool_defs
|
||||
};
|
||||
|
||||
// Call LLM with current context
|
||||
let context = ReasoningContext::new()
|
||||
// Call LLM with current context; force_text drops tools to guarantee a
|
||||
// text response on the final iteration.
|
||||
let mut context = ReasoningContext::new()
|
||||
.with_messages(context_messages.clone())
|
||||
.with_tools(tool_defs)
|
||||
.with_metadata({
|
||||
@@ -171,6 +203,14 @@ impl Agent {
|
||||
m.insert("thread_id".to_string(), thread_id.to_string());
|
||||
m
|
||||
});
|
||||
context.force_text = force_text;
|
||||
|
||||
if force_text {
|
||||
tracing::info!(
|
||||
iteration,
|
||||
"Forcing text-only response (iteration limit reached)"
|
||||
);
|
||||
}
|
||||
|
||||
let output = reasoning.respond_with_tools(&context).await?;
|
||||
|
||||
@@ -255,31 +295,8 @@ impl Agent {
|
||||
for (idx, original_tc) in tool_calls.iter().enumerate() {
|
||||
let mut tc = original_tc.clone();
|
||||
|
||||
// Check if tool requires approval
|
||||
if let Some(tool) = self.tools().get(&tc.name).await
|
||||
&& tool.requires_approval()
|
||||
{
|
||||
let mut is_auto_approved = {
|
||||
let sess = session.lock().await;
|
||||
sess.is_tool_auto_approved(&tc.name)
|
||||
};
|
||||
|
||||
// Override auto-approval for destructive parameters
|
||||
if is_auto_approved && tool.requires_approval_for(&tc.arguments) {
|
||||
tracing::info!(
|
||||
tool = %tc.name,
|
||||
"Parameters require explicit approval despite auto-approve"
|
||||
);
|
||||
is_auto_approved = false;
|
||||
}
|
||||
|
||||
if !is_auto_approved {
|
||||
approval_needed = Some((idx, tc, tool));
|
||||
break; // remaining tools are deferred
|
||||
}
|
||||
}
|
||||
|
||||
// Hook: BeforeToolCall
|
||||
// Hook: BeforeToolCall (runs before approval so hooks can
|
||||
// modify parameters — approval is checked on final params)
|
||||
let event = crate::hooks::HookEvent::ToolCall {
|
||||
tool_name: tc.name.clone(),
|
||||
parameters: tc.arguments.clone(),
|
||||
@@ -322,6 +339,27 @@ impl Agent {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Check if tool requires approval on the final (post-hook)
|
||||
// parameters. Skipped when auto_approve_tools is set.
|
||||
if !self.config.auto_approve_tools
|
||||
&& let Some(tool) = self.tools().get(&tc.name).await
|
||||
{
|
||||
use crate::tools::ApprovalRequirement;
|
||||
let needs_approval = match tool.requires_approval(&tc.arguments) {
|
||||
ApprovalRequirement::Never => false,
|
||||
ApprovalRequirement::UnlessAutoApproved => {
|
||||
let sess = session.lock().await;
|
||||
!sess.is_tool_auto_approved(&tc.name)
|
||||
}
|
||||
ApprovalRequirement::Always => true,
|
||||
};
|
||||
|
||||
if needs_approval {
|
||||
approval_needed = Some((idx, tc, tool));
|
||||
break; // remaining tools are deferred
|
||||
}
|
||||
}
|
||||
|
||||
let preflight_idx = preflight.len();
|
||||
preflight.push((tc.clone(), PreflightOutcome::Runnable));
|
||||
runnable.push((preflight_idx, tc));
|
||||
@@ -799,7 +837,6 @@ mod tests {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -813,7 +850,6 @@ mod tests {
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -850,6 +886,8 @@ mod tests {
|
||||
allow_local_tools: false,
|
||||
max_cost_per_day_cents: None,
|
||||
max_actions_per_hour: None,
|
||||
max_tool_iterations: 50,
|
||||
auto_approve_tools: false,
|
||||
},
|
||||
deps,
|
||||
ChannelManager::new(),
|
||||
@@ -880,9 +918,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shell_destructive_command_requires_approval_for() {
|
||||
// ShellTool::requires_approval_for should detect destructive commands.
|
||||
// This exercises the same code path used inline in run_agentic_loop.
|
||||
fn test_shell_destructive_command_requires_explicit_approval() {
|
||||
// requires_explicit_approval() detects destructive commands that
|
||||
// should return ApprovalRequirement::Always from ShellTool.
|
||||
use crate::tools::builtin::shell::requires_explicit_approval;
|
||||
|
||||
let destructive_cmds = [
|
||||
|
||||
+11
-13
@@ -29,7 +29,8 @@ use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::channels::OutgoingResponse;
|
||||
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
||||
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider, Reasoning};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::workspace::Workspace;
|
||||
use crate::workspace::hygiene::HygieneConfig;
|
||||
|
||||
@@ -100,6 +101,7 @@ pub struct HeartbeatRunner {
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
|
||||
consecutive_failures: u32,
|
||||
}
|
||||
@@ -111,12 +113,14 @@ impl HeartbeatRunner {
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
config,
|
||||
hygiene_config,
|
||||
workspace,
|
||||
llm,
|
||||
safety,
|
||||
response_tx: None,
|
||||
consecutive_failures: 0,
|
||||
}
|
||||
@@ -258,25 +262,18 @@ impl HeartbeatRunner {
|
||||
.with_max_tokens(max_tokens)
|
||||
.with_temperature(0.3);
|
||||
|
||||
let response = match self.llm.complete(request).await {
|
||||
let reasoning = Reasoning::new(self.llm.clone(), self.safety.clone());
|
||||
let (content, _usage) = match reasoning.complete(request).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)),
|
||||
};
|
||||
|
||||
let content = response.content.trim();
|
||||
let content = content.trim();
|
||||
|
||||
// Guard against empty content. Reasoning models (e.g. GLM-4.7) may
|
||||
// burn all output tokens on chain-of-thought and return content: null.
|
||||
if content.is_empty() {
|
||||
return if response.finish_reason == FinishReason::Length {
|
||||
HeartbeatResult::Failed(
|
||||
"LLM response was truncated (finish_reason=length) with no content. \
|
||||
The model may have exhausted its token budget on reasoning."
|
||||
.to_string(),
|
||||
)
|
||||
} else {
|
||||
HeartbeatResult::Failed("LLM returned empty content.".to_string())
|
||||
};
|
||||
return HeartbeatResult::Failed("LLM returned empty content.".to_string());
|
||||
}
|
||||
|
||||
// Check if nothing needs attention
|
||||
@@ -355,9 +352,10 @@ pub fn spawn_heartbeat(
|
||||
hygiene_config: HygieneConfig,
|
||||
workspace: Arc<Workspace>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm);
|
||||
let mut runner = HeartbeatRunner::new(config, hygiene_config, workspace, llm, safety);
|
||||
if let Some(tx) = response_tx {
|
||||
runner = runner.with_response_channel(tx);
|
||||
}
|
||||
|
||||
@@ -357,7 +357,7 @@ impl Scheduler {
|
||||
.into());
|
||||
}
|
||||
|
||||
if tool.requires_approval() {
|
||||
if tool.requires_approval(¶ms).is_required() {
|
||||
return Err(crate::error::ToolError::AuthRequired {
|
||||
name: tool_name.to_string(),
|
||||
}
|
||||
|
||||
@@ -185,10 +185,6 @@ pub struct Thread {
|
||||
/// Pending auth token request (thread is in auth mode).
|
||||
#[serde(default)]
|
||||
pub pending_auth: Option<PendingAuth>,
|
||||
/// Last NEAR AI response ID for response chaining. Persisted to DB
|
||||
/// metadata so we can resume chaining across restarts.
|
||||
#[serde(default)]
|
||||
pub last_response_id: Option<String>,
|
||||
}
|
||||
|
||||
impl Thread {
|
||||
@@ -205,7 +201,6 @@ impl Thread {
|
||||
metadata: serde_json::Value::Null,
|
||||
pending_approval: None,
|
||||
pending_auth: None,
|
||||
last_response_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,7 +217,6 @@ impl Thread {
|
||||
metadata: serde_json::Value::Null,
|
||||
pending_approval: None,
|
||||
pending_auth: None,
|
||||
last_response_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -863,7 +857,6 @@ mod tests {
|
||||
|
||||
thread.start_turn("hello");
|
||||
thread.complete_turn("world");
|
||||
thread.last_response_id = Some("resp_abc123".to_string());
|
||||
|
||||
let json = serde_json::to_string(&thread).unwrap();
|
||||
let restored: Thread = serde_json::from_str(&json).unwrap();
|
||||
@@ -873,7 +866,6 @@ mod tests {
|
||||
assert_eq!(restored.turns.len(), 1);
|
||||
assert_eq!(restored.turns[0].user_input, "hello");
|
||||
assert_eq!(restored.turns[0].response, Some("world".to_string()));
|
||||
assert_eq!(restored.last_response_id, Some("resp_abc123".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+48
-100
@@ -87,20 +87,6 @@ impl Agent {
|
||||
thread.restore_from_messages(chat_messages);
|
||||
}
|
||||
|
||||
// Restore response chain from conversation metadata
|
||||
if let Some(store) = self.store()
|
||||
&& let Ok(Some(metadata)) = store.get_conversation_metadata(thread_uuid).await
|
||||
&& let Some(rid) = metadata
|
||||
.get("last_response_id")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(String::from)
|
||||
{
|
||||
thread.last_response_id = Some(rid.clone());
|
||||
self.llm()
|
||||
.seed_response_chain(&thread_uuid.to_string(), rid);
|
||||
tracing::debug!("Restored response chain for thread {}", thread_uuid);
|
||||
}
|
||||
|
||||
// Insert into session and register with session manager
|
||||
{
|
||||
let mut sess = session.lock().await;
|
||||
@@ -228,7 +214,7 @@ impl Agent {
|
||||
)
|
||||
.await;
|
||||
|
||||
let compactor = ContextCompactor::new(self.llm().clone());
|
||||
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
|
||||
if let Err(e) = compactor
|
||||
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
|
||||
.await
|
||||
@@ -325,7 +311,6 @@ impl Agent {
|
||||
};
|
||||
|
||||
thread.complete_turn(&response);
|
||||
self.persist_response_chain(thread);
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
@@ -335,8 +320,10 @@ impl Agent {
|
||||
)
|
||||
.await;
|
||||
|
||||
// Fire-and-forget: persist turn to DB
|
||||
self.persist_turn(thread_id, &message.user_id, content, Some(&response));
|
||||
// Persist turn to DB before returning so the write
|
||||
// completes even if the process shuts down right after.
|
||||
self.persist_turn(thread_id, &message.user_id, content, Some(&response))
|
||||
.await;
|
||||
|
||||
Ok(SubmissionResult::response(response))
|
||||
}
|
||||
@@ -366,15 +353,16 @@ impl Agent {
|
||||
thread.fail_turn(e.to_string());
|
||||
|
||||
// Persist the user message even on failure
|
||||
self.persist_turn(thread_id, &message.user_id, content, None);
|
||||
self.persist_turn(thread_id, &message.user_id, content, None)
|
||||
.await;
|
||||
|
||||
Ok(SubmissionResult::error(e.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fire-and-forget: persist a turn (user message + optional assistant response) to the DB.
|
||||
pub(super) fn persist_turn(
|
||||
/// Persist a turn (user message + optional assistant response) to the DB.
|
||||
pub(super) async fn persist_turn(
|
||||
&self,
|
||||
thread_id: Uuid,
|
||||
user_id: &str,
|
||||
@@ -386,70 +374,29 @@ impl Agent {
|
||||
None => return,
|
||||
};
|
||||
|
||||
let user_id = user_id.to_string();
|
||||
let user_input = user_input.to_string();
|
||||
let response = response.map(String::from);
|
||||
if let Err(e) = store
|
||||
.ensure_conversation(thread_id, "gateway", user_id, None)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = store
|
||||
.ensure_conversation(thread_id, "gateway", &user_id, None)
|
||||
if let Err(e) = store
|
||||
.add_conversation_message(thread_id, "user", user_input)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to persist user message: {}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(resp) = response
|
||||
&& let Err(e) = store
|
||||
.add_conversation_message(thread_id, "assistant", resp)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to ensure conversation {}: {}", thread_id, e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Err(e) = store
|
||||
.add_conversation_message(thread_id, "user", &user_input)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to persist user message: {}", e);
|
||||
return;
|
||||
}
|
||||
|
||||
if let Some(ref resp) = response
|
||||
&& let Err(e) = store
|
||||
.add_conversation_message(thread_id, "assistant", resp)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("Failed to persist assistant message: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Sync the provider's response chain ID to the thread and DB metadata.
|
||||
///
|
||||
/// Call after a successful agentic loop to persist the latest
|
||||
/// `previous_response_id` so chaining survives restarts.
|
||||
pub(super) fn persist_response_chain(&self, thread: &mut crate::agent::session::Thread) {
|
||||
let tid = thread.id.to_string();
|
||||
let response_id = match self.llm().get_response_chain_id(&tid) {
|
||||
Some(rid) => rid,
|
||||
None => return,
|
||||
};
|
||||
|
||||
// Update in-memory thread
|
||||
thread.last_response_id = Some(response_id.clone());
|
||||
|
||||
// Fire-and-forget DB write
|
||||
let store = match self.store() {
|
||||
Some(s) => Arc::clone(s),
|
||||
None => return,
|
||||
};
|
||||
let thread_id = thread.id;
|
||||
tokio::spawn(async move {
|
||||
let val = serde_json::json!(response_id);
|
||||
if let Err(e) = store
|
||||
.update_conversation_metadata_field(thread_id, "last_response_id", &val)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
"Failed to persist response chain for thread {}: {}",
|
||||
thread_id,
|
||||
e
|
||||
);
|
||||
}
|
||||
});
|
||||
{
|
||||
tracing::warn!("Failed to persist assistant message: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn process_undo(
|
||||
@@ -562,7 +509,7 @@ impl Agent {
|
||||
crate::agent::context_monitor::CompactionStrategy::Summarize { keep_recent: 5 },
|
||||
);
|
||||
|
||||
let compactor = ContextCompactor::new(self.llm().clone());
|
||||
let compactor = ContextCompactor::new(self.llm().clone(), self.safety().clone());
|
||||
match compactor
|
||||
.compact(thread, strategy, self.workspace().map(|w| w.as_ref()))
|
||||
.await
|
||||
@@ -799,19 +746,18 @@ impl Agent {
|
||||
)> = None;
|
||||
|
||||
for (idx, tc) in deferred_tool_calls.iter().enumerate() {
|
||||
if let Some(tool) = self.tools().get(&tc.name).await
|
||||
&& tool.requires_approval()
|
||||
{
|
||||
let is_auto_approved = {
|
||||
let sess = session.lock().await;
|
||||
let mut approved = sess.is_tool_auto_approved(&tc.name);
|
||||
if approved && tool.requires_approval_for(&tc.arguments) {
|
||||
approved = false;
|
||||
if let Some(tool) = self.tools().get(&tc.name).await {
|
||||
use crate::tools::ApprovalRequirement;
|
||||
let needs_approval = match tool.requires_approval(&tc.arguments) {
|
||||
ApprovalRequirement::Never => false,
|
||||
ApprovalRequirement::UnlessAutoApproved => {
|
||||
let sess = session.lock().await;
|
||||
!sess.is_tool_auto_approved(&tc.name)
|
||||
}
|
||||
approved
|
||||
ApprovalRequirement::Always => true,
|
||||
};
|
||||
|
||||
if !is_auto_approved {
|
||||
if needs_approval {
|
||||
approval_needed = Some((idx, tc.clone(), tool));
|
||||
break; // remaining tools stay deferred
|
||||
}
|
||||
@@ -1072,9 +1018,9 @@ impl Agent {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.complete_turn(&response);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&response));
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&response))
|
||||
.await;
|
||||
}
|
||||
self.persist_response_chain(thread);
|
||||
let _ = self
|
||||
.channels
|
||||
.send_status(
|
||||
@@ -1112,7 +1058,8 @@ impl Agent {
|
||||
let user_input = thread.last_turn().map(|t| t.user_input.clone());
|
||||
thread.fail_turn(e.to_string());
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, None);
|
||||
self.persist_turn(thread_id, &message.user_id, &input, None)
|
||||
.await;
|
||||
}
|
||||
Ok(SubmissionResult::error(e.to_string()))
|
||||
}
|
||||
@@ -1131,7 +1078,8 @@ impl Agent {
|
||||
thread.clear_pending_approval();
|
||||
thread.complete_turn(&rejection);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&rejection));
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&rejection))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1171,9 +1119,9 @@ impl Agent {
|
||||
thread.enter_auth_mode(ext_name.clone());
|
||||
thread.complete_turn(&instructions);
|
||||
if let Some(input) = user_input {
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&instructions));
|
||||
self.persist_turn(thread_id, &message.user_id, &input, Some(&instructions))
|
||||
.await;
|
||||
}
|
||||
self.persist_response_chain(thread);
|
||||
}
|
||||
}
|
||||
let _ = self
|
||||
|
||||
+18
-2
@@ -18,6 +18,7 @@ use crate::llm::{
|
||||
};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::tools::ToolRegistry;
|
||||
use crate::tools::rate_limiter::RateLimitResult;
|
||||
|
||||
/// Shared dependencies for worker execution.
|
||||
///
|
||||
@@ -432,16 +433,31 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
})?;
|
||||
|
||||
// Tools requiring approval are blocked in autonomous jobs
|
||||
if tool.requires_approval() {
|
||||
if tool.requires_approval(params).is_required() {
|
||||
return Err(crate::error::ToolError::AuthRequired {
|
||||
name: tool_name.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
// Fetch job context early so we have the real user_id for hooks
|
||||
// Fetch job context early so we have the real user_id for hooks and rate limiting
|
||||
let job_ctx = deps.context_manager.get_context(job_id).await?;
|
||||
|
||||
// Check per-tool rate limit before running hooks or executing (cheaper check first)
|
||||
if let Some(config) = tool.rate_limit_config()
|
||||
&& let RateLimitResult::Limited { retry_after, .. } = deps
|
||||
.tools
|
||||
.rate_limiter()
|
||||
.check_and_record(&job_ctx.user_id, tool_name, &config)
|
||||
.await
|
||||
{
|
||||
return Err(crate::error::ToolError::RateLimited {
|
||||
name: tool_name.to_string(),
|
||||
retry_after: Some(retry_after),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
// Run BeforeToolCall hook
|
||||
let params = {
|
||||
use crate::hooks::{HookError, HookEvent, HookOutcome};
|
||||
|
||||
+64
-113
@@ -200,9 +200,13 @@ impl AppBuilder {
|
||||
|
||||
self.session.attach_store(db.clone(), "default").await;
|
||||
|
||||
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
// Fire-and-forget housekeeping — no need to block startup.
|
||||
let db_cleanup = db.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = db_cleanup.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
self.db = Some(db);
|
||||
Ok(())
|
||||
@@ -285,94 +289,14 @@ impl AppBuilder {
|
||||
|
||||
/// Phase 3: Initialize LLM provider chain.
|
||||
///
|
||||
/// Creates the primary provider, then wraps with failover, circuit
|
||||
/// breaker, and response cache as configured.
|
||||
/// Delegates to `build_provider_chain` which applies all decorators
|
||||
/// (retry, smart routing, failover, circuit breaker, response cache).
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn init_llm(
|
||||
&self,
|
||||
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), anyhow::Error> {
|
||||
use crate::llm::{
|
||||
CachedProvider, CircuitBreakerConfig, CircuitBreakerProvider, CooldownConfig,
|
||||
FailoverProvider, ResponseCacheConfig, create_cheap_llm_provider, create_llm_provider,
|
||||
create_llm_provider_with_config,
|
||||
};
|
||||
|
||||
let llm = create_llm_provider(&self.config.llm, self.session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
|
||||
// Wrap in failover if a fallback model is configured
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(fallback_model) =
|
||||
self.config.llm.nearai.fallback_model.as_ref()
|
||||
{
|
||||
if fallback_model == &self.config.llm.nearai.model {
|
||||
tracing::warn!(
|
||||
"fallback_model is the same as primary model, failover may not be effective"
|
||||
);
|
||||
}
|
||||
let mut fallback_config = self.config.llm.nearai.clone();
|
||||
fallback_config.model = fallback_model.clone();
|
||||
let fallback = create_llm_provider_with_config(&fallback_config, self.session.clone())?;
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(
|
||||
self.config.llm.nearai.failover_cooldown_secs,
|
||||
),
|
||||
failure_threshold: self.config.llm.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in circuit breaker if configured
|
||||
let llm: Arc<dyn LlmProvider> =
|
||||
if let Some(threshold) = self.config.llm.nearai.circuit_breaker_threshold {
|
||||
let cb_config = CircuitBreakerConfig {
|
||||
failure_threshold: threshold,
|
||||
recovery_timeout: std::time::Duration::from_secs(
|
||||
self.config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
),
|
||||
..CircuitBreakerConfig::default()
|
||||
};
|
||||
tracing::info!(
|
||||
threshold,
|
||||
recovery_secs = self.config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
"LLM circuit breaker enabled"
|
||||
);
|
||||
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in response cache if configured
|
||||
let llm: Arc<dyn LlmProvider> = if self.config.llm.nearai.response_cache_enabled {
|
||||
let rc_config = ResponseCacheConfig {
|
||||
ttl: std::time::Duration::from_secs(self.config.llm.nearai.response_cache_ttl_secs),
|
||||
max_entries: self.config.llm.nearai.response_cache_max_entries,
|
||||
};
|
||||
tracing::info!(
|
||||
ttl_secs = self.config.llm.nearai.response_cache_ttl_secs,
|
||||
max_entries = self.config.llm.nearai.response_cache_max_entries,
|
||||
"LLM response cache enabled"
|
||||
);
|
||||
Arc::new(CachedProvider::new(llm, rc_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Cheap LLM for lightweight tasks
|
||||
let cheap_llm = create_cheap_llm_provider(&self.config.llm, self.session.clone())?;
|
||||
if let Some(ref cheap) = cheap_llm {
|
||||
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
|
||||
}
|
||||
|
||||
let (llm, cheap_llm) =
|
||||
crate::llm::build_provider_chain(&self.config.llm, self.session.clone())?;
|
||||
Ok((llm, cheap_llm))
|
||||
}
|
||||
|
||||
@@ -396,7 +320,6 @@ impl AppBuilder {
|
||||
|
||||
let tools = Arc::new(ToolRegistry::new());
|
||||
tools.register_builtin_tools();
|
||||
tracing::info!("Registered {} built-in tools", tools.count());
|
||||
|
||||
// Create embeddings provider if configured
|
||||
let embeddings: Option<Arc<dyn EmbeddingProvider>> = if self.config.embeddings.enabled {
|
||||
@@ -656,11 +579,44 @@ impl AppBuilder {
|
||||
|
||||
tokio::join!(wasm_tools_future, mcp_servers_future);
|
||||
|
||||
// Create extension manager
|
||||
let extension_manager = if let Some(ref secrets) = self.secrets_store {
|
||||
// Load registry catalog entries for extension discovery
|
||||
let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() {
|
||||
Ok(catalog) => {
|
||||
let entries: Vec<_> = catalog
|
||||
.all()
|
||||
.iter()
|
||||
.map(|m| m.to_registry_entry())
|
||||
.collect();
|
||||
tracing::info!(
|
||||
count = entries.len(),
|
||||
"Loaded registry catalog entries for extension discovery"
|
||||
);
|
||||
entries
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to load registry catalog: {}", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Create extension manager. Use ephemeral in-memory secrets if no
|
||||
// persistent store is configured (listing/install/activate still work).
|
||||
let ext_secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = if let Some(ref s) =
|
||||
self.secrets_store
|
||||
{
|
||||
Arc::clone(s)
|
||||
} else {
|
||||
use crate::secrets::{InMemorySecretsStore, SecretsCrypto};
|
||||
let ephemeral_key =
|
||||
secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex());
|
||||
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
|
||||
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
|
||||
Arc::new(InMemorySecretsStore::new(crypto))
|
||||
};
|
||||
let extension_manager = {
|
||||
let manager = Arc::new(ExtensionManager::new(
|
||||
Arc::clone(&mcp_session_manager),
|
||||
Arc::clone(secrets),
|
||||
ext_secrets,
|
||||
Arc::clone(tools),
|
||||
Some(Arc::clone(hooks)),
|
||||
wasm_tool_runtime.clone(),
|
||||
@@ -669,24 +625,19 @@ impl AppBuilder {
|
||||
self.config.tunnel.public_url.clone(),
|
||||
"default".to_string(),
|
||||
self.db.clone(),
|
||||
catalog_entries.clone(),
|
||||
));
|
||||
tools.register_extension_tools(Arc::clone(&manager));
|
||||
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
||||
Some(manager)
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"Extension manager not available (no secrets store). \
|
||||
Extension tools won't be registered."
|
||||
);
|
||||
None
|
||||
};
|
||||
|
||||
// Register dev tools if local tools are enabled
|
||||
if self.config.agent.allow_local_tools {
|
||||
// register_builder_tool() already calls register_dev_tools() internally,
|
||||
// so only register them here when the builder didn't already do it.
|
||||
let builder_registered_dev_tools = self.config.builder.enabled
|
||||
&& (self.config.agent.allow_local_tools || !self.config.sandbox.enabled);
|
||||
if self.config.agent.allow_local_tools && !builder_registered_dev_tools {
|
||||
tools.register_dev_tools();
|
||||
tracing::info!(
|
||||
"Local tools enabled (allow_local_tools=true), dev tools registered directly"
|
||||
);
|
||||
}
|
||||
|
||||
Ok((mcp_session_manager, wasm_tool_runtime, extension_manager))
|
||||
@@ -709,9 +660,6 @@ impl AppBuilder {
|
||||
// Seed workspace and backfill embeddings
|
||||
if let Some(ref ws) = workspace {
|
||||
match ws.seed_if_empty().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Workspace seeded with {} core files", count);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to seed workspace: {}", e);
|
||||
@@ -719,15 +667,18 @@ impl AppBuilder {
|
||||
}
|
||||
|
||||
if embeddings.is_some() {
|
||||
match ws.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
let ws_bg = Arc::clone(ws);
|
||||
tokio::spawn(async move {
|
||||
match ws_bg.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,32 @@ use uuid::Uuid;
|
||||
|
||||
use crate::error::ChannelError;
|
||||
|
||||
/// Kind of attachment carried on an incoming message.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AttachmentKind {
|
||||
/// Audio content (voice notes, audio files).
|
||||
Audio,
|
||||
/// Image content (photos, screenshots).
|
||||
Image,
|
||||
/// Document content (PDFs, files).
|
||||
Document,
|
||||
}
|
||||
|
||||
/// Binary attachment on a message (e.g., voice note, photo).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attachment {
|
||||
/// What kind of content this is.
|
||||
pub kind: AttachmentKind,
|
||||
/// MIME type (e.g., "audio/ogg", "image/jpeg").
|
||||
pub mime_type: String,
|
||||
/// Raw bytes of the attachment.
|
||||
pub data: Vec<u8>,
|
||||
/// Optional filename.
|
||||
pub filename: Option<String>,
|
||||
/// Duration in seconds (for audio/video).
|
||||
pub duration_secs: Option<u32>,
|
||||
}
|
||||
|
||||
/// A message received from an external channel.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IncomingMessage {
|
||||
@@ -28,6 +54,8 @@ pub struct IncomingMessage {
|
||||
pub received_at: DateTime<Utc>,
|
||||
/// Channel-specific metadata.
|
||||
pub metadata: serde_json::Value,
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl IncomingMessage {
|
||||
@@ -46,6 +74,7 @@ impl IncomingMessage {
|
||||
thread_id: None,
|
||||
received_at: Utc::now(),
|
||||
metadata: serde_json::Value::Null,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +95,12 @@ impl IncomingMessage {
|
||||
self.user_name = Some(name.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of incoming messages.
|
||||
|
||||
+4
-1
@@ -35,7 +35,10 @@ pub mod wasm;
|
||||
pub mod web;
|
||||
mod webhook_server;
|
||||
|
||||
pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
pub use channel::{
|
||||
Attachment, AttachmentKind, Channel, IncomingMessage, MessageStream, OutgoingResponse,
|
||||
StatusUpdate,
|
||||
};
|
||||
pub use http::HttpChannel;
|
||||
pub use manager::ChannelManager;
|
||||
pub use repl::ReplChannel;
|
||||
|
||||
+41
-4
@@ -15,6 +15,7 @@
|
||||
//! - `/compact` - Compact the context
|
||||
//! - `/new` - Start a new thread
|
||||
//! - `yes`/`no`/`always` - Respond to tool approval prompts
|
||||
//! - `Esc` - Interrupt current operation
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::io::{self, Write};
|
||||
@@ -28,7 +29,10 @@ use rustyline::error::ReadlineError;
|
||||
use rustyline::highlight::Highlighter;
|
||||
use rustyline::hint::Hinter;
|
||||
use rustyline::validate::Validator;
|
||||
use rustyline::{CompletionType, Editor, Helper};
|
||||
use rustyline::{
|
||||
Cmd as ReadlineCmd, CompletionType, ConditionalEventHandler, Editor, Event, EventContext,
|
||||
EventHandler, Helper, KeyCode, KeyEvent, Modifiers, RepeatCount,
|
||||
};
|
||||
use termimad::MadSkin;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
@@ -121,6 +125,23 @@ impl Highlighter for ReplHelper {
|
||||
impl Validator for ReplHelper {}
|
||||
impl Helper for ReplHelper {}
|
||||
|
||||
struct EscInterruptHandler {
|
||||
triggered: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ConditionalEventHandler for EscInterruptHandler {
|
||||
fn handle(
|
||||
&self,
|
||||
_evt: &Event,
|
||||
_n: RepeatCount,
|
||||
_positive: bool,
|
||||
_ctx: &EventContext,
|
||||
) -> Option<ReadlineCmd> {
|
||||
self.triggered.store(true, Ordering::Relaxed);
|
||||
Some(ReadlineCmd::Interrupt)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a termimad skin with our color scheme.
|
||||
fn make_skin() -> MadSkin {
|
||||
let mut skin = MadSkin::default();
|
||||
@@ -247,6 +268,7 @@ fn print_help() {
|
||||
println!(" {c}/compact{r} {d}compact context window{r}");
|
||||
println!(" {c}/new{r} {d}new conversation thread{r}");
|
||||
println!(" {c}/interrupt{r} {d}stop current operation{r}");
|
||||
println!(" {c}esc{r} {d}stop current operation{r}");
|
||||
println!();
|
||||
println!(" {h}Approval responses{r}");
|
||||
println!(" {c}yes{r} ({c}y{r}) {d}approve tool execution{r}");
|
||||
@@ -274,6 +296,7 @@ impl Channel for ReplChannel {
|
||||
let single_message = self.single_message.clone();
|
||||
let debug_mode = Arc::clone(&self.debug_mode);
|
||||
let suppress_banner = Arc::clone(&self.suppress_banner);
|
||||
let esc_interrupt_triggered_for_thread = Arc::new(AtomicBool::new(false));
|
||||
|
||||
std::thread::spawn(move || {
|
||||
// Single message mode: send it and return
|
||||
@@ -301,6 +324,13 @@ impl Channel for ReplChannel {
|
||||
|
||||
rl.set_helper(Some(ReplHelper));
|
||||
|
||||
rl.bind_sequence(
|
||||
KeyEvent(KeyCode::Esc, Modifiers::NONE),
|
||||
EventHandler::Conditional(Box::new(EscInterruptHandler {
|
||||
triggered: Arc::clone(&esc_interrupt_triggered_for_thread),
|
||||
})),
|
||||
);
|
||||
|
||||
// Load history
|
||||
let hist_path = history_path();
|
||||
if let Some(parent) = hist_path.parent() {
|
||||
@@ -360,9 +390,16 @@ impl Channel for ReplChannel {
|
||||
}
|
||||
}
|
||||
Err(ReadlineError::Interrupted) => {
|
||||
// Ctrl+C: send /interrupt
|
||||
let msg = IncomingMessage::new("repl", "default", "/interrupt");
|
||||
if tx.blocking_send(msg).is_err() {
|
||||
if esc_interrupt_triggered_for_thread.swap(false, Ordering::Relaxed) {
|
||||
// Esc: interrupt current operation and keep REPL open.
|
||||
let msg = IncomingMessage::new("repl", "default", "/interrupt");
|
||||
if tx.blocking_send(msg).is_err() {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Ctrl+C (VINTR): request graceful shutdown.
|
||||
let msg = IncomingMessage::new("repl", "default", "/quit");
|
||||
let _ = tx.blocking_send(msg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
|
||||
const KNOWN_CHANNELS: &[(&str, &str)] = &[
|
||||
("telegram", "telegram_channel"),
|
||||
("slack", "slack_channel"),
|
||||
("discord", "discord_channel"),
|
||||
("whatsapp", "whatsapp_channel"),
|
||||
];
|
||||
|
||||
@@ -42,6 +43,10 @@ fn channels_src_dir() -> PathBuf {
|
||||
|
||||
/// Locate the build artifacts for a channel.
|
||||
///
|
||||
/// Checks two layouts:
|
||||
/// 1. **Flat** (Docker/packaged): `<channels_src>/<name>/<name>.wasm`
|
||||
/// 2. **Build tree** (dev): `<channels_src>/<name>/target/wasm32-wasip2/release/<crate_name>.wasm`
|
||||
///
|
||||
/// Returns (wasm_path, capabilities_path) or an error if files are missing.
|
||||
fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
|
||||
let (_, crate_name) = KNOWN_CHANNELS
|
||||
@@ -52,31 +57,34 @@ fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
|
||||
let src_dir = channels_src_dir();
|
||||
let channel_dir = src_dir.join(name);
|
||||
|
||||
let wasm_path = channel_dir
|
||||
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
// Check flat layout first (Docker/packaged deployments)
|
||||
let flat_wasm = channel_dir.join(format!("{}.wasm", name));
|
||||
if flat_wasm.exists() && caps_path.exists() {
|
||||
return Ok((flat_wasm, caps_path));
|
||||
}
|
||||
|
||||
// Fall back to build tree layout (dev builds)
|
||||
let build_wasm = channel_dir
|
||||
.join("target/wasm32-wasip2/release")
|
||||
.join(format!("{}.wasm", crate_name));
|
||||
|
||||
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if !wasm_path.exists() {
|
||||
return Err(format!(
|
||||
"Channel '{}' WASM not found at {}. Build it first:\n \
|
||||
cd {} && cargo build --target wasm32-wasip2 --release",
|
||||
name,
|
||||
wasm_path.display(),
|
||||
channel_dir.display()
|
||||
));
|
||||
if build_wasm.exists() && caps_path.exists() {
|
||||
return Ok((build_wasm, caps_path));
|
||||
}
|
||||
|
||||
if !caps_path.exists() {
|
||||
return Err(format!(
|
||||
"Channel '{}' capabilities not found at {}",
|
||||
name,
|
||||
caps_path.display()
|
||||
));
|
||||
}
|
||||
|
||||
Ok((wasm_path, caps_path))
|
||||
Err(format!(
|
||||
"Channel '{}' WASM not found. Checked:\n \
|
||||
- {} (flat/packaged)\n \
|
||||
- {} (build tree)\n \
|
||||
Build it first:\n \
|
||||
cd {} && cargo build --target wasm32-wasip2 --release",
|
||||
name,
|
||||
flat_wasm.display(),
|
||||
build_wasm.display(),
|
||||
channel_dir.display()
|
||||
))
|
||||
}
|
||||
|
||||
/// Install a channel from build artifacts into the channels directory.
|
||||
@@ -130,10 +138,11 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_known_channels_includes_all_three() {
|
||||
fn test_known_channels_includes_all_four() {
|
||||
let names = bundled_channel_names();
|
||||
assert!(names.contains(&"telegram"));
|
||||
assert!(names.contains(&"slack"));
|
||||
assert!(names.contains(&"discord"));
|
||||
assert!(names.contains(&"whatsapp"));
|
||||
}
|
||||
|
||||
|
||||
+128
-1
@@ -7,6 +7,7 @@
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::channels::channel::Attachment;
|
||||
use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig};
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::tools::wasm::{HostState, LogLevel};
|
||||
@@ -17,6 +18,9 @@ const MAX_EMITS_PER_EXECUTION: usize = 100;
|
||||
/// Maximum message content size (64 KB).
|
||||
const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024;
|
||||
|
||||
/// Maximum size for a single attachment (10 MB).
|
||||
const MAX_ATTACHMENT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// A message emitted by a WASM channel to be sent to the agent.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EmittedMessage {
|
||||
@@ -37,6 +41,9 @@ pub struct EmittedMessage {
|
||||
|
||||
/// Timestamp when the message was emitted.
|
||||
pub emitted_at_millis: u64,
|
||||
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl EmittedMessage {
|
||||
@@ -52,6 +59,7 @@ impl EmittedMessage {
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +80,12 @@ impl EmittedMessage {
|
||||
self.metadata_json = metadata_json.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A pending workspace write operation.
|
||||
@@ -168,7 +182,7 @@ impl ChannelHostState {
|
||||
///
|
||||
/// Messages are queued and delivered after callback execution completes.
|
||||
/// Rate limiting is enforced per-execution and globally.
|
||||
pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
pub fn emit_message(&mut self, mut msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
// Check per-execution limit
|
||||
if !self.emit_enabled {
|
||||
self.emits_dropped += 1;
|
||||
@@ -186,6 +200,22 @@ impl ChannelHostState {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Validate attachment sizes — drop only oversized attachments, not the whole message
|
||||
msg.attachments.retain(|attachment| {
|
||||
if attachment.data.len() > MAX_ATTACHMENT_SIZE {
|
||||
tracing::warn!(
|
||||
channel = %self.channel_name,
|
||||
size = attachment.data.len(),
|
||||
max = MAX_ATTACHMENT_SIZE,
|
||||
mime = %attachment.mime_type,
|
||||
"Attachment too large, dropping attachment (message still delivered)"
|
||||
);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
// Validate message content size
|
||||
if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE {
|
||||
tracing::warn!(
|
||||
@@ -300,6 +330,51 @@ impl ChannelHostState {
|
||||
}
|
||||
}
|
||||
|
||||
/// In-memory workspace store for WASM channels.
|
||||
///
|
||||
/// Persists workspace writes across callback invocations within a single
|
||||
/// channel lifetime. This allows WASM channels to maintain state (e.g.,
|
||||
/// Telegram polling offsets) between poll ticks without requiring a
|
||||
/// full database-backed workspace.
|
||||
///
|
||||
/// Uses `std::sync::RwLock` (not tokio) because WASM execution runs
|
||||
/// inside `spawn_blocking`.
|
||||
pub struct ChannelWorkspaceStore {
|
||||
data: std::sync::RwLock<std::collections::HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl ChannelWorkspaceStore {
|
||||
/// Create a new empty workspace store.
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
data: std::sync::RwLock::new(std::collections::HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Commit pending writes from a callback execution into the store.
|
||||
pub fn commit_writes(&self, writes: &[PendingWorkspaceWrite]) {
|
||||
if writes.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Ok(mut data) = self.data.write() {
|
||||
for write in writes {
|
||||
tracing::debug!(
|
||||
path = %write.path,
|
||||
content_len = write.content.len(),
|
||||
"Committing workspace write to channel store"
|
||||
);
|
||||
data.insert(write.path.clone(), write.content.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::tools::wasm::WorkspaceReader for ChannelWorkspaceStore {
|
||||
fn read(&self, path: &str) -> Option<String> {
|
||||
self.data.read().ok()?.get(path).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
/// Rate limiter for channel message emission.
|
||||
///
|
||||
/// Tracks emission rates across multiple executions.
|
||||
@@ -497,4 +572,56 @@ mod tests {
|
||||
|
||||
assert_eq!(state.channel_name(), "telegram");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_channel_workspace_store_commit_and_read() {
|
||||
use crate::channels::wasm::host::{ChannelWorkspaceStore, PendingWorkspaceWrite};
|
||||
use crate::tools::wasm::WorkspaceReader;
|
||||
|
||||
let store = ChannelWorkspaceStore::new();
|
||||
|
||||
// Initially empty
|
||||
assert!(store.read("channels/telegram/offset").is_none());
|
||||
|
||||
// Commit some writes
|
||||
let writes = vec![
|
||||
PendingWorkspaceWrite {
|
||||
path: "channels/telegram/offset".to_string(),
|
||||
content: "103".to_string(),
|
||||
},
|
||||
PendingWorkspaceWrite {
|
||||
path: "channels/telegram/state.json".to_string(),
|
||||
content: r#"{"ok":true}"#.to_string(),
|
||||
},
|
||||
];
|
||||
store.commit_writes(&writes);
|
||||
|
||||
// Should be readable
|
||||
assert_eq!(
|
||||
store.read("channels/telegram/offset"),
|
||||
Some("103".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
store.read("channels/telegram/state.json"),
|
||||
Some(r#"{"ok":true}"#.to_string())
|
||||
);
|
||||
|
||||
// Overwrite a value
|
||||
let writes2 = vec![PendingWorkspaceWrite {
|
||||
path: "channels/telegram/offset".to_string(),
|
||||
content: "200".to_string(),
|
||||
}];
|
||||
store.commit_writes(&writes2);
|
||||
assert_eq!(
|
||||
store.read("channels/telegram/offset"),
|
||||
Some("200".to_string())
|
||||
);
|
||||
|
||||
// Empty writes are a no-op
|
||||
store.commit_writes(&[]);
|
||||
assert_eq!(
|
||||
store.read("channels/telegram/offset"),
|
||||
Some("200".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -488,7 +488,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: name.to_string(),
|
||||
description: format!("Test channel: {}", name),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
|
||||
@@ -68,38 +68,51 @@ impl WasmChannelRuntimeConfig {
|
||||
}
|
||||
|
||||
/// A compiled WASM channel component ready for instantiation.
|
||||
#[derive(Debug)]
|
||||
///
|
||||
/// Stores the pre-compiled `Component` directly so instantiation
|
||||
/// doesn't require recompilation.
|
||||
pub struct PreparedChannelModule {
|
||||
/// Channel name.
|
||||
pub name: String,
|
||||
/// Channel description.
|
||||
pub description: String,
|
||||
/// Compiled component bytes (public for testing, otherwise use component_bytes()).
|
||||
pub(crate) component_bytes: Vec<u8>,
|
||||
/// Pre-compiled component (cheaply cloneable via internal Arc).
|
||||
pub(crate) component: Option<wasmtime::component::Component>,
|
||||
/// Resource limits for this channel.
|
||||
pub limits: ResourceLimits,
|
||||
}
|
||||
|
||||
impl PreparedChannelModule {
|
||||
/// Get the compiled component bytes.
|
||||
pub fn component_bytes(&self) -> &[u8] {
|
||||
&self.component_bytes
|
||||
/// Get the pre-compiled component for instantiation.
|
||||
pub fn component(&self) -> Option<&wasmtime::component::Component> {
|
||||
self.component.as_ref()
|
||||
}
|
||||
|
||||
/// Create a PreparedChannelModule for testing purposes.
|
||||
///
|
||||
/// Creates a module with no actual WASM bytes, suitable for testing
|
||||
/// Creates a module with no actual WASM component, suitable for testing
|
||||
/// channel infrastructure without requiring a real WASM component.
|
||||
pub fn for_testing(name: impl Into<String>, description: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PreparedChannelModule {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PreparedChannelModule")
|
||||
.field("name", &self.name)
|
||||
.field("description", &self.description)
|
||||
.field("has_component", &self.component.is_some())
|
||||
.field("limits", &self.limits)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// WASM channel runtime.
|
||||
///
|
||||
/// Manages the Wasmtime engine and a cache of prepared channel modules.
|
||||
@@ -137,6 +150,13 @@ impl WasmChannelRuntime {
|
||||
// Disable debug info in production
|
||||
wasmtime_config.debug_info(false);
|
||||
|
||||
// Enable persistent compilation cache. Wasmtime serializes compiled native
|
||||
// code to disk (~/.cache/wasmtime by default), so subsequent startups
|
||||
// deserialize instead of recompiling — typically 10-50x faster.
|
||||
if let Err(e) = wasmtime_config.cache_config_load_default() {
|
||||
tracing::warn!("Failed to enable wasmtime compilation cache: {}", e);
|
||||
}
|
||||
|
||||
let engine = Engine::new(&wasmtime_config).map_err(|e| {
|
||||
WasmChannelError::Config(format!("Failed to create Wasmtime engine: {}", e))
|
||||
})?;
|
||||
@@ -183,13 +203,13 @@ impl WasmChannelRuntime {
|
||||
// Compile in blocking task (Wasmtime compilation is synchronous)
|
||||
let prepared = tokio::task::spawn_blocking(move || {
|
||||
// Validate and compile the component
|
||||
let _component = wasmtime::component::Component::new(&engine, &wasm_bytes)
|
||||
let component = wasmtime::component::Component::new(&engine, &wasm_bytes)
|
||||
.map_err(|e| WasmChannelError::Compilation(e.to_string()))?;
|
||||
|
||||
Ok::<_, WasmChannelError>(PreparedChannelModule {
|
||||
name: name.clone(),
|
||||
description: desc,
|
||||
component_bytes: wasm_bytes,
|
||||
component: Some(component),
|
||||
limits: limits.unwrap_or(default_limits),
|
||||
})
|
||||
})
|
||||
|
||||
+207
-61
@@ -37,12 +37,15 @@ use tokio::sync::{RwLock, mpsc, oneshot};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use uuid::Uuid;
|
||||
use wasmtime::Store;
|
||||
use wasmtime::component::{Component, Linker};
|
||||
use wasmtime::component::Linker;
|
||||
use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView};
|
||||
|
||||
use crate::channels::channel::{Attachment, AttachmentKind};
|
||||
use crate::channels::wasm::capabilities::ChannelCapabilities;
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::channels::wasm::host::{ChannelEmitRateLimiter, ChannelHostState, EmittedMessage};
|
||||
use crate::channels::wasm::host::{
|
||||
ChannelEmitRateLimiter, ChannelHostState, ChannelWorkspaceStore, EmittedMessage,
|
||||
};
|
||||
use crate::channels::wasm::router::RegisteredEndpoint;
|
||||
use crate::channels::wasm::runtime::{PreparedChannelModule, WasmChannelRuntime};
|
||||
use crate::channels::wasm::schema::ChannelConfig;
|
||||
@@ -434,6 +437,7 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
user_id = %msg.user_id,
|
||||
user_name = ?msg.user_name,
|
||||
content_len = msg.content.len(),
|
||||
attachment_count = msg.attachments.len(),
|
||||
"WASM emit_message called"
|
||||
);
|
||||
|
||||
@@ -446,6 +450,31 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
}
|
||||
emitted = emitted.with_metadata(msg.metadata_json);
|
||||
|
||||
// Convert WIT attachments to Rust types
|
||||
if !msg.attachments.is_empty() {
|
||||
let attachments = msg
|
||||
.attachments
|
||||
.into_iter()
|
||||
.map(|a| {
|
||||
let kind = match a.kind {
|
||||
near::agent::channel_host::AttachmentKind::Audio => AttachmentKind::Audio,
|
||||
near::agent::channel_host::AttachmentKind::Image => AttachmentKind::Image,
|
||||
near::agent::channel_host::AttachmentKind::Document => {
|
||||
AttachmentKind::Document
|
||||
}
|
||||
};
|
||||
Attachment {
|
||||
kind,
|
||||
mime_type: a.mime_type,
|
||||
data: a.data,
|
||||
filename: a.filename,
|
||||
duration_secs: a.duration_secs,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
emitted = emitted.with_attachments(attachments);
|
||||
}
|
||||
|
||||
match self.host_state.emit_message(emitted) {
|
||||
Ok(()) => {
|
||||
tracing::info!("Message emitted to host state successfully");
|
||||
@@ -547,6 +576,13 @@ pub struct WasmChannel {
|
||||
|
||||
/// Pairing store for DM pairing (guest access control).
|
||||
pairing_store: Arc<PairingStore>,
|
||||
|
||||
/// In-memory workspace store persisting writes across callback invocations.
|
||||
/// Ensures WASM channels can maintain state (e.g., polling offsets) between ticks.
|
||||
workspace_store: Arc<ChannelWorkspaceStore>,
|
||||
|
||||
/// Optional transcription middleware for audio attachments.
|
||||
transcription_middleware: Option<Arc<crate::transcription::TranscriptionMiddleware>>,
|
||||
}
|
||||
|
||||
impl WasmChannel {
|
||||
@@ -577,9 +613,19 @@ impl WasmChannel {
|
||||
credentials: Arc::new(RwLock::new(HashMap::new())),
|
||||
typing_task: RwLock::new(None),
|
||||
pairing_store,
|
||||
workspace_store: Arc::new(ChannelWorkspaceStore::new()),
|
||||
transcription_middleware: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the transcription middleware for audio attachment processing.
|
||||
pub fn set_transcription_middleware(
|
||||
&mut self,
|
||||
middleware: Arc<crate::transcription::TranscriptionMiddleware>,
|
||||
) {
|
||||
self.transcription_middleware = Some(middleware);
|
||||
}
|
||||
|
||||
/// Update the channel config before starting.
|
||||
///
|
||||
/// Merges the provided values into the existing config JSON.
|
||||
@@ -634,6 +680,26 @@ impl WasmChannel {
|
||||
self.endpoints.read().await.clone()
|
||||
}
|
||||
|
||||
/// Inject the workspace store as the reader into a capabilities clone.
|
||||
///
|
||||
/// Ensures `workspace_read` capability is present with the store as its reader,
|
||||
/// so WASM callbacks can read previously written workspace state.
|
||||
fn inject_workspace_reader(
|
||||
capabilities: &ChannelCapabilities,
|
||||
store: &Arc<ChannelWorkspaceStore>,
|
||||
) -> ChannelCapabilities {
|
||||
let mut caps = capabilities.clone();
|
||||
let ws_cap = caps
|
||||
.tool_capabilities
|
||||
.workspace_read
|
||||
.get_or_insert_with(|| crate::tools::wasm::WorkspaceCapability {
|
||||
allowed_prefixes: Vec::new(),
|
||||
reader: None,
|
||||
});
|
||||
ws_cap.reader = Some(Arc::clone(store) as Arc<dyn crate::tools::wasm::WorkspaceReader>);
|
||||
caps
|
||||
}
|
||||
|
||||
/// Add channel host functions to the linker using generated bindings.
|
||||
///
|
||||
/// Uses the wasmtime::component::bindgen! generated `add_to_linker` function
|
||||
@@ -698,9 +764,13 @@ impl WasmChannel {
|
||||
) -> Result<SandboxedChannel, WasmChannelError> {
|
||||
let engine = runtime.engine();
|
||||
|
||||
// Compile the component (uses cached bytes)
|
||||
let component = Component::new(engine, prepared.component_bytes())
|
||||
.map_err(|e| WasmChannelError::Compilation(e.to_string()))?;
|
||||
// Use the pre-compiled component (no recompilation needed)
|
||||
let component = prepared
|
||||
.component()
|
||||
.ok_or_else(|| {
|
||||
WasmChannelError::Compilation("No compiled component available".to_string())
|
||||
})?
|
||||
.clone();
|
||||
|
||||
// Create linker and add host functions
|
||||
let mut linker = Linker::new(engine);
|
||||
@@ -751,7 +821,7 @@ impl WasmChannel {
|
||||
/// Returns the channel configuration for HTTP endpoint registration.
|
||||
async fn call_on_start(&self) -> Result<ChannelConfig, WasmChannelError> {
|
||||
// If no WASM bytes, return default config (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::info!(
|
||||
channel = %self.name,
|
||||
"WASM channel on_start called (no WASM module, returning defaults)"
|
||||
@@ -765,12 +835,13 @@ impl WasmChannel {
|
||||
|
||||
let runtime = Arc::clone(&self.runtime);
|
||||
let prepared = Arc::clone(&self.prepared);
|
||||
let capabilities = self.capabilities.clone();
|
||||
let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store);
|
||||
let config_json = self.config_json.read().await.clone();
|
||||
let timeout = self.runtime.config().callback_timeout;
|
||||
let channel_name = self.name.clone();
|
||||
let credentials = self.get_credentials().await;
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
|
||||
// Execute in blocking task with timeout
|
||||
let result = tokio::time::timeout(timeout, async move {
|
||||
@@ -801,8 +872,13 @@ impl WasmChannel {
|
||||
}
|
||||
};
|
||||
|
||||
let host_state =
|
||||
let mut host_state =
|
||||
Self::extract_host_state(&mut store, &prepared.name, &capabilities);
|
||||
|
||||
// Commit pending workspace writes to the persistent store
|
||||
let pending_writes = host_state.take_pending_writes();
|
||||
workspace_store.commit_writes(&pending_writes);
|
||||
|
||||
Ok((config, host_state))
|
||||
})
|
||||
.await
|
||||
@@ -885,7 +961,7 @@ impl WasmChannel {
|
||||
);
|
||||
|
||||
// If no WASM bytes, return 200 OK (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
method = method,
|
||||
@@ -897,10 +973,11 @@ impl WasmChannel {
|
||||
|
||||
let runtime = Arc::clone(&self.runtime);
|
||||
let prepared = Arc::clone(&self.prepared);
|
||||
let capabilities = self.capabilities.clone();
|
||||
let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store);
|
||||
let timeout = self.runtime.config().callback_timeout;
|
||||
let credentials = self.get_credentials().await;
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
|
||||
// Prepare request data
|
||||
let method = method.to_string();
|
||||
@@ -940,8 +1017,13 @@ impl WasmChannel {
|
||||
.map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?;
|
||||
|
||||
let response = convert_http_response(wit_response);
|
||||
let host_state =
|
||||
let mut host_state =
|
||||
Self::extract_host_state(&mut store, &prepared.name, &capabilities);
|
||||
|
||||
// Commit pending workspace writes to the persistent store
|
||||
let pending_writes = host_state.take_pending_writes();
|
||||
workspace_store.commit_writes(&pending_writes);
|
||||
|
||||
Ok((response, host_state))
|
||||
})
|
||||
.await
|
||||
@@ -979,7 +1061,7 @@ impl WasmChannel {
|
||||
/// Called periodically if polling is configured.
|
||||
pub async fn call_on_poll(&self) -> Result<(), WasmChannelError> {
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
"WASM channel on_poll called (no WASM module)"
|
||||
@@ -989,11 +1071,12 @@ impl WasmChannel {
|
||||
|
||||
let runtime = Arc::clone(&self.runtime);
|
||||
let prepared = Arc::clone(&self.prepared);
|
||||
let capabilities = self.capabilities.clone();
|
||||
let capabilities = Self::inject_workspace_reader(&self.capabilities, &self.workspace_store);
|
||||
let timeout = self.runtime.config().callback_timeout;
|
||||
let channel_name = self.name.clone();
|
||||
let credentials = self.get_credentials().await;
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
|
||||
// Execute in blocking task with timeout
|
||||
let result = tokio::time::timeout(timeout, async move {
|
||||
@@ -1013,8 +1096,13 @@ impl WasmChannel {
|
||||
.call_on_poll(&mut store)
|
||||
.map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?;
|
||||
|
||||
let host_state =
|
||||
let mut host_state =
|
||||
Self::extract_host_state(&mut store, &prepared.name, &capabilities);
|
||||
|
||||
// Commit pending workspace writes to the persistent store
|
||||
let pending_writes = host_state.take_pending_writes();
|
||||
workspace_store.commit_writes(&pending_writes);
|
||||
|
||||
Ok(((), host_state))
|
||||
})
|
||||
.await
|
||||
@@ -1073,7 +1161,7 @@ impl WasmChannel {
|
||||
);
|
||||
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
message_id = %message_id,
|
||||
@@ -1191,7 +1279,7 @@ impl WasmChannel {
|
||||
metadata: &serde_json::Value,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1262,7 +1350,7 @@ impl WasmChannel {
|
||||
timeout: Duration,
|
||||
wit_update: wit_channel::StatusUpdate,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
if prepared.component_bytes.is_empty() {
|
||||
if prepared.component().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1445,27 +1533,18 @@ impl WasmChannel {
|
||||
});
|
||||
}
|
||||
|
||||
// Convert to IncomingMessage
|
||||
let mut msg = IncomingMessage::new(&self.name, &emitted.user_id, &emitted.content);
|
||||
let msg = Self::convert_emitted_to_incoming(
|
||||
&self.name,
|
||||
emitted,
|
||||
self.transcription_middleware.as_deref(),
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %self.name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending emitted message to agent"
|
||||
);
|
||||
|
||||
@@ -1501,6 +1580,8 @@ impl WasmChannel {
|
||||
let credentials = self.credentials.clone();
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let callback_timeout = self.runtime.config().callback_timeout;
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
let transcription_middleware = self.transcription_middleware.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval_timer = tokio::time::interval(interval);
|
||||
@@ -1523,6 +1604,7 @@ impl WasmChannel {
|
||||
&credentials,
|
||||
pairing_store.clone(),
|
||||
callback_timeout,
|
||||
&workspace_store,
|
||||
).await;
|
||||
|
||||
match result {
|
||||
@@ -1534,6 +1616,7 @@ impl WasmChannel {
|
||||
emitted_messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
transcription_middleware.as_deref(),
|
||||
).await {
|
||||
tracing::warn!(
|
||||
channel = %channel_name,
|
||||
@@ -1565,7 +1648,10 @@ impl WasmChannel {
|
||||
|
||||
/// Execute a single poll callback with a fresh WASM instance.
|
||||
///
|
||||
/// Returns any emitted messages from the callback.
|
||||
/// Returns any emitted messages from the callback. Pending workspace writes
|
||||
/// are committed to the shared `ChannelWorkspaceStore` so state persists
|
||||
/// across poll ticks (e.g., Telegram polling offset).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_poll(
|
||||
channel_name: &str,
|
||||
runtime: &Arc<WasmChannelRuntime>,
|
||||
@@ -1574,9 +1660,10 @@ impl WasmChannel {
|
||||
credentials: &RwLock<HashMap<String, String>>,
|
||||
pairing_store: Arc<PairingStore>,
|
||||
timeout: Duration,
|
||||
workspace_store: &Arc<ChannelWorkspaceStore>,
|
||||
) -> Result<Vec<EmittedMessage>, WasmChannelError> {
|
||||
// Skip if no WASM bytes (testing mode)
|
||||
if prepared.component_bytes.is_empty() {
|
||||
if prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %channel_name,
|
||||
"WASM channel on_poll called (no WASM module)"
|
||||
@@ -1586,9 +1673,10 @@ impl WasmChannel {
|
||||
|
||||
let runtime = Arc::clone(runtime);
|
||||
let prepared = Arc::clone(prepared);
|
||||
let capabilities = capabilities.clone();
|
||||
let capabilities = Self::inject_workspace_reader(capabilities, workspace_store);
|
||||
let credentials_snapshot = credentials.read().await.clone();
|
||||
let channel_name_owned = channel_name.to_string();
|
||||
let workspace_store = Arc::clone(workspace_store);
|
||||
|
||||
// Execute in blocking task with timeout
|
||||
let result = tokio::time::timeout(timeout, async move {
|
||||
@@ -1608,8 +1696,13 @@ impl WasmChannel {
|
||||
.call_on_poll(&mut store)
|
||||
.map_err(|e| Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel))?;
|
||||
|
||||
let host_state =
|
||||
let mut host_state =
|
||||
Self::extract_host_state(&mut store, &prepared.name, &capabilities);
|
||||
|
||||
// Commit pending workspace writes to the persistent store
|
||||
let pending_writes = host_state.take_pending_writes();
|
||||
workspace_store.commit_writes(&pending_writes);
|
||||
|
||||
Ok(host_state)
|
||||
})
|
||||
.await
|
||||
@@ -1647,6 +1740,7 @@ impl WasmChannel {
|
||||
messages: Vec<EmittedMessage>,
|
||||
message_tx: &RwLock<Option<mpsc::Sender<IncomingMessage>>>,
|
||||
rate_limiter: &RwLock<ChannelEmitRateLimiter>,
|
||||
transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
@@ -1678,27 +1772,15 @@ impl WasmChannel {
|
||||
});
|
||||
}
|
||||
|
||||
// Convert to IncomingMessage
|
||||
let mut msg = IncomingMessage::new(channel_name, &emitted.user_id, &emitted.content);
|
||||
let msg =
|
||||
Self::convert_emitted_to_incoming(channel_name, emitted, transcription_middleware)
|
||||
.await;
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending polled message to agent"
|
||||
);
|
||||
|
||||
@@ -1718,6 +1800,65 @@ impl WasmChannel {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convert an `EmittedMessage` to an `IncomingMessage`, applying transcription
|
||||
/// middleware with a timeout if available.
|
||||
///
|
||||
/// Shared by both `process_emitted_messages` (HTTP callback path) and
|
||||
/// `dispatch_emitted_messages` (polling path) to avoid duplication.
|
||||
async fn convert_emitted_to_incoming(
|
||||
channel_name: &str,
|
||||
emitted: EmittedMessage,
|
||||
transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>,
|
||||
) -> IncomingMessage {
|
||||
// Save user_id before partial moves for potential timeout fallback
|
||||
let user_id = emitted.user_id.clone();
|
||||
|
||||
let mut msg = IncomingMessage::new(channel_name, &emitted.user_id, &emitted.content);
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Carry attachments through (moves emitted.attachments into msg)
|
||||
if !emitted.attachments.is_empty() {
|
||||
msg = msg.with_attachments(emitted.attachments);
|
||||
}
|
||||
|
||||
// Apply transcription middleware with a 30-second timeout to prevent
|
||||
// a slow/hanging provider from blocking the message pipeline indefinitely.
|
||||
if let Some(middleware) = transcription_middleware {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(30), middleware.process(msg))
|
||||
.await
|
||||
{
|
||||
Ok(processed) => return processed,
|
||||
Err(_) => {
|
||||
tracing::error!(
|
||||
channel = %channel_name,
|
||||
"Transcription timed out after 30s, delivering message without transcript"
|
||||
);
|
||||
// Timeout: `msg` was moved into the timed-out future, so
|
||||
// reconstruct a fallback message from the saved user_id.
|
||||
return IncomingMessage::new(
|
||||
channel_name,
|
||||
&user_id,
|
||||
"[Voice note: transcription timed out]",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
msg
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -2149,7 +2290,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "test".to_string(),
|
||||
description: "Test channel".to_string(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
@@ -2214,7 +2355,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_poll_no_wasm_returns_empty() {
|
||||
// When there's no WASM module (empty component_bytes), execute_poll
|
||||
// When there's no WASM module (None component), execute_poll
|
||||
// should return an empty vector of messages
|
||||
let config = WasmChannelRuntimeConfig::for_testing();
|
||||
let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap());
|
||||
@@ -2222,7 +2363,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "poll-test".to_string(),
|
||||
description: "Test channel".to_string(),
|
||||
component_bytes: Vec::new(), // No WASM bytes
|
||||
component: None, // No WASM module
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
@@ -2230,6 +2371,8 @@ mod tests {
|
||||
let credentials = Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new()));
|
||||
let timeout = std::time::Duration::from_secs(5);
|
||||
|
||||
let workspace_store = Arc::new(crate::channels::wasm::host::ChannelWorkspaceStore::new());
|
||||
|
||||
let result = WasmChannel::execute_poll(
|
||||
"poll-test",
|
||||
&runtime,
|
||||
@@ -2238,6 +2381,7 @@ mod tests {
|
||||
&credentials,
|
||||
Arc::new(PairingStore::new()),
|
||||
timeout,
|
||||
&workspace_store,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2268,6 +2412,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2306,6 +2451,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2321,7 +2467,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "poll-channel".to_string(),
|
||||
description: "Polling test channel".to_string(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
|
||||
@@ -22,7 +22,9 @@ use std::sync::{Arc, Mutex};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing_subscriber::Layer;
|
||||
use tracing_subscriber::layer::SubscriberExt;
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
use tracing_subscriber::{EnvFilter, Layer, reload};
|
||||
|
||||
use crate::safety::LeakDetector;
|
||||
|
||||
@@ -102,6 +104,115 @@ impl Default for LogBroadcaster {
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle for changing the tracing `EnvFilter` at runtime.
|
||||
///
|
||||
/// Wraps a `reload::Handle` so the gateway can switch between log levels
|
||||
/// (e.g. `ironclaw=debug`) without restarting the process.
|
||||
pub struct LogLevelHandle {
|
||||
handle: reload::Handle<EnvFilter, tracing_subscriber::Registry>,
|
||||
current_level: Mutex<String>,
|
||||
base_filter: String,
|
||||
}
|
||||
|
||||
impl LogLevelHandle {
|
||||
pub fn new(
|
||||
handle: reload::Handle<EnvFilter, tracing_subscriber::Registry>,
|
||||
initial_level: String,
|
||||
base_filter: String,
|
||||
) -> Self {
|
||||
Self {
|
||||
handle,
|
||||
current_level: Mutex::new(initial_level),
|
||||
base_filter,
|
||||
}
|
||||
}
|
||||
|
||||
/// Change the `ironclaw=<level>` directive at runtime.
|
||||
///
|
||||
/// `level` must be one of: trace, debug, info, warn, error.
|
||||
pub fn set_level(&self, level: &str) -> Result<(), String> {
|
||||
const VALID: &[&str] = &["trace", "debug", "info", "warn", "error"];
|
||||
let level = level.to_lowercase();
|
||||
if !VALID.contains(&level.as_str()) {
|
||||
return Err(format!(
|
||||
"invalid level '{}', must be one of: {}",
|
||||
level,
|
||||
VALID.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
let filter_str = if self.base_filter.is_empty() {
|
||||
format!("ironclaw={}", level)
|
||||
} else {
|
||||
format!("ironclaw={},{}", level, self.base_filter)
|
||||
};
|
||||
|
||||
let new_filter = EnvFilter::new(&filter_str);
|
||||
self.handle
|
||||
.reload(new_filter)
|
||||
.map_err(|e| format!("failed to reload filter: {}", e))?;
|
||||
|
||||
if let Ok(mut current) = self.current_level.lock() {
|
||||
*current = level;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the current ironclaw log level (e.g. "info", "debug").
|
||||
pub fn current_level(&self) -> String {
|
||||
self.current_level
|
||||
.lock()
|
||||
.map(|l| l.clone())
|
||||
.unwrap_or_else(|_| "info".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialise the tracing subscriber with a reloadable `EnvFilter`.
|
||||
///
|
||||
/// Returns the `LogLevelHandle` so callers can swap the filter at runtime.
|
||||
/// The fmt layer and `WebLogLayer` are attached alongside the reloadable filter.
|
||||
pub fn init_tracing(log_broadcaster: Arc<LogBroadcaster>) -> Arc<LogLevelHandle> {
|
||||
let raw_filter =
|
||||
std::env::var("RUST_LOG").unwrap_or_else(|_| "ironclaw=info,tower_http=warn".to_string());
|
||||
|
||||
// Split into the ironclaw directive and "everything else" (base_filter).
|
||||
let mut ironclaw_level = String::from("info");
|
||||
let mut base_parts: Vec<&str> = Vec::new();
|
||||
|
||||
for part in raw_filter.split(',') {
|
||||
let trimmed = part.trim();
|
||||
if trimmed.starts_with("ironclaw=") {
|
||||
if let Some(lvl) = trimmed.strip_prefix("ironclaw=") {
|
||||
ironclaw_level = lvl.to_string();
|
||||
}
|
||||
} else if !trimmed.is_empty() {
|
||||
base_parts.push(trimmed);
|
||||
}
|
||||
}
|
||||
let base_filter = base_parts.join(",");
|
||||
|
||||
let env_filter = EnvFilter::new(&raw_filter);
|
||||
let (reload_layer, reload_handle) = reload::Layer::new(env_filter);
|
||||
|
||||
let handle = Arc::new(LogLevelHandle::new(
|
||||
reload_handle,
|
||||
ironclaw_level,
|
||||
base_filter,
|
||||
));
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(reload_layer)
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_target(false)
|
||||
.with_writer(crate::tracing_fmt::TruncatingStderr::default()),
|
||||
)
|
||||
.with(WebLogLayer::new(log_broadcaster))
|
||||
.init();
|
||||
|
||||
handle
|
||||
}
|
||||
|
||||
/// Visitor that extracts the `message` field and all extra key-value
|
||||
/// fields from a tracing event.
|
||||
///
|
||||
|
||||
+27
-1
@@ -41,7 +41,7 @@ use crate::skills::registry::SkillRegistry;
|
||||
use crate::tools::ToolRegistry;
|
||||
use crate::workspace::Workspace;
|
||||
|
||||
use self::log_layer::LogBroadcaster;
|
||||
use self::log_layer::{LogBroadcaster, LogLevelHandle};
|
||||
|
||||
use self::server::GatewayState;
|
||||
use self::sse::SseManager;
|
||||
@@ -76,6 +76,7 @@ impl GatewayChannel {
|
||||
workspace: None,
|
||||
session_manager: None,
|
||||
log_broadcaster: None,
|
||||
log_level_handle: None,
|
||||
extension_manager: None,
|
||||
tool_registry: None,
|
||||
store: None,
|
||||
@@ -88,6 +89,9 @@ impl GatewayChannel {
|
||||
skill_registry: None,
|
||||
skill_catalog: None,
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
registry_entries: Vec::new(),
|
||||
cost_guard: None,
|
||||
startup_time: std::time::Instant::now(),
|
||||
});
|
||||
|
||||
Self {
|
||||
@@ -105,6 +109,7 @@ impl GatewayChannel {
|
||||
workspace: self.state.workspace.clone(),
|
||||
session_manager: self.state.session_manager.clone(),
|
||||
log_broadcaster: self.state.log_broadcaster.clone(),
|
||||
log_level_handle: self.state.log_level_handle.clone(),
|
||||
extension_manager: self.state.extension_manager.clone(),
|
||||
tool_registry: self.state.tool_registry.clone(),
|
||||
store: self.state.store.clone(),
|
||||
@@ -117,6 +122,9 @@ impl GatewayChannel {
|
||||
skill_registry: self.state.skill_registry.clone(),
|
||||
skill_catalog: self.state.skill_catalog.clone(),
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
registry_entries: self.state.registry_entries.clone(),
|
||||
cost_guard: self.state.cost_guard.clone(),
|
||||
startup_time: self.state.startup_time,
|
||||
};
|
||||
mutate(&mut new_state);
|
||||
self.state = Arc::new(new_state);
|
||||
@@ -140,6 +148,12 @@ impl GatewayChannel {
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject the log level handle for runtime log level control.
|
||||
pub fn with_log_level_handle(mut self, h: Arc<LogLevelHandle>) -> Self {
|
||||
self.rebuild_state(|s| s.log_level_handle = Some(h));
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject the extension manager for the extensions API.
|
||||
pub fn with_extension_manager(mut self, em: Arc<ExtensionManager>) -> Self {
|
||||
self.rebuild_state(|s| s.extension_manager = Some(em));
|
||||
@@ -198,6 +212,18 @@ impl GatewayChannel {
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject registry catalog entries for the available extensions API.
|
||||
pub fn with_registry_entries(mut self, entries: Vec<crate::extensions::RegistryEntry>) -> Self {
|
||||
self.rebuild_state(|s| s.registry_entries = entries);
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject the cost guard for token/cost tracking in the status popover.
|
||||
pub fn with_cost_guard(mut self, cg: Arc<crate::agent::cost_guard::CostGuard>) -> Self {
|
||||
self.rebuild_state(|s| s.cost_guard = Some(cg));
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the auth token (for printing to console on startup).
|
||||
pub fn auth_token(&self) -> &str {
|
||||
&self.auth_token
|
||||
|
||||
+307
-20
@@ -13,7 +13,7 @@ use axum::{
|
||||
http::{StatusCode, header},
|
||||
middleware,
|
||||
response::{
|
||||
Html, IntoResponse,
|
||||
IntoResponse,
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
},
|
||||
routing::{get, post},
|
||||
@@ -122,6 +122,8 @@ pub struct GatewayState {
|
||||
pub session_manager: Option<Arc<SessionManager>>,
|
||||
/// Log broadcaster for the logs SSE endpoint.
|
||||
pub log_broadcaster: Option<Arc<LogBroadcaster>>,
|
||||
/// Handle for changing the tracing log level at runtime.
|
||||
pub log_level_handle: Option<Arc<crate::channels::web::log_layer::LogLevelHandle>>,
|
||||
/// Extension manager for extension management API.
|
||||
pub extension_manager: Option<Arc<ExtensionManager>>,
|
||||
/// Tool registry for listing registered tools.
|
||||
@@ -146,6 +148,13 @@ pub struct GatewayState {
|
||||
pub skill_catalog: Option<Arc<crate::skills::catalog::SkillCatalog>>,
|
||||
/// Rate limiter for chat endpoints (30 messages per 60 seconds).
|
||||
pub chat_rate_limiter: RateLimiter,
|
||||
/// Registry catalog entries for the available extensions API.
|
||||
/// Populated at startup from `registry/` manifests, independent of extension manager.
|
||||
pub registry_entries: Vec<crate::extensions::RegistryEntry>,
|
||||
/// Cost guard for token/cost tracking.
|
||||
pub cost_guard: Option<Arc<crate::agent::cost_guard::CostGuard>>,
|
||||
/// Server startup time for uptime calculation.
|
||||
pub startup_time: std::time::Instant,
|
||||
}
|
||||
|
||||
/// Start the gateway HTTP server.
|
||||
@@ -204,9 +213,15 @@ pub async fn start_server(
|
||||
.route("/api/jobs/{id}/files/read", get(job_files_read_handler))
|
||||
// Logs
|
||||
.route("/api/logs/events", get(logs_events_handler))
|
||||
.route("/api/logs/level", get(logs_level_get_handler))
|
||||
.route(
|
||||
"/api/logs/level",
|
||||
axum::routing::put(logs_level_set_handler),
|
||||
)
|
||||
// Extensions
|
||||
.route("/api/extensions", get(extensions_list_handler))
|
||||
.route("/api/extensions/tools", get(extensions_tools_handler))
|
||||
.route("/api/extensions/registry", get(extensions_registry_handler))
|
||||
.route("/api/extensions/install", post(extensions_install_handler))
|
||||
.route(
|
||||
"/api/extensions/{name}/activate",
|
||||
@@ -216,6 +231,16 @@ pub async fn start_server(
|
||||
"/api/extensions/{name}/remove",
|
||||
post(extensions_remove_handler),
|
||||
)
|
||||
.route(
|
||||
"/api/extensions/{name}/setup",
|
||||
get(extensions_setup_handler).post(extensions_setup_submit_handler),
|
||||
)
|
||||
// Pairing
|
||||
.route("/api/pairing/{channel}", get(pairing_list_handler))
|
||||
.route(
|
||||
"/api/pairing/{channel}/approve",
|
||||
post(pairing_approve_handler),
|
||||
)
|
||||
// Routines
|
||||
.route("/api/routines", get(routines_list_handler))
|
||||
.route("/api/routines/summary", get(routines_summary_handler))
|
||||
@@ -337,20 +362,32 @@ pub async fn start_server(
|
||||
|
||||
// --- Static file handlers ---
|
||||
|
||||
async fn index_handler() -> Html<&'static str> {
|
||||
Html(include_str!("static/index.html"))
|
||||
async fn index_handler() -> impl IntoResponse {
|
||||
(
|
||||
[
|
||||
(header::CONTENT_TYPE, "text/html; charset=utf-8"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/index.html"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn css_handler() -> impl IntoResponse {
|
||||
(
|
||||
[(header::CONTENT_TYPE, "text/css")],
|
||||
[
|
||||
(header::CONTENT_TYPE, "text/css"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/style.css"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn js_handler() -> impl IntoResponse {
|
||||
(
|
||||
[(header::CONTENT_TYPE, "application/javascript")],
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/javascript"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/app.js"),
|
||||
)
|
||||
}
|
||||
@@ -557,9 +594,13 @@ pub async fn clear_auth_mode(state: &GatewayState) {
|
||||
async fn chat_events_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
state.sse.subscribe().ok_or((
|
||||
let sse = state.sse.subscribe().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Too many connections".to_string(),
|
||||
))?;
|
||||
Ok((
|
||||
[("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")],
|
||||
sse,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1585,10 +1626,7 @@ async fn job_files_read_handler(
|
||||
|
||||
async fn logs_events_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<
|
||||
Sse<impl futures::Stream<Item = Result<Event, Infallible>> + Send + 'static>,
|
||||
(StatusCode, String),
|
||||
> {
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let broadcaster = state.log_broadcaster.as_ref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Log broadcaster not available".to_string(),
|
||||
@@ -1601,25 +1639,60 @@ async fn logs_events_handler(
|
||||
|
||||
let history_stream = futures::stream::iter(history).map(|entry| {
|
||||
let data = serde_json::to_string(&entry).unwrap_or_default();
|
||||
Ok(Event::default().event("log").data(data))
|
||||
Ok::<_, Infallible>(Event::default().event("log").data(data))
|
||||
});
|
||||
|
||||
let live_stream = tokio_stream::wrappers::BroadcastStream::new(rx)
|
||||
.filter_map(|result| result.ok())
|
||||
.map(|entry| {
|
||||
let data = serde_json::to_string(&entry).unwrap_or_default();
|
||||
Ok(Event::default().event("log").data(data))
|
||||
Ok::<_, Infallible>(Event::default().event("log").data(data))
|
||||
});
|
||||
|
||||
let stream = history_stream.chain(live_stream);
|
||||
|
||||
Ok(Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(std::time::Duration::from_secs(30))
|
||||
.text(""),
|
||||
Ok((
|
||||
[("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")],
|
||||
Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(std::time::Duration::from_secs(30))
|
||||
.text(""),
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
async fn logs_level_get_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||
let handle = state.log_level_handle.as_ref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Log level control not available".to_string(),
|
||||
))?;
|
||||
Ok(Json(serde_json::json!({ "level": handle.current_level() })))
|
||||
}
|
||||
|
||||
async fn logs_level_set_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Json(body): Json<serde_json::Value>,
|
||||
) -> Result<Json<serde_json::Value>, (StatusCode, String)> {
|
||||
let handle = state.log_level_handle.as_ref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Log level control not available".to_string(),
|
||||
))?;
|
||||
|
||||
let level = body
|
||||
.get("level")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or((StatusCode::BAD_REQUEST, "missing 'level' field".to_string()))?;
|
||||
|
||||
handle
|
||||
.set_level(level)
|
||||
.map_err(|e| (StatusCode::BAD_REQUEST, e))?;
|
||||
|
||||
tracing::info!("Log level changed to '{}'", handle.current_level());
|
||||
Ok(Json(serde_json::json!({ "level": handle.current_level() })))
|
||||
}
|
||||
|
||||
// --- Extension handlers ---
|
||||
|
||||
async fn extensions_list_handler(
|
||||
@@ -1645,6 +1718,7 @@ async fn extensions_list_handler(
|
||||
authenticated: ext.authenticated,
|
||||
active: ext.active,
|
||||
tools: ext.tools,
|
||||
needs_setup: ext.needs_setup,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -1675,10 +1749,30 @@ async fn extensions_install_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Json(req): Json<InstallExtensionRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
// When extension manager isn't available, check registry entries for a helpful message
|
||||
let Some(ext_mgr) = state.extension_manager.as_ref() else {
|
||||
// Look up the entry in the catalog to give a specific error
|
||||
if let Some(entry) = state.registry_entries.iter().find(|e| e.name == req.name) {
|
||||
let msg = match &entry.source {
|
||||
crate::extensions::ExtensionSource::WasmBuildable { .. } => {
|
||||
format!(
|
||||
"'{}' requires building from source. \
|
||||
Run `ironclaw registry install {}` from the CLI.",
|
||||
req.name, req.name
|
||||
)
|
||||
}
|
||||
_ => format!(
|
||||
"Extension manager not available (secrets store required). \
|
||||
Configure DATABASE_URL or a secrets backend to enable installation of '{}'.",
|
||||
req.name
|
||||
),
|
||||
};
|
||||
return Ok(Json(ActionResponse::fail(msg)));
|
||||
}
|
||||
return Ok(Json(ActionResponse::fail(
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
)));
|
||||
};
|
||||
|
||||
let kind_hint = req.kind.as_deref().and_then(|k| match k {
|
||||
"mcp_server" => Some(crate::extensions::ExtensionKind::McpServer),
|
||||
@@ -1827,6 +1921,160 @@ async fn extensions_remove_handler(
|
||||
}
|
||||
}
|
||||
|
||||
async fn extensions_registry_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Query(params): Query<RegistrySearchQuery>,
|
||||
) -> Json<RegistrySearchResponse> {
|
||||
let query = params.query.unwrap_or_default();
|
||||
let query_lower = query.to_lowercase();
|
||||
let tokens: Vec<&str> = query_lower.split_whitespace().collect();
|
||||
|
||||
// Filter registry entries by query (or return all if empty)
|
||||
let matching: Vec<&crate::extensions::RegistryEntry> = if tokens.is_empty() {
|
||||
state.registry_entries.iter().collect()
|
||||
} else {
|
||||
state
|
||||
.registry_entries
|
||||
.iter()
|
||||
.filter(|e| {
|
||||
let name = e.name.to_lowercase();
|
||||
let display = e.display_name.to_lowercase();
|
||||
let desc = e.description.to_lowercase();
|
||||
tokens.iter().any(|t| {
|
||||
name.contains(t)
|
||||
|| display.contains(t)
|
||||
|| desc.contains(t)
|
||||
|| e.keywords.iter().any(|k| k.to_lowercase().contains(t))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
// Cross-reference with installed extensions by (name, kind) to avoid
|
||||
// false positives when the same name exists as different kinds.
|
||||
let installed: std::collections::HashSet<(String, String)> =
|
||||
if let Some(ext_mgr) = state.extension_manager.as_ref() {
|
||||
ext_mgr
|
||||
.list(None)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|ext| (ext.name, ext.kind.to_string()))
|
||||
.collect()
|
||||
} else {
|
||||
std::collections::HashSet::new()
|
||||
};
|
||||
|
||||
let entries = matching
|
||||
.into_iter()
|
||||
.map(|e| {
|
||||
let kind_str = e.kind.to_string();
|
||||
RegistryEntryInfo {
|
||||
name: e.name.clone(),
|
||||
display_name: e.display_name.clone(),
|
||||
installed: installed.contains(&(e.name.clone(), kind_str.clone())),
|
||||
kind: kind_str,
|
||||
description: e.description.clone(),
|
||||
keywords: e.keywords.clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Json(RegistrySearchResponse { entries })
|
||||
}
|
||||
|
||||
async fn extensions_setup_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Path(name): Path<String>,
|
||||
) -> Result<Json<ExtensionSetupResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
|
||||
let secrets = ext_mgr
|
||||
.get_setup_schema(&name)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
let kind = ext_mgr
|
||||
.list(None)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|list| list.into_iter().find(|e| e.name == name))
|
||||
.map(|e| e.kind.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(Json(ExtensionSetupResponse {
|
||||
name,
|
||||
kind,
|
||||
secrets,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn extensions_setup_submit_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Path(name): Path<String>,
|
||||
Json(req): Json<ExtensionSetupRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
|
||||
match ext_mgr.save_setup_secrets(&name, &req.secrets).await {
|
||||
Ok(message) => Ok(Json(ActionResponse::ok(message))),
|
||||
Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Pairing handlers ---
|
||||
|
||||
async fn pairing_list_handler(
|
||||
Path(channel): Path<String>,
|
||||
) -> Result<Json<PairingListResponse>, (StatusCode, String)> {
|
||||
let store = crate::pairing::PairingStore::new();
|
||||
let requests = store
|
||||
.list_pending(&channel)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
let infos = requests
|
||||
.into_iter()
|
||||
.map(|r| PairingRequestInfo {
|
||||
code: r.code,
|
||||
sender_id: r.id,
|
||||
meta: r.meta,
|
||||
created_at: r.created_at,
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Json(PairingListResponse {
|
||||
channel,
|
||||
requests: infos,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn pairing_approve_handler(
|
||||
Path(channel): Path<String>,
|
||||
Json(req): Json<PairingApproveRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let store = crate::pairing::PairingStore::new();
|
||||
match store.approve(&channel, &req.code) {
|
||||
Ok(Some(approved)) => Ok(Json(ActionResponse::ok(format!(
|
||||
"Pairing approved for sender '{}'",
|
||||
approved.id
|
||||
)))),
|
||||
Ok(None) => Ok(Json(ActionResponse::fail(
|
||||
"Invalid or expired pairing code".to_string(),
|
||||
))),
|
||||
Err(crate::pairing::PairingStoreError::ApproveRateLimited) => Err((
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"Too many failed approve attempts; try again later".to_string(),
|
||||
)),
|
||||
Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Skills handlers ---
|
||||
|
||||
async fn skills_list_handler(
|
||||
@@ -2526,18 +2774,57 @@ async fn gateway_status_handler(
|
||||
.map(|t| t.connection_count())
|
||||
.unwrap_or(0);
|
||||
|
||||
let uptime_secs = state.startup_time.elapsed().as_secs();
|
||||
|
||||
let (daily_cost, actions_this_hour, model_usage) = if let Some(ref cg) = state.cost_guard {
|
||||
let cost = cg.daily_spend().await;
|
||||
let actions = cg.actions_this_hour().await;
|
||||
let usage = cg.model_usage().await;
|
||||
let models: Vec<ModelUsageEntry> = usage
|
||||
.into_iter()
|
||||
.map(|(model, tokens)| ModelUsageEntry {
|
||||
model,
|
||||
input_tokens: tokens.input_tokens,
|
||||
output_tokens: tokens.output_tokens,
|
||||
cost: format!("{:.6}", tokens.cost),
|
||||
})
|
||||
.collect();
|
||||
(Some(format!("{:.4}", cost)), Some(actions), Some(models))
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
Json(GatewayStatusResponse {
|
||||
sse_connections,
|
||||
ws_connections,
|
||||
total_connections: sse_connections + ws_connections,
|
||||
uptime_secs,
|
||||
daily_cost,
|
||||
actions_this_hour,
|
||||
model_usage,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct ModelUsageEntry {
|
||||
model: String,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cost: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct GatewayStatusResponse {
|
||||
sse_connections: u64,
|
||||
ws_connections: u64,
|
||||
total_connections: u64,
|
||||
uptime_secs: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
daily_cost: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
actions_this_hour: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
model_usage: Option<Vec<ModelUsageEntry>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+623
-28
@@ -29,16 +29,25 @@ function authenticate() {
|
||||
sessionStorage.setItem('ironclaw_token', token);
|
||||
document.getElementById('auth-screen').style.display = 'none';
|
||||
document.getElementById('app').style.display = 'flex';
|
||||
// Strip token from URL so it's not visible in the address bar
|
||||
// Strip token and log_level from URL so they're not visible in the address bar
|
||||
const cleaned = new URL(window.location);
|
||||
const urlLogLevel = cleaned.searchParams.get('log_level');
|
||||
cleaned.searchParams.delete('token');
|
||||
cleaned.searchParams.delete('log_level');
|
||||
window.history.replaceState({}, '', cleaned.pathname + cleaned.search);
|
||||
connectSSE();
|
||||
connectLogSSE();
|
||||
startGatewayStatusPolling();
|
||||
checkTeeStatus();
|
||||
loadThreads();
|
||||
loadMemoryTree();
|
||||
loadJobs();
|
||||
// Apply URL log_level param if present, otherwise just sync the dropdown
|
||||
if (urlLogLevel) {
|
||||
setServerLogLevel(urlLogLevel);
|
||||
} else {
|
||||
loadServerLogLevel();
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
sessionStorage.removeItem('ironclaw_token');
|
||||
@@ -1167,19 +1176,46 @@ function applyLogFilters() {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Server-side log level control ---
|
||||
|
||||
function setServerLogLevel(level) {
|
||||
apiFetch('/api/logs/level', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ level: level }),
|
||||
})
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
document.getElementById('logs-server-level').value = data.level;
|
||||
})
|
||||
.catch(err => console.error('Failed to set server log level:', err));
|
||||
}
|
||||
|
||||
function loadServerLogLevel() {
|
||||
apiFetch('/api/logs/level')
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
document.getElementById('logs-server-level').value = data.level;
|
||||
})
|
||||
.catch(() => {}); // ignore if not available
|
||||
}
|
||||
|
||||
// --- Extensions ---
|
||||
|
||||
function loadExtensions() {
|
||||
const extList = document.getElementById('extensions-list');
|
||||
const wasmList = document.getElementById('available-wasm-list');
|
||||
const mcpList = document.getElementById('mcp-servers-list');
|
||||
const toolsTbody = document.getElementById('tools-tbody');
|
||||
const toolsEmpty = document.getElementById('tools-empty');
|
||||
|
||||
// Fetch both in parallel
|
||||
// Fetch all three in parallel
|
||||
Promise.all([
|
||||
apiFetch('/api/extensions').catch(() => ({ extensions: [] })),
|
||||
apiFetch('/api/extensions/tools').catch(() => ({ tools: [] })),
|
||||
]).then(([extData, toolData]) => {
|
||||
// Render extensions
|
||||
apiFetch('/api/extensions/registry').catch(function(err) { console.warn('registry fetch failed:', err); return { entries: [] }; }),
|
||||
]).then(([extData, toolData, registryData]) => {
|
||||
// Render installed extensions
|
||||
if (extData.extensions.length === 0) {
|
||||
extList.innerHTML = '<div class="empty-state">No extensions installed</div>';
|
||||
} else {
|
||||
@@ -1189,6 +1225,31 @@ function loadExtensions() {
|
||||
}
|
||||
}
|
||||
|
||||
// Split registry entries by kind
|
||||
var wasmEntries = registryData.entries.filter(function(e) { return e.kind !== 'mcp_server' && !e.installed; });
|
||||
var mcpEntries = registryData.entries.filter(function(e) { return e.kind === 'mcp_server'; });
|
||||
|
||||
// Available WASM extensions
|
||||
if (wasmEntries.length === 0) {
|
||||
wasmList.innerHTML = '<div class="empty-state">No additional WASM extensions available</div>';
|
||||
} else {
|
||||
wasmList.innerHTML = '';
|
||||
for (const entry of wasmEntries) {
|
||||
wasmList.appendChild(renderAvailableExtensionCard(entry));
|
||||
}
|
||||
}
|
||||
|
||||
// MCP servers (show both installed and uninstalled)
|
||||
if (mcpEntries.length === 0) {
|
||||
mcpList.innerHTML = '<div class="empty-state">No MCP servers available</div>';
|
||||
} else {
|
||||
mcpList.innerHTML = '';
|
||||
for (const entry of mcpEntries) {
|
||||
var installedExt = extData.extensions.find(function(e) { return e.name === entry.name; });
|
||||
mcpList.appendChild(renderMcpServerCard(entry, installedExt));
|
||||
}
|
||||
}
|
||||
|
||||
// Render tools
|
||||
if (toolData.tools.length === 0) {
|
||||
toolsTbody.innerHTML = '';
|
||||
@@ -1202,6 +1263,148 @@ function loadExtensions() {
|
||||
});
|
||||
}
|
||||
|
||||
function renderAvailableExtensionCard(entry) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'ext-card ext-available';
|
||||
|
||||
const header = document.createElement('div');
|
||||
header.className = 'ext-header';
|
||||
|
||||
const name = document.createElement('span');
|
||||
name.className = 'ext-name';
|
||||
name.textContent = entry.display_name;
|
||||
header.appendChild(name);
|
||||
|
||||
const kind = document.createElement('span');
|
||||
kind.className = 'ext-kind kind-' + entry.kind;
|
||||
kind.textContent = entry.kind;
|
||||
header.appendChild(kind);
|
||||
|
||||
card.appendChild(header);
|
||||
|
||||
const desc = document.createElement('div');
|
||||
desc.className = 'ext-desc';
|
||||
desc.textContent = entry.description;
|
||||
card.appendChild(desc);
|
||||
|
||||
if (entry.keywords && entry.keywords.length > 0) {
|
||||
const kw = document.createElement('div');
|
||||
kw.className = 'ext-keywords';
|
||||
kw.textContent = entry.keywords.join(', ');
|
||||
card.appendChild(kw);
|
||||
}
|
||||
|
||||
const actions = document.createElement('div');
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
const installBtn = document.createElement('button');
|
||||
installBtn.className = 'btn-ext install';
|
||||
installBtn.textContent = 'Install';
|
||||
installBtn.addEventListener('click', function() {
|
||||
installBtn.disabled = true;
|
||||
installBtn.textContent = 'Installing...';
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: entry.name, kind: entry.kind },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + entry.display_name, 'success');
|
||||
} else {
|
||||
showToast('Install: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
loadExtensions();
|
||||
});
|
||||
});
|
||||
actions.appendChild(installBtn);
|
||||
|
||||
card.appendChild(actions);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderMcpServerCard(entry, installedExt) {
|
||||
var card = document.createElement('div');
|
||||
card.className = 'ext-card' + (installedExt ? '' : ' ext-available');
|
||||
|
||||
var header = document.createElement('div');
|
||||
header.className = 'ext-header';
|
||||
|
||||
var name = document.createElement('span');
|
||||
name.className = 'ext-name';
|
||||
name.textContent = entry.display_name;
|
||||
header.appendChild(name);
|
||||
|
||||
var kind = document.createElement('span');
|
||||
kind.className = 'ext-kind kind-mcp_server';
|
||||
kind.textContent = 'mcp_server';
|
||||
header.appendChild(kind);
|
||||
|
||||
if (installedExt) {
|
||||
var authDot = document.createElement('span');
|
||||
authDot.className = 'ext-auth-dot ' + (installedExt.authenticated ? 'authed' : 'unauthed');
|
||||
authDot.title = installedExt.authenticated ? 'Authenticated' : 'Not authenticated';
|
||||
header.appendChild(authDot);
|
||||
}
|
||||
|
||||
card.appendChild(header);
|
||||
|
||||
var desc = document.createElement('div');
|
||||
desc.className = 'ext-desc';
|
||||
desc.textContent = entry.description;
|
||||
card.appendChild(desc);
|
||||
|
||||
var actions = document.createElement('div');
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
if (installedExt) {
|
||||
if (!installedExt.active) {
|
||||
var activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', function() { activateExtension(installedExt.name); });
|
||||
actions.appendChild(activateBtn);
|
||||
} else {
|
||||
var activeLabel = document.createElement('span');
|
||||
activeLabel.className = 'ext-active-label';
|
||||
activeLabel.textContent = 'Active';
|
||||
actions.appendChild(activeLabel);
|
||||
}
|
||||
var removeBtn = document.createElement('button');
|
||||
removeBtn.className = 'btn-ext remove';
|
||||
removeBtn.textContent = 'Remove';
|
||||
removeBtn.addEventListener('click', function() { removeExtension(installedExt.name); });
|
||||
actions.appendChild(removeBtn);
|
||||
} else {
|
||||
var installBtn = document.createElement('button');
|
||||
installBtn.className = 'btn-ext install';
|
||||
installBtn.textContent = 'Install';
|
||||
installBtn.addEventListener('click', function() {
|
||||
installBtn.disabled = true;
|
||||
installBtn.textContent = 'Installing...';
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: entry.name, kind: entry.kind },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + entry.display_name, 'success');
|
||||
} else {
|
||||
showToast('Install: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
loadExtensions();
|
||||
});
|
||||
});
|
||||
actions.appendChild(installBtn);
|
||||
}
|
||||
|
||||
card.appendChild(actions);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderExtensionCard(ext) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'ext-card';
|
||||
@@ -1252,11 +1455,18 @@ function renderExtensionCard(ext) {
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
if (!ext.active) {
|
||||
const activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', () => activateExtension(ext.name));
|
||||
actions.appendChild(activateBtn);
|
||||
if (ext.kind === 'wasm_channel') {
|
||||
const restartLabel = document.createElement('span');
|
||||
restartLabel.className = 'ext-restart-label';
|
||||
restartLabel.textContent = 'Restart to activate';
|
||||
actions.appendChild(restartLabel);
|
||||
} else {
|
||||
const activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', () => activateExtension(ext.name));
|
||||
actions.appendChild(activateBtn);
|
||||
}
|
||||
} else {
|
||||
const activeLabel = document.createElement('span');
|
||||
activeLabel.className = 'ext-active-label';
|
||||
@@ -1264,6 +1474,14 @@ function renderExtensionCard(ext) {
|
||||
actions.appendChild(activeLabel);
|
||||
}
|
||||
|
||||
if (ext.needs_setup) {
|
||||
const configBtn = document.createElement('button');
|
||||
configBtn.className = 'btn-ext configure';
|
||||
configBtn.textContent = ext.authenticated ? 'Reconfigure' : 'Configure';
|
||||
configBtn.addEventListener('click', () => showConfigureModal(ext.name));
|
||||
actions.appendChild(configBtn);
|
||||
}
|
||||
|
||||
const removeBtn = document.createElement('button');
|
||||
removeBtn.className = 'btn-ext remove';
|
||||
removeBtn.textContent = 'Remove';
|
||||
@@ -1271,6 +1489,15 @@ function renderExtensionCard(ext) {
|
||||
actions.appendChild(removeBtn);
|
||||
|
||||
card.appendChild(actions);
|
||||
|
||||
// For active WASM channels, check for pending pairing requests
|
||||
if (ext.active && ext.kind === 'wasm_channel') {
|
||||
const pairingSection = document.createElement('div');
|
||||
pairingSection.className = 'ext-pairing';
|
||||
card.appendChild(pairingSection);
|
||||
loadPairingRequests(ext.name, pairingSection);
|
||||
}
|
||||
|
||||
return card;
|
||||
}
|
||||
|
||||
@@ -1286,7 +1513,7 @@ function activateExtension(name) {
|
||||
showToast('Opening authentication for ' + name, 'info');
|
||||
window.open(res.auth_url, '_blank');
|
||||
} else if (res.awaiting_token) {
|
||||
showToast(res.instructions || 'Please provide an API token for ' + name, 'info');
|
||||
showConfigureModal(name);
|
||||
} else {
|
||||
showToast('Activate failed: ' + res.message, 'error');
|
||||
}
|
||||
@@ -1309,6 +1536,189 @@ function removeExtension(name) {
|
||||
.catch((err) => showToast('Remove failed: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
function showConfigureModal(name) {
|
||||
apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup')
|
||||
.then((setup) => {
|
||||
if (!setup.secrets || setup.secrets.length === 0) {
|
||||
showToast('No configuration needed for ' + name, 'info');
|
||||
return;
|
||||
}
|
||||
renderConfigureModal(name, setup.secrets);
|
||||
})
|
||||
.catch((err) => showToast('Failed to load setup: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
function renderConfigureModal(name, secrets) {
|
||||
closeConfigureModal();
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'configure-overlay';
|
||||
overlay.addEventListener('click', (e) => {
|
||||
if (e.target === overlay) closeConfigureModal();
|
||||
});
|
||||
|
||||
const modal = document.createElement('div');
|
||||
modal.className = 'configure-modal';
|
||||
|
||||
const header = document.createElement('h3');
|
||||
header.textContent = 'Configure ' + name;
|
||||
modal.appendChild(header);
|
||||
|
||||
const form = document.createElement('div');
|
||||
form.className = 'configure-form';
|
||||
|
||||
const fields = [];
|
||||
for (const secret of secrets) {
|
||||
const field = document.createElement('div');
|
||||
field.className = 'configure-field';
|
||||
|
||||
const label = document.createElement('label');
|
||||
label.textContent = secret.prompt;
|
||||
if (secret.optional) {
|
||||
const opt = document.createElement('span');
|
||||
opt.className = 'field-optional';
|
||||
opt.textContent = ' (optional)';
|
||||
label.appendChild(opt);
|
||||
}
|
||||
field.appendChild(label);
|
||||
|
||||
const inputRow = document.createElement('div');
|
||||
inputRow.className = 'configure-input-row';
|
||||
|
||||
const input = document.createElement('input');
|
||||
input.type = 'password';
|
||||
input.name = secret.name;
|
||||
input.placeholder = secret.provided ? '(already set — leave empty to keep)' : '';
|
||||
input.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter') submitConfigureModal(name, fields);
|
||||
});
|
||||
inputRow.appendChild(input);
|
||||
|
||||
if (secret.provided) {
|
||||
const badge = document.createElement('span');
|
||||
badge.className = 'field-provided';
|
||||
badge.textContent = 'Set';
|
||||
inputRow.appendChild(badge);
|
||||
}
|
||||
if (secret.auto_generate && !secret.provided) {
|
||||
const hint = document.createElement('span');
|
||||
hint.className = 'field-autogen';
|
||||
hint.textContent = 'Auto-generated if empty';
|
||||
inputRow.appendChild(hint);
|
||||
}
|
||||
|
||||
field.appendChild(inputRow);
|
||||
form.appendChild(field);
|
||||
fields.push({ name: secret.name, input: input });
|
||||
}
|
||||
|
||||
modal.appendChild(form);
|
||||
|
||||
const actions = document.createElement('div');
|
||||
actions.className = 'configure-actions';
|
||||
|
||||
const submitBtn = document.createElement('button');
|
||||
submitBtn.className = 'btn-ext activate';
|
||||
submitBtn.textContent = 'Save';
|
||||
submitBtn.addEventListener('click', () => submitConfigureModal(name, fields));
|
||||
actions.appendChild(submitBtn);
|
||||
|
||||
const cancelBtn = document.createElement('button');
|
||||
cancelBtn.className = 'btn-ext remove';
|
||||
cancelBtn.textContent = 'Cancel';
|
||||
cancelBtn.addEventListener('click', closeConfigureModal);
|
||||
actions.appendChild(cancelBtn);
|
||||
|
||||
modal.appendChild(actions);
|
||||
overlay.appendChild(modal);
|
||||
document.body.appendChild(overlay);
|
||||
|
||||
if (fields.length > 0) fields[0].input.focus();
|
||||
}
|
||||
|
||||
function submitConfigureModal(name, fields) {
|
||||
const secrets = {};
|
||||
for (const f of fields) {
|
||||
if (f.input.value.trim()) {
|
||||
secrets[f.name] = f.input.value.trim();
|
||||
}
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup', {
|
||||
method: 'POST',
|
||||
body: { secrets },
|
||||
})
|
||||
.then((res) => {
|
||||
closeConfigureModal();
|
||||
if (res.success) {
|
||||
showToast(res.message, 'success');
|
||||
} else {
|
||||
showToast(res.message || 'Configuration failed', 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
})
|
||||
.catch((err) => {
|
||||
showToast('Configuration failed: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
function closeConfigureModal() {
|
||||
const existing = document.querySelector('.configure-overlay');
|
||||
if (existing) existing.remove();
|
||||
}
|
||||
|
||||
// --- Pairing ---
|
||||
|
||||
function loadPairingRequests(channel, container) {
|
||||
apiFetch('/api/pairing/' + encodeURIComponent(channel))
|
||||
.then(data => {
|
||||
container.innerHTML = '';
|
||||
if (!data.requests || data.requests.length === 0) return;
|
||||
|
||||
const heading = document.createElement('div');
|
||||
heading.className = 'pairing-heading';
|
||||
heading.textContent = 'Pending pairing requests';
|
||||
container.appendChild(heading);
|
||||
|
||||
data.requests.forEach(req => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'pairing-row';
|
||||
|
||||
const code = document.createElement('span');
|
||||
code.className = 'pairing-code';
|
||||
code.textContent = req.code;
|
||||
row.appendChild(code);
|
||||
|
||||
const sender = document.createElement('span');
|
||||
sender.className = 'pairing-sender';
|
||||
sender.textContent = 'from ' + req.sender_id;
|
||||
row.appendChild(sender);
|
||||
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'btn-ext activate';
|
||||
btn.textContent = 'Approve';
|
||||
btn.addEventListener('click', () => approvePairing(channel, req.code, container));
|
||||
row.appendChild(btn);
|
||||
|
||||
container.appendChild(row);
|
||||
});
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
function approvePairing(channel, code, container) {
|
||||
apiFetch('/api/pairing/' + encodeURIComponent(channel) + '/approve', {
|
||||
method: 'POST',
|
||||
body: { code },
|
||||
}).then(res => {
|
||||
if (res.success) {
|
||||
showToast('Pairing approved', 'success');
|
||||
loadPairingRequests(channel, container);
|
||||
} else {
|
||||
showToast(res.message || 'Approve failed', 'error');
|
||||
}
|
||||
}).catch(err => showToast('Error: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
// --- Jobs ---
|
||||
|
||||
let currentJobId = null;
|
||||
@@ -2049,13 +2459,72 @@ function startGatewayStatusPolling() {
|
||||
gatewayStatusInterval = setInterval(fetchGatewayStatus, 30000);
|
||||
}
|
||||
|
||||
function formatTokenCount(n) {
|
||||
if (n == null || n === 0) return '0';
|
||||
if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M';
|
||||
if (n >= 1000) return (n / 1000).toFixed(1) + 'k';
|
||||
return '' + n;
|
||||
}
|
||||
|
||||
function formatCost(costStr) {
|
||||
if (!costStr) return '$0.00';
|
||||
var n = parseFloat(costStr);
|
||||
if (n < 0.01) return '$' + n.toFixed(4);
|
||||
return '$' + n.toFixed(2);
|
||||
}
|
||||
|
||||
function shortModelName(model) {
|
||||
// Strip provider prefix and shorten common model names
|
||||
var m = model.indexOf('/') >= 0 ? model.split('/').pop() : model;
|
||||
// Shorten dated suffixes
|
||||
m = m.replace(/-20\d{6}$/, '');
|
||||
return m;
|
||||
}
|
||||
|
||||
function fetchGatewayStatus() {
|
||||
apiFetch('/api/gateway/status').then((data) => {
|
||||
const popover = document.getElementById('gateway-popover');
|
||||
popover.innerHTML = '<div class="gw-stat"><span>SSE clients</span><span>' + (data.sse_clients || 0) + '</span></div>'
|
||||
+ '<div class="gw-stat"><span>Log clients</span><span>' + (data.log_clients || 0) + '</span></div>'
|
||||
+ '<div class="gw-stat"><span>Uptime</span><span>' + formatDuration(data.uptime_secs) + '</span></div>';
|
||||
}).catch(() => {});
|
||||
apiFetch('/api/gateway/status').then(function(data) {
|
||||
var popover = document.getElementById('gateway-popover');
|
||||
var html = '';
|
||||
|
||||
// Connection info
|
||||
html += '<div class="gw-section-label">Connections</div>';
|
||||
html += '<div class="gw-stat"><span>SSE</span><span>' + (data.sse_connections || 0) + '</span></div>';
|
||||
html += '<div class="gw-stat"><span>WebSocket</span><span>' + (data.ws_connections || 0) + '</span></div>';
|
||||
html += '<div class="gw-stat"><span>Uptime</span><span>' + formatDuration(data.uptime_secs) + '</span></div>';
|
||||
|
||||
// Cost tracker
|
||||
if (data.daily_cost != null) {
|
||||
html += '<div class="gw-divider"></div>';
|
||||
html += '<div class="gw-section-label">Cost Today</div>';
|
||||
html += '<div class="gw-stat"><span>Spent</span><span>' + formatCost(data.daily_cost) + '</span></div>';
|
||||
if (data.actions_this_hour != null) {
|
||||
html += '<div class="gw-stat"><span>Actions/hr</span><span>' + data.actions_this_hour + '</span></div>';
|
||||
}
|
||||
}
|
||||
|
||||
// Per-model token usage
|
||||
if (data.model_usage && data.model_usage.length > 0) {
|
||||
html += '<div class="gw-divider"></div>';
|
||||
html += '<div class="gw-section-label">Token Usage</div>';
|
||||
data.model_usage.sort(function(a, b) {
|
||||
return (b.input_tokens + b.output_tokens) - (a.input_tokens + a.output_tokens);
|
||||
});
|
||||
for (var i = 0; i < data.model_usage.length; i++) {
|
||||
var m = data.model_usage[i];
|
||||
var name = escapeHtml(shortModelName(m.model));
|
||||
html += '<div class="gw-model-row">'
|
||||
+ '<span class="gw-model-name">' + name + '</span>'
|
||||
+ '<span class="gw-model-cost">' + escapeHtml(formatCost(m.cost)) + '</span>'
|
||||
+ '</div>';
|
||||
html += '<div class="gw-token-detail">'
|
||||
+ '<span>in: ' + formatTokenCount(m.input_tokens) + '</span>'
|
||||
+ '<span>out: ' + formatTokenCount(m.output_tokens) + '</span>'
|
||||
+ '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
popover.innerHTML = html;
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
// Show/hide popover on hover
|
||||
@@ -2066,34 +2535,160 @@ document.getElementById('gateway-status-trigger').addEventListener('mouseleave',
|
||||
document.getElementById('gateway-popover').classList.remove('visible');
|
||||
});
|
||||
|
||||
// --- TEE attestation ---
|
||||
|
||||
let teeInfo = null;
|
||||
let teeReportCache = null;
|
||||
let teeReportLoading = false;
|
||||
|
||||
function teeApiBase() {
|
||||
var parts = window.location.hostname.split('.');
|
||||
if (parts.length < 2) return null;
|
||||
var domain = parts.slice(1).join('.');
|
||||
return window.location.protocol + '//api.' + domain;
|
||||
}
|
||||
|
||||
function teeInstanceName() {
|
||||
return window.location.hostname.split('.')[0];
|
||||
}
|
||||
|
||||
function checkTeeStatus() {
|
||||
var base = teeApiBase();
|
||||
if (!base) return;
|
||||
var name = teeInstanceName();
|
||||
fetch(base + '/instances/' + encodeURIComponent(name) + '/attestation').then(function(res) {
|
||||
if (!res.ok) throw new Error(res.status);
|
||||
return res.json();
|
||||
}).then(function(data) {
|
||||
teeInfo = data;
|
||||
document.getElementById('tee-shield').style.display = 'flex';
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
function fetchTeeReport() {
|
||||
if (teeReportCache) {
|
||||
renderTeePopover(teeReportCache);
|
||||
return;
|
||||
}
|
||||
if (teeReportLoading) return;
|
||||
teeReportLoading = true;
|
||||
var base = teeApiBase();
|
||||
if (!base) return;
|
||||
var popover = document.getElementById('tee-popover');
|
||||
popover.innerHTML = '<div class="tee-popover-loading">Loading attestation report...</div>';
|
||||
fetch(base + '/attestation/report').then(function(res) {
|
||||
if (!res.ok) throw new Error(res.status);
|
||||
return res.json();
|
||||
}).then(function(data) {
|
||||
teeReportCache = data;
|
||||
renderTeePopover(data);
|
||||
}).catch(function() {
|
||||
popover.innerHTML = '<div class="tee-popover-loading">Could not load attestation report</div>';
|
||||
}).finally(function() {
|
||||
teeReportLoading = false;
|
||||
});
|
||||
}
|
||||
|
||||
function renderTeePopover(report) {
|
||||
var popover = document.getElementById('tee-popover');
|
||||
var digest = (teeInfo && teeInfo.image_digest) || 'N/A';
|
||||
var fingerprint = report.tls_certificate_fingerprint || 'N/A';
|
||||
var reportData = report.report_data || '';
|
||||
var vmConfig = report.vm_config || 'N/A';
|
||||
var truncated = reportData.length > 32 ? reportData.slice(0, 32) + '...' : reportData;
|
||||
popover.innerHTML = '<div class="tee-popover-title">'
|
||||
+ '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z"/></svg>'
|
||||
+ 'TEE Attestation</div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">Image Digest</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(digest) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">TLS Certificate Fingerprint</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(fingerprint) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">Report Data</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(truncated) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">VM Config</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(vmConfig) + '</div></div>'
|
||||
+ '<div class="tee-popover-actions">'
|
||||
+ '<button class="tee-btn-copy" onclick="copyTeeReport()">Copy Full Report</button></div>';
|
||||
}
|
||||
|
||||
function copyTeeReport() {
|
||||
if (!teeReportCache) return;
|
||||
var combined = Object.assign({}, teeReportCache, teeInfo || {});
|
||||
navigator.clipboard.writeText(JSON.stringify(combined, null, 2)).then(function() {
|
||||
showToast('Attestation report copied', 'success');
|
||||
}).catch(function() {
|
||||
showToast('Failed to copy report', 'error');
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('tee-shield').addEventListener('mouseenter', function() {
|
||||
fetchTeeReport();
|
||||
document.getElementById('tee-popover').classList.add('visible');
|
||||
});
|
||||
document.getElementById('tee-shield').addEventListener('mouseleave', function() {
|
||||
document.getElementById('tee-popover').classList.remove('visible');
|
||||
});
|
||||
|
||||
// --- Extension install ---
|
||||
|
||||
function installExtension() {
|
||||
const name = document.getElementById('ext-install-name').value.trim();
|
||||
function installWasmExtension() {
|
||||
var name = document.getElementById('wasm-install-name').value.trim();
|
||||
if (!name) {
|
||||
showToast('Extension name is required', 'error');
|
||||
return;
|
||||
}
|
||||
const url = document.getElementById('ext-install-url').value.trim();
|
||||
const kind = document.getElementById('ext-install-kind').value;
|
||||
var url = document.getElementById('wasm-install-url').value.trim();
|
||||
if (!url) {
|
||||
showToast('URL to .tar.gz bundle is required', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name, url: url || undefined, kind },
|
||||
}).then((res) => {
|
||||
body: { name: name, url: url, kind: 'wasm_tool' },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + name, 'success');
|
||||
document.getElementById('ext-install-name').value = '';
|
||||
document.getElementById('ext-install-url').value = '';
|
||||
document.getElementById('wasm-install-name').value = '';
|
||||
document.getElementById('wasm-install-url').value = '';
|
||||
loadExtensions();
|
||||
} else {
|
||||
showToast('Install failed: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
}).catch((err) => {
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
function addMcpServer() {
|
||||
var name = document.getElementById('mcp-install-name').value.trim();
|
||||
if (!name) {
|
||||
showToast('Server name is required', 'error');
|
||||
return;
|
||||
}
|
||||
var url = document.getElementById('mcp-install-url').value.trim();
|
||||
if (!url) {
|
||||
showToast('MCP server URL is required', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: name, url: url, kind: 'mcp_server' },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Added MCP server ' + name, 'success');
|
||||
document.getElementById('mcp-install-name').value = '';
|
||||
document.getElementById('mcp-install-url').value = '';
|
||||
loadExtensions();
|
||||
} else {
|
||||
showToast('Failed to add MCP server: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
}).catch(function(err) {
|
||||
showToast('Failed to add MCP server: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
// --- Keyboard shortcuts ---
|
||||
|
||||
document.addEventListener('keydown', (e) => {
|
||||
@@ -2101,10 +2696,10 @@ document.addEventListener('keydown', (e) => {
|
||||
const tag = (e.target.tagName || '').toLowerCase();
|
||||
const inInput = tag === 'input' || tag === 'textarea';
|
||||
|
||||
// Mod+1-6: switch tabs
|
||||
if (mod && e.key >= '1' && e.key <= '6') {
|
||||
// Mod+1-5: switch tabs
|
||||
if (mod && e.key >= '1' && e.key <= '5') {
|
||||
e.preventDefault();
|
||||
const tabs = ['chat', 'memory', 'jobs', 'routines', 'logs', 'extensions'];
|
||||
const tabs = ['chat', 'memory', 'jobs', 'routines', 'extensions'];
|
||||
const idx = parseInt(e.key) - 1;
|
||||
if (tabs[idx]) switchTab(tabs[idx]);
|
||||
return;
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>IronClaw</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
<script
|
||||
src="https://cdn.jsdelivr.net/npm/[email protected]/lib/marked.umd.min.js"
|
||||
@@ -36,10 +39,17 @@
|
||||
<button class="active" data-tab="chat">Chat</button>
|
||||
<button data-tab="memory">Memory</button>
|
||||
<button data-tab="jobs">Jobs</button>
|
||||
<button data-tab="logs">Logs</button>
|
||||
<button data-tab="routines">Routines</button>
|
||||
<button data-tab="extensions">Extensions</button>
|
||||
<div class="spacer"></div>
|
||||
<button class="status-logs-btn" data-tab="logs" title="Logs">Logs</button>
|
||||
<div class="tee-shield" id="tee-shield" style="display:none" title="Running in a Trusted Execution Environment">
|
||||
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z"/>
|
||||
</svg>
|
||||
<span id="tee-shield-label">TEE Verified</span>
|
||||
<div class="tee-popover" id="tee-popover"></div>
|
||||
</div>
|
||||
<div class="status" id="gateway-status-trigger">
|
||||
<div class="dot" id="sse-dot"></div>
|
||||
<span id="sse-status">Connected</span>
|
||||
@@ -127,6 +137,12 @@
|
||||
<div class="tab-panel" id="tab-logs">
|
||||
<div class="logs-container">
|
||||
<div class="logs-toolbar">
|
||||
<select id="logs-server-level" onchange="setServerLogLevel(this.value)" title="Server-side log level (changes what the server emits)">
|
||||
<option value="error">Server: ERROR</option>
|
||||
<option value="warn">Server: WARN</option>
|
||||
<option value="info" selected>Server: INFO</option>
|
||||
<option value="debug">Server: DEBUG</option>
|
||||
</select>
|
||||
<select id="logs-level-filter">
|
||||
<option value="all">All Levels</option>
|
||||
<option value="ERROR">Error</option>
|
||||
@@ -172,34 +188,42 @@
|
||||
<!-- Extensions Tab -->
|
||||
<div class="tab-panel" id="tab-extensions">
|
||||
<div class="extensions-container">
|
||||
<div class="extensions-section">
|
||||
<h3>Install Extension</h3>
|
||||
<div class="ext-install-form" id="ext-install-form">
|
||||
<input type="text" id="ext-install-name" placeholder="Extension name (required)">
|
||||
<input type="text" id="ext-install-url" placeholder="URL (optional)">
|
||||
<select id="ext-install-kind">
|
||||
<option value="mcp_server">MCP Server</option>
|
||||
<option value="wasm_tool">WASM Tool</option>
|
||||
<option value="wasm_channel">WASM Channel</option>
|
||||
</select>
|
||||
<button onclick="installExtension()">Install</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Installed Extensions</h3>
|
||||
<div class="extensions-list" id="extensions-list">
|
||||
<div class="empty-state">Loading extensions...</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section" id="available-wasm-section">
|
||||
<h3>Available WASM Extensions</h3>
|
||||
<div class="extensions-list" id="available-wasm-list">
|
||||
<div class="empty-state">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Install WASM Extension</h3>
|
||||
<div class="ext-install-form">
|
||||
<input type="text" id="wasm-install-name" placeholder="Extension name">
|
||||
<input type="text" id="wasm-install-url" placeholder="URL to .tar.gz bundle">
|
||||
<button onclick="installWasmExtension()">Install</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>MCP Servers</h3>
|
||||
<div class="extensions-list" id="mcp-servers-list">
|
||||
<div class="empty-state">Loading...</div>
|
||||
</div>
|
||||
<h4>Add Custom MCP Server</h4>
|
||||
<div class="ext-install-form">
|
||||
<input type="text" id="mcp-install-name" placeholder="Server name">
|
||||
<input type="text" id="mcp-install-url" placeholder="MCP server URL (https://...)">
|
||||
<button onclick="addMcpServer()">Add</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Registered Tools</h3>
|
||||
<table class="tools-table" id="tools-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<thead><tr><th>Name</th><th>Description</th></tr></thead>
|
||||
<tbody id="tools-tbody"></tbody>
|
||||
</table>
|
||||
<div class="empty-state" id="tools-empty" style="display:none">No tools registered</div>
|
||||
|
||||
+535
-105
File diff suppressed because it is too large
Load Diff
@@ -346,6 +346,9 @@ pub struct ExtensionInfo {
|
||||
pub authenticated: bool,
|
||||
pub active: bool,
|
||||
pub tools: Vec<String>,
|
||||
/// Whether this extension has configurable secrets (setup schema).
|
||||
#[serde(default)]
|
||||
pub needs_setup: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -371,6 +374,31 @@ pub struct InstallExtensionRequest {
|
||||
pub kind: Option<String>,
|
||||
}
|
||||
|
||||
// --- Extension Setup ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ExtensionSetupResponse {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub secrets: Vec<SecretFieldInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SecretFieldInfo {
|
||||
pub name: String,
|
||||
pub prompt: String,
|
||||
pub optional: bool,
|
||||
/// Whether this secret is already stored.
|
||||
pub provided: bool,
|
||||
/// Whether the secret will be auto-generated if left empty.
|
||||
pub auto_generate: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ExtensionSetupRequest {
|
||||
pub secrets: std::collections::HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ActionResponse {
|
||||
pub success: bool,
|
||||
@@ -408,6 +436,50 @@ impl ActionResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Registry ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RegistryEntryInfo {
|
||||
pub name: String,
|
||||
pub display_name: String,
|
||||
pub kind: String,
|
||||
pub description: String,
|
||||
pub keywords: Vec<String>,
|
||||
pub installed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RegistrySearchResponse {
|
||||
pub entries: Vec<RegistryEntryInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct RegistrySearchQuery {
|
||||
pub query: Option<String>,
|
||||
}
|
||||
|
||||
// --- Pairing ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct PairingListResponse {
|
||||
pub channel: String,
|
||||
pub requests: Vec<PairingRequestInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct PairingRequestInfo {
|
||||
pub code: String,
|
||||
pub sender_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub meta: Option<serde_json::Value>,
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PairingApproveRequest {
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
// --- Skills ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
|
||||
@@ -477,6 +477,7 @@ mod tests {
|
||||
workspace: None,
|
||||
session_manager: None,
|
||||
log_broadcaster: None,
|
||||
log_level_handle: None,
|
||||
extension_manager: None,
|
||||
tool_registry: None,
|
||||
store: None,
|
||||
@@ -489,6 +490,9 @@ mod tests {
|
||||
skill_registry: None,
|
||||
skill_catalog: None,
|
||||
chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60),
|
||||
registry_entries: Vec::new(),
|
||||
cost_guard: None,
|
||||
startup_time: std::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+145
-31
@@ -66,12 +66,47 @@ pub const OAUTH_CALLBACK_PORT: u16 = 9876;
|
||||
///
|
||||
/// Checks `IRONCLAW_OAUTH_CALLBACK_URL` env var first (useful for remote/VPS
|
||||
/// deployments where `127.0.0.1` is unreachable from the user's browser),
|
||||
/// then falls back to `http://127.0.0.1:{OAUTH_CALLBACK_PORT}`.
|
||||
/// then falls back to `http://{callback_host()}:{OAUTH_CALLBACK_PORT}`.
|
||||
pub fn callback_url() -> String {
|
||||
std::env::var("IRONCLAW_OAUTH_CALLBACK_URL")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| format!("http://127.0.0.1:{}", OAUTH_CALLBACK_PORT))
|
||||
.unwrap_or_else(|| format!("http://{}:{}", callback_host(), OAUTH_CALLBACK_PORT))
|
||||
}
|
||||
|
||||
/// Returns the hostname used in OAuth callback URLs.
|
||||
///
|
||||
/// Reads `OAUTH_CALLBACK_HOST` from the environment (default: `127.0.0.1`).
|
||||
///
|
||||
/// **Remote server usage:** set `OAUTH_CALLBACK_HOST` to the network interface
|
||||
/// address you want to listen on (e.g. the server's LAN IP or `0.0.0.0`).
|
||||
/// The callback listener will bind to that specific address instead of the
|
||||
/// loopback interface, so the OAuth redirect can reach an external browser.
|
||||
/// Note: this transmits the session token over plain HTTP — prefer SSH port
|
||||
/// forwarding (`ssh -L 9876:127.0.0.1:9876 user@host`) when possible.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```bash
|
||||
/// export OAUTH_CALLBACK_HOST=203.0.113.10
|
||||
/// ironclaw login
|
||||
/// # Opens: http://203.0.113.10:9876/auth/callback
|
||||
/// ```
|
||||
pub fn callback_host() -> String {
|
||||
std::env::var("OAUTH_CALLBACK_HOST").unwrap_or_else(|_| "127.0.0.1".to_string())
|
||||
}
|
||||
|
||||
/// Returns `true` if `host` is a loopback address that only accepts local connections.
|
||||
///
|
||||
/// Covers `localhost` (case-insensitive), the full `127.0.0.0/8` IPv4 loopback
|
||||
/// range, and `::1` for IPv6.
|
||||
pub fn is_loopback_host(host: &str) -> bool {
|
||||
if host.eq_ignore_ascii_case("localhost") {
|
||||
return true;
|
||||
}
|
||||
host.parse::<std::net::IpAddr>()
|
||||
.map(|ip| ip.is_loopback())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Error from the OAuth callback listener.
|
||||
@@ -90,35 +125,50 @@ pub enum OAuthCallbackError {
|
||||
Io(String),
|
||||
}
|
||||
|
||||
/// Map a `std::io::Error` from a bind attempt to an `OAuthCallbackError`.
|
||||
fn bind_error(e: std::io::Error) -> OAuthCallbackError {
|
||||
if e.kind() == std::io::ErrorKind::AddrInUse {
|
||||
OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string())
|
||||
} else {
|
||||
OAuthCallbackError::Io(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind the OAuth callback listener on the fixed port.
|
||||
///
|
||||
/// Binds to IPv4 `127.0.0.1` first because callback URLs use `127.0.0.1`
|
||||
/// explicitly (e.g., NEAR AI redirects to `http://127.0.0.1:9876/auth/callback`).
|
||||
/// Falls back to IPv6 `[::1]` only if IPv4 binding fails for a reason other
|
||||
/// than `AddrInUse`. If the port is already occupied, fails immediately.
|
||||
/// When `OAUTH_CALLBACK_HOST` is a loopback address (the default `127.0.0.1`),
|
||||
/// binds to `127.0.0.1` first and falls back to `[::1]` so local-only auth
|
||||
/// flows remain restricted to the local machine.
|
||||
///
|
||||
/// When `OAUTH_CALLBACK_HOST` is set to a remote address, binds to that
|
||||
/// specific address so only connections directed to it are accepted.
|
||||
pub async fn bind_callback_listener() -> Result<TcpListener, OAuthCallbackError> {
|
||||
let ipv4_addr = format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||
match TcpListener::bind(&ipv4_addr).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||
return Err(OAuthCallbackError::PortInUse(
|
||||
OAUTH_CALLBACK_PORT,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
Err(_) => {
|
||||
// IPv4 not available, fall back to IPv6
|
||||
}
|
||||
}
|
||||
TcpListener::bind(format!("[::1]:{}", OAUTH_CALLBACK_PORT))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::AddrInUse {
|
||||
OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string())
|
||||
} else {
|
||||
OAuthCallbackError::Io(e.to_string())
|
||||
let host = callback_host();
|
||||
|
||||
if is_loopback_host(&host) {
|
||||
// Local mode: prefer IPv4 loopback, fall back to IPv6.
|
||||
let ipv4_addr = format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||
match TcpListener::bind(&ipv4_addr).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||
return Err(OAuthCallbackError::PortInUse(
|
||||
OAUTH_CALLBACK_PORT,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
})
|
||||
Err(_) => {
|
||||
// IPv4 not available, fall back to IPv6
|
||||
}
|
||||
}
|
||||
TcpListener::bind(format!("[::1]:{}", OAUTH_CALLBACK_PORT))
|
||||
.await
|
||||
.map_err(bind_error)
|
||||
} else {
|
||||
// Remote mode: bind to the specific configured host address only,
|
||||
// not 0.0.0.0, to limit exposure to the intended interface.
|
||||
let addr = format!("{}:{}", host, OAUTH_CALLBACK_PORT);
|
||||
TcpListener::bind(&addr).await.map_err(bind_error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for an OAuth callback and extract a query parameter value.
|
||||
@@ -311,27 +361,91 @@ pub fn landing_html(provider_name: &str, success: bool) -> String {
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::cli::oauth_defaults::{builtin_credentials, callback_url, landing_html};
|
||||
use crate::cli::oauth_defaults::{
|
||||
builtin_credentials, callback_host, callback_url, is_loopback_host, landing_html,
|
||||
};
|
||||
|
||||
/// Serializes env-mutating tests to prevent parallel races.
|
||||
static ENV_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn test_is_loopback_host() {
|
||||
assert!(is_loopback_host("127.0.0.1"));
|
||||
assert!(is_loopback_host("127.0.0.2")); // full 127.0.0.0/8 range
|
||||
assert!(is_loopback_host("127.255.255.254"));
|
||||
assert!(is_loopback_host("::1"));
|
||||
assert!(is_loopback_host("localhost"));
|
||||
assert!(is_loopback_host("LOCALHOST"));
|
||||
assert!(!is_loopback_host("203.0.113.10"));
|
||||
assert!(!is_loopback_host("my-server.example.com"));
|
||||
assert!(!is_loopback_host("0.0.0.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_host_default() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
let original = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
assert_eq!(callback_host(), "127.0.0.1");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_host_env_override() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", "203.0.113.10");
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
}
|
||||
assert_eq!(callback_host(), "203.0.113.10");
|
||||
// callback_url() fallback should incorporate the custom host
|
||||
let url = callback_url();
|
||||
assert!(url.contains("203.0.113.10"), "url was: {url}");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original_host {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
} else {
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
if let Some(val) = original_url {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_url_default() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
// Clear the env var to test default behavior
|
||||
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// Clear both env vars to test default behavior
|
||||
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
let url = callback_url();
|
||||
assert_eq!(url, "http://127.0.0.1:9876");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
if let Some(val) = original_url {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
}
|
||||
if let Some(val) = original_host {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-51
@@ -1,7 +1,5 @@
|
||||
//! Registry CLI commands for discovering and installing extensions.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Subcommand;
|
||||
|
||||
use crate::registry::catalog::RegistryCatalog;
|
||||
@@ -59,8 +57,20 @@ pub enum RegistryCommand {
|
||||
|
||||
/// Run a registry command.
|
||||
pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
|
||||
let registry_dir = find_registry_dir()?;
|
||||
let catalog = RegistryCatalog::load(®istry_dir)?;
|
||||
// For install commands that need to build from source, a disk registry is required.
|
||||
// For list/info, embedded manifests suffice.
|
||||
let registry_dir = RegistryCatalog::find_dir();
|
||||
let catalog = if let Some(ref dir) = registry_dir {
|
||||
RegistryCatalog::load(dir)?
|
||||
} else {
|
||||
RegistryCatalog::load_or_embedded()?
|
||||
};
|
||||
|
||||
// Resolve repo root for installer (empty path when running from binary)
|
||||
let repo_root = registry_dir
|
||||
.as_ref()
|
||||
.and_then(|d| d.parent().map(|p| p.to_path_buf()))
|
||||
.unwrap_or_default();
|
||||
|
||||
match cmd {
|
||||
RegistryCommand::List { kind, tag, verbose } => {
|
||||
@@ -68,53 +78,14 @@ pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
|
||||
}
|
||||
RegistryCommand::Info { name } => cmd_info(&catalog, &name),
|
||||
RegistryCommand::Install { name, force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, &name, force, build).await
|
||||
cmd_install(&catalog, &repo_root, &name, force, build).await
|
||||
}
|
||||
RegistryCommand::InstallDefaults { force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, "default", force, build).await
|
||||
cmd_install(&catalog, &repo_root, "default", force, build).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the registry directory by looking relative to the current executable or cwd.
|
||||
fn find_registry_dir() -> anyhow::Result<PathBuf> {
|
||||
// Try relative to current directory (for dev usage)
|
||||
let cwd = std::env::current_dir()?;
|
||||
let candidate = cwd.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
// Try relative to executable (covers installed binary, target/debug/, target/release/)
|
||||
if let Ok(exe) = std::env::current_exe()
|
||||
&& let Some(parent) = exe.parent()
|
||||
{
|
||||
// Walk up to 3 levels: exe dir, parent (target/release → target), grandparent (→ repo root)
|
||||
let mut dir = Some(parent);
|
||||
for _ in 0..3 {
|
||||
if let Some(d) = dir {
|
||||
let candidate = d.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
dir = d.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let candidate = manifest_dir.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
anyhow::bail!(
|
||||
"Could not find registry/ directory. Run from the ironclaw repo root, \
|
||||
or ensure registry/ is next to the ironclaw binary."
|
||||
)
|
||||
}
|
||||
|
||||
fn cmd_list(
|
||||
catalog: &RegistryCatalog,
|
||||
kind: Option<&str>,
|
||||
@@ -254,16 +225,11 @@ fn cmd_info(catalog: &RegistryCatalog, name: &str) -> anyhow::Result<()> {
|
||||
|
||||
async fn cmd_install(
|
||||
catalog: &RegistryCatalog,
|
||||
registry_dir: &std::path::Path,
|
||||
repo_root: &std::path::Path,
|
||||
name: &str,
|
||||
force: bool,
|
||||
prefer_build: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
// Registry dir parent is the repo root
|
||||
let repo_root = registry_dir
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("Cannot determine repo root from registry dir"))?;
|
||||
|
||||
let installer = RegistryInstaller::with_defaults(repo_root.to_path_buf());
|
||||
|
||||
let (manifests, bundle) = catalog.resolve(name)?;
|
||||
|
||||
@@ -23,6 +23,10 @@ pub struct AgentConfig {
|
||||
pub max_cost_per_day_cents: Option<u64>,
|
||||
/// Maximum LLM/tool actions per hour. None = unlimited.
|
||||
pub max_actions_per_hour: Option<u64>,
|
||||
/// Maximum tool-call iterations per agentic loop invocation. Default 50.
|
||||
pub max_tool_iterations: usize,
|
||||
/// When true, skip tool approval checks entirely. For benchmarks/CI.
|
||||
pub auto_approve_tools: bool,
|
||||
}
|
||||
|
||||
impl AgentConfig {
|
||||
@@ -115,6 +119,22 @@ impl AgentConfig {
|
||||
key: "MAX_ACTIONS_PER_HOUR".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?,
|
||||
max_tool_iterations: optional_env("AGENT_MAX_TOOL_ITERATIONS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "AGENT_MAX_TOOL_ITERATIONS".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.agent.max_tool_iterations),
|
||||
auto_approve_tools: optional_env("AGENT_AUTO_APPROVE_TOOLS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "AGENT_AUTO_APPROVE_TOOLS".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.agent.auto_approve_tools),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+116
-49
@@ -90,6 +90,9 @@ pub struct OpenAiCompatibleConfig {
|
||||
pub base_url: String,
|
||||
pub api_key: Option<SecretString>,
|
||||
pub model: String,
|
||||
/// Extra HTTP headers injected into every LLM request.
|
||||
/// Parsed from `LLM_EXTRA_HEADERS` env var (format: `Key:Value,Key2:Value2`).
|
||||
pub extra_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Configuration for Tinfoil private inference.
|
||||
@@ -121,37 +124,7 @@ pub struct LlmConfig {
|
||||
pub tinfoil: Option<TinfoilConfig>,
|
||||
}
|
||||
|
||||
/// API mode for NEAR AI.
|
||||
///
|
||||
/// - `Responses` = **NEAR AI Chat** (`private.near.ai`, session token auth)
|
||||
/// - `ChatCompletions` = **NEAR AI Cloud** (`cloud-api.near.ai`, API key auth)
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum NearAiApiMode {
|
||||
/// NEAR AI Chat: Responses API with session token auth
|
||||
#[default]
|
||||
Responses,
|
||||
/// NEAR AI Cloud: Chat Completions API with API key auth
|
||||
ChatCompletions,
|
||||
}
|
||||
|
||||
impl std::str::FromStr for NearAiApiMode {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"responses" | "response" => Ok(Self::Responses),
|
||||
"chat_completions" | "chatcompletions" | "chat" | "completions" => {
|
||||
Ok(Self::ChatCompletions)
|
||||
}
|
||||
_ => Err(format!(
|
||||
"invalid API mode '{}', expected 'responses' or 'chat_completions'",
|
||||
s
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// NEAR AI configuration (shared by Chat and Cloud modes).
|
||||
/// NEAR AI configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NearAiConfig {
|
||||
/// Model to use (e.g., "claude-3-5-sonnet-20241022", "gpt-4o")
|
||||
@@ -160,16 +133,13 @@ pub struct NearAiConfig {
|
||||
/// Falls back to the main model if not set.
|
||||
pub cheap_model: Option<String>,
|
||||
/// Base URL for the NEAR AI API.
|
||||
/// Chat mode default: `https://private.near.ai`
|
||||
/// Cloud mode default: `https://cloud-api.near.ai`
|
||||
/// Default: `https://private.near.ai` (session token) or `https://cloud-api.near.ai` (API key)
|
||||
pub base_url: String,
|
||||
/// Base URL for auth/refresh endpoints (default: https://private.near.ai)
|
||||
pub auth_base_url: String,
|
||||
/// Path to session file (default: ~/.ironclaw/session.json)
|
||||
pub session_path: PathBuf,
|
||||
/// API mode: NEAR AI Chat (Responses) or NEAR AI Cloud (ChatCompletions)
|
||||
pub api_mode: NearAiApiMode,
|
||||
/// API key for NEAR AI Cloud (required for ChatCompletions mode)
|
||||
/// API key for NEAR AI Cloud. When set, uses API key auth; otherwise uses session token auth.
|
||||
pub api_key: Option<SecretString>,
|
||||
/// Optional fallback model for failover (default: None).
|
||||
/// When set, a secondary provider is created with this model and wrapped
|
||||
@@ -200,6 +170,10 @@ pub struct NearAiConfig {
|
||||
/// Number of consecutive retryable failures before a provider enters
|
||||
/// cooldown (default: 3).
|
||||
pub failover_cooldown_threshold: u32,
|
||||
/// Enable cascade mode for smart routing: when a moderate-complexity task
|
||||
/// gets an uncertain response from the cheap model, re-send to primary.
|
||||
/// Default: true.
|
||||
pub smart_routing_cascade: bool,
|
||||
}
|
||||
|
||||
impl LlmConfig {
|
||||
@@ -229,17 +203,6 @@ impl LlmConfig {
|
||||
// Resolve NEAR AI config only when backend is NearAi (or when explicitly configured)
|
||||
let nearai_api_key = optional_env("NEARAI_API_KEY")?.map(SecretString::from);
|
||||
|
||||
let api_mode = if let Some(mode_str) = optional_env("NEARAI_API_MODE")? {
|
||||
mode_str.parse().map_err(|e| ConfigError::InvalidValue {
|
||||
key: "NEARAI_API_MODE".to_string(),
|
||||
message: e,
|
||||
})?
|
||||
} else if nearai_api_key.is_some() {
|
||||
NearAiApiMode::ChatCompletions
|
||||
} else {
|
||||
NearAiApiMode::Responses
|
||||
};
|
||||
|
||||
let nearai = NearAiConfig {
|
||||
model: optional_env("NEARAI_MODEL")?
|
||||
.or_else(|| settings.selected_model.clone())
|
||||
@@ -249,7 +212,7 @@ impl LlmConfig {
|
||||
}),
|
||||
cheap_model: optional_env("NEARAI_CHEAP_MODEL")?,
|
||||
base_url: optional_env("NEARAI_BASE_URL")?.unwrap_or_else(|| {
|
||||
if api_mode == NearAiApiMode::ChatCompletions {
|
||||
if nearai_api_key.is_some() {
|
||||
"https://cloud-api.near.ai".to_string()
|
||||
} else {
|
||||
"https://private.near.ai".to_string()
|
||||
@@ -260,7 +223,6 @@ impl LlmConfig {
|
||||
session_path: optional_env("NEARAI_SESSION_PATH")?
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(default_session_path),
|
||||
api_mode,
|
||||
api_key: nearai_api_key,
|
||||
fallback_model: optional_env("NEARAI_FALLBACK_MODEL")?,
|
||||
max_retries: parse_optional_env("NEARAI_MAX_RETRIES", 3)?,
|
||||
@@ -277,6 +239,7 @@ impl LlmConfig {
|
||||
response_cache_max_entries: parse_optional_env("RESPONSE_CACHE_MAX_ENTRIES", 1000)?,
|
||||
failover_cooldown_secs: parse_optional_env("LLM_FAILOVER_COOLDOWN_SECS", 300)?,
|
||||
failover_cooldown_threshold: parse_optional_env("LLM_FAILOVER_THRESHOLD", 3)?,
|
||||
smart_routing_cascade: parse_optional_env("SMART_ROUTING_CASCADE", true)?,
|
||||
};
|
||||
|
||||
// Resolve provider-specific configs based on backend
|
||||
@@ -338,10 +301,15 @@ impl LlmConfig {
|
||||
let model = optional_env("LLM_MODEL")?
|
||||
.or_else(|| settings.selected_model.clone())
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
let extra_headers = optional_env("LLM_EXTRA_HEADERS")?
|
||||
.map(|val| parse_extra_headers(&val))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
Some(OpenAiCompatibleConfig {
|
||||
base_url,
|
||||
api_key,
|
||||
model,
|
||||
extra_headers,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -372,6 +340,40 @@ impl LlmConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `LLM_EXTRA_HEADERS` value into a list of (key, value) pairs.
|
||||
///
|
||||
/// Format: `Key1:Value1,Key2:Value2` — colon-separated key:value, comma-separated pairs.
|
||||
/// Colon is used as the separator (not `=`) because header values often contain `=`
|
||||
/// (e.g., base64 tokens).
|
||||
fn parse_extra_headers(val: &str) -> Result<Vec<(String, String)>, ConfigError> {
|
||||
if val.trim().is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut headers = Vec::new();
|
||||
for pair in val.split(',') {
|
||||
let pair = pair.trim();
|
||||
if pair.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some((key, value)) = pair.split_once(':') else {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "LLM_EXTRA_HEADERS".to_string(),
|
||||
message: format!("malformed header entry '{}', expected Key:Value", pair),
|
||||
});
|
||||
};
|
||||
let key = key.trim();
|
||||
if key.is_empty() {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "LLM_EXTRA_HEADERS".to_string(),
|
||||
message: format!("empty header name in entry '{}'", pair),
|
||||
});
|
||||
}
|
||||
headers.push((key.to_string(), value.trim().to_string()));
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Get the default session file path (~/.ironclaw/session.json).
|
||||
fn default_session_path() -> PathBuf {
|
||||
dirs::home_dir()
|
||||
@@ -444,4 +446,69 @@ mod tests {
|
||||
std::env::remove_var("LLM_MODEL");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_parsed() {
|
||||
let result = parse_extra_headers("HTTP-Referer:https://myapp.com,X-Title:MyApp").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
("HTTP-Referer".to_string(), "https://myapp.com".to_string()),
|
||||
("X-Title".to_string(), "MyApp".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_empty_string() {
|
||||
let result = parse_extra_headers("").unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_whitespace_only() {
|
||||
let result = parse_extra_headers(" ").unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_malformed() {
|
||||
let result = parse_extra_headers("NoColonHere");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_empty_key() {
|
||||
let result = parse_extra_headers(":value");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_value_with_colons() {
|
||||
// Values can contain colons (e.g., URLs)
|
||||
let result = parse_extra_headers("Authorization:Bearer abc:def").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![("Authorization".to_string(), "Bearer abc:def".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_trailing_comma() {
|
||||
let result = parse_extra_headers("X-Title:MyApp,").unwrap();
|
||||
assert_eq!(result, vec![("X-Title".to_string(), "MyApp".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_with_spaces() {
|
||||
let result =
|
||||
parse_extra_headers(" HTTP-Referer : https://myapp.com , X-Title : MyApp ").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
("HTTP-Referer".to_string(), "https://myapp.com".to_string()),
|
||||
("X-Title".to_string(), "MyApp".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -19,6 +19,7 @@ mod safety;
|
||||
mod sandbox;
|
||||
mod secrets;
|
||||
mod skills;
|
||||
mod transcription;
|
||||
mod tunnel;
|
||||
mod wasm;
|
||||
|
||||
@@ -37,7 +38,7 @@ pub use self::embeddings::EmbeddingsConfig;
|
||||
pub use self::heartbeat::HeartbeatConfig;
|
||||
pub use self::hygiene::HygieneConfig;
|
||||
pub use self::llm::{
|
||||
AnthropicDirectConfig, LlmBackend, LlmConfig, NearAiApiMode, NearAiConfig, OllamaConfig,
|
||||
AnthropicDirectConfig, LlmBackend, LlmConfig, NearAiConfig, OllamaConfig,
|
||||
OpenAiCompatibleConfig, OpenAiDirectConfig, TinfoilConfig,
|
||||
};
|
||||
pub use self::routines::RoutineConfig;
|
||||
@@ -45,6 +46,7 @@ pub use self::safety::SafetyConfig;
|
||||
pub use self::sandbox::{ClaudeCodeConfig, SandboxModeConfig};
|
||||
pub use self::secrets::SecretsConfig;
|
||||
pub use self::skills::SkillsConfig;
|
||||
pub use self::transcription::TranscriptionConfig;
|
||||
pub use self::tunnel::TunnelConfig;
|
||||
pub use self::wasm::WasmConfig;
|
||||
|
||||
@@ -74,6 +76,7 @@ pub struct Config {
|
||||
pub sandbox: SandboxModeConfig,
|
||||
pub claude_code: ClaudeCodeConfig,
|
||||
pub skills: SkillsConfig,
|
||||
pub transcription: TranscriptionConfig,
|
||||
pub observability: crate::observability::ObservabilityConfig,
|
||||
}
|
||||
|
||||
@@ -198,6 +201,7 @@ impl Config {
|
||||
sandbox: SandboxModeConfig::resolve()?,
|
||||
claude_code: ClaudeCodeConfig::resolve()?,
|
||||
skills: SkillsConfig::resolve()?,
|
||||
transcription: TranscriptionConfig::resolve(settings)?,
|
||||
observability: crate::observability::ObservabilityConfig {
|
||||
backend: std::env::var("OBSERVABILITY_BACKEND").unwrap_or_else(|_| "none".into()),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
use secrecy::SecretString;
|
||||
|
||||
use crate::config::helpers::optional_env;
|
||||
use crate::error::ConfigError;
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Transcription provider configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TranscriptionConfig {
|
||||
/// Whether transcription is enabled.
|
||||
pub enabled: bool,
|
||||
/// Provider to use: "openai".
|
||||
pub provider: String,
|
||||
/// OpenAI API key (reused from embeddings/LLM config).
|
||||
pub openai_api_key: Option<SecretString>,
|
||||
/// Model to use for transcription.
|
||||
pub model: String,
|
||||
/// Optional language hint (ISO-639-1, e.g., "en").
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for TranscriptionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
provider: "openai".to_string(),
|
||||
openai_api_key: None,
|
||||
model: "whisper-1".to_string(),
|
||||
language: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TranscriptionConfig {
|
||||
pub(crate) fn resolve(settings: &Settings) -> Result<Self, ConfigError> {
|
||||
let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from);
|
||||
|
||||
let provider = optional_env("TRANSCRIPTION_PROVIDER")?
|
||||
.unwrap_or_else(|| settings.transcription.provider.clone());
|
||||
|
||||
let model = optional_env("TRANSCRIPTION_MODEL")?
|
||||
.unwrap_or_else(|| settings.transcription.model.clone());
|
||||
|
||||
let language = optional_env("TRANSCRIPTION_LANGUAGE")?
|
||||
.or_else(|| settings.transcription.language.clone());
|
||||
|
||||
let enabled = optional_env("TRANSCRIPTION_ENABLED")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_ENABLED".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.transcription.enabled);
|
||||
|
||||
// Only "openai" is currently supported
|
||||
if enabled && provider != "openai" {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_PROVIDER".to_string(),
|
||||
message: format!(
|
||||
"unsupported provider '{}', only 'openai' is currently supported",
|
||||
provider
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
provider,
|
||||
openai_api_key,
|
||||
model,
|
||||
language,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::helpers::ENV_MUTEX;
|
||||
use crate::settings::{Settings, TranscriptionSettings};
|
||||
|
||||
fn clear_transcription_env() {
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_PROVIDER");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
std::env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_defaults_from_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(!config.enabled);
|
||||
assert_eq!(config.provider, "openai");
|
||||
assert_eq!(config.model, "whisper-1");
|
||||
assert!(config.language.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_env_overrides_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
unsafe {
|
||||
std::env::set_var("TRANSCRIPTION_ENABLED", "true");
|
||||
std::env::set_var("TRANSCRIPTION_MODEL", "whisper-large-v3");
|
||||
std::env::set_var("TRANSCRIPTION_LANGUAGE", "en");
|
||||
}
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("en".to_string()));
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_settings_with_custom_values() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
model: "whisper-large-v3".to_string(),
|
||||
language: Some("fr".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("fr".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_rejects_unsupported_provider() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
provider: "deepgram".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = TranscriptionConfig::resolve(&settings);
|
||||
assert!(result.is_err());
|
||||
let err_msg = result.unwrap_err().to_string();
|
||||
assert!(
|
||||
err_msg.contains("unsupported provider"),
|
||||
"Error should mention unsupported provider, got: {err_msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_disabled_skips_provider_validation() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
// When disabled, any provider string is accepted (never used)
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: false,
|
||||
provider: "nonexistent".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("should succeed when disabled");
|
||||
assert!(!config.enabled);
|
||||
}
|
||||
}
|
||||
@@ -49,9 +49,10 @@ impl ConversationStore for LibSqlBackend {
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
let conn = self.connect().await?;
|
||||
let id = Uuid::new_v4();
|
||||
let now = fmt_ts(&Utc::now());
|
||||
conn.execute(
|
||||
"INSERT INTO conversation_messages (id, conversation_id, role, content) VALUES (?1, ?2, ?3, ?4)",
|
||||
params![id.to_string(), conversation_id.to_string(), role, content],
|
||||
"INSERT INTO conversation_messages (id, conversation_id, role, content, created_at) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![id.to_string(), conversation_id.to_string(), role, content, now],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| DatabaseError::Query(e.to_string()))?;
|
||||
@@ -100,7 +101,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
(SELECT substr(m2.content, 1, 100)
|
||||
FROM conversation_messages m2
|
||||
WHERE m2.conversation_id = c.id AND m2.role = 'user'
|
||||
ORDER BY m2.created_at ASC
|
||||
ORDER BY m2.created_at ASC, m2.rowid ASC
|
||||
LIMIT 1
|
||||
) AS title
|
||||
FROM conversations c
|
||||
@@ -216,7 +217,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1 AND created_at < ?2
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, rowid DESC
|
||||
LIMIT ?3
|
||||
"#,
|
||||
params![cid, fmt_ts(&before_ts), fetch_limit],
|
||||
@@ -228,7 +229,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, rowid DESC
|
||||
LIMIT ?2
|
||||
"#,
|
||||
params![cid, fetch_limit],
|
||||
@@ -309,7 +310,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1
|
||||
ORDER BY created_at ASC
|
||||
ORDER BY created_at ASC, rowid ASC
|
||||
"#,
|
||||
params![conversation_id.to_string()],
|
||||
)
|
||||
|
||||
@@ -202,6 +202,12 @@ pub enum ToolError {
|
||||
#[error("Tool {name} requires authentication")]
|
||||
AuthRequired { name: String },
|
||||
|
||||
#[error("Tool {name} is rate limited, retry after {retry_after:?}")]
|
||||
RateLimited {
|
||||
name: String,
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
|
||||
#[error("Tool builder failed: {0}")]
|
||||
BuilderFailed(String),
|
||||
}
|
||||
|
||||
@@ -246,6 +246,7 @@ fn extract_url(source: &ExtensionSource) -> String {
|
||||
ExtensionSource::Discovered { url } => url.clone(),
|
||||
ExtensionSource::WasmDownload { wasm_url, .. } => wasm_url.clone(),
|
||||
ExtensionSource::WasmBuildable { repo_url, .. } => repo_url.clone(),
|
||||
ExtensionSource::Bundled { name } => name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+655
-58
@@ -4,7 +4,7 @@
|
||||
//! and tool registry. All extension operations (search, install, auth, activate,
|
||||
//! list, remove) flow through here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -60,6 +60,8 @@ pub struct ExtensionManager {
|
||||
user_id: String,
|
||||
/// Optional database store for DB-backed MCP config.
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
/// Names of WASM channels that were successfully loaded at startup.
|
||||
active_channel_names: RwLock<HashSet<String>>,
|
||||
}
|
||||
|
||||
impl ExtensionManager {
|
||||
@@ -75,9 +77,15 @@ impl ExtensionManager {
|
||||
tunnel_url: Option<String>,
|
||||
user_id: String,
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
catalog_entries: Vec<RegistryEntry>,
|
||||
) -> Self {
|
||||
let registry = if catalog_entries.is_empty() {
|
||||
ExtensionRegistry::new()
|
||||
} else {
|
||||
ExtensionRegistry::new_with_catalog(catalog_entries)
|
||||
};
|
||||
Self {
|
||||
registry: ExtensionRegistry::new(),
|
||||
registry,
|
||||
discovery: OnlineDiscovery::new(),
|
||||
mcp_session_manager,
|
||||
mcp_clients: RwLock::new(HashMap::new()),
|
||||
@@ -91,9 +99,17 @@ impl ExtensionManager {
|
||||
_tunnel_url: tunnel_url,
|
||||
user_id,
|
||||
store,
|
||||
active_channel_names: RwLock::new(HashSet::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register channel names that were loaded at startup.
|
||||
/// Called after WASM channels are loaded so `list()` reports accurate active status.
|
||||
pub async fn set_active_channels(&self, names: Vec<String>) {
|
||||
let mut active = self.active_channel_names.write().await;
|
||||
active.extend(names);
|
||||
}
|
||||
|
||||
/// Search for extensions. If `discover` is true, also searches online.
|
||||
pub async fn search(
|
||||
&self,
|
||||
@@ -131,9 +147,14 @@ impl ExtensionManager {
|
||||
url: Option<&str>,
|
||||
kind_hint: Option<ExtensionKind>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension");
|
||||
|
||||
// If we have a registry entry, use it
|
||||
if let Some(entry) = self.registry.get(name).await {
|
||||
return self.install_from_entry(&entry).await;
|
||||
return self.install_from_entry(&entry).await.map_err(|e| {
|
||||
tracing::error!(extension = %name, error = %e, "Extension install failed");
|
||||
e
|
||||
});
|
||||
}
|
||||
|
||||
// If a URL was provided, determine kind and install
|
||||
@@ -143,19 +164,21 @@ impl ExtensionManager {
|
||||
ExtensionKind::McpServer => self.install_mcp_from_url(name, url).await,
|
||||
ExtensionKind::WasmTool => self.install_wasm_tool_from_url(name, url).await,
|
||||
ExtensionKind::WasmChannel => {
|
||||
Err(ExtensionError::InstallFailed(
|
||||
"WASM channel installation from URL not yet supported. \
|
||||
Place the .wasm and .capabilities.json files in ~/.ironclaw/channels/ and restart."
|
||||
.to_string(),
|
||||
))
|
||||
self.install_wasm_channel_from_url(name, url, None).await
|
||||
}
|
||||
};
|
||||
}
|
||||
.map_err(|e| {
|
||||
tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed");
|
||||
e
|
||||
});
|
||||
}
|
||||
|
||||
Err(ExtensionError::NotFound(format!(
|
||||
let err = ExtensionError::NotFound(format!(
|
||||
"'{}' not found in registry. Try searching with discover:true or provide a URL.",
|
||||
name
|
||||
)))
|
||||
));
|
||||
tracing::warn!(extension = %name, "Extension not found in registry");
|
||||
Err(err)
|
||||
}
|
||||
|
||||
/// Authenticate an installed extension.
|
||||
@@ -173,7 +196,7 @@ impl ExtensionManager {
|
||||
match kind {
|
||||
ExtensionKind::McpServer => self.auth_mcp(name, token).await,
|
||||
ExtensionKind::WasmTool => self.auth_wasm_tool(name, token).await,
|
||||
ExtensionKind::WasmChannel => self.auth_wasm_tool(name, token).await,
|
||||
ExtensionKind::WasmChannel => self.auth_wasm_channel(name, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,6 +248,7 @@ impl ExtensionManager {
|
||||
authenticated,
|
||||
active,
|
||||
tools,
|
||||
needs_setup: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -251,6 +275,7 @@ impl ExtensionManager {
|
||||
authenticated: true, // WASM tools don't always need auth
|
||||
active,
|
||||
tools: if active { vec![name] } else { Vec::new() },
|
||||
needs_setup: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -266,15 +291,20 @@ impl ExtensionManager {
|
||||
{
|
||||
match crate::channels::wasm::discover_channels(&self.wasm_channels_dir).await {
|
||||
Ok(channels) => {
|
||||
let active_names = self.active_channel_names.read().await;
|
||||
for (name, _discovered) in channels {
|
||||
let active = active_names.contains(&name);
|
||||
let (authenticated, needs_setup) =
|
||||
self.check_channel_auth_status(&name).await;
|
||||
extensions.push(InstalledExtension {
|
||||
name,
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: None,
|
||||
url: None,
|
||||
authenticated: true,
|
||||
active: true, // If loaded at startup, they're active
|
||||
authenticated,
|
||||
active,
|
||||
tools: Vec::new(),
|
||||
needs_setup,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -356,10 +386,27 @@ impl ExtensionManager {
|
||||
|
||||
Ok(format!("Removed WASM tool '{}'", name))
|
||||
}
|
||||
ExtensionKind::WasmChannel => Err(ExtensionError::Other(
|
||||
"Channel removal requires restart. Delete the .wasm file from ~/.ironclaw/channels/ and restart."
|
||||
.to_string(),
|
||||
)),
|
||||
ExtensionKind::WasmChannel => {
|
||||
// Delete channel files
|
||||
let wasm_path = self.wasm_channels_dir.join(format!("{}.wasm", name));
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if wasm_path.exists() {
|
||||
tokio::fs::remove_file(&wasm_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
}
|
||||
if cap_path.exists() {
|
||||
let _ = tokio::fs::remove_file(&cap_path).await;
|
||||
}
|
||||
|
||||
Ok(format!(
|
||||
"Removed channel '{}'. Restart IronClaw for the change to take effect.",
|
||||
name
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,16 +480,54 @@ impl ExtensionManager {
|
||||
self.install_mcp_from_url(&entry.name, &url).await
|
||||
}
|
||||
ExtensionKind::WasmTool => match &entry.source {
|
||||
ExtensionSource::WasmDownload { wasm_url, .. } => {
|
||||
self.install_wasm_tool_from_url(&entry.name, wasm_url).await
|
||||
ExtensionSource::WasmDownload {
|
||||
wasm_url,
|
||||
capabilities_url,
|
||||
} => {
|
||||
self.install_wasm_tool_from_url_with_caps(
|
||||
&entry.name,
|
||||
wasm_url,
|
||||
capabilities_url.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ExtensionSource::WasmBuildable { .. } => {
|
||||
Err(ExtensionError::InstallFailed(format!(
|
||||
"'{}' requires building from source. Run `ironclaw registry install {}` \
|
||||
from the CLI (requires cargo-component).",
|
||||
entry.name, entry.name
|
||||
)))
|
||||
}
|
||||
_ => Err(ExtensionError::InstallFailed(
|
||||
"WASM tool entry has no download URL".to_string(),
|
||||
)),
|
||||
},
|
||||
ExtensionKind::WasmChannel => Err(ExtensionError::InstallFailed(
|
||||
"WASM channel installation not yet supported via this flow".to_string(),
|
||||
)),
|
||||
ExtensionKind::WasmChannel => match &entry.source {
|
||||
ExtensionSource::WasmDownload {
|
||||
wasm_url,
|
||||
capabilities_url,
|
||||
} => {
|
||||
self.install_wasm_channel_from_url(
|
||||
&entry.name,
|
||||
wasm_url,
|
||||
capabilities_url.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ExtensionSource::WasmBuildable { .. } => {
|
||||
Err(ExtensionError::InstallFailed(format!(
|
||||
"'{}' requires building from source. Run `ironclaw registry install {}` \
|
||||
from the CLI (requires cargo-component).",
|
||||
entry.name, entry.name
|
||||
)))
|
||||
}
|
||||
ExtensionSource::Bundled { name } => {
|
||||
self.install_bundled_channel_from_artifacts(name).await
|
||||
}
|
||||
_ => Err(ExtensionError::InstallFailed(
|
||||
"WASM channel entry has no download URL".to_string(),
|
||||
)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -482,6 +567,57 @@ impl ExtensionManager {
|
||||
name: &str,
|
||||
url: &str,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.install_wasm_tool_from_url_with_caps(name, url, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn install_wasm_tool_from_url_with_caps(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_tools_dir)
|
||||
.await?;
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
|
||||
})
|
||||
}
|
||||
|
||||
async fn install_wasm_channel_from_url(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_channels_dir)
|
||||
.await?;
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
message: format!(
|
||||
"WASM channel '{}' installed to {}. Restart to activate.",
|
||||
name,
|
||||
self.wasm_channels_dir.display()
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/// Download a WASM extension (tool or channel) from URL and install to target directory.
|
||||
///
|
||||
/// Handles both tar.gz bundles (containing `.wasm` + `.capabilities.json`) and bare
|
||||
/// `.wasm` files. Validates HTTPS, size limits, and file format.
|
||||
async fn download_and_install_wasm(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
target_dir: &std::path::Path,
|
||||
) -> Result<(), ExtensionError> {
|
||||
// Require HTTPS to prevent downgrade attacks
|
||||
if !url.starts_with("https://") {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
@@ -490,33 +626,41 @@ impl ExtensionManager {
|
||||
}
|
||||
|
||||
// 50 MB cap to prevent disk-fill DoS
|
||||
const MAX_WASM_SIZE: usize = 50 * 1024 * 1024;
|
||||
const MAX_DOWNLOAD_SIZE: usize = 50 * 1024 * 1024;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(60))
|
||||
.build()
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
|
||||
let response = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
tracing::debug!(extension = %name, url = %url, "Downloading WASM extension");
|
||||
|
||||
let response = client.get(url).send().await.map_err(|e| {
|
||||
tracing::error!(extension = %name, url = %url, error = %e, "Download request failed");
|
||||
ExtensionError::DownloadFailed(e.to_string())
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
tracing::error!(
|
||||
extension = %name,
|
||||
url = %url,
|
||||
status = %status,
|
||||
"Download returned non-success HTTP status"
|
||||
);
|
||||
return Err(ExtensionError::DownloadFailed(format!(
|
||||
"HTTP {}",
|
||||
response.status()
|
||||
"HTTP {} from {}",
|
||||
status, url
|
||||
)));
|
||||
}
|
||||
|
||||
// Check Content-Length header before downloading the full body
|
||||
if let Some(len) = response.content_length()
|
||||
&& len as usize > MAX_WASM_SIZE
|
||||
&& len as usize > MAX_DOWNLOAD_SIZE
|
||||
{
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
len, MAX_WASM_SIZE
|
||||
"Download too large ({} bytes, max {} bytes)",
|
||||
len, MAX_DOWNLOAD_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -525,44 +669,196 @@ impl ExtensionManager {
|
||||
.await
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
|
||||
if bytes.len() > MAX_WASM_SIZE {
|
||||
if bytes.len() > MAX_DOWNLOAD_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
"Download too large ({} bytes, max {} bytes)",
|
||||
bytes.len(),
|
||||
MAX_WASM_SIZE
|
||||
MAX_DOWNLOAD_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Basic WASM magic number check (\0asm)
|
||||
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
|
||||
));
|
||||
// Ensure target directory exists
|
||||
tokio::fs::create_dir_all(target_dir)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
let wasm_path = target_dir.join(format!("{}.wasm", name));
|
||||
let caps_path = target_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
// Detect format: gzip (tar.gz bundle) or bare WASM
|
||||
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
||||
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
|
||||
self.extract_wasm_tar_gz(name, &bytes, &wasm_path, &caps_path)?;
|
||||
} else {
|
||||
// Bare WASM file: validate magic number
|
||||
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
tokio::fs::write(&wasm_path, &bytes)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
// Download capabilities separately if URL provided
|
||||
if let Some(caps_url) = capabilities_url {
|
||||
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
match client.get(caps_url).send().await {
|
||||
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
|
||||
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
|
||||
if let Err(e) = tokio::fs::write(&caps_path, &caps_bytes).await {
|
||||
tracing::warn!(
|
||||
"Failed to write capabilities for '{}': {}",
|
||||
name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(caps_bytes) => {
|
||||
tracing::warn!(
|
||||
"Capabilities file for '{}' too large ({} bytes, max {})",
|
||||
name,
|
||||
caps_bytes.len(),
|
||||
MAX_CAPS_SIZE
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to download capabilities for '{}': {}", name, e);
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"Failed to download capabilities for '{}' from {}",
|
||||
name,
|
||||
caps_url
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure tools directory exists
|
||||
tokio::fs::create_dir_all(&self.wasm_tools_dir)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
// Write the WASM file
|
||||
let wasm_path = self.wasm_tools_dir.join(format!("{}.wasm", name));
|
||||
tokio::fs::write(&wasm_path, &bytes)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
tracing::info!(
|
||||
"Installed WASM tool '{}' ({} bytes) from {} to {}",
|
||||
"Installed WASM extension '{}' from {} to {}",
|
||||
name,
|
||||
bytes.len(),
|
||||
url,
|
||||
wasm_path.display()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract a tar.gz bundle into the WASM tools directory.
|
||||
fn extract_wasm_tar_gz(
|
||||
&self,
|
||||
name: &str,
|
||||
bytes: &[u8],
|
||||
target_wasm: &std::path::Path,
|
||||
target_caps: &std::path::Path,
|
||||
) -> Result<(), ExtensionError> {
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
use std::io::Read as _;
|
||||
|
||||
let decoder = GzDecoder::new(bytes);
|
||||
let mut archive = Archive::new(decoder);
|
||||
// Defense-in-depth: do not preserve permissions or extended attributes
|
||||
archive.set_preserve_permissions(false);
|
||||
#[cfg(any(unix, target_os = "redox"))]
|
||||
archive.set_unpack_xattrs(false);
|
||||
|
||||
// 100 MB cap on decompressed entry size to prevent decompression bombs
|
||||
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
|
||||
|
||||
let wasm_filename = format!("{}.wasm", name);
|
||||
let caps_filename = format!("{}.capabilities.json", name);
|
||||
let mut found_wasm = false;
|
||||
|
||||
let entries = archive
|
||||
.entries()
|
||||
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz archive: {}", e)))?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry
|
||||
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz entry: {}", e)))?;
|
||||
|
||||
if entry.size() > MAX_ENTRY_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"Archive entry too large ({} bytes, max {} bytes)",
|
||||
entry.size(),
|
||||
MAX_ENTRY_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
let entry_path = entry
|
||||
.path()
|
||||
.map_err(|e| {
|
||||
ExtensionError::InstallFailed(format!("Invalid path in tar.gz: {}", e))
|
||||
})?
|
||||
.to_path_buf();
|
||||
|
||||
let filename = entry_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("");
|
||||
|
||||
if filename == wasm_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
std::fs::write(target_wasm, &data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
found_wasm = true;
|
||||
} else if filename == caps_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
std::fs::write(target_caps, &data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
if !found_wasm {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"tar.gz archive does not contain '{}'",
|
||||
wasm_filename
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn install_bundled_channel_from_artifacts(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
// Check if already installed
|
||||
let channel_wasm = self.wasm_channels_dir.join(format!("{}.wasm", name));
|
||||
if channel_wasm.exists() {
|
||||
return Err(ExtensionError::AlreadyInstalled(name.to_string()));
|
||||
}
|
||||
|
||||
crate::channels::wasm::install_bundled_channel(name, &self.wasm_channels_dir, false)
|
||||
.await
|
||||
.map_err(ExtensionError::InstallFailed)?;
|
||||
|
||||
tracing::info!(
|
||||
"Installed bundled channel '{}' to {}",
|
||||
name,
|
||||
self.wasm_channels_dir.display()
|
||||
);
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
message: format!(
|
||||
"Channel '{}' installed to {}. Restart IronClaw for the channel to activate. \
|
||||
Run tool_auth('{}') to configure authentication before restarting.",
|
||||
name,
|
||||
self.wasm_channels_dir.display(),
|
||||
name,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -868,6 +1164,169 @@ impl ExtensionManager {
|
||||
})
|
||||
}
|
||||
|
||||
/// Check whether a WASM channel has all required secrets stored.
|
||||
/// Returns `(authenticated, needs_setup)`.
|
||||
async fn check_channel_auth_status(&self, name: &str) -> (bool, bool) {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return (true, false);
|
||||
}
|
||||
let Ok(cap_bytes) = tokio::fs::read(&cap_path).await else {
|
||||
return (true, false);
|
||||
};
|
||||
let Ok(cap_file) = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
else {
|
||||
return (true, false);
|
||||
};
|
||||
let required = &cap_file.setup.required_secrets;
|
||||
if required.is_empty() {
|
||||
return (true, false);
|
||||
}
|
||||
let mut all_provided = true;
|
||||
for secret in required {
|
||||
if secret.optional {
|
||||
continue;
|
||||
}
|
||||
if !self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
all_provided = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
(all_provided, true)
|
||||
}
|
||||
|
||||
async fn auth_wasm_channel(
|
||||
&self,
|
||||
name: &str,
|
||||
token: Option<&str>,
|
||||
) -> Result<AuthResult, ExtensionError> {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if !cap_path.exists() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "no_auth_required".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
let cap_file = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
// Get required secrets from the setup section
|
||||
let required_secrets = &cap_file.setup.required_secrets;
|
||||
if required_secrets.is_empty() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "no_auth_required".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Find the first non-optional secret that isn't yet stored
|
||||
let mut missing = Vec::new();
|
||||
for secret in required_secrets {
|
||||
if secret.optional {
|
||||
continue;
|
||||
}
|
||||
if !self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
missing.push(secret);
|
||||
}
|
||||
}
|
||||
|
||||
if missing.is_empty() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "authenticated".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// If a token was provided, store it for the first missing secret
|
||||
if let Some(token_value) = token {
|
||||
let secret = &missing[0];
|
||||
let params =
|
||||
CreateSecretParams::new(&secret.name, token_value).with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
|
||||
// Check if there are more missing secrets
|
||||
if missing.len() <= 1 {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "authenticated".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// More secrets needed; prompt for the next one
|
||||
let next = &missing[1];
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: Some(next.prompt.clone()),
|
||||
setup_url: cap_file.setup.validation_endpoint.clone(),
|
||||
awaiting_token: true,
|
||||
status: "awaiting_token".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Prompt for the first missing secret
|
||||
let secret = &missing[0];
|
||||
Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: Some(secret.prompt.clone()),
|
||||
setup_url: cap_file.setup.validation_endpoint.clone(),
|
||||
awaiting_token: true,
|
||||
status: "awaiting_token".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn activate_mcp(&self, name: &str) -> Result<ActivateResult, ExtensionError> {
|
||||
// Check if already activated
|
||||
{
|
||||
@@ -1056,6 +1515,140 @@ impl ExtensionManager {
|
||||
pending.retain(|_, auth| auth.created_at.elapsed() < std::time::Duration::from_secs(300));
|
||||
}
|
||||
|
||||
/// Get the setup schema for an extension (secret fields and their status).
|
||||
pub async fn get_setup_schema(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<Vec<crate::channels::web::types::SecretFieldInfo>, ExtensionError> {
|
||||
let kind = self.determine_installed_kind(name).await?;
|
||||
match kind {
|
||||
ExtensionKind::WasmChannel => {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
let cap_file =
|
||||
crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
let mut fields = Vec::new();
|
||||
for secret in &cap_file.setup.required_secrets {
|
||||
let provided = self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
fields.push(crate::channels::web::types::SecretFieldInfo {
|
||||
name: secret.name.clone(),
|
||||
prompt: secret.prompt.clone(),
|
||||
optional: secret.optional,
|
||||
provided,
|
||||
auto_generate: secret.auto_generate.is_some(),
|
||||
});
|
||||
}
|
||||
Ok(fields)
|
||||
}
|
||||
_ => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Save setup secrets for an extension, validating names against the capabilities schema.
|
||||
pub async fn save_setup_secrets(
|
||||
&self,
|
||||
name: &str,
|
||||
secrets: &std::collections::HashMap<String, String>,
|
||||
) -> Result<String, ExtensionError> {
|
||||
let kind = self.determine_installed_kind(name).await?;
|
||||
if kind != ExtensionKind::WasmChannel {
|
||||
return Err(ExtensionError::Other(
|
||||
"Setup is only supported for WASM channels".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return Err(ExtensionError::Other(format!(
|
||||
"Capabilities file not found for '{}'",
|
||||
name
|
||||
)));
|
||||
}
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
let cap_file = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
// Build allowed secret names from capabilities
|
||||
let allowed: std::collections::HashSet<String> = cap_file
|
||||
.setup
|
||||
.required_secrets
|
||||
.iter()
|
||||
.map(|s| s.name.clone())
|
||||
.collect();
|
||||
|
||||
// Validate and store each submitted secret
|
||||
for (secret_name, secret_value) in secrets {
|
||||
if !allowed.contains(secret_name.as_str()) {
|
||||
return Err(ExtensionError::Other(format!(
|
||||
"Unknown secret '{}' for extension '{}'",
|
||||
secret_name, name
|
||||
)));
|
||||
}
|
||||
if secret_value.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let params =
|
||||
CreateSecretParams::new(secret_name, secret_value).with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Auto-generate any missing secrets that have auto_generate set
|
||||
for secret_def in &cap_file.setup.required_secrets {
|
||||
if let Some(ref auto_gen) = secret_def.auto_generate {
|
||||
let already_provided = secrets
|
||||
.get(&secret_def.name)
|
||||
.is_some_and(|v| !v.trim().is_empty());
|
||||
let already_stored = self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret_def.name)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if !already_provided && !already_stored {
|
||||
use rand::RngCore;
|
||||
let mut bytes = vec![0u8; auto_gen.length];
|
||||
rand::thread_rng().fill_bytes(&mut bytes);
|
||||
let hex_value: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
|
||||
let params = CreateSecretParams::new(&secret_def.name, &hex_value)
|
||||
.with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
tracing::info!(
|
||||
"Auto-generated secret '{}' for channel '{}'",
|
||||
secret_def.name,
|
||||
name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(format!(
|
||||
"Configuration saved for '{}'. Restart IronClaw for changes to take effect.",
|
||||
name
|
||||
))
|
||||
}
|
||||
|
||||
async fn unregister_hook_prefix(&self, prefix: &str) -> usize {
|
||||
let Some(ref hooks) = self.hooks else {
|
||||
return 0;
|
||||
@@ -1074,7 +1667,7 @@ impl ExtensionManager {
|
||||
|
||||
/// Infer the extension kind from a URL.
|
||||
fn infer_kind_from_url(url: &str) -> ExtensionKind {
|
||||
if url.ends_with(".wasm") {
|
||||
if url.ends_with(".wasm") || url.ends_with(".tar.gz") {
|
||||
ExtensionKind::WasmTool
|
||||
} else {
|
||||
ExtensionKind::McpServer
|
||||
@@ -1092,6 +1685,10 @@ mod tests {
|
||||
infer_kind_from_url("https://example.com/tool.wasm"),
|
||||
ExtensionKind::WasmTool
|
||||
);
|
||||
assert_eq!(
|
||||
infer_kind_from_url("https://example.com/tool-wasm32-wasip2.tar.gz"),
|
||||
ExtensionKind::WasmTool
|
||||
);
|
||||
assert_eq!(
|
||||
infer_kind_from_url("https://mcp.notion.com"),
|
||||
ExtensionKind::McpServer
|
||||
|
||||
@@ -85,6 +85,11 @@ pub enum ExtensionSource {
|
||||
},
|
||||
/// Discovered online (not yet validated for a specific source type).
|
||||
Discovered { url: String },
|
||||
/// Bundled with the application (pre-built WASM, copied from build artifacts).
|
||||
Bundled {
|
||||
/// Channel or tool name used to locate build artifacts.
|
||||
name: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Hint about what authentication method is needed.
|
||||
@@ -184,6 +189,9 @@ pub struct InstalledExtension {
|
||||
/// Tool names if active.
|
||||
#[serde(default)]
|
||||
pub tools: Vec<String>,
|
||||
/// Whether this extension has a setup schema (required_secrets) that can be configured.
|
||||
#[serde(default)]
|
||||
pub needs_setup: bool,
|
||||
}
|
||||
|
||||
/// Error type for extension operations.
|
||||
|
||||
+204
-3
@@ -26,6 +26,26 @@ impl ExtensionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new registry merging builtin entries with catalog-provided entries.
|
||||
///
|
||||
/// Deduplicates by `(name, kind)` pair -- a builtin MCP "slack" and a registry
|
||||
/// WASM "slack" can coexist since they're different kinds.
|
||||
pub fn new_with_catalog(catalog_entries: Vec<RegistryEntry>) -> Self {
|
||||
let mut entries = builtin_entries();
|
||||
for entry in catalog_entries {
|
||||
if !entries
|
||||
.iter()
|
||||
.any(|e| e.name == entry.name && e.kind == entry.kind)
|
||||
{
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
entries,
|
||||
discovery_cache: RwLock::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Search the registry by query string. Returns results sorted by relevance.
|
||||
///
|
||||
/// Splits the query into lowercase tokens and scores each entry by matches
|
||||
@@ -250,11 +270,11 @@ fn builtin_entries() -> Vec<RegistryEntry> {
|
||||
auth_hint: AuthHint::Dcr,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "slack".to_string(),
|
||||
display_name: "Slack".to_string(),
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP".to_string(),
|
||||
kind: ExtensionKind::McpServer,
|
||||
description:
|
||||
"Connect to Slack for messaging, channel management, and team communication"
|
||||
"Connect to Slack via MCP for messaging, channel management, and team communication"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"messaging".into(),
|
||||
@@ -360,6 +380,72 @@ fn builtin_entries() -> Vec<RegistryEntry> {
|
||||
},
|
||||
auth_hint: AuthHint::Dcr,
|
||||
},
|
||||
// -- WASM Channels (bundled) --
|
||||
RegistryEntry {
|
||||
name: "telegram".to_string(),
|
||||
display_name: "Telegram".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Telegram Bot API channel for receiving and sending messages via Telegram"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"bot".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "telegram".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "slack".to_string(),
|
||||
display_name: "Slack".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Slack Events API channel for receiving and sending messages via Slack"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"team".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "slack".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "discord".to_string(),
|
||||
display_name: "Discord".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description:
|
||||
"Discord Gateway channel for handling slash commands, buttons, and messages"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"gaming".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "discord".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "whatsapp".to_string(),
|
||||
display_name: "WhatsApp".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description:
|
||||
"WhatsApp Business API channel for receiving and sending WhatsApp messages"
|
||||
.to_string(),
|
||||
keywords: vec!["chat".into(), "messaging".into(), "channel".into()],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "whatsapp".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -542,4 +628,119 @@ mod tests {
|
||||
let results = registry.search("dup").await;
|
||||
assert_eq!(results.len(), 1, "Should not duplicate cached entries");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_with_catalog() {
|
||||
let catalog_entries = vec![
|
||||
RegistryEntry {
|
||||
name: "telegram".to_string(),
|
||||
display_name: "Telegram".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Telegram Bot API channel".to_string(),
|
||||
keywords: vec!["messaging".into(), "bot".into()],
|
||||
source: ExtensionSource::WasmBuildable {
|
||||
repo_url: "channels-src/telegram".to_string(),
|
||||
build_dir: Some("channels-src/telegram".to_string()),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
// This shares a name with the builtin slack-mcp but has a different kind, so both should appear
|
||||
RegistryEntry {
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP WASM".to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
description: "Slack WASM tool".to_string(),
|
||||
keywords: vec!["messaging".into()],
|
||||
source: ExtensionSource::WasmBuildable {
|
||||
repo_url: "tools-src/slack".to_string(),
|
||||
build_dir: Some("tools-src/slack".to_string()),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
];
|
||||
|
||||
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
|
||||
|
||||
// Should find the new telegram entry
|
||||
let results = registry.search("telegram").await;
|
||||
assert!(!results.is_empty(), "Should find telegram from catalog");
|
||||
assert_eq!(results[0].entry.name, "telegram");
|
||||
|
||||
// Should have both builtin MCP slack-mcp and catalog WASM slack-mcp
|
||||
let results = registry.search("slack").await;
|
||||
let slack_mcp = results
|
||||
.iter()
|
||||
.any(|r| r.entry.name == "slack-mcp" && r.entry.kind == ExtensionKind::McpServer);
|
||||
let slack_wasm = results
|
||||
.iter()
|
||||
.any(|r| r.entry.name == "slack-mcp" && r.entry.kind == ExtensionKind::WasmTool);
|
||||
assert!(slack_mcp, "Should have builtin MCP slack-mcp");
|
||||
assert!(slack_wasm, "Should have catalog WASM slack-mcp");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_with_catalog_dedup_same_kind() {
|
||||
// A catalog entry with same name AND kind as a builtin should be skipped
|
||||
let catalog_entries = vec![RegistryEntry {
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP Override".to_string(),
|
||||
kind: ExtensionKind::McpServer, // same kind as builtin slack-mcp
|
||||
description: "Should be skipped".to_string(),
|
||||
keywords: vec![],
|
||||
source: ExtensionSource::McpUrl {
|
||||
url: "https://other.slack.com".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::Dcr,
|
||||
}];
|
||||
|
||||
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
|
||||
|
||||
let entry = registry.get("slack-mcp").await;
|
||||
assert!(entry.is_some());
|
||||
// Should still be the builtin, not the override
|
||||
assert_eq!(entry.unwrap().display_name, "Slack MCP");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_search_finds_telegram_channel() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
let results = registry.search("telegram").await;
|
||||
|
||||
assert!(!results.is_empty(), "Should find telegram in registry");
|
||||
assert_eq!(results[0].entry.name, "telegram");
|
||||
assert_eq!(results[0].entry.kind, ExtensionKind::WasmChannel);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_search_channel_by_keyword() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
let results = registry.search("bot messaging").await;
|
||||
|
||||
let has_telegram = results.iter().any(|r| r.entry.name == "telegram");
|
||||
assert!(
|
||||
has_telegram,
|
||||
"Telegram should appear in bot messaging search"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bundled_channels() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
|
||||
let telegram = registry.get("telegram").await;
|
||||
assert!(telegram.is_some());
|
||||
assert_eq!(telegram.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let slack = registry.get("slack").await;
|
||||
assert!(slack.is_some());
|
||||
assert_eq!(slack.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let discord = registry.get("discord").await;
|
||||
assert!(discord.is_some());
|
||||
assert_eq!(discord.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let whatsapp = registry.get("whatsapp").await;
|
||||
assert!(whatsapp.is_some());
|
||||
assert_eq!(whatsapp.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ pub mod setup;
|
||||
pub mod skills;
|
||||
pub mod tools;
|
||||
pub mod tracing_fmt;
|
||||
pub mod transcription;
|
||||
pub mod tunnel;
|
||||
pub mod util;
|
||||
pub mod worker;
|
||||
|
||||
@@ -296,14 +296,6 @@ impl LlmProvider for CircuitBreakerProvider {
|
||||
self.inner.set_model(model)
|
||||
}
|
||||
|
||||
fn seed_response_chain(&self, thread_id: &str, response_id: String) {
|
||||
self.inner.seed_response_chain(thread_id, response_id)
|
||||
}
|
||||
|
||||
fn get_response_chain_id(&self, thread_id: &str) -> Option<String> {
|
||||
self.inner.get_response_chain_id(thread_id)
|
||||
}
|
||||
|
||||
fn calculate_cost(&self, input_tokens: u32, output_tokens: u32) -> Decimal {
|
||||
self.inner.calculate_cost(input_tokens, output_tokens)
|
||||
}
|
||||
|
||||
@@ -359,15 +359,6 @@ impl LlmProvider for FailoverProvider {
|
||||
.await
|
||||
}
|
||||
|
||||
fn seed_response_chain(&self, thread_id: &str, response_id: String) {
|
||||
self.providers[self.last_used.load(Ordering::Relaxed)]
|
||||
.seed_response_chain(thread_id, response_id);
|
||||
}
|
||||
|
||||
fn get_response_chain_id(&self, thread_id: &str) -> Option<String> {
|
||||
self.providers[self.last_used.load(Ordering::Relaxed)].get_response_chain_id(thread_id)
|
||||
}
|
||||
|
||||
fn calculate_cost(&self, input_tokens: u32, output_tokens: u32) -> Decimal {
|
||||
self.providers[self.last_used.load(Ordering::Relaxed)]
|
||||
.calculate_cost(input_tokens, output_tokens)
|
||||
@@ -413,7 +404,6 @@ mod tests {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
}))),
|
||||
tool_complete_result: Mutex::new(Some(Ok(ToolCompletionResponse {
|
||||
content: Some(content.to_string()),
|
||||
@@ -421,7 +411,6 @@ mod tests {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
}))),
|
||||
}
|
||||
}
|
||||
@@ -803,7 +792,6 @@ mod tests {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -829,7 +817,6 @@ mod tests {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+184
-33
@@ -1,7 +1,7 @@
|
||||
//! LLM integration for the agent.
|
||||
//!
|
||||
//! Supports multiple backends:
|
||||
//! - **NEAR AI** (default): Session-based or API key auth via NEAR AI proxy
|
||||
//! - **NEAR AI** (default): Session token or API key auth via Chat Completions API
|
||||
//! - **OpenAI**: Direct API access with your own key
|
||||
//! - **Anthropic**: Direct API access with your own key
|
||||
//! - **Ollama**: Local model inference
|
||||
@@ -10,7 +10,6 @@
|
||||
pub mod circuit_breaker;
|
||||
pub mod costs;
|
||||
pub mod failover;
|
||||
mod nearai;
|
||||
mod nearai_chat;
|
||||
mod provider;
|
||||
mod reasoning;
|
||||
@@ -18,30 +17,31 @@ pub mod response_cache;
|
||||
pub mod retry;
|
||||
mod rig_adapter;
|
||||
pub mod session;
|
||||
pub mod smart_routing;
|
||||
|
||||
pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider};
|
||||
pub use failover::{CooldownConfig, FailoverProvider};
|
||||
pub use nearai::{ModelInfo, NearAiProvider};
|
||||
pub use nearai_chat::NearAiChatProvider;
|
||||
pub use nearai_chat::{ModelInfo, NearAiChatProvider};
|
||||
pub use provider::{
|
||||
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata,
|
||||
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult,
|
||||
};
|
||||
pub use reasoning::{
|
||||
ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, TokenUsage,
|
||||
ToolSelection,
|
||||
ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, SILENT_REPLY_TOKEN,
|
||||
TokenUsage, ToolSelection, is_silent_reply,
|
||||
};
|
||||
pub use response_cache::{CachedProvider, ResponseCacheConfig};
|
||||
pub use retry::{RetryConfig, RetryProvider};
|
||||
pub use rig_adapter::RigAdapter;
|
||||
pub use session::{SessionConfig, SessionManager, create_session_manager};
|
||||
pub use smart_routing::{SmartRoutingConfig, SmartRoutingProvider, TaskComplexity};
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rig::client::CompletionClient;
|
||||
use secrecy::ExposeSecret;
|
||||
|
||||
use crate::config::{LlmBackend, LlmConfig, NearAiApiMode, NearAiConfig};
|
||||
use crate::config::{LlmBackend, LlmConfig, NearAiConfig};
|
||||
use crate::error::LlmError;
|
||||
|
||||
/// Create an LLM provider based on configuration.
|
||||
@@ -71,24 +71,18 @@ pub fn create_llm_provider_with_config(
|
||||
config: &NearAiConfig,
|
||||
session: Arc<SessionManager>,
|
||||
) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
||||
match config.api_mode {
|
||||
NearAiApiMode::Responses => {
|
||||
tracing::info!(
|
||||
model = %config.model,
|
||||
base_url = %config.base_url,
|
||||
"Using NEAR AI Chat (Responses API, session token auth)"
|
||||
);
|
||||
Ok(Arc::new(NearAiProvider::new(config.clone(), session)?))
|
||||
}
|
||||
NearAiApiMode::ChatCompletions => {
|
||||
tracing::info!(
|
||||
model = %config.model,
|
||||
base_url = %config.base_url,
|
||||
"Using NEAR AI Cloud (Chat Completions API, API key auth)"
|
||||
);
|
||||
Ok(Arc::new(NearAiChatProvider::new(config.clone())?))
|
||||
}
|
||||
}
|
||||
let auth_mode = if config.api_key.is_some() {
|
||||
"API key"
|
||||
} else {
|
||||
"session token"
|
||||
};
|
||||
tracing::info!(
|
||||
model = %config.model,
|
||||
base_url = %config.base_url,
|
||||
auth = auth_mode,
|
||||
"Using NEAR AI (Chat Completions API)"
|
||||
);
|
||||
Ok(Arc::new(NearAiChatProvider::new(config.clone(), session)?))
|
||||
}
|
||||
|
||||
fn create_openai_provider(config: &LlmConfig) -> Result<Arc<dyn LlmProvider>, LlmError> {
|
||||
@@ -226,6 +220,25 @@ fn create_openai_compatible_provider(config: &LlmConfig) -> Result<Arc<dyn LlmPr
|
||||
|
||||
use rig::providers::openai;
|
||||
|
||||
let mut extra_headers = reqwest::header::HeaderMap::new();
|
||||
for (key, value) in &compat.extra_headers {
|
||||
let name = match reqwest::header::HeaderName::from_bytes(key.as_bytes()) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::warn!(header = %key, error = %e, "Skipping LLM_EXTRA_HEADERS entry: invalid header name");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let val = match reqwest::header::HeaderValue::from_str(value) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(header = %key, error = %e, "Skipping LLM_EXTRA_HEADERS entry: invalid header value");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
extra_headers.insert(name, val);
|
||||
}
|
||||
|
||||
let client: openai::CompletionsClient = openai::Client::builder()
|
||||
.base_url(&compat.base_url)
|
||||
.api_key(
|
||||
@@ -235,6 +248,7 @@ fn create_openai_compatible_provider(config: &LlmConfig) -> Result<Arc<dyn LlmPr
|
||||
.map(|k| k.expose_secret().to_string())
|
||||
.unwrap_or_else(|| "no-key".to_string()),
|
||||
)
|
||||
.http_headers(extra_headers)
|
||||
.build()
|
||||
.map_err(|e| LlmError::RequestFailed {
|
||||
provider: "openai_compatible".to_string(),
|
||||
@@ -254,7 +268,7 @@ fn create_openai_compatible_provider(config: &LlmConfig) -> Result<Arc<dyn LlmPr
|
||||
/// Create a cheap/fast LLM provider for lightweight tasks (heartbeat, routing, evaluation).
|
||||
///
|
||||
/// Uses `NEARAI_CHEAP_MODEL` if set, otherwise falls back to the main provider.
|
||||
/// Currently only supports NEAR AI backends (Responses and ChatCompletions modes).
|
||||
/// Currently only supports NEAR AI backend.
|
||||
pub fn create_cheap_llm_provider(
|
||||
config: &LlmConfig,
|
||||
session: Arc<SessionManager>,
|
||||
@@ -275,20 +289,157 @@ pub fn create_cheap_llm_provider(
|
||||
let mut cheap_config = config.nearai.clone();
|
||||
cheap_config.model = cheap_model.clone();
|
||||
|
||||
tracing::info!("Cheap LLM provider: {}", cheap_model);
|
||||
Ok(Some(Arc::new(NearAiChatProvider::new(
|
||||
cheap_config,
|
||||
session,
|
||||
)?)))
|
||||
}
|
||||
|
||||
match cheap_config.api_mode {
|
||||
NearAiApiMode::Responses => Ok(Some(Arc::new(NearAiProvider::new(cheap_config, session)?))),
|
||||
NearAiApiMode::ChatCompletions => {
|
||||
Ok(Some(Arc::new(NearAiChatProvider::new(cheap_config)?)))
|
||||
/// Build the full LLM provider chain with all configured wrappers.
|
||||
///
|
||||
/// Applies decorators in this order:
|
||||
/// 1. Raw provider (from config)
|
||||
/// 2. RetryProvider (per-provider retry with exponential backoff)
|
||||
/// 3. SmartRoutingProvider (cheap/primary split when cheap model is configured)
|
||||
/// 4. FailoverProvider (fallback model when primary fails)
|
||||
/// 5. CircuitBreakerProvider (fast-fail when backend is degraded)
|
||||
/// 6. CachedProvider (in-memory response cache)
|
||||
///
|
||||
/// Also returns a separate cheap LLM provider for heartbeat/evaluation (not
|
||||
/// part of the chain — it's a standalone provider for explicitly cheap tasks).
|
||||
///
|
||||
/// This is the single source of truth for provider chain construction,
|
||||
/// called by both `main.rs` and `app.rs`.
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn build_provider_chain(
|
||||
config: &LlmConfig,
|
||||
session: Arc<SessionManager>,
|
||||
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), LlmError> {
|
||||
let llm = create_llm_provider(config, session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
|
||||
// 1. Retry
|
||||
let retry_config = RetryConfig {
|
||||
max_retries: config.nearai.max_retries,
|
||||
};
|
||||
let llm: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
tracing::info!(
|
||||
max_retries = retry_config.max_retries,
|
||||
"LLM retry wrapper enabled"
|
||||
);
|
||||
Arc::new(RetryProvider::new(llm, retry_config.clone()))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 2. Smart routing (cheap/primary split)
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(ref cheap_model) = config.nearai.cheap_model {
|
||||
let mut cheap_config = config.nearai.clone();
|
||||
cheap_config.model = cheap_model.clone();
|
||||
let cheap = create_llm_provider_with_config(&cheap_config, session.clone())?;
|
||||
let cheap: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
Arc::new(RetryProvider::new(cheap, retry_config.clone()))
|
||||
} else {
|
||||
cheap
|
||||
};
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
cheap = %cheap.model_name(),
|
||||
"Smart routing enabled"
|
||||
);
|
||||
Arc::new(SmartRoutingProvider::new(
|
||||
llm,
|
||||
cheap,
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: config.nearai.smart_routing_cascade,
|
||||
..SmartRoutingConfig::default()
|
||||
},
|
||||
))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 3. Failover
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(ref fallback_model) = config.nearai.fallback_model {
|
||||
if fallback_model == &config.nearai.model {
|
||||
tracing::warn!(
|
||||
"fallback_model is the same as primary model, failover may not be effective"
|
||||
);
|
||||
}
|
||||
let mut fallback_config = config.nearai.clone();
|
||||
fallback_config.model = fallback_model.clone();
|
||||
let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?;
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
let fallback: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
Arc::new(RetryProvider::new(fallback, retry_config.clone()))
|
||||
} else {
|
||||
fallback
|
||||
};
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(config.nearai.failover_cooldown_secs),
|
||||
failure_threshold: config.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 4. Circuit breaker
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(threshold) = config.nearai.circuit_breaker_threshold
|
||||
{
|
||||
let cb_config = CircuitBreakerConfig {
|
||||
failure_threshold: threshold,
|
||||
recovery_timeout: std::time::Duration::from_secs(
|
||||
config.nearai.circuit_breaker_recovery_secs,
|
||||
),
|
||||
..CircuitBreakerConfig::default()
|
||||
};
|
||||
tracing::info!(
|
||||
threshold,
|
||||
recovery_secs = config.nearai.circuit_breaker_recovery_secs,
|
||||
"LLM circuit breaker enabled"
|
||||
);
|
||||
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 5. Response cache
|
||||
let llm: Arc<dyn LlmProvider> = if config.nearai.response_cache_enabled {
|
||||
let rc_config = ResponseCacheConfig {
|
||||
ttl: std::time::Duration::from_secs(config.nearai.response_cache_ttl_secs),
|
||||
max_entries: config.nearai.response_cache_max_entries,
|
||||
};
|
||||
tracing::info!(
|
||||
ttl_secs = config.nearai.response_cache_ttl_secs,
|
||||
max_entries = config.nearai.response_cache_max_entries,
|
||||
"LLM response cache enabled"
|
||||
);
|
||||
Arc::new(CachedProvider::new(llm, rc_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Standalone cheap LLM for heartbeat/evaluation (not part of the chain)
|
||||
let cheap_llm = create_cheap_llm_provider(config, session)?;
|
||||
if let Some(ref cheap) = cheap_llm {
|
||||
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
|
||||
}
|
||||
|
||||
Ok((llm, cheap_llm))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{LlmBackend, NearAiApiMode, NearAiConfig};
|
||||
use crate::config::{LlmBackend, NearAiConfig};
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn test_nearai_config() -> NearAiConfig {
|
||||
@@ -298,7 +449,6 @@ mod tests {
|
||||
base_url: "https://api.near.ai".to_string(),
|
||||
auth_base_url: "https://private.near.ai".to_string(),
|
||||
session_path: PathBuf::from("/tmp/test-session.json"),
|
||||
api_mode: NearAiApiMode::Responses,
|
||||
api_key: None,
|
||||
fallback_model: None,
|
||||
max_retries: 3,
|
||||
@@ -309,6 +459,7 @@ mod tests {
|
||||
response_cache_max_entries: 1000,
|
||||
failover_cooldown_secs: 300,
|
||||
failover_cooldown_threshold: 3,
|
||||
smart_routing_cascade: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
-1205
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user