mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
497b93cebb | ||
|
|
37c0158765 | ||
|
|
5a70e3e1ef | ||
|
|
bbb2d5c4dd | ||
|
|
0a30c95ee1 | ||
|
|
b3bf50f10e | ||
|
|
48b5323ec9 | ||
|
|
3124ab2b7f | ||
|
|
dbd3e0807f | ||
|
|
436066415b | ||
|
|
3d4c647216 | ||
|
|
b68d67bd35 | ||
|
|
493e4578d0 | ||
|
|
250551799b | ||
|
|
c038c7705b | ||
|
|
98ee648fcb | ||
|
|
2cdd1acb1e | ||
|
|
542268fde5 | ||
|
|
3b6105d5ea | ||
|
|
df8616b604 | ||
|
|
e8dcb52fda |
@@ -286,8 +286,8 @@ impl Tool for <Name>Tool {
|
||||
false // Set true if tool processes external data
|
||||
}
|
||||
|
||||
fn requires_approval(&self) -> bool {
|
||||
false // Set true if tool is destructive or contacts external services
|
||||
fn requires_approval(&self, _params: &serde_json::Value) -> crate::tools::tool::ApprovalRequirement {
|
||||
crate::tools::tool::ApprovalRequirement::Never // Set to UnlessAutoApproved or Always as needed
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
+17
-2
@@ -33,11 +33,26 @@ NEARAI_AUTH_URL=https://private.near.ai
|
||||
# LLM_BASE_URL=http://localhost:1234/v1
|
||||
# LLM_API_KEY=sk-... # optional for local servers
|
||||
|
||||
# === OpenRouter (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=anthropic/claude-sonnet-4
|
||||
# === OpenRouter (300+ models via OpenAI-compatible) ===
|
||||
# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
# LLM_API_KEY=sk-or-...
|
||||
# LLM_EXTRA_HEADERS=HTTP-Referer:https://myapp.com,X-Title:MyApp
|
||||
|
||||
# === Together AI (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://api.together.xyz/v1
|
||||
# LLM_API_KEY=...
|
||||
|
||||
# === Fireworks AI (via OpenAI-compatible) ===
|
||||
# LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
|
||||
# LLM_BACKEND=openai_compatible
|
||||
# LLM_BASE_URL=https://api.fireworks.ai/inference/v1
|
||||
# LLM_API_KEY=fw_...
|
||||
|
||||
# For full provider setup guide see docs/LLM_PROVIDERS.md
|
||||
|
||||
|
||||
# Channel Configuration
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
tests/test-pages/**/*.html linguist-generated=true
|
||||
@@ -214,14 +214,113 @@ jobs:
|
||||
path: |
|
||||
${{ steps.cargo-dist.outputs.paths }}
|
||||
${{ env.BUILD_MANIFEST_NAME }}
|
||||
# Build WASM extension bundles (tar.gz with .wasm + .capabilities.json)
|
||||
build-wasm-extensions:
|
||||
needs:
|
||||
- plan
|
||||
if: ${{ needs.plan.outputs.publishing == 'true' }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: recursive
|
||||
- name: Install Rust toolchain + wasm target
|
||||
run: |
|
||||
rustup target add wasm32-wasip2
|
||||
cargo install cargo-component --locked || true
|
||||
- uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
key: wasm-extensions
|
||||
- name: Build and package WASM extensions
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p target/wasm-bundles
|
||||
|
||||
# Process each manifest in registry/tools/ and registry/channels/
|
||||
for manifest in registry/tools/*.json registry/channels/*.json; do
|
||||
[ -f "$manifest" ] || continue
|
||||
|
||||
name=$(jq -r '.name' "$manifest")
|
||||
source_dir=$(jq -r '.source.dir' "$manifest")
|
||||
caps_file=$(jq -r '.source.capabilities' "$manifest")
|
||||
crate_name=$(jq -r '.source.crate_name' "$manifest")
|
||||
|
||||
if [ ! -d "$source_dir" ]; then
|
||||
echo "::warning::Source dir '$source_dir' not found for '$name', skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "=== Building $name from $source_dir ==="
|
||||
|
||||
# Build WASM component
|
||||
cargo component build --release --manifest-path "$source_dir/Cargo.toml" || {
|
||||
echo "::warning::Build failed for '$name', skipping"
|
||||
continue
|
||||
}
|
||||
|
||||
# Find the built WASM file (Cargo uses underscores in artifact names)
|
||||
wasm_artifact="${crate_name//-/_}"
|
||||
wasm_path=""
|
||||
for target_dir in wasm32-wasip2 wasm32-wasip1 wasm32-wasi; do
|
||||
candidate="$source_dir/target/$target_dir/release/${wasm_artifact}.wasm"
|
||||
if [ -f "$candidate" ]; then
|
||||
wasm_path="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$wasm_path" ]; then
|
||||
echo "::warning::No WASM output found for '$name', skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Copy files with standardized names for the archive
|
||||
cp "$wasm_path" "target/wasm-bundles/${name}.wasm"
|
||||
|
||||
caps_path="$source_dir/$caps_file"
|
||||
if [ -f "$caps_path" ]; then
|
||||
cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json"
|
||||
else
|
||||
echo "::warning::No capabilities file at '$caps_path' for '$name'"
|
||||
fi
|
||||
|
||||
# Create tar.gz bundle
|
||||
bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz"
|
||||
(cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi)
|
||||
|
||||
# Compute SHA256
|
||||
sha256=$(sha256sum "$bundle" | cut -d' ' -f1)
|
||||
echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt
|
||||
|
||||
# Clean up intermediate files
|
||||
rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json"
|
||||
|
||||
echo " -> $bundle ($sha256)"
|
||||
done
|
||||
|
||||
echo "=== WASM bundles built ==="
|
||||
ls -la target/wasm-bundles/
|
||||
- name: "Upload WASM bundles"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: artifacts-wasm-extensions
|
||||
path: |
|
||||
target/wasm-bundles/*.tar.gz
|
||||
target/wasm-bundles/checksums.txt
|
||||
|
||||
# Determines if we should publish/announce
|
||||
host:
|
||||
needs:
|
||||
- plan
|
||||
- build-local-artifacts
|
||||
- build-global-artifacts
|
||||
# Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
|
||||
- build-wasm-extensions
|
||||
# Only run if we're "publishing", and only if plan, local, global, and wasm didn't fail (skipped is fine)
|
||||
if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
runs-on: "ubuntu-22.04"
|
||||
|
||||
@@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.9.0](https://github.com/nearai/ironclaw/compare/v0.8.0...v0.9.0) - 2026-02-21
|
||||
|
||||
### Added
|
||||
|
||||
- add TEE attestation shield to web gateway UI ([#275](https://github.com/nearai/ironclaw/pull/275))
|
||||
- configurable tool iterations, auto-approve, and policy fix ([#251](https://github.com/nearai/ironclaw/pull/251))
|
||||
|
||||
### Fixed
|
||||
|
||||
- add X-Accel-Buffering header to SSE endpoints ([#277](https://github.com/nearai/ironclaw/pull/277))
|
||||
|
||||
## [0.8.0](https://github.com/nearai/ironclaw/compare/ironclaw-v0.7.0...ironclaw-v0.8.0) - 2026-02-20
|
||||
|
||||
### Added
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
# Format code
|
||||
cargo fmt
|
||||
|
||||
# Lint (address warnings before committing)
|
||||
# Lint (fix ALL warnings before committing, including pre-existing ones)
|
||||
cargo clippy --all --benches --tests --examples --all-features
|
||||
|
||||
# Run all tests
|
||||
@@ -321,7 +321,10 @@ cargo check --all-features # all features
|
||||
```
|
||||
Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature.
|
||||
|
||||
**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate.
|
||||
|
||||
**Mechanical verification before committing:** Run these checks on changed files before committing:
|
||||
- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings
|
||||
- `grep -rnE '\.unwrap\(|\.expect\(' <files>` -- no panics in production
|
||||
- `grep -rn 'super::' <files>` -- use `crate::` imports
|
||||
- If you fixed a pattern bug, `grep` for other instances of that pattern across `src/`
|
||||
@@ -408,6 +411,10 @@ IronClaw supports multiple LLM backends via the `LLM_BACKEND` env var: `nearai`
|
||||
|
||||
**NEAR AI** -- Uses the Chat Completions API with dual auth support. Session token auth (default): authenticates with session tokens (`sess_xxx`) obtained via browser OAuth (GitHub/Google), base URL defaults to `https://private.near.ai`. API key auth: set `NEARAI_API_KEY` (from `cloud.near.ai`), base URL defaults to `https://cloud-api.near.ai`. Both modes use the same Chat Completions endpoint. Tool messages are flattened to plain text for compatibility. Set `NEARAI_SESSION_TOKEN` env var for hosting providers that inject tokens via environment.
|
||||
|
||||
**NEAR AI Cloud** -- Uses the OpenAI-compatible Chat Completions API (`https://cloud-api.near.ai/v1/chat/completions`). Authenticates with API keys from `cloud.near.ai`. Auto-selected when `NEARAI_API_KEY` is set (or explicitly via `NEARAI_API_MODE=chat_completions`). Tool messages are flattened to plain text for compatibility. Configure with `NEARAI_API_KEY` and `NEARAI_BASE_URL` (default: `https://cloud-api.near.ai`).
|
||||
|
||||
**OpenAI-compatible** -- Any endpoint that speaks the OpenAI API (vLLM, LiteLLM, OpenRouter, etc.). Configure with `LLM_BASE_URL`, `LLM_API_KEY` (optional), `LLM_MODEL`. Set `LLM_EXTRA_HEADERS` to inject custom HTTP headers into every request (format: `Key:Value,Key2:Value2`), useful for OpenRouter attribution headers like `HTTP-Referer` and `X-Title`.
|
||||
|
||||
**Tinfoil** -- Private inference via `https://inference.tinfoil.sh/v1`. Runs models inside hardware-attested TEEs so neither Tinfoil nor the cloud provider can see prompts or responses. Uses the OpenAI-compatible Chat Completions API. Configure with `TINFOIL_API_KEY` and `TINFOIL_MODEL` (default: `kimi-k2-5`).
|
||||
|
||||
## Database
|
||||
|
||||
Generated
+480
-28
@@ -11,6 +11,12 @@ dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aead"
|
||||
version = "0.5.2"
|
||||
@@ -64,6 +70,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"const-random",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy 0.8.37",
|
||||
@@ -188,6 +195,15 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063"
|
||||
|
||||
[[package]]
|
||||
name = "astral-tl"
|
||||
version = "0.7.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-broadcast"
|
||||
version = "0.7.2"
|
||||
@@ -924,6 +940,26 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
|
||||
dependencies = [
|
||||
"const-random-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random-macro"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"once_cell",
|
||||
"tiny-keccak",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
@@ -1099,6 +1135,21 @@ dependencies = [
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
@@ -1244,6 +1295,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
@@ -1255,6 +1312,29 @@ dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa",
|
||||
"phf 0.13.1",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser-macros"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctr"
|
||||
version = "0.9.2"
|
||||
@@ -1497,12 +1577,33 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa-short"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
|
||||
dependencies = [
|
||||
"dtoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ego-tree"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
@@ -1680,6 +1781,16 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
@@ -1692,6 +1803,12 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
@@ -1743,6 +1860,16 @@ version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||
dependencies = [
|
||||
"mac",
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.31"
|
||||
@@ -1883,6 +2010,15 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getopts"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
|
||||
dependencies = [
|
||||
"unicode-width 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -2001,7 +2137,7 @@ version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"foldhash 0.1.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -2010,6 +2146,11 @@ name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
@@ -2065,6 +2206,54 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-escape"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
|
||||
dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-to-markdown-rs"
|
||||
version = "2.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb31d75f2fdbc8d889d78a912e10c22c30451afb44ee3310f5bfcabf79a31a17"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"astral-tl",
|
||||
"base64 0.22.1",
|
||||
"html-escape",
|
||||
"html5ever 0.38.0",
|
||||
"lru",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.38.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.12"
|
||||
@@ -2490,7 +2679,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ironclaw"
|
||||
version = "0.8.0"
|
||||
version = "0.9.0"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"aho-corasick",
|
||||
@@ -2508,9 +2697,11 @@ dependencies = [
|
||||
"deadpool-postgres",
|
||||
"dirs 6.0.0",
|
||||
"dotenvy",
|
||||
"flate2",
|
||||
"fs4",
|
||||
"futures",
|
||||
"hkdf",
|
||||
"html-to-markdown-rs",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper-util",
|
||||
@@ -2521,6 +2712,7 @@ dependencies = [
|
||||
"postgres-types",
|
||||
"pretty_assertions",
|
||||
"rand 0.8.5",
|
||||
"readabilityrs",
|
||||
"refinery",
|
||||
"regex",
|
||||
"reqwest",
|
||||
@@ -2536,6 +2728,7 @@ dependencies = [
|
||||
"serde_yml",
|
||||
"sha2",
|
||||
"subtle",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"termimad",
|
||||
"testcontainers-modules",
|
||||
@@ -2559,30 +2752,6 @@ dependencies = [
|
||||
"zbus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ironclaw-bench"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"chrono",
|
||||
"clap",
|
||||
"futures",
|
||||
"ironclaw",
|
||||
"regex",
|
||||
"rust_decimal",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"toml",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-docker"
|
||||
version = "0.2.0"
|
||||
@@ -2663,6 +2832,21 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuchikikiki"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b73885c6a3cefdf7a1db0327cefbe4b9b72cac94cae4b19ede4fa492d8af02a0"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"crc",
|
||||
"cssparser",
|
||||
"html5ever 0.38.0",
|
||||
"indexmap 2.13.0",
|
||||
"precomputed-hash",
|
||||
"selectors 0.35.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex"
|
||||
version = "3.5.1"
|
||||
@@ -2813,7 +2997,7 @@ dependencies = [
|
||||
"log",
|
||||
"memchr",
|
||||
"phf 0.11.3",
|
||||
"phf_codegen",
|
||||
"phf_codegen 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
"uncased",
|
||||
]
|
||||
@@ -2907,12 +3091,27 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.4.3"
|
||||
@@ -2922,6 +3121,28 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.4.3",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.5.0",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -3014,6 +3235,16 @@ version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.1"
|
||||
@@ -3052,6 +3283,12 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
@@ -3422,6 +3659,7 @@ version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.13.1",
|
||||
"serde",
|
||||
]
|
||||
@@ -3432,10 +3670,20 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
@@ -3446,6 +3694,29 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
@@ -3609,6 +3880,12 @@ dependencies = [
|
||||
"zerocopy 0.8.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "pretty_assertions"
|
||||
version = "1.4.1"
|
||||
@@ -3876,6 +4153,24 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "readabilityrs"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eb174b0af6c181a87d68b42800806657bfbdf88b566f819aaadb9d2a7b7699d"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"kuchikikiki",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"url",
|
||||
"v_htmlescape",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
@@ -4418,6 +4713,21 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
"getopts",
|
||||
"html5ever 0.36.1",
|
||||
"precomputed-hash",
|
||||
"selectors 0.33.0",
|
||||
"tendril 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seahash"
|
||||
version = "4.1.0"
|
||||
@@ -4489,6 +4799,44 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
@@ -4627,6 +4975,15 @@ dependencies = [
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yml"
|
||||
version = "0.0.12"
|
||||
@@ -4719,6 +5076,12 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
@@ -4790,6 +5153,30 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared 0.13.1",
|
||||
"precomputed-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.5"
|
||||
@@ -4927,6 +5314,17 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.16"
|
||||
@@ -4946,6 +5344,27 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
@@ -5089,6 +5508,15 @@ dependencies = [
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiny-keccak"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
|
||||
dependencies = [
|
||||
"crunchy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.2"
|
||||
@@ -5730,6 +6158,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -5754,6 +6188,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "v_htmlescape"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
@@ -6289,6 +6729,18 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
|
||||
dependencies = [
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
|
||||
+18
-5
@@ -1,5 +1,5 @@
|
||||
[workspace]
|
||||
members = [".", "benchmarks"]
|
||||
members = ["."]
|
||||
exclude = [
|
||||
"channels-src/discord",
|
||||
"channels-src/telegram",
|
||||
@@ -19,7 +19,7 @@ exclude = [
|
||||
|
||||
[package]
|
||||
name = "ironclaw"
|
||||
version = "0.8.0"
|
||||
version = "0.9.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.92"
|
||||
description = "Secure personal AI assistant that protects your data and expands its capabilities on the fly"
|
||||
@@ -41,7 +41,7 @@ tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -82,7 +82,7 @@ clap = { version = "4", features = ["derive", "env"] }
|
||||
|
||||
# Terminal
|
||||
crossterm = "0.28"
|
||||
rustyline = { version = "17", features = ["derive", "with-file-history"] }
|
||||
rustyline = { version = "17", features = ["custom-bindings", "derive", "with-file-history"] }
|
||||
termimad = "0.34"
|
||||
|
||||
# Channel integrations
|
||||
@@ -137,6 +137,10 @@ rig-core = "0.30"
|
||||
# Docker sandbox
|
||||
bollard = "0.18"
|
||||
|
||||
# Archive extraction for WASM extension bundles
|
||||
flate2 = "1"
|
||||
tar = "0.4"
|
||||
|
||||
# HTTP proxy for sandboxed network access
|
||||
hyper = { version = "1.5", features = ["server", "http1", "http2"] }
|
||||
hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] }
|
||||
@@ -145,6 +149,10 @@ bytes = "1"
|
||||
base64 = "0.22.1"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
# HTML to Markdown conversion (feature gated)
|
||||
html-to-markdown-rs = { version = "2.3", optional = true }
|
||||
readabilityrs = { version = "0.1.2", optional = true }
|
||||
|
||||
# macOS keychain
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
security-framework = "3"
|
||||
@@ -162,7 +170,7 @@ pretty_assertions = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = ["postgres", "libsql"]
|
||||
default = ["postgres", "libsql", "html-to-markdown"]
|
||||
postgres = [
|
||||
"dep:deadpool-postgres",
|
||||
"dep:tokio-postgres",
|
||||
@@ -173,6 +181,11 @@ postgres = [
|
||||
]
|
||||
libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
|
||||
|
||||
[[test]]
|
||||
name = "html_to_markdown"
|
||||
required-features = ["html-to-markdown"]
|
||||
|
||||
# The profile that 'cargo dist' will build with
|
||||
[profile.dist]
|
||||
|
||||
@@ -143,6 +143,23 @@ and secrets encryption (using your system keychain). Settings are persisted in t
|
||||
connected database; bootstrap variables (e.g. `DATABASE_URL`, `LLM_BACKEND`) are
|
||||
written to `~/.ironclaw/.env` so they are available before the database connects.
|
||||
|
||||
### Alternative LLM Providers
|
||||
|
||||
IronClaw defaults to NEAR AI but works with any OpenAI-compatible endpoint.
|
||||
Popular options include **OpenRouter** (300+ models), **Together AI**, **Fireworks AI**,
|
||||
**Ollama** (local), and self-hosted servers like **vLLM** or **LiteLLM**.
|
||||
|
||||
Select *"OpenAI-compatible"* in the wizard, or set environment variables directly:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
LLM_API_KEY=sk-or-...
|
||||
LLM_MODEL=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
See [docs/LLM_PROVIDERS.md](docs/LLM_PROVIDERS.md) for a full provider guide.
|
||||
|
||||
## Security
|
||||
|
||||
IronClaw implements defense in depth to protect your data and prevent misuse.
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
[package]
|
||||
name = "ironclaw-bench"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
description = "Benchmarking harness for IronClaw agent"
|
||||
license = "MIT OR Apache-2.0"
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "ironclaw-bench"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
ironclaw = { path = ".." }
|
||||
|
||||
# Async runtime
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
toml = "0.8"
|
||||
|
||||
# CLI
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
||||
# Core types
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
|
||||
# Async traits
|
||||
async-trait = "0.1"
|
||||
|
||||
# Logging
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
|
||||
# Scoring
|
||||
regex = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"run_id": "8a48de1f-09ce-4c35-9ad1-dad98fb83a1a",
|
||||
"suite_id": "spot",
|
||||
"config_label": "default",
|
||||
"model": "openai/gpt-5.2",
|
||||
"commit_hash": "2c43b83",
|
||||
"pass_rate": 1.0,
|
||||
"avg_score": 1.0,
|
||||
"total_tasks": 21,
|
||||
"completed_tasks": 21,
|
||||
"total_cost_usd": 0.307053,
|
||||
"total_wall_time_ms": 111009,
|
||||
"started_at": "2026-02-17T22:02:08.206112Z",
|
||||
"finished_at": "2026-02-17T22:03:59.270325Z"
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
{"task_id":"smoke-greeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2037,"llm_calls":1,"input_tokens":1359,"output_tokens":65,"estimated_cost_usd":0.005052,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"I’m NEAR AI Agent—an autonomous assistant that helps you plan, write, debug, and execute tasks using available tools (like reading/writing files, running shell commands, and making HTTP requests). Tell me what you’re working on and what outcome you want, and I’ll drive it to completion.","started_at":"2026-02-17T22:02:08.206165Z","finished_at":"2026-02-17T22:02:10.243552Z","config_label":"default","error":null}
|
||||
{"task_id":"smoke-math","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2038,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"1081","started_at":"2026-02-17T22:02:10.243933Z","finished_at":"2026-02-17T22:02:12.282133Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2646,"llm_calls":2,"input_tokens":2786,"output_tokens":24,"estimated_cost_usd":0.008718,"tool_calls":[{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Spot check passed","started_at":"2026-02-17T22:02:12.282294Z","finished_at":"2026-02-17T22:02:14.928540Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-time","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2839,"output_tokens":78,"estimated_cost_usd":0.009687,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Current date/time (UTC): **2026-02-17 22:02:16** \nISO 8601: **2026-02-17T22:02:16.045933+00:00**","started_at":"2026-02-17T22:02:14.928810Z","finished_at":"2026-02-17T22:02:18.389188Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-json-query","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2751,"llm_calls":2,"input_tokens":2839,"output_tokens":43,"estimated_cost_usd":0.009162,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Bob","started_at":"2026-02-17T22:02:18.389812Z","finished_at":"2026-02-17T22:02:21.141146Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-shell-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3460,"llm_calls":2,"input_tokens":2811,"output_tokens":25,"estimated_cost_usd":0.008808,"tool_calls":[{"name":"shell","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"benchmark test","started_at":"2026-02-17T22:02:21.141384Z","finished_at":"2026-02-17T22:02:24.601812Z","config_label":"default","error":null}
|
||||
{"task_id":"tool-list-dir","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":19146,"llm_calls":2,"input_tokens":4370,"output_tokens":1337,"estimated_cost_usd":0.033165,"tool_calls":[{"name":"list_dir","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```\nclaude-501/\nclaude-505/\nclaude/\ncodex/\ncom.apple.launchd.HqYyNM7hoK/\ncom.apple.launchd.Z0bTtlsPEN/\ncom.apple.launchd.oY8y5gkzN4/\ncom.apple.launchd.sdo3WGu0S7/\ndata-gym-cache/\ndocker-desktop-privileged2203342862/\ngrammers-test/\nnear-agent-target/\nnear-market-bot-push/\nnear-market-bot/\nopenclaw/\npowerlog/\ntest_grpo/\ntest_grpo2/\ntest_grpo_entry/\ntest_grpo_mix/\ntest_grpo_singleton/\ntmpybtjiuc0/\n.d8bd37babaf2f1f7-00000000.node (5.9MB)\n.d8bd37fbfbb0f7ff-00000000.node (5.9MB)\n.d8bd3d9bdf6dd3f7-00000000.node (5.9MB)\n.d8bd3fbfbf21ddf7-00000000.node (5.9MB)\n.d8bd7dfebb38d5ff-00000000.node (5.9MB)\n.d8bdf5deba3bf1f7-00000000.node (5.9MB)\n.d8bdf7cfff76f3f7-00000000.node (5.9MB)\n.d8bdfd8ffe26d5ff-00000000.node (5.9MB)\n.d8bdfdeb9ba8dbff-00000000.node (5.9MB)\n.d8bdfffe9ab2ddff-00000000.node (5.9MB)\n.s.PGSQL.5432 (0B)\n.s.PGSQL.5432.lock (56B)\n__KMP_REGISTERED_LIB_75079 (1.0KB)\nagent_loop_new.rs (28.5KB)\nagent_mod.rs (1.6KB)\nauth_trace.md (10.8KB)\nbench-daily.md (72B)\nbench-log.md (109B)\nbench-meeting.md (160B)\nbench-monday.md (44B)\nbench-prefs.md (61B)\nbench-project.md (213B)\nbench-reminder.md (68B)\nbench-todo.md (153B)\nbench-tuesday.md (40B)\ncac-deck.html (151.5KB)\ncircuit_breaker.rs (22.1KB)\ncli_config.rs (9.1KB)\ncli_service.rs (1.1KB)\ncommands.rs (17.5KB)\nconfig.rs (58.4KB)\nconflicts_summary.md (21.4KB)\ncost_guard.rs (11.3KB)\ndebug_forc2.py (2.8KB)\ndebug_forc3.py (3.0KB)\ndebug_forc4.py (3.0KB)\ndebug_forc5.py (4.0KB)\ndebug_forc6.py (3.9KB)\ndebug_forc7.py (2.7KB)\ndebug_forc8.py (2.9KB)\ndebug_forc_prove.py (3.0KB)\ndispatcher.rs (26.2KB)\ndoctor.rs (8.4KB)\nhygiene.rs (7.3KB)\nironclaw_blog_test.png (21.4KB)\nironclaw_browser_test_viewport.png (156.1KB)\nironclaw_linkedin_debug.png (6.5KB)\nironclaw_spot_test.txt (19B)\nkeys_chain_signatures.rs (6.1KB)\nkeys_error.rs (1.6KB)\nkeys_intents.rs (5.8KB)\nkeys_mod.rs (31.6KB)\nkeys_policy.rs (31.4KB)\nkeys_rpc.rs (8.5KB)\nkeys_signer.rs (8.0KB)\nkeys_spending.rs (5.9KB)\nkeys_transaction.rs (13.7KB)\nkeys_types.rs (17.4KB)\nleak_detection_research_summary.md (13.0KB)\nleak_detector.rs (25.3KB)\nlib_new.rs (5.0KB)\nllm_mod.rs (10.0KB)\nmain.rs (56.8KB)\nmain_bootstrap.rs (11.0KB)\nmain_rs.txt (34.1KB)\nnear_resp.json (39B)\nobs_log.rs (5.9KB)\nobs_mod.rs (2.7KB)\nobs_multi.rs (4.0KB)\nobs_noop.rs (1.5KB)\nobs_traits.rs (4.0KB)\nour_bootstrap.rs (12.6KB)\nour_config.rs (45.2KB)\npost_final_review.py (6.4KB)\npost_review.py (4.6KB)\npost_review2.py (4.8KB)\npost_review3.py (2.2KB)\npr-35.diff (137.4KB)\npr115.diff (1.6MB)\npr126.diff (336.9KB)\npr14_full_diff.txt (193.4KB)\npr39_files.txt (243B)\npr51_files.txt (548B)\npr93_diff.txt (160.4KB)\nprs.json (4.6KB)\nresponse_cache.rs (15.1KB)\nrun_hard_proof_search.sh (545B)\nrun_hard_proof_search2.sh (515B)\nrun_proof_search_v3.sh (523B)\nrun_proof_search_v4.sh (524B)\nservice.rs (11.9KB)\nsettings.rs (34.7KB)\ntest_line.py (789B)\ntest_search.txt (1.8KB)\nthread_ops.rs (38.5KB)\ntool_wit.txt (6.0KB)\ntunnel_cf.rs (3.9KB)\ntunnel_custom.rs (6.2KB)\ntunnel_mod.rs (10.4KB)\ntunnel_ngrok.rs (4.2KB)\ntunnel_none.rs (1.3KB)\ntunnel_ts.rs (3.5KB)\nwasm_capabilities.rs (15.2KB)\nwasm_capabilities_schema.rs (26.2KB)\nwasm_host.rs (25.3KB)\nwasm_wrapper.rs (14.9KB)\n```","started_at":"2026-02-17T22:02:24.602248Z","finished_at":"2026-02-17T22:02:43.748463Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-write-read","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5712,"llm_calls":3,"input_tokens":4489,"output_tokens":73,"estimated_cost_usd":0.014562,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It says: `ironclaw spot check`.","started_at":"2026-02-17T22:02:43.749933Z","finished_at":"2026-02-17T22:02:49.462007Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-shell-json","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4582,"llm_calls":3,"input_tokens":4433,"output_tokens":94,"estimated_cost_usd":0.014709,"tool_calls":[{"name":"shell","duration_ms":0,"success":true},{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"[Called tool `json` with arguments: {\"data\":{\"code\":200,\"status\":\"ok\"},\"operation\":\"query\",\"path\":\"status\"}]","started_at":"2026-02-17T22:02:49.463794Z","finished_at":"2026-02-17T22:02:54.046092Z","config_label":"default","error":null}
|
||||
{"task_id":"chain-time-echo","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4583,"llm_calls":3,"input_tokens":4404,"output_tokens":76,"estimated_cost_usd":0.014352,"tool_calls":[{"name":"time","duration_ms":0,"success":true},{"name":"echo","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"2026-02-17T22:02:55.786456+00:00","started_at":"2026-02-17T22:02:54.046306Z","finished_at":"2026-02-17T22:02:58.629389Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-no-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":1020,"llm_calls":1,"input_tokens":1366,"output_tokens":5,"estimated_cost_usd":0.004173,"tool_calls":[],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Paris.","started_at":"2026-02-17T22:02:58.630502Z","finished_at":"2026-02-17T22:02:59.650655Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-correct-tool","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":3980,"llm_calls":2,"input_tokens":2825,"output_tokens":50,"estimated_cost_usd":0.009225,"tool_calls":[{"name":"time","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"It’s **22:03:01 UTC** right now (**2026-02-17**).","started_at":"2026-02-17T22:02:59.650901Z","finished_at":"2026-02-17T22:03:03.631122Z","config_label":"default","error":null}
|
||||
{"task_id":"robust-json-validate","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":2954,"llm_calls":2,"input_tokens":2822,"output_tokens":43,"estimated_cost_usd":0.009111,"tool_calls":[{"name":"json","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The JSON is valid (`valid: true`).","started_at":"2026-02-17T22:03:03.631527Z","finished_at":"2026-02-17T22:03:06.585971Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-daily","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4890,"llm_calls":3,"input_tokens":4547,"output_tokens":114,"estimated_cost_usd":0.015351,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"```md\n# Daily Tasks\n\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\n```","started_at":"2026-02-17T22:03:06.586556Z","finished_at":"2026-02-17T22:03:11.476648Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-reminder","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5607,"llm_calls":3,"input_tokens":4531,"output_tokens":95,"estimated_cost_usd":0.015018,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The appointment is on **March 5th at 2pm**, with **Dr. Smith**.","started_at":"2026-02-17T22:03:11.476873Z","finished_at":"2026-02-17T22:03:17.084343Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-save-meeting","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6113,"llm_calls":3,"input_tokens":4750,"output_tokens":122,"estimated_cost_usd":0.01608,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Frontend owner: **Bob** \nLaunch date: **April 15th**","started_at":"2026-02-17T22:03:17.085432Z","finished_at":"2026-02-17T22:03:23.198905Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-append-log","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9880,"llm_calls":6,"input_tokens":9941,"output_tokens":208,"estimated_cost_usd":0.032943,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"shell","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Contents of `/tmp/bench-log.md`:\n1. Morning standup: discussed blockers on auth service \n2. Afternoon: resolved API timeout issue in payment gateway \n\nSummary: You discussed blockers affecting the auth service during the morning standup, and later in the afternoon you resolved an API timeout issue in the payment gateway.","started_at":"2026-02-17T22:03:23.199155Z","finished_at":"2026-02-17T22:03:33.079396Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-recall-detail","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":4989,"llm_calls":3,"input_tokens":4721,"output_tokens":121,"estimated_cost_usd":0.015978,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"The project uses **PostgreSQL** as the database, and the PM is **James**.","started_at":"2026-02-17T22:03:33.080611Z","finished_at":"2026-02-17T22:03:38.069896Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-todo-priority","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":5916,"llm_calls":3,"input_tokens":4676,"output_tokens":110,"estimated_cost_usd":0.015678,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"High priority tasks:\n- Fix login bug\n- Security audit","started_at":"2026-02-17T22:03:38.070162Z","finished_at":"2026-02-17T22:03:43.986831Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-multifile","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":6081,"llm_calls":4,"input_tokens":6442,"output_tokens":196,"estimated_cost_usd":0.022266,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"On Tuesday you have **Sprint retro at 11am**, then **team lunch at noon**.","started_at":"2026-02-17T22:03:43.988013Z","finished_at":"2026-02-17T22:03:50.069452Z","config_label":"default","error":null}
|
||||
{"task_id":"memory-update-context","suite_id":"spot","score":{"value":1.0,"label":"pass","details":null},"trace":{"wall_time_ms":9164,"llm_calls":5,"input_tokens":8354,"output_tokens":252,"estimated_cost_usd":0.028842,"tool_calls":[{"name":"write_file","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true},{"name":"apply_patch","duration_ms":0,"success":true},{"name":"read_file","duration_ms":0,"success":true}],"turns":1,"hit_iteration_limit":false,"hit_timeout":false},"response":"Final read confirms the file now says `timezone: EST`.","started_at":"2026-02-17T22:03:50.070277Z","finished_at":"2026-02-17T22:03:59.235196Z","config_label":"default","error":null}
|
||||
@@ -1,21 +0,0 @@
|
||||
{"id": "smoke-greeting", "prompt": "Hello! Introduce yourself briefly.", "tags": ["smoke"], "assertions": {"response_matches": "(?i)(hello|hi|hey|assistant|agent|help)", "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "smoke-math", "prompt": "What is 47 * 23? Reply with just the number.", "tags": ["smoke"], "assertions": {"response_contains": ["1081"], "no_error": true, "max_tool_calls": 0}}
|
||||
{"id": "tool-echo", "prompt": "Use the echo tool to repeat the message: 'Spot check passed'", "tags": ["tool"], "assertions": {"tools_used": ["echo"], "response_contains": ["Spot check passed"], "no_error": true}}
|
||||
{"id": "tool-time", "prompt": "What is the current date and time? Use the time tool.", "tags": ["tool"], "assertions": {"tools_used": ["time"], "response_matches": "20\\d{2}", "no_error": true}}
|
||||
{"id": "tool-json-query", "prompt": "Given this JSON: {\"users\": [{\"name\": \"Alice\"}, {\"name\": \"Bob\"}]}, use the json tool to extract the second user's name.", "tags": ["tool"], "assertions": {"tools_used": ["json"], "response_contains": ["Bob"], "no_error": true}}
|
||||
{"id": "tool-shell-echo", "prompt": "Use the shell tool to run: echo 'benchmark test'", "tags": ["tool"], "assertions": {"tools_used": ["shell"], "response_contains": ["benchmark test"], "no_error": true}}
|
||||
{"id": "tool-list-dir", "prompt": "Use the list_dir tool to list the contents of the /tmp directory.", "tags": ["tool"], "assertions": {"tools_used": ["list_dir"], "no_error": true}}
|
||||
{"id": "chain-write-read", "prompt": "Write the text 'ironclaw spot check' to /tmp/ironclaw_spot_test.txt using the write_file tool, then read it back using the read_file tool and tell me what it says.", "tags": ["chain"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["ironclaw spot check"], "no_error": true}}
|
||||
{"id": "chain-shell-json", "prompt": "Run a shell command to output the JSON string '{\"status\": \"ok\", \"code\": 200}', then use the json tool to extract the status field.", "tags": ["chain"], "assertions": {"response_contains": ["ok"], "min_tool_calls": 1, "no_error": true}}
|
||||
{"id": "chain-time-echo", "prompt": "First get the current time using the time tool, then use the echo tool to repeat it back.", "tags": ["chain"], "assertions": {"tools_used": ["time", "echo"], "no_error": true}}
|
||||
{"id": "robust-no-tool", "prompt": "What is the capital of France? Answer directly without using any tools.", "tags": ["robust"], "assertions": {"response_contains": ["Paris"], "max_tool_calls": 0, "no_error": true}}
|
||||
{"id": "robust-correct-tool", "prompt": "What time is it right now?", "tags": ["robust"], "assertions": {"tools_used": ["time"], "tools_not_used": ["shell", "echo"], "no_error": true}}
|
||||
{"id": "robust-json-validate", "prompt": "Use the json tool to validate whether this is valid JSON: {\"key\": \"value\", \"num\": 42}", "tags": ["robust"], "assertions": {"tools_used": ["json"], "tools_not_used": ["shell"], "no_error": true}}
|
||||
{"id": "memory-save-daily", "prompt": "Save these daily tasks to /tmp/bench-daily.md:\n1. Review PR #42\n2. Update API docs\n3. Deploy to staging\nThen read the file back and confirm what was saved.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PR #42", "docs", "staging"], "no_error": true}}
|
||||
{"id": "memory-save-reminder", "prompt": "Write a reminder to /tmp/bench-reminder.md: Dentist appointment on March 5th at 2pm with Dr. Smith. Then read the file back and tell me when the appointment is and with whom.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["March 5", "Smith"], "response_matches": "2(:00)?\\s*[Pp][Mm]", "no_error": true}}
|
||||
{"id": "memory-save-meeting", "prompt": "Save these meeting notes to /tmp/bench-meeting.md:\nMeeting: Project Phoenix sync\nAttendees: Alice, Bob, Carol\nDecisions:\n- Launch date: April 15th\n- Budget: $50k approved\n- Bob owns frontend, Carol owns backend\nThen read the file back and tell me who owns the frontend and what the launch date is.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["Bob", "frontend", "April 15"], "no_error": true}}
|
||||
{"id": "memory-append-log", "prompt": "Write 'Morning standup: discussed blockers on auth service' to /tmp/bench-log.md. Then append a new line 'Afternoon: resolved API timeout issue in payment gateway' to the same file. Finally read the full file and summarize what happened.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["auth", "timeout"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-recall-detail", "prompt": "Save the following project context to /tmp/bench-project.md:\nProject Ironclad uses Rust for the backend, React for the frontend, and PostgreSQL for the database. The API is deployed on AWS ECS. The lead developer is Sarah and the PM is James. The sprint ends on March 20th.\nThen read it back and answer: What database does the project use, and who is the PM?", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["PostgreSQL", "James"], "no_error": true}}
|
||||
{"id": "memory-todo-priority", "prompt": "Write the following to /tmp/bench-todo.md:\n- [ ] Fix login bug (priority: HIGH)\n- [ ] Write unit tests (priority: medium)\n- [ ] Update README (priority: low)\n- [ ] Security audit (priority: HIGH)\nThen read it back and tell me which tasks are high priority.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["login bug", "security audit"], "no_error": true}}
|
||||
{"id": "memory-multifile", "prompt": "Save 'Team standup at 9am, then client demo at 2pm' to /tmp/bench-monday.md and 'Sprint retro at 11am, team lunch at noon' to /tmp/bench-tuesday.md. Then read both files and tell me what's happening on Tuesday.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["retro", "lunch"], "min_tool_calls": 3, "no_error": true}}
|
||||
{"id": "memory-update-context", "prompt": "Write 'User preference: dark mode, timezone: PST, language: English' to /tmp/bench-prefs.md. Then read it back, and rewrite the file changing the timezone to EST. Finally read it one more time and confirm the timezone is now EST.", "tags": ["memory"], "assertions": {"tools_used": ["write_file", "read_file"], "response_contains": ["EST"], "min_tool_calls": 4, "no_error": true}}
|
||||
@@ -1,8 +0,0 @@
|
||||
task_timeout = "120s"
|
||||
parallelism = 1
|
||||
|
||||
[[matrix]]
|
||||
label = "default"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "benchmarks/data/spot.jsonl"
|
||||
@@ -1,243 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// A single entry in the custom JSONL format.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CustomEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
expected: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_contains: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_regex: Option<String>,
|
||||
/// "exact", "contains", "regex", or "llm" (default: "exact")
|
||||
#[serde(default = "default_scorer")]
|
||||
scorer: String,
|
||||
}
|
||||
|
||||
fn default_scorer() -> String {
|
||||
"exact".to_string()
|
||||
}
|
||||
|
||||
/// Custom JSONL benchmark suite.
|
||||
///
|
||||
/// Each line of the JSONL file is a task with `id`, `prompt`, and scoring
|
||||
/// criteria (`expected`, `expected_contains`, `expected_regex`).
|
||||
pub struct CustomSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl CustomSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for CustomSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Custom JSONL"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"custom"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: CustomEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut metadata = serde_json::json!({
|
||||
"scorer": entry.scorer,
|
||||
});
|
||||
if let Some(ref expected) = entry.expected {
|
||||
metadata["expected"] = serde_json::Value::String(expected.clone());
|
||||
}
|
||||
if let Some(ref expected_contains) = entry.expected_contains {
|
||||
metadata["expected_contains"] =
|
||||
serde_json::Value::String(expected_contains.clone());
|
||||
}
|
||||
if let Some(ref expected_regex) = entry.expected_regex {
|
||||
metadata["expected_regex"] = serde_json::Value::String(expected_regex.clone());
|
||||
}
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let scorer = task
|
||||
.metadata
|
||||
.get("scorer")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("exact");
|
||||
|
||||
match scorer {
|
||||
"exact" => {
|
||||
if let Some(expected) = task.metadata.get("expected").and_then(|v| v.as_str()) {
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected' field for exact scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"contains" => {
|
||||
if let Some(expected) = task
|
||||
.metadata
|
||||
.get("expected_contains")
|
||||
.and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::contains_match(expected, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_contains' field for contains scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"regex" => {
|
||||
if let Some(pattern) = task.metadata.get("expected_regex").and_then(|v| v.as_str())
|
||||
{
|
||||
Ok(scoring::regex_match(pattern, &submission.response))
|
||||
} else {
|
||||
Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "no 'expected_regex' field for regex scoring".to_string(),
|
||||
})
|
||||
}
|
||||
}
|
||||
"llm" => {
|
||||
// TODO: LLM-as-judge scoring
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"LLM-as-judge scoring not implemented, returning placeholder 0.5"
|
||||
);
|
||||
Ok(BenchScore::partial(0.5, "LLM scoring not yet implemented"))
|
||||
}
|
||||
other => Err(BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("unknown scorer: {other}"),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t2", "prompt": "Say hello", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "t1");
|
||||
assert_eq!(tasks[1].id, "t2");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_exact_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "What is 2+2?", "expected": "4"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "4".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_custom_contains_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "prompt": "Greet me", "expected_contains": "hello", "scorer": "contains"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = CustomSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: "Hello there!".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::scoring;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskResource, TaskSubmission};
|
||||
|
||||
/// GAIA dataset entry (Hugging Face JSONL format).
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct GaiaEntry {
|
||||
task_id: String,
|
||||
#[serde(alias = "Question")]
|
||||
question: String,
|
||||
#[serde(alias = "Final answer", alias = "final_answer")]
|
||||
final_answer: String,
|
||||
#[serde(alias = "Level", default)]
|
||||
level: Option<u32>,
|
||||
#[serde(alias = "file_name", default)]
|
||||
file_name: Option<String>,
|
||||
}
|
||||
|
||||
/// GAIA benchmark suite.
|
||||
///
|
||||
/// Tasks are loaded from HuggingFace JSONL exports. Scoring uses normalized
|
||||
/// exact match against the `final_answer` field.
|
||||
pub struct GaiaSuite {
|
||||
dataset_path: PathBuf,
|
||||
attachments_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl GaiaSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
attachments_dir: Option<impl Into<PathBuf>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
attachments_dir: attachments_dir.map(|d| d.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for GaiaSuite {
|
||||
fn name(&self) -> &str {
|
||||
"GAIA"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"gaia"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: GaiaEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("GAIA line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let mut resources = Vec::new();
|
||||
if let Some(ref file_name) = entry.file_name {
|
||||
if !file_name.is_empty() {
|
||||
if let Some(ref dir) = self.attachments_dir {
|
||||
resources.push(TaskResource {
|
||||
name: file_name.clone(),
|
||||
path: dir.join(file_name).to_string_lossy().to_string(),
|
||||
resource_type: crate::suite::ResourceType::File,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut tags = Vec::new();
|
||||
if let Some(level) = entry.level {
|
||||
tags.push(format!("level-{level}"));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"expected": entry.final_answer,
|
||||
"level": entry.level,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.task_id,
|
||||
prompt: entry.question,
|
||||
context: None,
|
||||
resources,
|
||||
tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let expected = task
|
||||
.metadata
|
||||
.get("expected")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing expected answer in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
Ok(scoring::exact_match(expected, &submission.response))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "What is the capital of France?", "final_answer": "Paris", "Level": 1}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "g1");
|
||||
assert!(tasks[0].tags.contains(&"level-1".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_gaia_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("gaia.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"task_id": "g1", "question": "Capital of France?", "final_answer": "Paris"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = GaiaSuite::new(&path, None::<PathBuf>);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Exact match (case insensitive)
|
||||
let submission = TaskSubmission {
|
||||
response: "paris".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
|
||||
// Wrong answer
|
||||
let submission = TaskSubmission {
|
||||
response: "London".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
pub mod custom;
|
||||
pub mod gaia;
|
||||
pub mod spot;
|
||||
pub mod swe_bench;
|
||||
pub mod tau_bench;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchSuite;
|
||||
|
||||
/// List of all known suite IDs.
|
||||
pub const KNOWN_SUITES: &[(&str, &str)] = &[
|
||||
("custom", "Custom JSONL tasks"),
|
||||
("gaia", "GAIA benchmark (knowledge & reasoning)"),
|
||||
("spot", "Spot checks (end-to-end user workflows)"),
|
||||
("tau_bench", "Tau-bench (multi-turn tool use)"),
|
||||
("swe_bench", "SWE-bench Pro (software engineering)"),
|
||||
];
|
||||
|
||||
/// Create a suite adapter by name.
|
||||
pub fn create_suite(name: &str, config: &BenchConfig) -> Result<Box<dyn BenchSuite>, BenchError> {
|
||||
let suite_map = config.suite_config_map();
|
||||
match name {
|
||||
"custom" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'custom' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(custom::CustomSuite::new(dataset_path)))
|
||||
}
|
||||
"gaia" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'gaia' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let attachments_dir = suite_map
|
||||
.get("attachments_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string());
|
||||
Ok(Box::new(gaia::GaiaSuite::new(
|
||||
dataset_path,
|
||||
attachments_dir,
|
||||
)))
|
||||
}
|
||||
"spot" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'spot' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
Ok(Box::new(spot::SpotSuite::new(dataset_path)))
|
||||
}
|
||||
"tau_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'tau_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let domain = suite_map
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("retail")
|
||||
.to_string();
|
||||
Ok(Box::new(tau_bench::TauBenchSuite::new(
|
||||
dataset_path,
|
||||
domain,
|
||||
)))
|
||||
}
|
||||
"swe_bench" => {
|
||||
let dataset_path = suite_map
|
||||
.get("dataset_path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.ok_or_else(|| {
|
||||
BenchError::Config(
|
||||
"suite_config.dataset_path is required for 'swe_bench' suite".to_string(),
|
||||
)
|
||||
})?;
|
||||
let workspace_dir = suite_map
|
||||
.get("workspace_dir")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("/tmp/swe-bench")
|
||||
.to_string();
|
||||
let use_docker = suite_map
|
||||
.get("use_docker")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
Ok(Box::new(swe_bench::SweBenchSuite::new(
|
||||
dataset_path,
|
||||
workspace_dir,
|
||||
use_docker,
|
||||
)))
|
||||
}
|
||||
_ => {
|
||||
let available = KNOWN_SUITES
|
||||
.iter()
|
||||
.map(|(id, _)| *id)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
Err(BenchError::SuiteNotFound {
|
||||
name: name.to_string(),
|
||||
available,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,504 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Multi-criterion assertions for a spot check scenario.
|
||||
///
|
||||
/// Each field generates one or more individual checks. The final score is
|
||||
/// `passed_checks / total_checks`, giving a value between 0.0 and 1.0.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct SpotAssertions {
|
||||
/// All must appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_contains: Vec<String>,
|
||||
|
||||
/// None may appear in the response (case-insensitive).
|
||||
#[serde(default)]
|
||||
pub response_not_contains: Vec<String>,
|
||||
|
||||
/// Each tool name must appear in the tool_calls list (checked by name,
|
||||
/// not by count; duplicates in tool_calls are collapsed).
|
||||
#[serde(default)]
|
||||
pub tools_used: Vec<String>,
|
||||
|
||||
/// None of these tool names may appear in the tool_calls list.
|
||||
#[serde(default)]
|
||||
pub tools_not_used: Vec<String>,
|
||||
|
||||
/// Regex pattern the response must match.
|
||||
#[serde(default)]
|
||||
pub response_matches: Option<String>,
|
||||
|
||||
/// Hard fail if the task produced an error.
|
||||
#[serde(default)]
|
||||
pub no_error: bool,
|
||||
|
||||
/// Minimum number of tool calls expected (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub min_tool_calls: Option<usize>,
|
||||
|
||||
/// Maximum number of tool calls allowed (counts duplicates).
|
||||
#[serde(default)]
|
||||
pub max_tool_calls: Option<usize>,
|
||||
}
|
||||
|
||||
impl SpotAssertions {
|
||||
/// Evaluate all assertions against a submission, returning (score, failure_details).
|
||||
pub fn evaluate(&self, submission: &TaskSubmission) -> (f64, Vec<String>) {
|
||||
let mut passed: usize = 0;
|
||||
let mut total: usize = 0;
|
||||
let mut failures: Vec<String> = Vec::new();
|
||||
|
||||
// Hard fail: error check
|
||||
if self.no_error {
|
||||
total += 1;
|
||||
if let Some(ref err) = submission.error {
|
||||
failures.push(format!("no_error: task errored with: {err}"));
|
||||
// Hard fail: return 0.0 immediately
|
||||
return (0.0, failures);
|
||||
}
|
||||
passed += 1;
|
||||
}
|
||||
|
||||
let response_lower = submission.response.to_lowercase();
|
||||
|
||||
// response_contains: all must appear
|
||||
for needle in &self.response_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_contains: missing \"{needle}\""));
|
||||
}
|
||||
}
|
||||
|
||||
// response_not_contains: none may appear
|
||||
for needle in &self.response_not_contains {
|
||||
total += 1;
|
||||
if response_lower.contains(&needle.to_lowercase()) {
|
||||
failures.push(format!("response_not_contains: found \"{needle}\""));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let tool_set: HashSet<&str> = submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
|
||||
// tools_used: each must appear
|
||||
for tool in &self.tools_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("tools_used: \"{tool}\" not called"));
|
||||
}
|
||||
}
|
||||
|
||||
// tools_not_used: none may appear
|
||||
for tool in &self.tools_not_used {
|
||||
total += 1;
|
||||
if tool_set.contains(tool.as_str()) {
|
||||
failures.push(format!("tools_not_used: \"{tool}\" was called"));
|
||||
} else {
|
||||
passed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// response_matches: regex pattern
|
||||
if let Some(ref pattern) = self.response_matches {
|
||||
total += 1;
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(&submission.response) {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!("response_matches: /{pattern}/ did not match"));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
failures.push(format!("response_matches: bad regex: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let call_count = submission.tool_calls.len();
|
||||
|
||||
// min_tool_calls
|
||||
if let Some(min) = self.min_tool_calls {
|
||||
total += 1;
|
||||
if call_count >= min {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"min_tool_calls: expected >= {min}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// max_tool_calls
|
||||
if let Some(max) = self.max_tool_calls {
|
||||
total += 1;
|
||||
if call_count <= max {
|
||||
passed += 1;
|
||||
} else {
|
||||
failures.push(format!(
|
||||
"max_tool_calls: expected <= {max}, got {call_count}"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if total == 0 {
|
||||
return (1.0, failures);
|
||||
}
|
||||
|
||||
let score = passed as f64 / total as f64;
|
||||
(score, failures)
|
||||
}
|
||||
}
|
||||
|
||||
/// JSONL entry for a spot check scenario.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SpotEntry {
|
||||
id: String,
|
||||
prompt: String,
|
||||
#[serde(default)]
|
||||
context: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
assertions: SpotAssertions,
|
||||
}
|
||||
|
||||
/// Spot benchmark suite: end-to-end checks for real user workflows.
|
||||
///
|
||||
/// Tests conversation, individual tool use, multi-tool chaining, and robustness.
|
||||
/// Each task declares multi-criterion assertions scored as passed/total.
|
||||
pub struct SpotSuite {
|
||||
dataset_path: PathBuf,
|
||||
}
|
||||
|
||||
impl SpotSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SpotSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Spot Checks"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"spot"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path).map_err(BenchError::Io)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SpotEntry = serde_json::from_str(trimmed)
|
||||
.map_err(|e| BenchError::Config(format!("spot line {}: {}", line_num + 1, e)))?;
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"assertions": serde_json::to_value(&entry.assertions)
|
||||
.map_err(|e| BenchError::Config(format!("spot {}: {}", entry.id, e)))?,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.prompt,
|
||||
context: entry.context,
|
||||
resources: vec![],
|
||||
tags: entry.tags,
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
let assertions: SpotAssertions = task
|
||||
.metadata
|
||||
.get("assertions")
|
||||
.ok_or_else(|| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing assertions in metadata".to_string(),
|
||||
})
|
||||
.and_then(|v| {
|
||||
serde_json::from_value(v.clone()).map_err(|e| BenchError::Scoring {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("bad assertions: {e}"),
|
||||
})
|
||||
})?;
|
||||
|
||||
let (score, failures) = assertions.evaluate(submission);
|
||||
|
||||
if score >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if score <= 0.0 {
|
||||
Ok(BenchScore::fail(failures.join("; ")))
|
||||
} else {
|
||||
Ok(BenchScore::partial(score, failures.join("; ")))
|
||||
}
|
||||
}
|
||||
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![
|
||||
Arc::new(ironclaw::tools::builtin::ShellTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ReadFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::WriteFileTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ListDirTool::new()),
|
||||
Arc::new(ironclaw::tools::builtin::ApplyPatchTool::new()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
fn make_submission(
|
||||
response: &str,
|
||||
tool_calls: Vec<&str>,
|
||||
error: Option<&str>,
|
||||
) -> TaskSubmission {
|
||||
TaskSubmission {
|
||||
response: response.to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: tool_calls.into_iter().map(|s| s.to_string()).collect(),
|
||||
error: error.map(|s| s.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_all_pass() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["hello".to_string()],
|
||||
tools_used: vec!["echo".to_string()],
|
||||
no_error: true,
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello, world!", vec!["echo"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hard_fail_on_error() {
|
||||
let assertions = SpotAssertions {
|
||||
no_error: true,
|
||||
response_contains: vec!["hello".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("Hello!", vec![], Some("timeout after 60s"));
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
assert!(failures[0].contains("no_error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_score() {
|
||||
let assertions = SpotAssertions {
|
||||
response_contains: vec!["alpha".to_string(), "beta".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("alpha is here but not the other", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("beta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_not_contains() {
|
||||
let assertions = SpotAssertions {
|
||||
response_not_contains: vec!["error".to_string(), "fail".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("This is an error message", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert!(failures[0].contains("error"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_used_and_not_used() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_used: vec!["time".to_string()],
|
||||
tools_not_used: vec!["shell".to_string(), "echo".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The time is now", vec!["time"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_not_used_fails() {
|
||||
let assertions = SpotAssertions {
|
||||
tools_not_used: vec!["shell".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("result", vec!["shell", "time"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"\d{4}".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("The year is 2026", vec![], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
assert!(failures.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_matches_regex_fail() {
|
||||
let assertions = SpotAssertions {
|
||||
response_matches: Some(r"^\d+$".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("not a number", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_min_max_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
min_tool_calls: Some(2),
|
||||
max_tool_calls: Some(4),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// Within range
|
||||
let sub = make_submission("ok", vec!["a", "b", "c"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
// Too few
|
||||
let sub = make_submission("ok", vec!["a"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("min_tool_calls"));
|
||||
|
||||
// Too many
|
||||
let sub = make_submission("ok", vec!["a", "b", "c", "d", "e"], None);
|
||||
let (score, failures) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.5);
|
||||
assert!(failures[0].contains("max_tool_calls"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_zero_tool_calls() {
|
||||
let assertions = SpotAssertions {
|
||||
max_tool_calls: Some(0),
|
||||
..Default::default()
|
||||
};
|
||||
let sub = make_submission("just talking", vec![], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
|
||||
let sub = make_submission("oops", vec!["echo"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_assertions() {
|
||||
let assertions = SpotAssertions::default();
|
||||
let sub = make_submission("anything", vec!["whatever"], None);
|
||||
let (score, _) = assertions.evaluate(&sub);
|
||||
assert_eq!(score, 1.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_load_tasks() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "tags": ["smoke"], "assertions": {{"response_contains": ["hello"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s2", "prompt": "Echo test", "assertions": {{"tools_used": ["echo"]}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 2);
|
||||
assert_eq!(tasks[0].id, "s1");
|
||||
assert_eq!(tasks[1].id, "s2");
|
||||
assert!(tasks[0].tags.contains(&"smoke".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_spot_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("spot.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "s1", "prompt": "Hello", "assertions": {{"response_contains": ["hello", "world"], "no_error": true}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SpotSuite::new(&path);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Full pass
|
||||
let sub = make_submission("Hello World!", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
|
||||
// Partial
|
||||
let sub = make_submission("Hello there", vec![], None);
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert!(score.value > 0.0 && score.value < 1.0);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Error hard fail
|
||||
let sub = make_submission("Hello World!", vec![], Some("boom"));
|
||||
let score = suite.score(&tasks[0], &sub).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
}
|
||||
@@ -1,416 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, TaskSubmission};
|
||||
|
||||
/// Validate that a string is safe for use as a filesystem path component.
|
||||
/// Allows alphanumerics, hyphens, underscores, dots, and forward slashes (for nested paths).
|
||||
/// Rejects absolute paths, `..` traversal, and shell metacharacters.
|
||||
fn is_safe_path_component(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& !s.starts_with('/')
|
||||
&& !s.contains("..")
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
}
|
||||
|
||||
/// Validate that a repo string matches the expected `owner/repo` GitHub format.
|
||||
fn is_valid_github_repo(repo: &str) -> bool {
|
||||
// Match "owner/repo" where both parts are alphanumeric with hyphens/underscores/dots
|
||||
static REPO_PATTERN: std::sync::LazyLock<Regex> =
|
||||
std::sync::LazyLock::new(|| Regex::new(r"^[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+$").unwrap());
|
||||
REPO_PATTERN.is_match(repo)
|
||||
}
|
||||
|
||||
/// Validate that a string looks like a git ref (hex SHA or valid ref name).
|
||||
fn is_valid_git_ref(s: &str) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '/'))
|
||||
&& !s.contains("..")
|
||||
}
|
||||
|
||||
/// SWE-bench dataset entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SweBenchEntry {
|
||||
instance_id: String,
|
||||
repo: String,
|
||||
base_commit: String,
|
||||
#[serde(default)]
|
||||
problem_statement: String,
|
||||
#[serde(default)]
|
||||
hints_text: Option<String>,
|
||||
#[serde(default)]
|
||||
test_patch: Option<String>,
|
||||
#[serde(default)]
|
||||
patch: Option<String>,
|
||||
}
|
||||
|
||||
/// SWE-bench Pro: real-world software engineering tasks.
|
||||
///
|
||||
/// Each task clones a repo at a specific commit, presents the problem statement,
|
||||
/// and expects the agent to produce a patch. Scoring runs the test suite.
|
||||
pub struct SweBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
workspace_dir: PathBuf,
|
||||
use_docker: bool,
|
||||
}
|
||||
|
||||
impl SweBenchSuite {
|
||||
pub fn new(
|
||||
dataset_path: impl Into<PathBuf>,
|
||||
workspace_dir: impl Into<PathBuf>,
|
||||
use_docker: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
workspace_dir: workspace_dir.into(),
|
||||
use_docker,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for SweBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"SWE-bench Pro"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"swe_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: SweBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("swe_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
if !is_safe_path_component(&entry.instance_id) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: unsafe instance_id \"{}\"",
|
||||
line_num + 1,
|
||||
entry.instance_id,
|
||||
)));
|
||||
}
|
||||
if !is_valid_github_repo(&entry.repo) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid repo format \"{}\"",
|
||||
line_num + 1,
|
||||
entry.repo,
|
||||
)));
|
||||
}
|
||||
if !is_valid_git_ref(&entry.base_commit) {
|
||||
return Err(BenchError::Config(format!(
|
||||
"swe_bench line {}: invalid base_commit \"{}\"",
|
||||
line_num + 1,
|
||||
entry.base_commit,
|
||||
)));
|
||||
}
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"repo": entry.repo,
|
||||
"base_commit": entry.base_commit,
|
||||
"test_patch": entry.test_patch,
|
||||
"gold_patch": entry.patch,
|
||||
"use_docker": self.use_docker,
|
||||
"workspace_dir": self.workspace_dir.to_string_lossy(),
|
||||
});
|
||||
|
||||
let prompt = if let Some(ref hints) = entry.hints_text {
|
||||
format!("{}\n\nHints:\n{}", entry.problem_statement, hints)
|
||||
} else {
|
||||
entry.problem_statement
|
||||
};
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.instance_id,
|
||||
prompt,
|
||||
context: Some(format!(
|
||||
"Repository: {}, Commit: {}",
|
||||
entry.repo, entry.base_commit
|
||||
)),
|
||||
resources: vec![],
|
||||
tags: vec![format!("repo-{}", entry.repo.replace('/', "-"))],
|
||||
expected_turns: None,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn setup_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let repo = task
|
||||
.metadata
|
||||
.get("repo")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing repo in metadata".to_string(),
|
||||
})?;
|
||||
let base_commit = task
|
||||
.metadata
|
||||
.get("base_commit")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: "missing base_commit in metadata".to_string(),
|
||||
})?;
|
||||
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
|
||||
// Clone repo if not already present
|
||||
if !task_dir.exists() {
|
||||
let repo_url = format!("https://github.com/{}.git", repo);
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
"--depth",
|
||||
"1",
|
||||
&repo_url,
|
||||
&task_dir.to_string_lossy(),
|
||||
])
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git clone failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Checkout the base commit
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
// Shallow clone might not have the commit; fetch more history
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["fetch", "--unshallow"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["checkout", base_commit])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout retry failed: {e}"),
|
||||
})?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(BenchError::TaskFailed {
|
||||
task_id: task.id.clone(),
|
||||
reason: format!("git checkout failed: {stderr}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn teardown_task(&self, task: &BenchTask) -> Result<(), BenchError> {
|
||||
let task_dir = self.workspace_dir.join(&task.id);
|
||||
if task_dir.exists() {
|
||||
// Reset any changes
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["checkout", "."])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["clean", "-fdx"])
|
||||
.current_dir(&task_dir)
|
||||
.output()
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// For SWE-bench, scoring requires running the test patch against the agent's changes.
|
||||
// This is a simplified version that checks if the agent produced any code changes.
|
||||
|
||||
let test_patch = task.metadata.get("test_patch").and_then(|v| v.as_str());
|
||||
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response from agent"));
|
||||
}
|
||||
|
||||
// If we have a test patch, try to verify the submission
|
||||
if let Some(_test_patch) = test_patch {
|
||||
// TODO: Apply agent's patch, then apply test patch, then run tests.
|
||||
// For now, give partial credit if the agent produced some output.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"SWE-bench test execution not implemented, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"test execution not yet implemented; partial credit for response",
|
||||
))
|
||||
} else {
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"no test_patch available, returning placeholder 0.25"
|
||||
);
|
||||
Ok(BenchScore::partial(
|
||||
0.25,
|
||||
"no test_patch available for automated scoring",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "django__django-12345", "repo": "django/django", "base_commit": "abc123", "problem_statement": "Fix the ORM bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].id, "django__django-12345");
|
||||
assert!(tasks[0].tags.contains(&"repo-django-django".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_scoring_no_response() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "s1", "repo": "org/repo", "base_commit": "abc", "problem_statement": "Fix bug"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
let submission = TaskSubmission {
|
||||
response: String::new(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec![],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_safe_path_component() {
|
||||
assert!(is_safe_path_component("django__django-12345"));
|
||||
assert!(is_safe_path_component("org/repo"));
|
||||
assert!(is_safe_path_component("abc123"));
|
||||
assert!(!is_safe_path_component(""));
|
||||
assert!(!is_safe_path_component("../../etc/passwd"));
|
||||
assert!(!is_safe_path_component("/etc/passwd"));
|
||||
assert!(!is_safe_path_component("foo;rm -rf /"));
|
||||
assert!(!is_safe_path_component("foo bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_github_repo() {
|
||||
assert!(is_valid_github_repo("django/django"));
|
||||
assert!(is_valid_github_repo("org/repo-name"));
|
||||
assert!(is_valid_github_repo("Org.Name/Repo_v2"));
|
||||
assert!(!is_valid_github_repo(""));
|
||||
assert!(!is_valid_github_repo("no-slash"));
|
||||
assert!(!is_valid_github_repo("too/many/slashes"));
|
||||
assert!(!is_valid_github_repo("spa ce/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_valid_git_ref() {
|
||||
assert!(is_valid_git_ref("abc123"));
|
||||
assert!(is_valid_git_ref("deadbeef0123456789abcdef0123456789abcdef"));
|
||||
assert!(is_valid_git_ref("v1.2.3"));
|
||||
assert!(is_valid_git_ref("main"));
|
||||
assert!(!is_valid_git_ref(""));
|
||||
assert!(!is_valid_git_ref("bad..ref"));
|
||||
assert!(!is_valid_git_ref("has space"));
|
||||
assert!(!is_valid_git_ref("semi;colon"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_path_traversal() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "../../etc/passwd", "repo": "org/repo", "base_commit": "abc", "problem_statement": "evil"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("unsafe instance_id"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_swe_bench_rejects_bad_repo() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("swe.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"instance_id": "task1", "repo": "not-a-repo-format", "base_commit": "abc", "problem_statement": "bad"}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = SweBenchSuite::new(&path, "/tmp/swe-test", false);
|
||||
let err = suite.load_tasks().await.unwrap_err();
|
||||
assert!(err.to_string().contains("invalid repo format"));
|
||||
}
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
use std::io::BufRead;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::{BenchScore, BenchSuite, BenchTask, ConversationTurn, TaskSubmission};
|
||||
|
||||
/// Tau-bench task entry.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TauBenchEntry {
|
||||
id: String,
|
||||
#[serde(default)]
|
||||
domain: String,
|
||||
instruction: String,
|
||||
#[serde(default)]
|
||||
user_persona: Option<String>,
|
||||
#[serde(default)]
|
||||
expected_state: Option<serde_json::Value>,
|
||||
#[serde(default)]
|
||||
expected_actions: Vec<String>,
|
||||
#[serde(default)]
|
||||
max_turns: Option<usize>,
|
||||
}
|
||||
|
||||
/// Tau-bench: multi-turn tool-calling dialog benchmark.
|
||||
///
|
||||
/// Tests agent ability to handle customer service scenarios with simulated
|
||||
/// domain APIs (retail, airline). Scoring compares final state against expected.
|
||||
pub struct TauBenchSuite {
|
||||
dataset_path: PathBuf,
|
||||
domain: String,
|
||||
}
|
||||
|
||||
impl TauBenchSuite {
|
||||
pub fn new(dataset_path: impl Into<PathBuf>, domain: impl Into<String>) -> Self {
|
||||
Self {
|
||||
dataset_path: dataset_path.into(),
|
||||
domain: domain.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BenchSuite for TauBenchSuite {
|
||||
fn name(&self) -> &str {
|
||||
"Tau-bench"
|
||||
}
|
||||
|
||||
fn id(&self) -> &str {
|
||||
"tau_bench"
|
||||
}
|
||||
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError> {
|
||||
let file = std::fs::File::open(&self.dataset_path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut tasks = Vec::new();
|
||||
|
||||
for (line_num, line) in reader.lines().enumerate() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry: TauBenchEntry = serde_json::from_str(trimmed).map_err(|e| {
|
||||
BenchError::Config(format!("tau_bench line {}: {}", line_num + 1, e))
|
||||
})?;
|
||||
|
||||
let domain = if entry.domain.is_empty() {
|
||||
self.domain.clone()
|
||||
} else {
|
||||
entry.domain.clone()
|
||||
};
|
||||
|
||||
let metadata = serde_json::json!({
|
||||
"domain": domain,
|
||||
"user_persona": entry.user_persona,
|
||||
"expected_state": entry.expected_state,
|
||||
"expected_actions": entry.expected_actions,
|
||||
});
|
||||
|
||||
tasks.push(BenchTask {
|
||||
id: entry.id,
|
||||
prompt: entry.instruction,
|
||||
context: entry.user_persona.clone(),
|
||||
resources: vec![],
|
||||
tags: vec![format!("domain-{domain}")],
|
||||
expected_turns: entry.max_turns,
|
||||
timeout: None,
|
||||
metadata,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError> {
|
||||
// Score based on expected actions completion
|
||||
let expected_actions: Vec<String> = task
|
||||
.metadata
|
||||
.get("expected_actions")
|
||||
.and_then(|v| serde_json::from_value(v.clone()).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if expected_actions.is_empty() {
|
||||
// No expected actions defined; score based on whether agent responded
|
||||
if submission.response.is_empty() {
|
||||
return Ok(BenchScore::fail("no response"));
|
||||
}
|
||||
return Ok(BenchScore::partial(
|
||||
0.5,
|
||||
"no expected_actions to evaluate against",
|
||||
));
|
||||
}
|
||||
|
||||
// Check which expected actions were actually called
|
||||
let called: std::collections::HashSet<&str> =
|
||||
submission.tool_calls.iter().map(|s| s.as_str()).collect();
|
||||
let matched = expected_actions
|
||||
.iter()
|
||||
.filter(|a| called.contains(a.as_str()))
|
||||
.count();
|
||||
|
||||
let ratio = matched as f64 / expected_actions.len() as f64;
|
||||
if ratio >= 1.0 {
|
||||
Ok(BenchScore::pass())
|
||||
} else if ratio > 0.0 {
|
||||
Ok(BenchScore::partial(
|
||||
ratio,
|
||||
format!(
|
||||
"{}/{} expected actions completed",
|
||||
matched,
|
||||
expected_actions.len()
|
||||
),
|
||||
))
|
||||
} else {
|
||||
Ok(BenchScore::fail(format!(
|
||||
"0/{} expected actions completed",
|
||||
expected_actions.len()
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
// Check if we've exceeded max turns
|
||||
if let Some(max) = task.expected_turns {
|
||||
let user_turns = conversation
|
||||
.iter()
|
||||
.filter(|t| matches!(t.role, crate::suite::TurnRole::User))
|
||||
.count();
|
||||
if user_turns >= max {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-turn simulation requires an LLM to play the customer role.
|
||||
// Until that's implemented, every scenario is single-turn only.
|
||||
// TODO: Use LLM to simulate customer based on user_persona.
|
||||
tracing::warn!(
|
||||
task_id = %task.id,
|
||||
"multi-turn simulation not implemented, ending after first turn"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_load() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return my order", "expected_actions": ["lookup_order", "process_return"], "max_turns": 3}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
assert_eq!(tasks.len(), 1);
|
||||
assert_eq!(tasks[0].expected_turns, Some(3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_tau_bench_scoring() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("tau.jsonl");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
writeln!(
|
||||
file,
|
||||
r#"{{"id": "t1", "instruction": "Return order", "expected_actions": ["lookup_order", "process_return"]}}"#
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let suite = TauBenchSuite::new(&path, "retail");
|
||||
let tasks = suite.load_tasks().await.unwrap();
|
||||
|
||||
// Partial completion
|
||||
let submission = TaskSubmission {
|
||||
response: "I found your order.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 0.5);
|
||||
assert_eq!(score.label, "partial");
|
||||
|
||||
// Full completion
|
||||
let submission = TaskSubmission {
|
||||
response: "Return processed.".to_string(),
|
||||
conversation: vec![],
|
||||
tool_calls: vec!["lookup_order".to_string(), "process_return".to_string()],
|
||||
error: None,
|
||||
};
|
||||
let score = suite.score(&tasks[0], &submission).await.unwrap();
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use tokio::sync::{Mutex, mpsc};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use ironclaw::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
use ironclaw::error::ChannelError;
|
||||
|
||||
use crate::results::TraceToolCall;
|
||||
use crate::suite::ConversationTurn;
|
||||
|
||||
/// Truncate a string to at most `max_bytes` without splitting a UTF-8 character.
|
||||
fn truncate_str(s: &str, max_bytes: usize) -> &str {
|
||||
if s.len() <= max_bytes {
|
||||
return s;
|
||||
}
|
||||
let mut end = max_bytes;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
}
|
||||
|
||||
/// Captured state from a benchmark channel run.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct ChannelCapture {
|
||||
/// All responses the agent sent back.
|
||||
pub responses: Vec<String>,
|
||||
/// Tool calls observed (name, success, duration_ms).
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
/// Full conversation turns for multi-turn scoring.
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
/// Status messages (for debugging).
|
||||
pub status_log: Vec<String>,
|
||||
}
|
||||
|
||||
/// A headless Channel implementation for benchmarking.
|
||||
///
|
||||
/// Modeled after `ReplChannel`: uses mpsc to inject messages and captures
|
||||
/// all responses and tool status events. Auto-approves tool execution
|
||||
/// so benchmarks run without user interaction.
|
||||
pub struct BenchChannel {
|
||||
/// Sender to inject messages into the agent loop.
|
||||
msg_tx: mpsc::Sender<IncomingMessage>,
|
||||
/// Receiver the agent loop reads from (taken once by `start()`).
|
||||
msg_rx: Mutex<Option<mpsc::Receiver<IncomingMessage>>>,
|
||||
/// Accumulated capture data.
|
||||
capture: Arc<Mutex<ChannelCapture>>,
|
||||
}
|
||||
|
||||
impl BenchChannel {
|
||||
pub fn new() -> (Self, mpsc::Sender<IncomingMessage>) {
|
||||
let (tx, rx) = mpsc::channel(64);
|
||||
let channel = Self {
|
||||
msg_tx: tx.clone(),
|
||||
msg_rx: Mutex::new(Some(rx)),
|
||||
capture: Arc::new(Mutex::new(ChannelCapture::default())),
|
||||
};
|
||||
(channel, tx)
|
||||
}
|
||||
|
||||
/// Get a handle to the capture data.
|
||||
pub fn capture(&self) -> Arc<Mutex<ChannelCapture>> {
|
||||
Arc::clone(&self.capture)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Channel for BenchChannel {
|
||||
fn name(&self) -> &str {
|
||||
"bench"
|
||||
}
|
||||
|
||||
async fn start(&self) -> Result<MessageStream, ChannelError> {
|
||||
let rx = self
|
||||
.msg_rx
|
||||
.lock()
|
||||
.await
|
||||
.take()
|
||||
.ok_or_else(|| ChannelError::StartupFailed {
|
||||
name: "bench".to_string(),
|
||||
reason: "start() already called".to_string(),
|
||||
})?;
|
||||
Ok(Box::pin(ReceiverStream::new(rx)))
|
||||
}
|
||||
|
||||
async fn respond(
|
||||
&self,
|
||||
_msg: &IncomingMessage,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.responses.push(response.content.clone());
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: crate::suite::TurnRole::Assistant,
|
||||
content: response.content,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_status(
|
||||
&self,
|
||||
status: StatusUpdate,
|
||||
_metadata: &serde_json::Value,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
|
||||
match status {
|
||||
StatusUpdate::ToolCompleted { ref name, success } => {
|
||||
cap.tool_calls.push(TraceToolCall {
|
||||
name: name.clone(),
|
||||
duration_ms: 0, // We don't have precise per-tool timing here
|
||||
success,
|
||||
});
|
||||
cap.status_log
|
||||
.push(format!("tool_completed: {name} success={success}"));
|
||||
}
|
||||
StatusUpdate::ApprovalNeeded { ref request_id, .. } => {
|
||||
// Auto-approve all tools during benchmarks
|
||||
cap.status_log.push(format!("auto_approved: {request_id}"));
|
||||
drop(cap); // Release lock before sending
|
||||
let approval = IncomingMessage::new("bench", "bench-user", "always");
|
||||
let _ = self.msg_tx.send(approval).await;
|
||||
return Ok(());
|
||||
}
|
||||
StatusUpdate::Thinking(ref msg) => {
|
||||
cap.status_log.push(format!("thinking: {msg}"));
|
||||
}
|
||||
StatusUpdate::ToolStarted { ref name } => {
|
||||
cap.status_log.push(format!("tool_started: {name}"));
|
||||
}
|
||||
StatusUpdate::ToolResult {
|
||||
ref name,
|
||||
ref preview,
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"tool_result: {name} -> {}",
|
||||
truncate_str(preview, 100)
|
||||
));
|
||||
}
|
||||
StatusUpdate::StreamChunk(_) => {}
|
||||
StatusUpdate::Status(ref msg) => {
|
||||
cap.status_log.push(format!("status: {msg}"));
|
||||
}
|
||||
StatusUpdate::JobStarted {
|
||||
ref job_id,
|
||||
ref title,
|
||||
..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("job_started: {job_id} ({title})"));
|
||||
}
|
||||
StatusUpdate::AuthRequired {
|
||||
ref extension_name, ..
|
||||
} => {
|
||||
cap.status_log
|
||||
.push(format!("auth_required: {extension_name} (auto-skipped)"));
|
||||
}
|
||||
StatusUpdate::AuthCompleted {
|
||||
ref extension_name,
|
||||
success,
|
||||
..
|
||||
} => {
|
||||
cap.status_log.push(format!(
|
||||
"auth_completed: {extension_name} success={success}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn broadcast(
|
||||
&self,
|
||||
_user_id: &str,
|
||||
response: OutgoingResponse,
|
||||
) -> Result<(), ChannelError> {
|
||||
let mut cap = self.capture.lock().await;
|
||||
cap.status_log.push(format!(
|
||||
"broadcast: {}",
|
||||
truncate_str(&response.content, 100)
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn shutdown(&self) -> Result<(), ChannelError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_responses() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
let capture = channel.capture();
|
||||
|
||||
let msg = IncomingMessage::new("bench", "user", "hello");
|
||||
let response = OutgoingResponse::text("world");
|
||||
channel.respond(&msg, response).await.unwrap();
|
||||
|
||||
let cap = capture.lock().await;
|
||||
assert_eq!(cap.responses.len(), 1);
|
||||
assert_eq!(cap.responses[0], "world");
|
||||
assert_eq!(cap.conversation.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_auto_approves() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
// start() to consume the receiver
|
||||
let _stream = channel.start().await.unwrap();
|
||||
|
||||
let status = StatusUpdate::ApprovalNeeded {
|
||||
request_id: "req-1".to_string(),
|
||||
tool_name: "shell".to_string(),
|
||||
description: "run ls".to_string(),
|
||||
parameters: serde_json::json!({}),
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The approval message was sent through msg_tx,
|
||||
// which means the stream would receive it.
|
||||
// We can't easily read from the stream in this test without
|
||||
// consuming it, but we can verify the status log.
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert!(cap.status_log.iter().any(|s| s.contains("auto_approved")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bench_channel_captures_tool_events() {
|
||||
let (channel, _tx) = BenchChannel::new();
|
||||
|
||||
let status = StatusUpdate::ToolCompleted {
|
||||
name: "echo".to_string(),
|
||||
success: true,
|
||||
};
|
||||
channel
|
||||
.send_status(status, &serde_json::Value::Null)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let capture_arc = channel.capture();
|
||||
let cap = capture_arc.lock().await;
|
||||
assert_eq!(cap.tool_calls.len(), 1);
|
||||
assert_eq!(cap.tool_calls[0].name, "echo");
|
||||
assert!(cap.tool_calls[0].success);
|
||||
}
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// Top-level bench configuration, loaded from TOML.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct BenchConfig {
|
||||
/// Where to write results. Default: "./bench-results".
|
||||
#[serde(default = "default_results_dir")]
|
||||
pub results_dir: PathBuf,
|
||||
|
||||
/// Per-task timeout. Default: "300s".
|
||||
#[serde(
|
||||
default = "default_task_timeout",
|
||||
deserialize_with = "deserialize_duration"
|
||||
)]
|
||||
pub task_timeout: Duration,
|
||||
|
||||
/// How many tasks to run in parallel. Default: 1.
|
||||
#[serde(default = "default_parallelism")]
|
||||
pub parallelism: usize,
|
||||
|
||||
/// Model/config matrix entries. At least one required.
|
||||
#[serde(default)]
|
||||
pub matrix: Vec<MatrixEntry>,
|
||||
|
||||
/// Suite-specific configuration (passed through to adapter).
|
||||
#[serde(default = "default_suite_config")]
|
||||
pub suite_config: toml::Value,
|
||||
}
|
||||
|
||||
/// A single model/config combination to benchmark.
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct MatrixEntry {
|
||||
/// Label for this configuration (used in results).
|
||||
pub label: String,
|
||||
|
||||
/// Model identifier.
|
||||
#[serde(default)]
|
||||
pub model: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchConfig {
|
||||
/// Load from a TOML file.
|
||||
pub fn from_file(path: &Path) -> Result<Self, BenchError> {
|
||||
if !path.exists() {
|
||||
return Err(BenchError::ConfigNotFound {
|
||||
path: path.to_path_buf(),
|
||||
});
|
||||
}
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let config: BenchConfig = toml::from_str(&content)?;
|
||||
if config.matrix.is_empty() {
|
||||
return Err(BenchError::Config(
|
||||
"config must have at least one [[matrix]] entry".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// Create a minimal config for when no config file is provided.
|
||||
/// Uses defaults and optional CLI overrides.
|
||||
pub fn minimal(model: Option<String>) -> Self {
|
||||
let label = model.as_deref().unwrap_or("default").to_string();
|
||||
Self {
|
||||
results_dir: default_results_dir(),
|
||||
task_timeout: default_task_timeout(),
|
||||
parallelism: default_parallelism(),
|
||||
matrix: vec![MatrixEntry { label, model }],
|
||||
suite_config: toml::Value::Table(toml::map::Map::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the suite_config as a generic map for adapter use.
|
||||
pub fn suite_config_map(&self) -> toml::map::Map<String, toml::Value> {
|
||||
match &self.suite_config {
|
||||
toml::Value::Table(map) => map.clone(),
|
||||
_ => toml::map::Map::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a string value from suite_config.
|
||||
pub fn suite_config_str(&self, key: &str) -> Option<String> {
|
||||
self.suite_config_map()
|
||||
.get(key)
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn default_suite_config() -> toml::Value {
|
||||
toml::Value::Table(toml::map::Map::new())
|
||||
}
|
||||
|
||||
fn default_results_dir() -> PathBuf {
|
||||
PathBuf::from("./bench-results")
|
||||
}
|
||||
|
||||
fn default_task_timeout() -> Duration {
|
||||
Duration::from_secs(300)
|
||||
}
|
||||
|
||||
fn default_parallelism() -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
/// Deserialize a duration from a string like "300s", "5m", etc.
|
||||
fn deserialize_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
parse_duration(&s).map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
fn parse_duration(s: &str) -> Result<Duration, String> {
|
||||
let s = s.trim();
|
||||
if let Some(secs) = s.strip_suffix('s') {
|
||||
secs.trim()
|
||||
.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid seconds: {e}"))
|
||||
} else if let Some(mins) = s.strip_suffix('m') {
|
||||
mins.trim()
|
||||
.parse::<u64>()
|
||||
.map(|m| Duration::from_secs(m * 60))
|
||||
.map_err(|e| format!("invalid minutes: {e}"))
|
||||
} else {
|
||||
// Assume seconds if no suffix
|
||||
s.parse::<u64>()
|
||||
.map(Duration::from_secs)
|
||||
.map_err(|e| format!("invalid duration '{s}': {e}"))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_duration() {
|
||||
assert_eq!(parse_duration("300s").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("5m").unwrap(), Duration::from_secs(300));
|
||||
assert_eq!(parse_duration("60").unwrap(), Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_minimal_config() {
|
||||
let config = BenchConfig::minimal(Some("test-model".to_string()));
|
||||
assert_eq!(config.matrix.len(), 1);
|
||||
assert_eq!(config.matrix[0].label, "test-model");
|
||||
assert_eq!(config.parallelism, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_rejects_empty_matrix() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("empty.toml");
|
||||
std::fs::write(
|
||||
&path,
|
||||
r#"
|
||||
results_dir = "./results"
|
||||
task_timeout = "60s"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let err = BenchConfig::from_file(&path).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("at least one [[matrix]]"),
|
||||
"got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_config_from_toml() {
|
||||
let toml_str = r#"
|
||||
results_dir = "./my-results"
|
||||
task_timeout = "60s"
|
||||
parallelism = 2
|
||||
|
||||
[[matrix]]
|
||||
label = "fast"
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
[[matrix]]
|
||||
label = "full"
|
||||
model = "claude-3-5-sonnet"
|
||||
|
||||
[suite_config]
|
||||
dataset_path = "./data/test.jsonl"
|
||||
"#;
|
||||
let config: BenchConfig = toml::from_str(toml_str).unwrap();
|
||||
assert_eq!(config.results_dir, PathBuf::from("./my-results"));
|
||||
assert_eq!(config.task_timeout, Duration::from_secs(60));
|
||||
assert_eq!(config.parallelism, 2);
|
||||
assert_eq!(config.matrix.len(), 2);
|
||||
assert_eq!(
|
||||
config.suite_config_str("dataset_path").unwrap(),
|
||||
"./data/test.jsonl"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum BenchError {
|
||||
#[error("Config error: {0}")]
|
||||
Config(String),
|
||||
|
||||
#[error("Config file not found: {path}")]
|
||||
ConfigNotFound { path: PathBuf },
|
||||
|
||||
#[error("Suite {name} not found. Available: {available}")]
|
||||
SuiteNotFound { name: String, available: String },
|
||||
|
||||
#[error("Task {task_id} failed: {reason}")]
|
||||
TaskFailed { task_id: String, reason: String },
|
||||
|
||||
#[error("Scoring error for task {task_id}: {reason}")]
|
||||
Scoring { task_id: String, reason: String },
|
||||
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("JSON error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("TOML parse error: {0}")]
|
||||
Toml(#[from] toml::de::Error),
|
||||
|
||||
#[error("Agent error: {0}")]
|
||||
Agent(#[from] ironclaw::Error),
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
use rust_decimal::prelude::ToPrimitive;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use ironclaw::error::LlmError;
|
||||
use ironclaw::llm::{
|
||||
CompletionRequest, CompletionResponse, LlmProvider, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
|
||||
/// Recorded metrics from a single LLM call.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct LlmCallRecord {
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub duration_ms: u64,
|
||||
pub had_tool_calls: bool,
|
||||
}
|
||||
|
||||
/// Wraps an `LlmProvider` to record per-call metrics.
|
||||
///
|
||||
/// The wrapper is transparent to the agent: it delegates every call
|
||||
/// to the inner provider and captures token counts and timings.
|
||||
pub struct InstrumentedLlm {
|
||||
inner: Arc<dyn LlmProvider>,
|
||||
records: Mutex<Vec<LlmCallRecord>>,
|
||||
total_input_tokens: AtomicU32,
|
||||
total_output_tokens: AtomicU32,
|
||||
call_count: AtomicU32,
|
||||
}
|
||||
|
||||
impl InstrumentedLlm {
|
||||
pub fn new(inner: Arc<dyn LlmProvider>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
records: Mutex::new(Vec::new()),
|
||||
total_input_tokens: AtomicU32::new(0),
|
||||
total_output_tokens: AtomicU32::new(0),
|
||||
call_count: AtomicU32::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Take all recorded call metrics, clearing the internal buffer.
|
||||
pub async fn take_records(&self) -> Vec<LlmCallRecord> {
|
||||
let mut records = self.records.lock().await;
|
||||
std::mem::take(&mut *records)
|
||||
}
|
||||
|
||||
/// Snapshot of total tokens without clearing.
|
||||
pub fn total_input_tokens(&self) -> u32 {
|
||||
self.total_input_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn total_output_tokens(&self) -> u32 {
|
||||
self.total_output_tokens.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn call_count(&self) -> u32 {
|
||||
self.call_count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Estimated cost using the inner provider's cost-per-token rates.
|
||||
pub fn estimated_cost(&self) -> f64 {
|
||||
let (input_rate, output_rate) = self.inner.cost_per_token();
|
||||
let input_cost =
|
||||
input_rate * Decimal::from(self.total_input_tokens.load(Ordering::Relaxed));
|
||||
let output_cost =
|
||||
output_rate * Decimal::from(self.total_output_tokens.load(Ordering::Relaxed));
|
||||
let total = input_cost + output_cost;
|
||||
total.to_f64().unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// Reset all counters and records.
|
||||
pub async fn reset(&self) {
|
||||
self.records.lock().await.clear();
|
||||
self.total_input_tokens.store(0, Ordering::Relaxed);
|
||||
self.total_output_tokens.store(0, Ordering::Relaxed);
|
||||
self.call_count.store(0, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn record(
|
||||
&self,
|
||||
input_tokens: u32,
|
||||
output_tokens: u32,
|
||||
duration_ms: u64,
|
||||
had_tool_calls: bool,
|
||||
) {
|
||||
self.total_input_tokens
|
||||
.fetch_add(input_tokens, Ordering::Relaxed);
|
||||
self.total_output_tokens
|
||||
.fetch_add(output_tokens, Ordering::Relaxed);
|
||||
self.call_count.fetch_add(1, Ordering::Relaxed);
|
||||
self.records.lock().await.push(LlmCallRecord {
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
duration_ms,
|
||||
had_tool_calls,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for InstrumentedLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
self.inner.cost_per_token()
|
||||
}
|
||||
|
||||
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
false,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
let start = Instant::now();
|
||||
let response = self.inner.complete_with_tools(request).await?;
|
||||
let elapsed = start.elapsed().as_millis() as u64;
|
||||
let had_tool_calls = !response.tool_calls.is_empty();
|
||||
self.record(
|
||||
response.input_tokens,
|
||||
response.output_tokens,
|
||||
elapsed,
|
||||
had_tool_calls,
|
||||
)
|
||||
.await;
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
self.inner.list_models().await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use ironclaw::llm::{ChatMessage, CompletionRequest, CompletionResponse, FinishReason};
|
||||
|
||||
/// Fake LLM that returns a canned response with known token counts.
|
||||
struct FakeLlm;
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for FakeLlm {
|
||||
fn model_name(&self) -> &str {
|
||||
"fake-model"
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
(
|
||||
Decimal::new(3, 6), // $0.000003 per input token
|
||||
Decimal::new(15, 6), // $0.000015 per output token
|
||||
)
|
||||
}
|
||||
|
||||
async fn complete(
|
||||
&self,
|
||||
_request: CompletionRequest,
|
||||
) -> Result<CompletionResponse, LlmError> {
|
||||
Ok(CompletionResponse {
|
||||
content: "test response".to_string(),
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
finish_reason: FinishReason::Stop,
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
_request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
Ok(ToolCompletionResponse {
|
||||
content: Some("tool response".to_string()),
|
||||
tool_calls: vec![],
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
finish_reason: FinishReason::Stop,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_records_metrics() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
assert_eq!(instrumented.total_input_tokens(), 100);
|
||||
assert_eq!(instrumented.total_output_tokens(), 50);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].input_tokens, 100);
|
||||
assert!(!records[0].had_tool_calls);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_cost_calculation() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
|
||||
// 100 * 0.000003 + 50 * 0.000015 = 0.0003 + 0.00075 = 0.00105
|
||||
let cost = instrumented.estimated_cost();
|
||||
assert!((cost - 0.00105).abs() < 0.0001);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_instrumented_reset() {
|
||||
let inner = Arc::new(FakeLlm);
|
||||
let instrumented = InstrumentedLlm::new(inner);
|
||||
|
||||
let request = CompletionRequest::new(vec![ChatMessage::user("hello")]);
|
||||
let _ = instrumented.complete(request).await.unwrap();
|
||||
assert_eq!(instrumented.call_count(), 1);
|
||||
|
||||
instrumented.reset().await;
|
||||
assert_eq!(instrumented.call_count(), 0);
|
||||
assert_eq!(instrumented.total_input_tokens(), 0);
|
||||
|
||||
let records = instrumented.take_records().await;
|
||||
assert!(records.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,313 +0,0 @@
|
||||
mod adapters;
|
||||
mod channel;
|
||||
mod config;
|
||||
mod error;
|
||||
mod instrumented_llm;
|
||||
mod results;
|
||||
mod runner;
|
||||
mod scoring;
|
||||
mod suite;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
use tracing_subscriber::{EnvFilter, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::BenchConfig;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "ironclaw-bench", about = "IronClaw benchmarking harness")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// Run a benchmark suite.
|
||||
Run {
|
||||
/// Suite to run (custom, gaia, spot, tau_bench, swe_bench).
|
||||
#[arg(long)]
|
||||
suite: String,
|
||||
|
||||
/// Path to bench config TOML.
|
||||
#[arg(long)]
|
||||
config: Option<PathBuf>,
|
||||
|
||||
/// Override model for all matrix entries.
|
||||
#[arg(long)]
|
||||
model: Option<String>,
|
||||
|
||||
/// Max tasks to run in parallel.
|
||||
#[arg(long)]
|
||||
parallelism: Option<usize>,
|
||||
|
||||
/// Sample N tasks from the suite (for quick testing).
|
||||
#[arg(long)]
|
||||
sample: Option<usize>,
|
||||
|
||||
/// Only run these task IDs (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
task_ids: Option<Vec<String>>,
|
||||
|
||||
/// Only run tasks with these tags (comma-separated).
|
||||
#[arg(long, value_delimiter = ',')]
|
||||
tags: Option<Vec<String>>,
|
||||
|
||||
/// Per-task timeout in seconds.
|
||||
#[arg(long)]
|
||||
timeout_secs: Option<u64>,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
|
||||
/// Resume a previous run by ID.
|
||||
#[arg(long)]
|
||||
resume: Option<Uuid>,
|
||||
},
|
||||
|
||||
/// Show results for a run.
|
||||
Results {
|
||||
/// Run ID or "latest".
|
||||
#[arg(default_value = "latest")]
|
||||
run_id: String,
|
||||
|
||||
/// Output format.
|
||||
#[arg(long, default_value = "table")]
|
||||
format: ResultsFormat,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// Compare two runs.
|
||||
Compare {
|
||||
/// Baseline run ID.
|
||||
baseline: Uuid,
|
||||
|
||||
/// Comparison run ID.
|
||||
comparison: Uuid,
|
||||
|
||||
/// Override results directory.
|
||||
#[arg(long)]
|
||||
results_dir: Option<PathBuf>,
|
||||
},
|
||||
|
||||
/// List available benchmark suites.
|
||||
List,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, clap::ValueEnum)]
|
||||
enum ResultsFormat {
|
||||
Table,
|
||||
Json,
|
||||
Csv,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("ironclaw_bench=info,ironclaw=warn")),
|
||||
)
|
||||
.with(tracing_subscriber::fmt::layer().with_target(false))
|
||||
.init();
|
||||
|
||||
match cli.command {
|
||||
Commands::List => {
|
||||
println!("Available benchmark suites:\n");
|
||||
for (id, desc) in adapters::KNOWN_SUITES {
|
||||
println!(" {:<15} {}", id, desc);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
Commands::Run {
|
||||
suite,
|
||||
config: config_path,
|
||||
model,
|
||||
parallelism,
|
||||
sample,
|
||||
task_ids,
|
||||
tags,
|
||||
timeout_secs,
|
||||
results_dir,
|
||||
resume,
|
||||
} => {
|
||||
// Load or create config
|
||||
let mut bench_config = if let Some(ref path) = config_path {
|
||||
BenchConfig::from_file(path)?
|
||||
} else {
|
||||
BenchConfig::minimal(model.clone())
|
||||
};
|
||||
|
||||
// Apply CLI overrides
|
||||
if let Some(p) = parallelism {
|
||||
bench_config.parallelism = p;
|
||||
}
|
||||
if let Some(t) = timeout_secs {
|
||||
bench_config.task_timeout = std::time::Duration::from_secs(t);
|
||||
}
|
||||
if let Some(ref dir) = results_dir {
|
||||
bench_config.results_dir = dir.clone();
|
||||
}
|
||||
|
||||
// If model override specified and we have matrix entries, update them
|
||||
if let Some(ref m) = model {
|
||||
for entry in &mut bench_config.matrix {
|
||||
entry.model = Some(m.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Create suite
|
||||
let bench_suite = adapters::create_suite(&suite, &bench_config)?;
|
||||
|
||||
// Initialize ironclaw LLM provider
|
||||
let ironclaw_config = ironclaw::Config::from_env().await.map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
"Failed to load ironclaw config: {}. Make sure .env is configured.",
|
||||
e
|
||||
)
|
||||
})?;
|
||||
|
||||
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
||||
auth_base_url: ironclaw_config.llm.nearai.auth_base_url.clone(),
|
||||
session_path: ironclaw_config.llm.nearai.session_path.clone(),
|
||||
})
|
||||
.await;
|
||||
session.ensure_authenticated().await?;
|
||||
|
||||
let llm = ironclaw::llm::create_llm_provider(&ironclaw_config.llm, session)?;
|
||||
let safety = Arc::new(ironclaw::safety::SafetyLayer::new(&ironclaw_config.safety));
|
||||
|
||||
let runner = runner::BenchRunner::new(bench_suite, bench_config.clone(), llm, safety);
|
||||
|
||||
// Run for each matrix entry
|
||||
for matrix_entry in &bench_config.matrix {
|
||||
let run_id = runner
|
||||
.run(
|
||||
matrix_entry,
|
||||
sample,
|
||||
task_ids.as_deref(),
|
||||
tags.as_deref(),
|
||||
resume,
|
||||
)
|
||||
.await?;
|
||||
println!("Run complete: {}", run_id);
|
||||
}
|
||||
}
|
||||
Commands::Results {
|
||||
run_id,
|
||||
format,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
let uuid = if run_id == "latest" {
|
||||
results::find_latest_run(&base)?
|
||||
.ok_or_else(|| anyhow::anyhow!("No runs found in {}", base.display()))?
|
||||
} else {
|
||||
Uuid::parse_str(&run_id)?
|
||||
};
|
||||
|
||||
let json_path = results::run_json_path(&base, uuid);
|
||||
let jsonl_path = results::tasks_jsonl_path(&base, uuid);
|
||||
|
||||
let run = results::read_run_result(&json_path)?;
|
||||
let tasks = results::read_task_results(&jsonl_path)?;
|
||||
|
||||
match format {
|
||||
ResultsFormat::Table => {
|
||||
results::print_results_table(&tasks, &run);
|
||||
}
|
||||
ResultsFormat::Json => {
|
||||
let output = serde_json::json!({
|
||||
"run": run,
|
||||
"tasks": tasks,
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output)?);
|
||||
}
|
||||
ResultsFormat::Csv => {
|
||||
println!("task_id,score,label,tokens,cost,turns,time_s");
|
||||
for task in &tasks {
|
||||
println!(
|
||||
"{},{:.3},{},{},{:.4},{},{:.1}",
|
||||
task.task_id,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
task.trace.input_tokens + task.trace.output_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Commands::Compare {
|
||||
baseline,
|
||||
comparison,
|
||||
results_dir,
|
||||
} => {
|
||||
let base = results_dir.unwrap_or_else(|| PathBuf::from("./bench-results"));
|
||||
|
||||
let baseline_run = results::read_run_result(&results::run_json_path(&base, baseline))?;
|
||||
let comparison_run =
|
||||
results::read_run_result(&results::run_json_path(&base, comparison))?;
|
||||
|
||||
println!("\nComparison: {} vs {}\n", baseline, comparison);
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12} {:>10}",
|
||||
"Metric", "Baseline", "Comparison", "Delta"
|
||||
);
|
||||
println!("{}", "-".repeat(58));
|
||||
|
||||
let pass_delta = comparison_run.pass_rate - baseline_run.pass_rate;
|
||||
println!(
|
||||
"{:<20} {:>11.1}% {:>11.1}% {:>+9.1}%",
|
||||
"Pass rate",
|
||||
baseline_run.pass_rate * 100.0,
|
||||
comparison_run.pass_rate * 100.0,
|
||||
pass_delta * 100.0,
|
||||
);
|
||||
|
||||
let score_delta = comparison_run.avg_score - baseline_run.avg_score;
|
||||
println!(
|
||||
"{:<20} {:>12.3} {:>12.3} {:>+10.3}",
|
||||
"Avg score", baseline_run.avg_score, comparison_run.avg_score, score_delta,
|
||||
);
|
||||
|
||||
let cost_delta = comparison_run.total_cost_usd - baseline_run.total_cost_usd;
|
||||
println!(
|
||||
"{:<20} {:>11.4}$ {:>11.4}$ {:>+9.4}$",
|
||||
"Total cost",
|
||||
baseline_run.total_cost_usd,
|
||||
comparison_run.total_cost_usd,
|
||||
cost_delta,
|
||||
);
|
||||
|
||||
let time_b = baseline_run.total_wall_time_ms as f64 / 1000.0;
|
||||
let time_c = comparison_run.total_wall_time_ms as f64 / 1000.0;
|
||||
println!(
|
||||
"{:<20} {:>11.1}s {:>11.1}s {:>+9.1}s",
|
||||
"Total time",
|
||||
time_b,
|
||||
time_c,
|
||||
time_c - time_b,
|
||||
);
|
||||
|
||||
println!(
|
||||
"{:<20} {:>12} {:>12}",
|
||||
"Model", baseline_run.model, comparison_run.model,
|
||||
);
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,473 +0,0 @@
|
||||
use std::collections::HashSet;
|
||||
use std::io::{BufRead, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::BenchError;
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Metrics from a single task run: LLM usage, timing, tool calls.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct Trace {
|
||||
pub wall_time_ms: u64,
|
||||
pub llm_calls: u32,
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
pub estimated_cost_usd: f64,
|
||||
pub tool_calls: Vec<TraceToolCall>,
|
||||
pub turns: u32,
|
||||
pub hit_iteration_limit: bool,
|
||||
pub hit_timeout: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TraceToolCall {
|
||||
pub name: String,
|
||||
pub duration_ms: u64,
|
||||
pub success: bool,
|
||||
}
|
||||
|
||||
/// Result of running a single benchmark task.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResult {
|
||||
pub task_id: String,
|
||||
pub suite_id: String,
|
||||
pub score: BenchScore,
|
||||
pub trace: Trace,
|
||||
pub response: String,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
pub config_label: String,
|
||||
#[serde(default)]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Aggregate results for a full benchmark run.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct RunResult {
|
||||
pub run_id: Uuid,
|
||||
pub suite_id: String,
|
||||
pub config_label: String,
|
||||
pub model: String,
|
||||
/// Short git commit hash at the time of the run.
|
||||
#[serde(default)]
|
||||
pub commit_hash: String,
|
||||
pub pass_rate: f64,
|
||||
pub avg_score: f64,
|
||||
pub total_tasks: usize,
|
||||
pub completed_tasks: usize,
|
||||
pub total_cost_usd: f64,
|
||||
pub total_wall_time_ms: u64,
|
||||
pub started_at: DateTime<Utc>,
|
||||
pub finished_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl RunResult {
|
||||
/// Build aggregate from individual task results.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn from_tasks(
|
||||
run_id: Uuid,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
model: &str,
|
||||
commit_hash: &str,
|
||||
total_tasks: usize,
|
||||
tasks: &[TaskResult],
|
||||
started_at: DateTime<Utc>,
|
||||
) -> Self {
|
||||
let pass_count = tasks.iter().filter(|t| t.score.value >= 1.0).count();
|
||||
let pass_rate = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
pass_count as f64 / tasks.len() as f64
|
||||
};
|
||||
let avg_score = if tasks.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
tasks.iter().map(|t| t.score.value).sum::<f64>() / tasks.len() as f64
|
||||
};
|
||||
let total_cost: f64 = tasks.iter().map(|t| t.trace.estimated_cost_usd).sum();
|
||||
let total_wall: u64 = tasks.iter().map(|t| t.trace.wall_time_ms).sum();
|
||||
|
||||
Self {
|
||||
run_id,
|
||||
suite_id: suite_id.to_string(),
|
||||
config_label: config_label.to_string(),
|
||||
model: model.to_string(),
|
||||
commit_hash: commit_hash.to_string(),
|
||||
pass_rate,
|
||||
avg_score,
|
||||
total_tasks,
|
||||
completed_tasks: tasks.len(),
|
||||
total_cost_usd: total_cost,
|
||||
total_wall_time_ms: total_wall,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a single task result as one JSON line to the JSONL file.
|
||||
pub fn append_task_result(path: &Path, result: &TaskResult) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(path)?;
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Overwrite the JSONL file with the given results (used after scoring).
|
||||
pub fn write_task_results(path: &Path, results: &[TaskResult]) -> Result<(), BenchError> {
|
||||
let mut file = std::fs::File::create(path)?;
|
||||
for result in results {
|
||||
let line = serde_json::to_string(result)?;
|
||||
writeln!(file, "{line}")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read all task results from a JSONL file.
|
||||
pub fn read_task_results(path: &Path) -> Result<Vec<TaskResult>, BenchError> {
|
||||
if !path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let file = std::fs::File::open(path)?;
|
||||
let reader = std::io::BufReader::new(file);
|
||||
let mut results = Vec::new();
|
||||
for line in reader.lines() {
|
||||
let line = line?;
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let result: TaskResult = serde_json::from_str(trimmed)?;
|
||||
results.push(result);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Write the aggregate run result as JSON.
|
||||
pub fn write_run_result(path: &Path, result: &RunResult) -> Result<(), BenchError> {
|
||||
let json = serde_json::to_string_pretty(result)?;
|
||||
std::fs::write(path, json)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read the aggregate run result from JSON.
|
||||
pub fn read_run_result(path: &Path) -> Result<RunResult, BenchError> {
|
||||
let json = std::fs::read_to_string(path)?;
|
||||
let result: RunResult = serde_json::from_str(&json)?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Get the set of already-completed task IDs from a JSONL file (for resume).
|
||||
///
|
||||
/// Only includes tasks that have been scored (label != "pending"). Tasks that
|
||||
/// were written but not scored (e.g., from an interrupted run) will be re-executed.
|
||||
pub fn completed_task_ids(path: &Path) -> Result<HashSet<String>, BenchError> {
|
||||
let results = read_task_results(path)?;
|
||||
Ok(results
|
||||
.into_iter()
|
||||
.filter(|r| r.score.label != "pending")
|
||||
.map(|r| r.task_id)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Get the results directory for a specific run.
|
||||
pub fn run_dir(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
base.join(run_id.to_string())
|
||||
}
|
||||
|
||||
/// Get the tasks JSONL path for a run.
|
||||
pub fn tasks_jsonl_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("tasks.jsonl")
|
||||
}
|
||||
|
||||
/// Get the run JSON path for a run.
|
||||
pub fn run_json_path(base: &Path, run_id: Uuid) -> PathBuf {
|
||||
run_dir(base, run_id).join("run.json")
|
||||
}
|
||||
|
||||
/// Find the latest run directory by the modification time of its `run.json`.
|
||||
///
|
||||
/// Falls back to `tasks.jsonl` mtime, then directory mtime. This avoids the
|
||||
/// issue where modifying files inside a directory doesn't update the directory's
|
||||
/// mtime on many filesystems.
|
||||
pub fn find_latest_run(base: &Path) -> Result<Option<Uuid>, BenchError> {
|
||||
if !base.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut entries: Vec<_> = std::fs::read_dir(base)?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_type().map(|ft| ft.is_dir()).unwrap_or(false))
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_string_lossy().to_string();
|
||||
let uuid = Uuid::parse_str(&name).ok()?;
|
||||
let dir_path = e.path();
|
||||
// Prefer run.json mtime, fall back to tasks.jsonl, then directory
|
||||
let modified = std::fs::metadata(dir_path.join("run.json"))
|
||||
.and_then(|m| m.modified())
|
||||
.or_else(|_| {
|
||||
std::fs::metadata(dir_path.join("tasks.jsonl")).and_then(|m| m.modified())
|
||||
})
|
||||
.or_else(|_| e.metadata().and_then(|m| m.modified()))
|
||||
.ok()?;
|
||||
Some((uuid, modified))
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by(|a, b| b.1.cmp(&a.1));
|
||||
Ok(entries.first().map(|(uuid, _)| *uuid))
|
||||
}
|
||||
|
||||
/// Print a summary table of task results.
|
||||
pub fn print_results_table(tasks: &[TaskResult], run: &RunResult) {
|
||||
println!();
|
||||
let commit_suffix = if run.commit_hash.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" | Commit: {}", run.commit_hash)
|
||||
};
|
||||
println!(
|
||||
"Run: {} | Suite: {} | Model: {}{}",
|
||||
run.run_id, run.suite_id, run.model, commit_suffix
|
||||
);
|
||||
println!(
|
||||
"Pass rate: {:.1}% | Avg score: {:.3} | Tasks: {}/{} | Cost: ${:.4} | Time: {:.1}s",
|
||||
run.pass_rate * 100.0,
|
||||
run.avg_score,
|
||||
run.completed_tasks,
|
||||
run.total_tasks,
|
||||
run.total_cost_usd,
|
||||
run.total_wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
println!();
|
||||
|
||||
// Header
|
||||
println!(
|
||||
"{:<30} {:>6} {:>7} {:>8} {:>10} {:>6} {:>8}",
|
||||
"Task ID", "Score", "Label", "Tokens", "Cost", "Turns", "Time"
|
||||
);
|
||||
println!("{}", "-".repeat(80));
|
||||
|
||||
for task in tasks {
|
||||
let total_tokens = task.trace.input_tokens + task.trace.output_tokens;
|
||||
let task_id_display = if task.task_id.len() > 28 {
|
||||
let truncated: String = task.task_id.chars().take(25).collect();
|
||||
format!("{truncated}...")
|
||||
} else {
|
||||
task.task_id.clone()
|
||||
};
|
||||
println!(
|
||||
"{:<30} {:>6.3} {:>7} {:>8} {:>10.4} {:>6} {:>7.1}s",
|
||||
task_id_display,
|
||||
task.score.value,
|
||||
task.score.label,
|
||||
total_tokens,
|
||||
task.trace.estimated_cost_usd,
|
||||
task.trace.turns,
|
||||
task.trace.wall_time_ms as f64 / 1000.0,
|
||||
);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_run_result_from_tasks() {
|
||||
let tasks = vec![
|
||||
TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 1000,
|
||||
llm_calls: 2,
|
||||
input_tokens: 100,
|
||||
output_tokens: 50,
|
||||
estimated_cost_usd: 0.01,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
TaskResult {
|
||||
task_id: "t2".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some("wrong".to_string()),
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 2000,
|
||||
llm_calls: 3,
|
||||
input_tokens: 200,
|
||||
output_tokens: 100,
|
||||
estimated_cost_usd: 0.02,
|
||||
tool_calls: vec![],
|
||||
turns: 2,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "wrong answer".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
},
|
||||
];
|
||||
|
||||
let run = RunResult::from_tasks(
|
||||
Uuid::new_v4(),
|
||||
"custom",
|
||||
"default",
|
||||
"test-model",
|
||||
"abc1234",
|
||||
2,
|
||||
&tasks,
|
||||
Utc::now(),
|
||||
);
|
||||
|
||||
assert_eq!(run.pass_rate, 0.5);
|
||||
assert_eq!(run.avg_score, 0.5);
|
||||
assert_eq!(run.total_tasks, 2);
|
||||
assert_eq!(run.completed_tasks, 2);
|
||||
assert!((run.total_cost_usd - 0.03).abs() < f64::EPSILON);
|
||||
assert_eq!(run.total_wall_time_ms, 3000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jsonl_roundtrip() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "round-trip-test".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 500,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "hello".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
|
||||
append_task_result(&path, &result).expect("append");
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let loaded = read_task_results(&path).expect("read");
|
||||
assert_eq!(loaded.len(), 2);
|
||||
assert_eq!(loaded[0].task_id, "round-trip-test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completed_task_ids() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
let result = TaskResult {
|
||||
task_id: "unique-id-1".to_string(),
|
||||
suite_id: "custom".to_string(),
|
||||
score: BenchScore::pass(),
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "x".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "test".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &result).expect("append");
|
||||
|
||||
let ids = completed_task_ids(&path).expect("ids");
|
||||
assert!(ids.contains("unique-id-1"));
|
||||
assert!(!ids.contains("unique-id-2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_write_task_results_overwrites() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let path = dir.path().join("tasks.jsonl");
|
||||
|
||||
// Write initial "pending" result via append
|
||||
let pending = TaskResult {
|
||||
task_id: "t1".to_string(),
|
||||
suite_id: "spot".to_string(),
|
||||
score: BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace: Trace {
|
||||
wall_time_ms: 100,
|
||||
llm_calls: 1,
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
estimated_cost_usd: 0.001,
|
||||
tool_calls: vec![],
|
||||
turns: 1,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: "42".to_string(),
|
||||
started_at: Utc::now(),
|
||||
finished_at: Utc::now(),
|
||||
config_label: "default".to_string(),
|
||||
error: None,
|
||||
};
|
||||
append_task_result(&path, &pending).expect("append");
|
||||
|
||||
// Verify pending score
|
||||
let before = read_task_results(&path).expect("read");
|
||||
assert_eq!(before.len(), 1);
|
||||
assert_eq!(before[0].score.label, "pending");
|
||||
|
||||
// Overwrite with scored result
|
||||
let mut scored = pending;
|
||||
scored.score = BenchScore::pass();
|
||||
write_task_results(&path, &[scored]).expect("write");
|
||||
|
||||
// Verify scored result replaced pending
|
||||
let after = read_task_results(&path).expect("read");
|
||||
assert_eq!(after.len(), 1);
|
||||
assert_eq!(after[0].score.label, "pass");
|
||||
assert_eq!(after[0].score.value, 1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,550 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::Utc;
|
||||
use tokio::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
use ironclaw::agent::{Agent, AgentDeps};
|
||||
use ironclaw::channels::{ChannelManager, IncomingMessage};
|
||||
use ironclaw::config::AgentConfig;
|
||||
use ironclaw::llm::LlmProvider;
|
||||
use ironclaw::safety::SafetyLayer;
|
||||
use ironclaw::tools::ToolRegistry;
|
||||
|
||||
use crate::channel::BenchChannel;
|
||||
use crate::config::{BenchConfig, MatrixEntry};
|
||||
use crate::error::BenchError;
|
||||
use crate::instrumented_llm::InstrumentedLlm;
|
||||
use crate::results::{
|
||||
RunResult, TaskResult, Trace, append_task_result, completed_task_ids, run_dir, run_json_path,
|
||||
tasks_jsonl_path, write_run_result, write_task_results,
|
||||
};
|
||||
use crate::suite::{BenchSuite, BenchTask, ConversationTurn, TaskSubmission, TurnRole};
|
||||
|
||||
/// Parameters for running a single task in isolation.
|
||||
struct TaskRunParams<'a> {
|
||||
task: &'a BenchTask,
|
||||
suite_id: &'a str,
|
||||
config_label: &'a str,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
timeout: std::time::Duration,
|
||||
additional_tools: &'a [Arc<dyn ironclaw::tools::Tool>],
|
||||
}
|
||||
|
||||
/// Orchestrates benchmark execution: loads tasks, runs agent per task,
|
||||
/// scores results, writes JSONL output.
|
||||
pub struct BenchRunner {
|
||||
suite: Arc<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
}
|
||||
|
||||
impl BenchRunner {
|
||||
pub fn new(
|
||||
suite: Box<dyn BenchSuite>,
|
||||
config: BenchConfig,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
) -> Self {
|
||||
Self {
|
||||
suite: Arc::from(suite),
|
||||
config,
|
||||
llm,
|
||||
safety,
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the benchmark for one matrix entry.
|
||||
///
|
||||
/// Returns the run_id for result retrieval.
|
||||
pub async fn run(
|
||||
&self,
|
||||
matrix: &MatrixEntry,
|
||||
sample: Option<usize>,
|
||||
task_filter: Option<&[String]>,
|
||||
tag_filter: Option<&[String]>,
|
||||
resume_run_id: Option<Uuid>,
|
||||
) -> Result<Uuid, BenchError> {
|
||||
let run_id = resume_run_id.unwrap_or_else(Uuid::new_v4);
|
||||
let results_base = &self.config.results_dir;
|
||||
let dir = run_dir(results_base, run_id);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
|
||||
let jsonl_path = tasks_jsonl_path(results_base, run_id);
|
||||
let json_path = run_json_path(results_base, run_id);
|
||||
|
||||
// Load completed task IDs for resume support
|
||||
let completed: HashSet<String> = if resume_run_id.is_some() {
|
||||
completed_task_ids(&jsonl_path)?
|
||||
} else {
|
||||
HashSet::new()
|
||||
};
|
||||
|
||||
if !completed.is_empty() {
|
||||
tracing::info!(
|
||||
"Resuming run {}: {} tasks already completed",
|
||||
run_id,
|
||||
completed.len()
|
||||
);
|
||||
}
|
||||
|
||||
// Load all tasks once (used for both execution and scoring)
|
||||
let all_tasks = self.suite.load_tasks().await?;
|
||||
let task_index: HashMap<String, BenchTask> = all_tasks
|
||||
.iter()
|
||||
.map(|t| (t.id.clone(), t.clone()))
|
||||
.collect();
|
||||
|
||||
// Filter tasks for execution
|
||||
let mut tasks = all_tasks;
|
||||
|
||||
if let Some(ids) = task_filter {
|
||||
let id_set: HashSet<&str> = ids.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| id_set.contains(t.id.as_str()));
|
||||
}
|
||||
|
||||
if let Some(tags) = tag_filter {
|
||||
let tag_set: HashSet<&str> = tags.iter().map(|s| s.as_str()).collect();
|
||||
tasks.retain(|t| t.tags.iter().any(|tag| tag_set.contains(tag.as_str())));
|
||||
}
|
||||
|
||||
// Filter out already-completed tasks
|
||||
tasks.retain(|t| !completed.contains(&t.id));
|
||||
|
||||
// Sample if requested
|
||||
if let Some(n) = sample {
|
||||
tasks.truncate(n);
|
||||
}
|
||||
|
||||
let total_tasks = tasks.len() + completed.len();
|
||||
let model_label = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
let commit_hash = git_short_hash();
|
||||
tracing::info!(
|
||||
"[{} @ {}] Running {} tasks for suite '{}' (run: {})",
|
||||
model_label,
|
||||
commit_hash,
|
||||
tasks.len(),
|
||||
self.suite.id(),
|
||||
run_id
|
||||
);
|
||||
|
||||
let started_at = Utc::now();
|
||||
let all_results: Arc<Mutex<Vec<TaskResult>>> =
|
||||
Arc::new(Mutex::new(Vec::with_capacity(tasks.len())));
|
||||
|
||||
if self.config.parallelism <= 1 {
|
||||
// Sequential execution
|
||||
let additional_tools = self.suite.additional_tools();
|
||||
for (i, task) in tasks.iter().enumerate() {
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed.len(),
|
||||
total_tasks,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = self.suite.setup_task(task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
task,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
continue;
|
||||
}
|
||||
let params = TaskRunParams {
|
||||
task,
|
||||
suite_id: self.suite.id(),
|
||||
config_label: &matrix.label,
|
||||
llm: Arc::clone(&self.llm),
|
||||
safety: Arc::clone(&self.safety),
|
||||
timeout: task.timeout.unwrap_or(self.config.task_timeout),
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = self.suite.teardown_task(task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
append_task_result(&jsonl_path, &result)?;
|
||||
all_results.lock().await.push(result);
|
||||
}
|
||||
} else {
|
||||
// Parallel execution with bounded concurrency
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.config.parallelism));
|
||||
let shared_tools: Arc<[Arc<dyn ironclaw::tools::Tool>]> =
|
||||
Arc::from(self.suite.additional_tools());
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for (i, task) in tasks.into_iter().enumerate() {
|
||||
let sem = Arc::clone(&semaphore);
|
||||
let suite = Arc::clone(&self.suite);
|
||||
let config_label = matrix.label.clone();
|
||||
let llm = Arc::clone(&self.llm);
|
||||
let safety = Arc::clone(&self.safety);
|
||||
let timeout = task.timeout.unwrap_or(self.config.task_timeout);
|
||||
let results_ref = Arc::clone(&all_results);
|
||||
let completed_count = completed.len();
|
||||
let total = total_tasks;
|
||||
let additional_tools = Arc::clone(&shared_tools);
|
||||
|
||||
handles.push(tokio::spawn(async move {
|
||||
let _permit = match sem.acquire().await {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
tracing::error!("Semaphore closed for task {}", task.id);
|
||||
return;
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
"[{}/{}] Running task: {}",
|
||||
i + 1 + completed_count,
|
||||
total,
|
||||
task.id
|
||||
);
|
||||
if let Err(e) = suite.setup_task(&task).await {
|
||||
tracing::warn!("setup_task failed for {}: {}", task.id, e);
|
||||
let result = make_error_result(
|
||||
&task,
|
||||
suite.id(),
|
||||
&config_label,
|
||||
Utc::now(),
|
||||
&format!("setup_task failed: {e}"),
|
||||
);
|
||||
results_ref.lock().await.push(result);
|
||||
return;
|
||||
}
|
||||
let suite_id = suite.id().to_string();
|
||||
let params = TaskRunParams {
|
||||
task: &task,
|
||||
suite_id: &suite_id,
|
||||
config_label: &config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools: &additional_tools,
|
||||
};
|
||||
let result = run_task_isolated(params).await;
|
||||
if let Err(e) = suite.teardown_task(&task).await {
|
||||
tracing::warn!("teardown_task failed for {}: {}", task.id, e);
|
||||
}
|
||||
results_ref.lock().await.push(result);
|
||||
}));
|
||||
}
|
||||
|
||||
for handle in handles {
|
||||
if let Err(e) = handle.await {
|
||||
tracing::error!("Task panicked: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Write all results to JSONL after parallel execution completes.
|
||||
// This avoids the race condition of concurrent file appends.
|
||||
let results = all_results.lock().await;
|
||||
for result in results.iter() {
|
||||
append_task_result(&jsonl_path, result)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Score all results using the cached task index
|
||||
let results = all_results.lock().await;
|
||||
let mut scored: Vec<TaskResult> = Vec::with_capacity(results.len());
|
||||
for result in results.iter() {
|
||||
if let Some(task) = task_index.get(&result.task_id) {
|
||||
let submission = TaskSubmission {
|
||||
response: result.response.clone(),
|
||||
conversation: vec![],
|
||||
tool_calls: result
|
||||
.trace
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|tc| tc.name.clone())
|
||||
.collect(),
|
||||
error: result.error.clone(),
|
||||
};
|
||||
match self.suite.score(task, &submission).await {
|
||||
Ok(score) => {
|
||||
let mut scored_result = result.clone();
|
||||
scored_result.score = score;
|
||||
scored.push(scored_result);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Scoring failed for {}: {}", result.task_id, e);
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
scored.push(result.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Combine with any previously completed results for the aggregate
|
||||
let mut all_for_aggregate = crate::results::read_task_results(&jsonl_path)?;
|
||||
// De-duplicate (prefer the newer scored versions)
|
||||
let scored_ids: HashSet<String> = scored.iter().map(|r| r.task_id.clone()).collect();
|
||||
all_for_aggregate.retain(|r| !scored_ids.contains(&r.task_id));
|
||||
all_for_aggregate.extend(scored);
|
||||
|
||||
// Rewrite JSONL with scored results so `results` command shows final scores
|
||||
write_task_results(&jsonl_path, &all_for_aggregate)?;
|
||||
|
||||
let model_name = matrix.model.as_deref().unwrap_or(self.llm.model_name());
|
||||
|
||||
let run_result = RunResult::from_tasks(
|
||||
run_id,
|
||||
self.suite.id(),
|
||||
&matrix.label,
|
||||
model_name,
|
||||
&commit_hash,
|
||||
total_tasks,
|
||||
&all_for_aggregate,
|
||||
started_at,
|
||||
);
|
||||
|
||||
write_run_result(&json_path, &run_result)?;
|
||||
|
||||
tracing::info!(
|
||||
"[{} @ {}] Run {} complete: {:.1}% pass rate, {:.3} avg score, ${:.4} cost",
|
||||
model_name,
|
||||
commit_hash,
|
||||
run_id,
|
||||
run_result.pass_rate * 100.0,
|
||||
run_result.avg_score,
|
||||
run_result.total_cost_usd,
|
||||
);
|
||||
|
||||
Ok(run_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a single benchmark task in complete isolation.
|
||||
///
|
||||
/// Creates a fresh Agent + BenchChannel + InstrumentedLlm for the task,
|
||||
/// injects the prompt, waits for the response, and returns the result.
|
||||
///
|
||||
/// # Current limitations
|
||||
///
|
||||
/// - **Single-turn only**: After the first assistant response, `/quit` is sent.
|
||||
/// Multi-turn suites (e.g., Tau-bench's `next_user_message()`) are not yet wired.
|
||||
/// - **Resources not injected**: `BenchTask.resources` (e.g., GAIA file attachments)
|
||||
/// are not included in the prompt or made available via the workspace.
|
||||
/// - **Conversation not captured**: `TaskSubmission.conversation` is always empty,
|
||||
/// which prevents multi-turn scoring hooks from working.
|
||||
async fn run_task_isolated(params: TaskRunParams<'_>) -> TaskResult {
|
||||
let TaskRunParams {
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
llm,
|
||||
safety,
|
||||
timeout,
|
||||
additional_tools,
|
||||
} = params;
|
||||
|
||||
let started_at = Utc::now();
|
||||
let start = Instant::now();
|
||||
|
||||
// Wrap LLM with instrumentation
|
||||
let instrumented = Arc::new(InstrumentedLlm::new(llm));
|
||||
|
||||
// Create bench channel
|
||||
let (bench_channel, msg_tx) = BenchChannel::new();
|
||||
let capture = bench_channel.capture();
|
||||
|
||||
// Build tool registry
|
||||
let tools = Arc::new(ToolRegistry::new());
|
||||
tools.register_builtin_tools();
|
||||
|
||||
// Register additional suite-specific tools
|
||||
for tool in additional_tools {
|
||||
tools.register(Arc::clone(tool)).await;
|
||||
}
|
||||
|
||||
// Build agent config (minimal, headless)
|
||||
let agent_config = AgentConfig {
|
||||
name: format!("bench-{}", task.id),
|
||||
max_parallel_jobs: 1,
|
||||
job_timeout: timeout,
|
||||
stuck_threshold: timeout,
|
||||
repair_check_interval: timeout + std::time::Duration::from_secs(999),
|
||||
max_repair_attempts: 0,
|
||||
use_planning: false,
|
||||
session_idle_timeout: timeout,
|
||||
allow_local_tools: true,
|
||||
max_cost_per_day_cents: None,
|
||||
max_actions_per_hour: None,
|
||||
};
|
||||
|
||||
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
|
||||
ironclaw::agent::cost_guard::CostGuardConfig::default(),
|
||||
));
|
||||
|
||||
let deps = AgentDeps {
|
||||
store: None,
|
||||
llm: instrumented.clone() as Arc<dyn LlmProvider>,
|
||||
cheap_llm: None,
|
||||
safety,
|
||||
tools,
|
||||
workspace: None,
|
||||
extension_manager: None,
|
||||
skill_registry: None,
|
||||
skills_config: ironclaw::config::SkillsConfig::default(),
|
||||
hooks: Arc::new(ironclaw::hooks::HookRegistry::new()),
|
||||
cost_guard,
|
||||
};
|
||||
|
||||
let mut channels = ChannelManager::new();
|
||||
channels.add(Box::new(bench_channel));
|
||||
|
||||
let agent = Agent::new(agent_config, deps, channels, None, None, None, None, None);
|
||||
|
||||
// Build the full prompt with context
|
||||
let full_prompt = if let Some(ref ctx) = task.context {
|
||||
format!("{}\n\nContext:\n{}", task.prompt, ctx)
|
||||
} else {
|
||||
task.prompt.clone()
|
||||
};
|
||||
|
||||
// Inject the task prompt
|
||||
let incoming = IncomingMessage::new("bench", "bench-user", &full_prompt);
|
||||
if msg_tx.send(incoming).await.is_err() {
|
||||
return make_error_result(
|
||||
task,
|
||||
suite_id,
|
||||
config_label,
|
||||
started_at,
|
||||
"failed to send prompt",
|
||||
);
|
||||
}
|
||||
|
||||
// Record prompt in conversation
|
||||
{
|
||||
let mut cap = capture.lock().await;
|
||||
cap.conversation.push(ConversationTurn {
|
||||
role: TurnRole::User,
|
||||
content: full_prompt,
|
||||
});
|
||||
}
|
||||
|
||||
// Run agent with timeout.
|
||||
// After the first response, send /quit to end the session.
|
||||
let quit_tx = msg_tx.clone();
|
||||
let capture_for_quit = Arc::clone(&capture);
|
||||
let quit_handle = tokio::spawn(async move {
|
||||
// Poll for first response
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
let cap = capture_for_quit.lock().await;
|
||||
if !cap.responses.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Give a small grace period for any final status events
|
||||
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
|
||||
let quit = IncomingMessage::new("bench", "bench-user", "/quit");
|
||||
let _ = quit_tx.send(quit).await;
|
||||
});
|
||||
|
||||
let agent_result = tokio::time::timeout(timeout, agent.run()).await;
|
||||
|
||||
quit_handle.abort();
|
||||
|
||||
let wall_time = start.elapsed();
|
||||
let hit_timeout = agent_result.is_err();
|
||||
|
||||
if let Ok(Err(e)) = &agent_result {
|
||||
tracing::warn!("Agent error for task {}: {}", task.id, e);
|
||||
}
|
||||
|
||||
// Extract results from capture
|
||||
let cap = capture.lock().await;
|
||||
let response = cap.responses.last().cloned().unwrap_or_default();
|
||||
|
||||
let trace = Trace {
|
||||
wall_time_ms: wall_time.as_millis() as u64,
|
||||
llm_calls: instrumented.call_count(),
|
||||
input_tokens: instrumented.total_input_tokens(),
|
||||
output_tokens: instrumented.total_output_tokens(),
|
||||
estimated_cost_usd: instrumented.estimated_cost(),
|
||||
tool_calls: cap.tool_calls.clone(),
|
||||
turns: cap.responses.len() as u32,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout,
|
||||
};
|
||||
|
||||
let error = if hit_timeout {
|
||||
Some(format!("timeout after {}s", timeout.as_secs()))
|
||||
} else if let Ok(Err(e)) = &agent_result {
|
||||
Some(e.to_string())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore {
|
||||
value: 0.0,
|
||||
label: "pending".to_string(),
|
||||
details: None,
|
||||
},
|
||||
trace,
|
||||
response,
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error,
|
||||
}
|
||||
}
|
||||
|
||||
fn make_error_result(
|
||||
task: &BenchTask,
|
||||
suite_id: &str,
|
||||
config_label: &str,
|
||||
started_at: chrono::DateTime<Utc>,
|
||||
reason: &str,
|
||||
) -> TaskResult {
|
||||
TaskResult {
|
||||
task_id: task.id.clone(),
|
||||
suite_id: suite_id.to_string(),
|
||||
score: crate::suite::BenchScore::fail(reason),
|
||||
trace: Trace {
|
||||
wall_time_ms: 0,
|
||||
llm_calls: 0,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
estimated_cost_usd: 0.0,
|
||||
tool_calls: vec![],
|
||||
turns: 0,
|
||||
hit_iteration_limit: false,
|
||||
hit_timeout: false,
|
||||
},
|
||||
response: String::new(),
|
||||
started_at,
|
||||
finished_at: Utc::now(),
|
||||
config_label: config_label.to_string(),
|
||||
error: Some(reason.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the short git commit hash of HEAD, or "unknown" if not in a repo.
|
||||
fn git_short_hash() -> String {
|
||||
std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| {
|
||||
if o.status.success() {
|
||||
Some(String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
use regex::Regex;
|
||||
|
||||
use crate::suite::BenchScore;
|
||||
|
||||
/// Normalize an answer string for comparison: lowercase, trim whitespace,
|
||||
/// strip trailing punctuation, collapse internal whitespace.
|
||||
pub fn normalize_answer(s: &str) -> String {
|
||||
let trimmed = s.trim().to_lowercase();
|
||||
let collapsed: String = trimmed.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
collapsed.trim_end_matches(['.', ',', ';', '!']).to_string()
|
||||
}
|
||||
|
||||
/// Exact match after normalization.
|
||||
pub fn exact_match(expected: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_expected == norm_actual {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!(
|
||||
"expected \"{norm_expected}\", got \"{norm_actual}\""
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer contains the expected substring (normalized).
|
||||
pub fn contains_match(expected_substring: &str, actual: &str) -> BenchScore {
|
||||
let norm_expected = normalize_answer(expected_substring);
|
||||
let norm_actual = normalize_answer(actual);
|
||||
if norm_actual.contains(&norm_expected) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not contain \"{norm_expected}\""))
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the actual answer matches a regex pattern.
|
||||
pub fn regex_match(pattern: &str, actual: &str) -> BenchScore {
|
||||
match Regex::new(pattern) {
|
||||
Ok(re) => {
|
||||
if re.is_match(actual) {
|
||||
BenchScore::pass()
|
||||
} else {
|
||||
BenchScore::fail(format!("response does not match pattern /{pattern}/"))
|
||||
}
|
||||
}
|
||||
Err(e) => BenchScore::fail(format!("invalid regex pattern: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_normalize_answer() {
|
||||
assert_eq!(normalize_answer(" Hello World. "), "hello world");
|
||||
assert_eq!(normalize_answer("Yes!"), "yes");
|
||||
assert_eq!(normalize_answer("42"), "42");
|
||||
assert_eq!(normalize_answer(" "), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_pass() {
|
||||
let score = exact_match("Hello World", " hello world. ");
|
||||
assert_eq!(score.value, 1.0);
|
||||
assert_eq!(score.label, "pass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exact_match_fail() {
|
||||
let score = exact_match("hello", "world");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert_eq!(score.label, "fail");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_pass() {
|
||||
let score = contains_match("world", "Hello World!");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_contains_match_fail() {
|
||||
let score = contains_match("xyz", "Hello World!");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_pass() {
|
||||
let score = regex_match(r"\d{4}", "The year is 2024.");
|
||||
assert_eq!(score.value, 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_fail() {
|
||||
let score = regex_match(r"\d{4}", "No numbers here.");
|
||||
assert_eq!(score.value, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_regex_match_invalid_pattern() {
|
||||
let score = regex_match(r"[invalid", "anything");
|
||||
assert_eq!(score.value, 0.0);
|
||||
assert!(
|
||||
score
|
||||
.details
|
||||
.as_deref()
|
||||
.unwrap_or("")
|
||||
.contains("invalid regex")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::error::BenchError;
|
||||
|
||||
/// A single task in a benchmark suite.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchTask {
|
||||
pub id: String,
|
||||
pub prompt: String,
|
||||
#[serde(default)]
|
||||
pub context: Option<String>,
|
||||
#[serde(default)]
|
||||
pub resources: Vec<TaskResource>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub expected_turns: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub timeout: Option<Duration>,
|
||||
#[serde(default)]
|
||||
pub metadata: serde_json::Value,
|
||||
}
|
||||
|
||||
/// A resource attached to a benchmark task (file, URL, etc.).
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TaskResource {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
#[serde(default)]
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ResourceType {
|
||||
#[default]
|
||||
File,
|
||||
Url,
|
||||
Directory,
|
||||
}
|
||||
|
||||
/// What the agent produced for scoring.
|
||||
#[derive(Debug, Clone)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TaskSubmission {
|
||||
pub response: String,
|
||||
pub conversation: Vec<ConversationTurn>,
|
||||
pub tool_calls: Vec<String>,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// A single turn in a multi-turn conversation.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ConversationTurn {
|
||||
pub role: TurnRole,
|
||||
pub content: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TurnRole {
|
||||
User,
|
||||
Assistant,
|
||||
System,
|
||||
}
|
||||
|
||||
/// Score for a single task.
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct BenchScore {
|
||||
/// 0.0 to 1.0 (1.0 = perfect).
|
||||
pub value: f64,
|
||||
/// "pass" / "fail" / "partial".
|
||||
pub label: String,
|
||||
#[serde(default)]
|
||||
pub details: Option<String>,
|
||||
}
|
||||
|
||||
impl BenchScore {
|
||||
pub fn pass() -> Self {
|
||||
Self {
|
||||
value: 1.0,
|
||||
label: "pass".to_string(),
|
||||
details: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fail(details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: 0.0,
|
||||
label: "fail".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn partial(value: f64, details: impl Into<String>) -> Self {
|
||||
Self {
|
||||
value: value.clamp(0.0, 1.0),
|
||||
label: "partial".to_string(),
|
||||
details: Some(details.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Trait for benchmark suite adapters.
|
||||
///
|
||||
/// Each suite (GAIA, Tau-bench, custom, etc.) implements this trait
|
||||
/// to provide task loading, scoring, and optional lifecycle hooks.
|
||||
#[async_trait]
|
||||
#[allow(dead_code)]
|
||||
pub trait BenchSuite: Send + Sync {
|
||||
/// Human-readable name (e.g., "GAIA Validation").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Machine ID (e.g., "gaia").
|
||||
fn id(&self) -> &str;
|
||||
|
||||
/// Load all tasks from the suite's data source.
|
||||
async fn load_tasks(&self) -> Result<Vec<BenchTask>, BenchError>;
|
||||
|
||||
/// Score the agent's submission against the expected answer.
|
||||
async fn score(
|
||||
&self,
|
||||
task: &BenchTask,
|
||||
submission: &TaskSubmission,
|
||||
) -> Result<BenchScore, BenchError>;
|
||||
|
||||
/// Optional: set up environment before running a task (clone repo, init DB, etc.).
|
||||
async fn setup_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: tear down environment after a task completes.
|
||||
async fn teardown_task(&self, _task: &BenchTask) -> Result<(), BenchError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Optional: additional tools to register for this suite's tasks.
|
||||
fn additional_tools(&self) -> Vec<Arc<dyn ironclaw::tools::Tool>> {
|
||||
vec![]
|
||||
}
|
||||
|
||||
/// Multi-turn: generate next simulated user message based on conversation so far.
|
||||
/// Return `None` to end the conversation.
|
||||
async fn next_user_message(
|
||||
&self,
|
||||
_task: &BenchTask,
|
||||
_conversation: &[ConversationTurn],
|
||||
) -> Result<Option<String>, BenchError> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -10,12 +10,17 @@
|
||||
//! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools
|
||||
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
fn main() {
|
||||
let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
|
||||
let root = PathBuf::from(&manifest_dir);
|
||||
|
||||
// ── Embed registry manifests ────────────────────────────────────────
|
||||
embed_registry_catalog(&root);
|
||||
|
||||
// ── Build Telegram channel WASM ─────────────────────────────────────
|
||||
let channel_dir = root.join("channels-src/telegram");
|
||||
let wasm_out = channel_dir.join("telegram.wasm");
|
||||
|
||||
@@ -104,3 +109,89 @@ fn main() {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect all registry manifests into a single JSON blob at compile time.
|
||||
///
|
||||
/// Output: `$OUT_DIR/embedded_catalog.json` with structure:
|
||||
/// ```json
|
||||
/// { "tools": [...], "channels": [...], "bundles": {...} }
|
||||
/// ```
|
||||
fn embed_registry_catalog(root: &Path) {
|
||||
use std::fs;
|
||||
|
||||
let registry_dir = root.join("registry");
|
||||
|
||||
// Rerun if the bundles file changes (per-file watches for tools/channels
|
||||
// are emitted inside collect_json_files to track content changes reliably).
|
||||
println!("cargo:rerun-if-changed=registry/_bundles.json");
|
||||
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
|
||||
let out_path = out_dir.join("embedded_catalog.json");
|
||||
|
||||
if !registry_dir.is_dir() {
|
||||
// No registry dir: write empty catalog
|
||||
fs::write(
|
||||
&out_path,
|
||||
r#"{"tools":[],"channels":[],"bundles":{"bundles":{}}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
return;
|
||||
}
|
||||
|
||||
let mut tools = Vec::new();
|
||||
let mut channels = Vec::new();
|
||||
|
||||
// Collect tool manifests
|
||||
let tools_dir = registry_dir.join("tools");
|
||||
if tools_dir.is_dir() {
|
||||
collect_json_files(&tools_dir, &mut tools);
|
||||
}
|
||||
|
||||
// Collect channel manifests
|
||||
let channels_dir = registry_dir.join("channels");
|
||||
if channels_dir.is_dir() {
|
||||
collect_json_files(&channels_dir, &mut channels);
|
||||
}
|
||||
|
||||
// Read bundles
|
||||
let bundles_path = registry_dir.join("_bundles.json");
|
||||
let bundles_raw = if bundles_path.is_file() {
|
||||
fs::read_to_string(&bundles_path).unwrap_or_else(|_| r#"{"bundles":{}}"#.to_string())
|
||||
} else {
|
||||
r#"{"bundles":{}}"#.to_string()
|
||||
};
|
||||
|
||||
// Build the combined JSON
|
||||
let catalog = format!(
|
||||
r#"{{"tools":[{}],"channels":[{}],"bundles":{}}}"#,
|
||||
tools.join(","),
|
||||
channels.join(","),
|
||||
bundles_raw,
|
||||
);
|
||||
|
||||
fs::write(&out_path, catalog).unwrap();
|
||||
}
|
||||
|
||||
/// Read all .json files from a directory and push their raw contents into `out`.
|
||||
fn collect_json_files(dir: &Path, out: &mut Vec<String>) {
|
||||
use std::fs;
|
||||
|
||||
let mut entries: Vec<_> = fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("json")
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Sort for deterministic output
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
|
||||
for entry in entries {
|
||||
// Emit per-file watch so Cargo reruns when file contents change
|
||||
println!("cargo:rerun-if-changed={}", entry.path().display());
|
||||
if let Ok(content) = fs::read_to_string(entry.path()) {
|
||||
out.push(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+401
@@ -0,0 +1,401 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.102"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "discord-channel"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"wit-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "equivalent"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "id-arena"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
|
||||
|
||||
[[package]]
|
||||
name = "leb128"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67"
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.106"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.228"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
|
||||
[[package]]
|
||||
name = "spdx"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.117"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-encoder"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e913f9242315ca39eff82aee0e19ee7a372155717ff0eb082c741e435ce25ed1"
|
||||
dependencies = [
|
||||
"leb128",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-metadata"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "185dfcd27fa5db2e6a23906b54c28199935f71d9a27a1a27b3a88d6fee2afae7"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"indexmap",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"spdx",
|
||||
"wasm-encoder",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d07b6a3b550fefa1a914b6d54fc175dd11c3392da11eee604e6ffc759805d25"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bitflags",
|
||||
"hashbrown 0.14.5",
|
||||
"indexmap",
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a2b3e15cd6068f233926e7d8c7c588b2ec4fb7cc7bf3824115e7c7e2a8485a3"
|
||||
dependencies = [
|
||||
"wit-bindgen-rt",
|
||||
"wit-bindgen-rust-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-core"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b632a5a0fa2409489bd49c9e6d99fcc61bb3d4ce9d1907d44662e75a28c71172"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rt"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7947d0131c7c9da3f01dfde0ab8bd4c4cf3c5bd49b6dba0ae640f1fa752572ea"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4329de4186ee30e2ef30a0533f9b3c123c019a237a7c82d692807bf1b3ee2697"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"heck",
|
||||
"indexmap",
|
||||
"prettyplease",
|
||||
"syn",
|
||||
"wasm-metadata",
|
||||
"wit-bindgen-core",
|
||||
"wit-component",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen-rust-macro"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "177fb7ee1484d113b4792cc480b1ba57664bbc951b42a4beebe573502135b1fc"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wit-bindgen-core",
|
||||
"wit-bindgen-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-component"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b505603761ed400c90ed30261f44a768317348e49f1864e82ecdc3b2744e5627"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags",
|
||||
"indexmap",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"wasm-encoder",
|
||||
"wasm-metadata",
|
||||
"wasmparser",
|
||||
"wit-parser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wit-parser"
|
||||
version = "0.220.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae2a7999ed18efe59be8de2db9cb2b7f84d88b27818c79353dfc53131840fe1a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"id-arena",
|
||||
"indexmap",
|
||||
"log",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"unicode-xid",
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db6d35d663eadb6c932438e763b262fe1a70987f9ae936e60158176d710cae4a"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4122cd3169e94605190e77839c9a40d40ed048d305bfdc146e7df40ab0f3e517"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
@@ -9,7 +9,7 @@ publish = false
|
||||
[dependencies]
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
wit-bindgen = "0.41.0"
|
||||
wit-bindgen = "0.36"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
@@ -2,6 +2,15 @@
|
||||
"type": "channel",
|
||||
"name": "discord",
|
||||
"description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "discord_bot_token",
|
||||
"prompt": "Enter your Discord Bot Token (from Developer Portal)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
@@ -10,7 +19,7 @@
|
||||
"credentials": {
|
||||
"discord_bot_token": {
|
||||
"secret_name": "discord_bot_token",
|
||||
"location": { "type": "header", "header_name": "Authorization", "prefix": "Bot " },
|
||||
"location": { "type": "header", "name": "Authorization", "prefix": "Bot " },
|
||||
"host_patterns": ["discord.com"]
|
||||
}
|
||||
},
|
||||
@@ -34,6 +43,9 @@
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"require_signature_verification": true
|
||||
"require_signature_verification": true,
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
+226
-16
@@ -124,12 +124,57 @@ struct DiscordMessageMetadata {
|
||||
thread_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "discord";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiscordConfig {
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)]
|
||||
require_signature_verification: bool,
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
struct DiscordChannel;
|
||||
|
||||
impl Guest for DiscordChannel {
|
||||
fn on_start(_config_json: String) -> Result<ChannelConfig, String> {
|
||||
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
|
||||
let config: DiscordConfig = serde_json::from_str(&config_json)
|
||||
.map_err(|e| format!("Failed to parse config: {}", e))?;
|
||||
|
||||
channel_host::log(channel_host::LogLevel::Info, "Discord channel starting");
|
||||
|
||||
// Persist owner_id so subsequent callbacks can read it
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
// Persist dm_policy and allow_from for DM pairing
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
Ok(ChannelConfig {
|
||||
display_name: "Discord".to_string(),
|
||||
http_endpoints: vec![HttpEndpointConfig {
|
||||
@@ -169,16 +214,21 @@ impl Guest for DiscordChannel {
|
||||
|
||||
// Application Command (slash command)
|
||||
2 => {
|
||||
handle_slash_command(&interaction);
|
||||
json_response(
|
||||
200,
|
||||
serde_json::json!({
|
||||
"type": 5,
|
||||
"data": {
|
||||
"content": "🤔 Thinking..."
|
||||
}
|
||||
}),
|
||||
)
|
||||
if handle_slash_command(&interaction) {
|
||||
json_response(200, serde_json::json!({"type": 5}))
|
||||
} else {
|
||||
// Permission denied — ephemeral response
|
||||
json_response(
|
||||
200,
|
||||
serde_json::json!({
|
||||
"type": 4,
|
||||
"data": {
|
||||
"content": "You are not authorized to use this bot.",
|
||||
"flags": 64
|
||||
}
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Message Component (buttons, selects)
|
||||
@@ -270,7 +320,8 @@ impl Guest for DiscordChannel {
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
/// Returns true if the message was emitted, false if permission denied.
|
||||
fn handle_slash_command(interaction: &DiscordInteraction) -> bool {
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
@@ -287,6 +338,22 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
// DM if no guild member context (only direct user field set)
|
||||
let is_dm = interaction.member.is_none();
|
||||
|
||||
// Permission check
|
||||
if !check_sender_permission(
|
||||
&user_id,
|
||||
Some(&user_name),
|
||||
is_dm,
|
||||
Some(&PairingReplyCtx {
|
||||
application_id: interaction.application_id.clone(),
|
||||
token: interaction.token.clone(),
|
||||
}),
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let channel_id = interaction.channel_id.clone().unwrap_or_default();
|
||||
|
||||
let command_name = interaction
|
||||
@@ -322,14 +389,13 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to serialize metadata: {}", e),
|
||||
);
|
||||
// Attempt to notify user of internal error
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
interaction.application_id, interaction.token
|
||||
);
|
||||
let payload = serde_json::json!({
|
||||
"content": "❌ Internal Error: Failed to process command metadata.",
|
||||
"flags": 64 // Ephemeral
|
||||
"flags": 64
|
||||
});
|
||||
let _ = channel_host::http_request(
|
||||
"POST",
|
||||
@@ -338,7 +404,7 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
Some(&serde_json::to_vec(&payload).unwrap_or_default()),
|
||||
None,
|
||||
);
|
||||
return;
|
||||
return true; // Error, but not a permission denial
|
||||
}
|
||||
};
|
||||
|
||||
@@ -349,10 +415,10 @@ fn handle_slash_command(interaction: &DiscordInteraction) {
|
||||
thread_id: None,
|
||||
metadata_json,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordMessage) {
|
||||
// Check member first (for server contexts), then user (for DMs)
|
||||
let user = interaction
|
||||
.member
|
||||
.as_ref()
|
||||
@@ -369,6 +435,11 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let is_dm = interaction.member.is_none();
|
||||
if !check_sender_permission(&user_id, Some(&user_name), is_dm, None) {
|
||||
return;
|
||||
}
|
||||
|
||||
let channel_id = message.channel_id.clone();
|
||||
|
||||
let metadata = DiscordMessageMetadata {
|
||||
@@ -399,6 +470,145 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Context needed to send a pairing reply via Discord webhook followup.
|
||||
struct PairingReplyCtx {
|
||||
application_id: String,
|
||||
token: String,
|
||||
}
|
||||
|
||||
/// Check if a sender is permitted to interact with the bot.
|
||||
/// Returns true if allowed, false if denied (pairing reply sent if applicable).
|
||||
fn check_sender_permission(
|
||||
user_id: &str,
|
||||
username: Option<&str>,
|
||||
is_dm: bool,
|
||||
reply_ctx: Option<&PairingReplyCtx>,
|
||||
) -> bool {
|
||||
// 1. Owner check (highest priority, applies to all contexts)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if user_id != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping interaction from non-owner user {} (owner: {})",
|
||||
user_id, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (only for DMs when no owner_id)
|
||||
if !is_dm {
|
||||
return true; // Guild interactions bypass DM policy
|
||||
}
|
||||
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list: config allow_from + pairing store
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender against allow list
|
||||
let is_allowed = allowed.contains(&"*".to_string())
|
||||
|| allowed.contains(&user_id.to_string())
|
||||
|| username.is_some_and(|u| allowed.contains(&u.to_string()));
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"user_id": user_id,
|
||||
"username": username,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for user {}: code {}",
|
||||
user_id, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
if let Some(ctx) = reply_ctx {
|
||||
let _ = send_pairing_reply(ctx, &result.code);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code as an ephemeral Discord followup message.
|
||||
fn send_pairing_reply(ctx: &PairingReplyCtx, code: &str) -> Result<(), String> {
|
||||
let url = format!(
|
||||
"https://discord.com/api/v10/webhooks/{}/{}",
|
||||
ctx.application_id, ctx.token
|
||||
);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"content": format!(
|
||||
"To pair with this bot, run: `ironclaw pairing approve discord {}`",
|
||||
code
|
||||
),
|
||||
"flags": 64 // Ephemeral — only visible to the sender
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"Discord API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
@@ -2,6 +2,20 @@
|
||||
"type": "channel",
|
||||
"name": "slack",
|
||||
"description": "Slack Events API channel for receiving and responding to Slack messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "slack_bot_token",
|
||||
"prompt": "Enter your Slack Bot OAuth Token (xoxb-...)",
|
||||
"optional": false
|
||||
},
|
||||
{
|
||||
"name": "slack_signing_secret",
|
||||
"prompt": "Enter your Slack Signing Secret (from App Credentials)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
@@ -33,6 +47,9 @@
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"signing_secret_name": "slack_signing_secret"
|
||||
"signing_secret_name": "slack_signing_secret",
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,15 +104,31 @@ struct SlackPostMessageResponse {
|
||||
ts: Option<String>,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "slack";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct SlackConfig {
|
||||
/// Name of secret containing signing secret (for verification by host).
|
||||
/// Parsed from config for forward compatibility; not yet used in WASM
|
||||
/// (host handles signature verification).
|
||||
#[serde(default = "default_signing_secret_name")]
|
||||
#[allow(dead_code)]
|
||||
signing_secret_name: String,
|
||||
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_signing_secret_name() -> String {
|
||||
@@ -123,12 +139,30 @@ struct SlackChannel;
|
||||
|
||||
impl Guest for SlackChannel {
|
||||
fn on_start(config_json: String) -> Result<ChannelConfig, String> {
|
||||
// Parse configuration
|
||||
let _config: SlackConfig = serde_json::from_str(&config_json)
|
||||
let config: SlackConfig = serde_json::from_str(&config_json)
|
||||
.map_err(|e| format!("Failed to parse config: {}", e))?;
|
||||
|
||||
channel_host::log(channel_host::LogLevel::Info, "Slack channel starting");
|
||||
|
||||
// Persist owner_id so subsequent callbacks can read it
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
// Persist dm_policy and allow_from for DM pairing
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
Ok(ChannelConfig {
|
||||
display_name: "Slack".to_string(),
|
||||
http_endpoints: vec![HttpEndpointConfig {
|
||||
@@ -136,7 +170,7 @@ impl Guest for SlackChannel {
|
||||
methods: vec!["POST".to_string()],
|
||||
require_secret: true,
|
||||
}],
|
||||
poll: None, // Slack uses push via webhooks, no polling needed
|
||||
poll: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -280,7 +314,7 @@ impl Guest for SlackChannel {
|
||||
/// Handle a Slack event and emit message if applicable.
|
||||
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
|
||||
match event.event_type.as_str() {
|
||||
// Direct mention of the bot
|
||||
// Direct mention of the bot (always in a channel, not a DM)
|
||||
"app_mention" => {
|
||||
if let (Some(user), Some(channel), Some(text), Some(ts)) = (
|
||||
event.user,
|
||||
@@ -288,6 +322,10 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
|
||||
event.text,
|
||||
event.ts.clone(),
|
||||
) {
|
||||
// app_mention is always in a channel (not DM)
|
||||
if !check_sender_permission(&user, &channel, false) {
|
||||
return;
|
||||
}
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
}
|
||||
}
|
||||
@@ -307,6 +345,9 @@ fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Opt
|
||||
) {
|
||||
// Only process DMs (channel IDs starting with D)
|
||||
if channel.starts_with('D') {
|
||||
if !check_sender_permission(&user, &channel, true) {
|
||||
return;
|
||||
}
|
||||
emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id);
|
||||
}
|
||||
}
|
||||
@@ -358,6 +399,126 @@ fn emit_message(
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Check if a sender is permitted. Returns true if allowed.
|
||||
/// For pairing mode, sends a pairing code DM if denied.
|
||||
fn check_sender_permission(user_id: &str, channel_id: &str, is_dm: bool) -> bool {
|
||||
// 1. Owner check (highest priority, applies to all contexts)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if user_id != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner user {} (owner: {})",
|
||||
user_id, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (only for DMs when no owner_id)
|
||||
if !is_dm {
|
||||
return true; // Channel messages bypass DM policy
|
||||
}
|
||||
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list: config allow_from + pairing store
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender (Slack events only have user ID, not username)
|
||||
let is_allowed =
|
||||
allowed.contains(&"*".to_string()) || allowed.contains(&user_id.to_string());
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"user_id": user_id,
|
||||
"channel_id": channel_id,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, user_id, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for user {}: code {}",
|
||||
user_id, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
let _ = send_pairing_reply(channel_id, &result.code);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code message via Slack chat.postMessage.
|
||||
fn send_pairing_reply(channel_id: &str, code: &str) -> Result<(), String> {
|
||||
let payload = serde_json::json!({
|
||||
"channel": channel_id,
|
||||
"text": format!(
|
||||
"To pair with this bot, run: `ironclaw pairing approve slack {}`",
|
||||
code
|
||||
),
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({"Content-Type": "application/json"});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
"https://slack.com/api/chat.postMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status == 200 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"Slack API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip leading bot mention from text.
|
||||
fn strip_bot_mention(text: &str) -> String {
|
||||
// Slack mentions look like <@U12345678>
|
||||
|
||||
@@ -25,5 +25,3 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -76,6 +76,9 @@ struct TelegramMessage {
|
||||
#[serde(default)]
|
||||
caption: Option<String>,
|
||||
|
||||
/// Voice message.
|
||||
voice: Option<TelegramVoice>,
|
||||
|
||||
/// Original message if this is a reply.
|
||||
reply_to_message: Option<Box<TelegramMessage>>,
|
||||
|
||||
@@ -139,6 +142,36 @@ struct MessageEntity {
|
||||
user: Option<TelegramUser>,
|
||||
}
|
||||
|
||||
/// Telegram Voice object.
|
||||
/// https://core.telegram.org/bots/api#voice
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramVoice {
|
||||
/// Identifier for this file, which can be used to download the file.
|
||||
file_id: String,
|
||||
|
||||
/// Duration of the audio in seconds.
|
||||
duration: u32,
|
||||
|
||||
/// MIME type of the file.
|
||||
#[serde(default)]
|
||||
mime_type: Option<String>,
|
||||
|
||||
/// File size in bytes.
|
||||
#[serde(default)]
|
||||
file_size: Option<i64>,
|
||||
}
|
||||
|
||||
/// Telegram File object returned by getFile.
|
||||
/// https://core.telegram.org/bots/api#file
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramFile {
|
||||
/// Identifier for this file.
|
||||
file_id: String,
|
||||
|
||||
/// File path for downloading. Use https://api.telegram.org/file/bot<token>/<file_path>.
|
||||
file_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Telegram API response wrapper.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramApiResponse<T> {
|
||||
@@ -867,6 +900,87 @@ fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Voice File Download
|
||||
// ============================================================================
|
||||
|
||||
/// Download a voice file from Telegram by file_id.
|
||||
///
|
||||
/// 1. Call getFile to get the file_path.
|
||||
/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}.
|
||||
fn download_voice_file(file_id: &str) -> Result<Vec<u8>, String> {
|
||||
// Reject file_id containing curly braces to prevent credential placeholder
|
||||
// injection (e.g., a malicious file_id like "{OPENAI_API_KEY}" would be
|
||||
// interpreted by the host-side credential injector).
|
||||
if file_id.contains('{') || file_id.contains('}') {
|
||||
return Err("invalid file_id: contains forbidden characters".to_string());
|
||||
}
|
||||
|
||||
// Step 1: Call getFile to get file_path
|
||||
// Double braces `{{...}}` produce a literal `{TELEGRAM_BOT_TOKEN}` placeholder
|
||||
// in the URL, which the host-side credential injector replaces with the real token.
|
||||
let get_file_url = format!(
|
||||
"https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}",
|
||||
file_id
|
||||
);
|
||||
|
||||
let headers = serde_json::json!({});
|
||||
let result = channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("getFile request failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
return Err(format!("getFile returned {}: {}", response.status, body_str));
|
||||
}
|
||||
|
||||
let api_response: TelegramApiResponse<TelegramFile> =
|
||||
serde_json::from_slice(&response.body)
|
||||
.map_err(|e| format!("Failed to parse getFile response: {}", e))?;
|
||||
|
||||
if !api_response.ok {
|
||||
return Err(format!(
|
||||
"getFile API error: {}",
|
||||
api_response
|
||||
.description
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
let file = api_response
|
||||
.result
|
||||
.ok_or_else(|| "getFile returned no result".to_string())?;
|
||||
|
||||
let file_path = file
|
||||
.file_path
|
||||
.ok_or_else(|| "getFile returned no file_path".to_string())?;
|
||||
|
||||
// Sanitize file_path against credential placeholder injection
|
||||
if file_path.contains('{') || file_path.contains('}') {
|
||||
return Err("invalid file_path: contains forbidden characters".to_string());
|
||||
}
|
||||
|
||||
// Step 2: Download the actual file bytes
|
||||
let download_url = format!(
|
||||
"https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}",
|
||||
file_path
|
||||
);
|
||||
|
||||
let result =
|
||||
channel_host::http_request("GET", &download_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("File download failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
return Err(format!(
|
||||
"File download returned status {}",
|
||||
response.status
|
||||
));
|
||||
}
|
||||
|
||||
Ok(response.body)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Update Handling
|
||||
// ============================================================================
|
||||
@@ -886,6 +1000,9 @@ fn handle_update(update: TelegramUpdate) {
|
||||
|
||||
/// Process a single message.
|
||||
fn handle_message(message: TelegramMessage) {
|
||||
// Check for voice note first (voice-only messages have no text)
|
||||
let is_voice = message.voice.is_some();
|
||||
|
||||
// Use text or caption (for media messages)
|
||||
let content = message
|
||||
.text
|
||||
@@ -893,7 +1010,8 @@ fn handle_message(message: TelegramMessage) {
|
||||
.or_else(|| message.caption.filter(|c| !c.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if content.is_empty() {
|
||||
// Allow voice notes through even when content is empty
|
||||
if content.is_empty() && !is_voice {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1038,19 +1156,65 @@ fn handle_message(message: TelegramMessage) {
|
||||
},
|
||||
);
|
||||
|
||||
// Handle voice notes: download and attach audio bytes.
|
||||
// Note: download is synchronous (two HTTP roundtrips to Telegram API).
|
||||
// This blocks the WASM execution for the current polling tick.
|
||||
let mut attachments = Vec::new();
|
||||
let mut voice_download_failed = false;
|
||||
if let Some(ref voice) = message.voice {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Voice note from user {} (duration: {}s, file_id: {})",
|
||||
from.id, voice.duration, voice.file_id
|
||||
),
|
||||
);
|
||||
|
||||
match download_voice_file(&voice.file_id) {
|
||||
Ok(audio_bytes) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Downloaded voice file: {} bytes", audio_bytes.len()),
|
||||
);
|
||||
attachments.push(channel_host::Attachment {
|
||||
kind: channel_host::AttachmentKind::Audio,
|
||||
mime_type: voice
|
||||
.mime_type
|
||||
.clone()
|
||||
.unwrap_or_else(|| "audio/ogg".to_string()),
|
||||
data: audio_bytes,
|
||||
filename: Some(format!("voice_{}.ogg", voice.file_id)),
|
||||
duration_secs: Some(voice.duration),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to download voice file: {}", e),
|
||||
);
|
||||
voice_download_failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine what to emit to the agent.
|
||||
// - Voice notes: use "[Voice note]" as content (transcription happens host-side)
|
||||
// - `/start` (no args): emit a welcome placeholder so the agent greets the user
|
||||
// - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through
|
||||
// so Submission::parse() can handle it
|
||||
// - Commands with args (e.g. `/start hello`): cleaned_text already has the args
|
||||
// - Plain text: pass through as-is
|
||||
let trimmed_content = content.trim();
|
||||
let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
let content_to_emit = if is_voice && voice_download_failed && content.is_empty() {
|
||||
"[Voice note: download failed]".to_string()
|
||||
} else if is_voice && content.is_empty() {
|
||||
"[Voice note]".to_string()
|
||||
} else if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
"[User started the bot]".to_string()
|
||||
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
|
||||
// Bare control command like /interrupt, /stop, /help — pass through raw
|
||||
trimmed_content.to_string()
|
||||
} else if cleaned_text.is_empty() {
|
||||
} else if cleaned_text.is_empty() && !is_voice {
|
||||
return;
|
||||
} else {
|
||||
cleaned_text
|
||||
@@ -1063,6 +1227,7 @@ fn handle_message(message: TelegramMessage) {
|
||||
content: content_to_emit,
|
||||
thread_id: None, // Telegram doesn't have threads in the same way
|
||||
metadata_json,
|
||||
attachments,
|
||||
});
|
||||
|
||||
channel_host::log(
|
||||
|
||||
@@ -1 +1,55 @@
|
||||
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
|
||||
{
|
||||
"type": "channel",
|
||||
"name": "telegram",
|
||||
"description": "Telegram Bot API channel for receiving and responding to Telegram messages",
|
||||
"setup": {
|
||||
"required_secrets": [
|
||||
{
|
||||
"name": "telegram_bot_token",
|
||||
"prompt": "Enter your Telegram Bot API token (from @BotFather)",
|
||||
"optional": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"capabilities": {
|
||||
"http": {
|
||||
"allowlist": [
|
||||
{ "host": "api.telegram.org", "path_prefix": "/bot" },
|
||||
{ "host": "api.telegram.org", "path_prefix": "/file/bot" }
|
||||
],
|
||||
"credentials": {
|
||||
"telegram_bot": {
|
||||
"secret_name": "telegram_bot_token",
|
||||
"location": { "type": "url_path", "placeholder": "{TELEGRAM_BOT_TOKEN}" },
|
||||
"host_patterns": ["api.telegram.org"]
|
||||
}
|
||||
},
|
||||
"rate_limit": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 1000
|
||||
}
|
||||
},
|
||||
"secrets": {
|
||||
"allowed_names": ["telegram_*"]
|
||||
},
|
||||
"channel": {
|
||||
"allowed_paths": ["/webhook/telegram"],
|
||||
"allow_polling": true,
|
||||
"min_poll_interval_ms": 30000,
|
||||
"workspace_prefix": "channels/telegram/",
|
||||
"emit_rate_limit": {
|
||||
"messages_per_minute": 100,
|
||||
"messages_per_hour": 5000
|
||||
}
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"bot_username": null,
|
||||
"owner_id": null,
|
||||
"respond_to_all_group_messages": false,
|
||||
"polling_enabled": false,
|
||||
"poll_interval_ms": 30000,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,15 @@ struct WhatsAppMessageMetadata {
|
||||
timestamp: String,
|
||||
}
|
||||
|
||||
/// Workspace path for persisting owner_id across WASM callbacks.
|
||||
const OWNER_ID_PATH: &str = "state/owner_id";
|
||||
/// Workspace path for persisting dm_policy across WASM callbacks.
|
||||
const DM_POLICY_PATH: &str = "state/dm_policy";
|
||||
/// Workspace path for persisting allow_from (JSON array) across WASM callbacks.
|
||||
const ALLOW_FROM_PATH: &str = "state/allow_from";
|
||||
/// Channel name for pairing store (used by pairing host APIs).
|
||||
const CHANNEL_NAME: &str = "whatsapp";
|
||||
|
||||
/// Channel configuration from capabilities file.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WhatsAppConfig {
|
||||
@@ -236,6 +245,15 @@ struct WhatsAppConfig {
|
||||
/// Whether to reply to the original message (thread context)
|
||||
#[serde(default = "default_reply_to_message")]
|
||||
reply_to_message: bool,
|
||||
|
||||
#[serde(default)]
|
||||
owner_id: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
dm_policy: Option<String>,
|
||||
|
||||
#[serde(default)]
|
||||
allow_from: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_api_version() -> String {
|
||||
@@ -264,6 +282,9 @@ impl Guest for WhatsAppChannel {
|
||||
WhatsAppConfig {
|
||||
api_version: default_api_version(),
|
||||
reply_to_message: default_reply_to_message(),
|
||||
owner_id: None,
|
||||
dm_policy: None,
|
||||
allow_from: None,
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -279,6 +300,24 @@ impl Guest for WhatsAppChannel {
|
||||
// Persist api_version in workspace so on_respond() can read it
|
||||
let _ = channel_host::workspace_write("channels/whatsapp/api_version", &config.api_version);
|
||||
|
||||
// Persist permission config for handle_message
|
||||
if let Some(ref owner_id) = config.owner_id {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, owner_id);
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Owner restriction enabled: user {}", owner_id),
|
||||
);
|
||||
} else {
|
||||
let _ = channel_host::workspace_write(OWNER_ID_PATH, "");
|
||||
}
|
||||
|
||||
let dm_policy = config.dm_policy.as_deref().unwrap_or("pairing");
|
||||
let _ = channel_host::workspace_write(DM_POLICY_PATH, dm_policy);
|
||||
|
||||
let allow_from_json = serde_json::to_string(&config.allow_from.unwrap_or_default())
|
||||
.unwrap_or_else(|_| "[]".to_string());
|
||||
let _ = channel_host::workspace_write(ALLOW_FROM_PATH, &allow_from_json);
|
||||
|
||||
// WhatsApp Cloud API is webhook-only, no polling available
|
||||
Ok(ChannelConfig {
|
||||
display_name: "WhatsApp".to_string(),
|
||||
@@ -604,6 +643,15 @@ fn handle_message(
|
||||
// Look up sender's name from contacts
|
||||
let user_name = contact_names.get(&message.from).cloned();
|
||||
|
||||
// Permission check (WhatsApp is always DM)
|
||||
if !check_sender_permission(
|
||||
&message.from,
|
||||
user_name.as_deref(),
|
||||
phone_number_id,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Build metadata for response routing
|
||||
// This is critical - the response handler uses this to know where to send
|
||||
let metadata = WhatsAppMessageMetadata {
|
||||
@@ -637,6 +685,149 @@ fn handle_message(
|
||||
// Utilities
|
||||
// ============================================================================
|
||||
|
||||
// ============================================================================
|
||||
// Permission & Pairing
|
||||
// ============================================================================
|
||||
|
||||
/// Check if a sender is permitted. Returns true if allowed.
|
||||
/// WhatsApp is always 1-to-1 (DM), so dm_policy always applies.
|
||||
fn check_sender_permission(
|
||||
sender_phone: &str,
|
||||
user_name: Option<&str>,
|
||||
phone_number_id: &str,
|
||||
) -> bool {
|
||||
// 1. Owner check (highest priority)
|
||||
let owner_id = channel_host::workspace_read(OWNER_ID_PATH).filter(|s| !s.is_empty());
|
||||
if let Some(ref owner) = owner_id {
|
||||
if sender_phone != owner {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Debug,
|
||||
&format!(
|
||||
"Dropping message from non-owner {} (owner: {})",
|
||||
sender_phone, owner
|
||||
),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. DM policy (WhatsApp is always DM)
|
||||
let dm_policy =
|
||||
channel_host::workspace_read(DM_POLICY_PATH).unwrap_or_else(|| "pairing".to_string());
|
||||
|
||||
if dm_policy == "open" {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 3. Build merged allow list
|
||||
let mut allowed: Vec<String> = channel_host::workspace_read(ALLOW_FROM_PATH)
|
||||
.and_then(|s| serde_json::from_str(&s).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Ok(store_allowed) = channel_host::pairing_read_allow_from(CHANNEL_NAME) {
|
||||
allowed.extend(store_allowed);
|
||||
}
|
||||
|
||||
// 4. Check sender (phone number or name)
|
||||
let is_allowed = allowed.contains(&"*".to_string())
|
||||
|| allowed.contains(&sender_phone.to_string())
|
||||
|| user_name.is_some_and(|u| allowed.contains(&u.to_string()));
|
||||
|
||||
if is_allowed {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 5. Not allowed — handle by policy
|
||||
if dm_policy == "pairing" {
|
||||
let meta = serde_json::json!({
|
||||
"phone": sender_phone,
|
||||
"name": user_name,
|
||||
})
|
||||
.to_string();
|
||||
|
||||
match channel_host::pairing_upsert_request(CHANNEL_NAME, sender_phone, &meta) {
|
||||
Ok(result) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Pairing request for {}: code {}",
|
||||
sender_phone, result.code
|
||||
),
|
||||
);
|
||||
if result.created {
|
||||
let _ = send_pairing_reply(sender_phone, phone_number_id, &result.code);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Pairing upsert failed: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Send a pairing code message via WhatsApp Cloud API.
|
||||
fn send_pairing_reply(
|
||||
recipient_phone: &str,
|
||||
phone_number_id: &str,
|
||||
code: &str,
|
||||
) -> Result<(), String> {
|
||||
let api_version = channel_host::workspace_read("channels/whatsapp/api_version")
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| "v18.0".to_string());
|
||||
|
||||
let url = format!(
|
||||
"https://graph.facebook.com/{}/{}/messages",
|
||||
api_version, phone_number_id
|
||||
);
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"messaging_product": "whatsapp",
|
||||
"recipient_type": "individual",
|
||||
"to": recipient_phone,
|
||||
"type": "text",
|
||||
"text": {
|
||||
"preview_url": false,
|
||||
"body": format!(
|
||||
"To pair with this bot, run: ironclaw pairing approve whatsapp {}",
|
||||
code
|
||||
)
|
||||
}
|
||||
});
|
||||
|
||||
let payload_bytes =
|
||||
serde_json::to_vec(&payload).map_err(|e| format!("Failed to serialize: {}", e))?;
|
||||
|
||||
let headers = serde_json::json!({
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer {WHATSAPP_ACCESS_TOKEN}"
|
||||
});
|
||||
|
||||
let result = channel_host::http_request(
|
||||
"POST",
|
||||
&url,
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(response) if response.status >= 200 && response.status < 300 => Ok(()),
|
||||
Ok(response) => {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
Err(format!(
|
||||
"WhatsApp API error: {} - {}",
|
||||
response.status, body_str
|
||||
))
|
||||
}
|
||||
Err(e) => Err(format!("HTTP request failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a JSON HTTP response.
|
||||
fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse {
|
||||
let body = serde_json::to_vec(&value).unwrap_or_default();
|
||||
|
||||
@@ -48,6 +48,9 @@
|
||||
},
|
||||
"config": {
|
||||
"api_version": "v18.0",
|
||||
"reply_to_message": true
|
||||
"reply_to_message": true,
|
||||
"owner_id": null,
|
||||
"dm_policy": "pairing",
|
||||
"allow_from": []
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# LLM Provider Configuration
|
||||
|
||||
IronClaw defaults to NEAR AI for model access, but supports any OpenAI-compatible
|
||||
endpoint as well as Anthropic and Ollama directly. This guide covers the most common
|
||||
configurations.
|
||||
|
||||
## Provider Overview
|
||||
|
||||
| Provider | Backend value | Requires API key | Notes |
|
||||
|---|---|---|---|
|
||||
| NEAR AI | `nearai` | OAuth (browser) | Default; multi-model |
|
||||
| Anthropic | `anthropic` | `ANTHROPIC_API_KEY` | Claude models |
|
||||
| OpenAI | `openai` | `OPENAI_API_KEY` | GPT models |
|
||||
| Ollama | `ollama` | No | Local inference |
|
||||
| OpenRouter | `openai_compatible` | `LLM_API_KEY` | 300+ models |
|
||||
| Together AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
|
||||
| Fireworks AI | `openai_compatible` | `LLM_API_KEY` | Fast inference |
|
||||
| vLLM / LiteLLM | `openai_compatible` | Optional | Self-hosted |
|
||||
| LM Studio | `openai_compatible` | No | Local GUI |
|
||||
|
||||
---
|
||||
|
||||
## NEAR AI (default)
|
||||
|
||||
No additional configuration required. On first run, `ironclaw onboard` opens a browser
|
||||
for OAuth authentication. Credentials are saved to `~/.ironclaw/session.json`.
|
||||
|
||||
```env
|
||||
NEARAI_MODEL=claude-3-5-sonnet-20241022
|
||||
NEARAI_BASE_URL=https://private.near.ai
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Anthropic (Claude)
|
||||
|
||||
```env
|
||||
LLM_BACKEND=anthropic
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
```
|
||||
|
||||
Popular models: `claude-sonnet-4-20250514`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-20241022`
|
||||
|
||||
---
|
||||
|
||||
## OpenAI (GPT)
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai
|
||||
OPENAI_API_KEY=sk-...
|
||||
```
|
||||
|
||||
Popular models: `gpt-4o`, `gpt-4o-mini`, `o3-mini`
|
||||
|
||||
---
|
||||
|
||||
## Ollama (local)
|
||||
|
||||
Install Ollama from [ollama.com](https://ollama.com), pull a model, then:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=ollama
|
||||
OLLAMA_MODEL=llama3.2
|
||||
# OLLAMA_BASE_URL=http://localhost:11434 # default
|
||||
```
|
||||
|
||||
Pull a model first: `ollama pull llama3.2`
|
||||
|
||||
---
|
||||
|
||||
## OpenAI-Compatible Endpoints
|
||||
|
||||
All providers below use `LLM_BACKEND=openai_compatible`. Set `LLM_BASE_URL` to the
|
||||
provider's OpenAI-compatible endpoint and `LLM_API_KEY` to your API key.
|
||||
|
||||
### OpenRouter
|
||||
|
||||
[OpenRouter](https://openrouter.ai) routes to 300+ models from a single API key.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
LLM_API_KEY=sk-or-...
|
||||
LLM_MODEL=anthropic/claude-sonnet-4
|
||||
```
|
||||
|
||||
Popular OpenRouter model IDs:
|
||||
|
||||
| Model | ID |
|
||||
|---|---|
|
||||
| Claude Sonnet 4 | `anthropic/claude-sonnet-4` |
|
||||
| GPT-4o | `openai/gpt-4o` |
|
||||
| Llama 4 Maverick | `meta-llama/llama-4-maverick` |
|
||||
| Gemini 2.0 Flash | `google/gemini-2.0-flash-001` |
|
||||
| Mistral Small | `mistralai/mistral-small-3.1-24b-instruct` |
|
||||
|
||||
Browse all models at [openrouter.ai/models](https://openrouter.ai/models).
|
||||
|
||||
### Together AI
|
||||
|
||||
[Together AI](https://www.together.ai) provides fast inference for open-source models.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://api.together.xyz/v1
|
||||
LLM_API_KEY=...
|
||||
LLM_MODEL=meta-llama/Llama-3.3-70B-Instruct-Turbo
|
||||
```
|
||||
|
||||
Popular Together AI model IDs:
|
||||
|
||||
| Model | ID |
|
||||
|---|---|
|
||||
| Llama 3.3 70B | `meta-llama/Llama-3.3-70B-Instruct-Turbo` |
|
||||
| DeepSeek R1 | `deepseek-ai/DeepSeek-R1` |
|
||||
| Qwen 2.5 72B | `Qwen/Qwen2.5-72B-Instruct-Turbo` |
|
||||
|
||||
### Fireworks AI
|
||||
|
||||
[Fireworks AI](https://fireworks.ai) offers fast inference with compound AI system support.
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=https://api.fireworks.ai/inference/v1
|
||||
LLM_API_KEY=fw_...
|
||||
LLM_MODEL=accounts/fireworks/models/llama4-maverick-instruct-basic
|
||||
```
|
||||
|
||||
### vLLM / LiteLLM (self-hosted)
|
||||
|
||||
For self-hosted inference servers:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:8000/v1
|
||||
LLM_API_KEY=token-abc123 # set to any string if auth is not configured
|
||||
LLM_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
||||
```
|
||||
|
||||
LiteLLM proxy (forwards to any backend, including Bedrock, Vertex, Azure):
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:4000/v1
|
||||
LLM_API_KEY=sk-...
|
||||
LLM_MODEL=gpt-4o # as configured in litellm config.yaml
|
||||
```
|
||||
|
||||
### LM Studio (local GUI)
|
||||
|
||||
Start LM Studio's local server, then:
|
||||
|
||||
```env
|
||||
LLM_BACKEND=openai_compatible
|
||||
LLM_BASE_URL=http://localhost:1234/v1
|
||||
LLM_MODEL=llama-3.2-3b-instruct-q4_K_M
|
||||
# LLM_API_KEY is not required for LM Studio
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Using the Setup Wizard
|
||||
|
||||
Instead of editing `.env` manually, run the onboarding wizard:
|
||||
|
||||
```bash
|
||||
ironclaw onboard
|
||||
```
|
||||
|
||||
Select **"OpenAI-compatible"** for OpenRouter, Together AI, Fireworks, vLLM, LiteLLM,
|
||||
or LM Studio. You will be prompted for the base URL and (optionally) an API key.
|
||||
The model name is configured in the following step.
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/okta-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
"artifacts": {
|
||||
"wasm32-wasip2": {
|
||||
"url": null,
|
||||
"url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz",
|
||||
"sha256": null
|
||||
}
|
||||
},
|
||||
|
||||
@@ -682,7 +682,15 @@ impl Agent {
|
||||
|
||||
// Convert SubmissionResult to response string
|
||||
match result? {
|
||||
SubmissionResult::Response { content } => Ok(Some(content)),
|
||||
SubmissionResult::Response { content } => {
|
||||
// Suppress silent replies (e.g. from group chat "nothing to say" responses)
|
||||
if crate::llm::is_silent_reply(&content) {
|
||||
tracing::debug!("Suppressing silent reply token");
|
||||
Ok(None)
|
||||
} else {
|
||||
Ok(Some(content))
|
||||
}
|
||||
}
|
||||
SubmissionResult::Ok { message } => Ok(message),
|
||||
SubmissionResult::Error { message } => Ok(Some(format!("Error: {}", message))),
|
||||
SubmissionResult::Interrupted => Ok(Some("Interrupted.".into())),
|
||||
|
||||
+60
-1
@@ -4,7 +4,7 @@
|
||||
//! to prevent runaway agents from burning through API credits. Especially
|
||||
//! important for daemon/heartbeat modes where the agent acts autonomously.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -53,6 +53,14 @@ impl std::fmt::Display for CostLimitExceeded {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-model token usage counters.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ModelTokens {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub cost: Decimal,
|
||||
}
|
||||
|
||||
/// Tracks costs and action rates, enforcing configurable limits.
|
||||
///
|
||||
/// Thread-safe; designed to be shared via `Arc<CostGuard>`.
|
||||
@@ -67,6 +75,9 @@ pub struct CostGuard {
|
||||
|
||||
/// Flag set when daily budget is exceeded to short-circuit checks.
|
||||
budget_exceeded: AtomicBool,
|
||||
|
||||
/// Per-model token usage since startup.
|
||||
model_tokens: Mutex<HashMap<String, ModelTokens>>,
|
||||
}
|
||||
|
||||
struct DailyCost {
|
||||
@@ -85,6 +96,7 @@ impl CostGuard {
|
||||
}),
|
||||
action_window: Mutex::new(VecDeque::new()),
|
||||
budget_exceeded: AtomicBool::new(false),
|
||||
model_tokens: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,6 +204,15 @@ impl CostGuard {
|
||||
window.push_back(Instant::now());
|
||||
}
|
||||
|
||||
// Track per-model token usage
|
||||
{
|
||||
let mut tokens = self.model_tokens.lock().await;
|
||||
let entry = tokens.entry(model.to_string()).or_default();
|
||||
entry.input_tokens += u64::from(input_tokens);
|
||||
entry.output_tokens += u64::from(output_tokens);
|
||||
entry.cost += cost;
|
||||
}
|
||||
|
||||
cost
|
||||
}
|
||||
|
||||
@@ -215,6 +236,11 @@ impl CostGuard {
|
||||
}
|
||||
window.len() as u64
|
||||
}
|
||||
|
||||
/// Per-model token usage since startup.
|
||||
pub async fn model_usage(&self) -> HashMap<String, ModelTokens> {
|
||||
self.model_tokens.lock().await.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a Decimal USD amount to whole cents (truncated).
|
||||
@@ -336,4 +362,37 @@ mod tests {
|
||||
assert!(rate.to_string().contains("101 actions"));
|
||||
assert!(rate.to_string().contains("100 allowed"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_model_usage_per_model_tracking() {
|
||||
let guard = CostGuard::new(CostGuardConfig::default());
|
||||
|
||||
// Initially empty
|
||||
assert!(guard.model_usage().await.is_empty());
|
||||
|
||||
// Record calls for two different models
|
||||
guard.record_llm_call("gpt-4o", 1000, 500).await;
|
||||
guard.record_llm_call("gpt-4o", 2000, 1000).await;
|
||||
guard
|
||||
.record_llm_call("claude-3-5-sonnet-20241022", 500, 200)
|
||||
.await;
|
||||
|
||||
let usage = guard.model_usage().await;
|
||||
assert_eq!(usage.len(), 2);
|
||||
|
||||
let gpt = usage.get("gpt-4o").expect("gpt-4o should be tracked");
|
||||
assert_eq!(gpt.input_tokens, 3000);
|
||||
assert_eq!(gpt.output_tokens, 1500);
|
||||
assert!(gpt.cost > Decimal::ZERO);
|
||||
|
||||
let claude = usage
|
||||
.get("claude-3-5-sonnet-20241022")
|
||||
.expect("claude should be tracked");
|
||||
assert_eq!(claude.input_tokens, 500);
|
||||
assert_eq!(claude.output_tokens, 200);
|
||||
assert!(claude.cost > Decimal::ZERO);
|
||||
|
||||
// Costs should differ since models have different pricing
|
||||
assert_ne!(gpt.cost, claude.cost);
|
||||
}
|
||||
}
|
||||
|
||||
+49
-38
@@ -40,9 +40,17 @@ impl Agent {
|
||||
thread_id: Uuid,
|
||||
initial_messages: Vec<ChatMessage>,
|
||||
) -> Result<AgenticLoopResult, Error> {
|
||||
// Detect group chat from channel metadata (needed before loading system prompt)
|
||||
let is_group_chat = message
|
||||
.metadata
|
||||
.get("chat_type")
|
||||
.and_then(|v| v.as_str())
|
||||
.is_some_and(|t| t == "group" || t == "channel" || t == "supergroup");
|
||||
|
||||
// Load workspace system prompt (identity files: AGENTS.md, SOUL.md, etc.)
|
||||
// In group chats, MEMORY.md is excluded to prevent leaking personal context.
|
||||
let system_prompt = if let Some(ws) = self.workspace() {
|
||||
match ws.system_prompt().await {
|
||||
match ws.system_prompt_for_context(is_group_chat).await {
|
||||
Ok(prompt) if !prompt.is_empty() => Some(prompt),
|
||||
Ok(_) => None,
|
||||
Err(e) => {
|
||||
@@ -94,7 +102,10 @@ impl Agent {
|
||||
None
|
||||
};
|
||||
|
||||
let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone());
|
||||
let mut reasoning = Reasoning::new(self.llm().clone(), self.safety().clone())
|
||||
.with_channel(message.channel.clone())
|
||||
.with_model_name(self.llm().active_model_name())
|
||||
.with_group_chat(is_group_chat);
|
||||
if let Some(prompt) = system_prompt {
|
||||
reasoning = reasoning.with_system_prompt(prompt);
|
||||
}
|
||||
@@ -108,21 +119,21 @@ impl Agent {
|
||||
// Create a JobContext for tool execution (chat doesn't have a real job)
|
||||
let job_ctx = JobContext::with_user(&message.user_id, "chat", "Interactive chat session");
|
||||
|
||||
const MAX_TOOL_ITERATIONS: usize = 10;
|
||||
let max_tool_iterations = self.config.max_tool_iterations;
|
||||
// Force a text-only response on the last iteration to guarantee termination
|
||||
// instead of hard-erroring. The penultimate iteration also gets a nudge
|
||||
// message so the LLM knows it should wrap up.
|
||||
const FORCE_TEXT_AT: usize = MAX_TOOL_ITERATIONS;
|
||||
const NUDGE_AT: usize = MAX_TOOL_ITERATIONS - 1;
|
||||
let force_text_at = max_tool_iterations;
|
||||
let nudge_at = max_tool_iterations.saturating_sub(1);
|
||||
let mut iteration = 0;
|
||||
loop {
|
||||
iteration += 1;
|
||||
// Hard ceiling one past the forced-text iteration (should never be reached
|
||||
// since FORCE_TEXT_AT guarantees a text response, but kept as a safety net).
|
||||
if iteration > MAX_TOOL_ITERATIONS + 1 {
|
||||
// since force_text_at guarantees a text response, but kept as a safety net).
|
||||
if iteration > max_tool_iterations + 1 {
|
||||
return Err(crate::error::LlmError::InvalidResponse {
|
||||
provider: "agent".to_string(),
|
||||
reason: format!("Exceeded maximum tool iterations ({})", MAX_TOOL_ITERATIONS),
|
||||
reason: format!("Exceeded maximum tool iterations ({max_tool_iterations})"),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
@@ -152,7 +163,7 @@ impl Agent {
|
||||
|
||||
// Inject a nudge message when approaching the iteration limit so the
|
||||
// LLM is aware it should produce a final answer on the next turn.
|
||||
if iteration == NUDGE_AT {
|
||||
if iteration == nudge_at {
|
||||
context_messages.push(ChatMessage::system(
|
||||
"You are approaching the tool call limit. \
|
||||
Provide your best final answer on the next response \
|
||||
@@ -161,7 +172,7 @@ impl Agent {
|
||||
));
|
||||
}
|
||||
|
||||
let force_text = iteration >= FORCE_TEXT_AT;
|
||||
let force_text = iteration >= force_text_at;
|
||||
|
||||
// Refresh tool definitions each iteration so newly built tools become visible
|
||||
let tool_defs = self.tools().tool_definitions().await;
|
||||
@@ -284,31 +295,8 @@ impl Agent {
|
||||
for (idx, original_tc) in tool_calls.iter().enumerate() {
|
||||
let mut tc = original_tc.clone();
|
||||
|
||||
// Check if tool requires approval
|
||||
if let Some(tool) = self.tools().get(&tc.name).await
|
||||
&& tool.requires_approval()
|
||||
{
|
||||
let mut is_auto_approved = {
|
||||
let sess = session.lock().await;
|
||||
sess.is_tool_auto_approved(&tc.name)
|
||||
};
|
||||
|
||||
// Override auto-approval for destructive parameters
|
||||
if is_auto_approved && tool.requires_approval_for(&tc.arguments) {
|
||||
tracing::info!(
|
||||
tool = %tc.name,
|
||||
"Parameters require explicit approval despite auto-approve"
|
||||
);
|
||||
is_auto_approved = false;
|
||||
}
|
||||
|
||||
if !is_auto_approved {
|
||||
approval_needed = Some((idx, tc, tool));
|
||||
break; // remaining tools are deferred
|
||||
}
|
||||
}
|
||||
|
||||
// Hook: BeforeToolCall
|
||||
// Hook: BeforeToolCall (runs before approval so hooks can
|
||||
// modify parameters — approval is checked on final params)
|
||||
let event = crate::hooks::HookEvent::ToolCall {
|
||||
tool_name: tc.name.clone(),
|
||||
parameters: tc.arguments.clone(),
|
||||
@@ -351,6 +339,27 @@ impl Agent {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Check if tool requires approval on the final (post-hook)
|
||||
// parameters. Skipped when auto_approve_tools is set.
|
||||
if !self.config.auto_approve_tools
|
||||
&& let Some(tool) = self.tools().get(&tc.name).await
|
||||
{
|
||||
use crate::tools::ApprovalRequirement;
|
||||
let needs_approval = match tool.requires_approval(&tc.arguments) {
|
||||
ApprovalRequirement::Never => false,
|
||||
ApprovalRequirement::UnlessAutoApproved => {
|
||||
let sess = session.lock().await;
|
||||
!sess.is_tool_auto_approved(&tc.name)
|
||||
}
|
||||
ApprovalRequirement::Always => true,
|
||||
};
|
||||
|
||||
if needs_approval {
|
||||
approval_needed = Some((idx, tc, tool));
|
||||
break; // remaining tools are deferred
|
||||
}
|
||||
}
|
||||
|
||||
let preflight_idx = preflight.len();
|
||||
preflight.push((tc.clone(), PreflightOutcome::Runnable));
|
||||
runnable.push((preflight_idx, tc));
|
||||
@@ -877,6 +886,8 @@ mod tests {
|
||||
allow_local_tools: false,
|
||||
max_cost_per_day_cents: None,
|
||||
max_actions_per_hour: None,
|
||||
max_tool_iterations: 50,
|
||||
auto_approve_tools: false,
|
||||
},
|
||||
deps,
|
||||
ChannelManager::new(),
|
||||
@@ -907,9 +918,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_shell_destructive_command_requires_approval_for() {
|
||||
// ShellTool::requires_approval_for should detect destructive commands.
|
||||
// This exercises the same code path used inline in run_agentic_loop.
|
||||
fn test_shell_destructive_command_requires_explicit_approval() {
|
||||
// requires_explicit_approval() detects destructive commands that
|
||||
// should return ApprovalRequirement::Always from ShellTool.
|
||||
use crate::tools::builtin::shell::requires_explicit_approval;
|
||||
|
||||
let destructive_cmds = [
|
||||
|
||||
@@ -357,7 +357,7 @@ impl Scheduler {
|
||||
.into());
|
||||
}
|
||||
|
||||
if tool.requires_approval() {
|
||||
if tool.requires_approval(¶ms).is_required() {
|
||||
return Err(crate::error::ToolError::AuthRequired {
|
||||
name: tool_name.to_string(),
|
||||
}
|
||||
|
||||
+9
-10
@@ -746,19 +746,18 @@ impl Agent {
|
||||
)> = None;
|
||||
|
||||
for (idx, tc) in deferred_tool_calls.iter().enumerate() {
|
||||
if let Some(tool) = self.tools().get(&tc.name).await
|
||||
&& tool.requires_approval()
|
||||
{
|
||||
let is_auto_approved = {
|
||||
let sess = session.lock().await;
|
||||
let mut approved = sess.is_tool_auto_approved(&tc.name);
|
||||
if approved && tool.requires_approval_for(&tc.arguments) {
|
||||
approved = false;
|
||||
if let Some(tool) = self.tools().get(&tc.name).await {
|
||||
use crate::tools::ApprovalRequirement;
|
||||
let needs_approval = match tool.requires_approval(&tc.arguments) {
|
||||
ApprovalRequirement::Never => false,
|
||||
ApprovalRequirement::UnlessAutoApproved => {
|
||||
let sess = session.lock().await;
|
||||
!sess.is_tool_auto_approved(&tc.name)
|
||||
}
|
||||
approved
|
||||
ApprovalRequirement::Always => true,
|
||||
};
|
||||
|
||||
if !is_auto_approved {
|
||||
if needs_approval {
|
||||
approval_needed = Some((idx, tc.clone(), tool));
|
||||
break; // remaining tools stay deferred
|
||||
}
|
||||
|
||||
+18
-2
@@ -18,6 +18,7 @@ use crate::llm::{
|
||||
};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::tools::ToolRegistry;
|
||||
use crate::tools::rate_limiter::RateLimitResult;
|
||||
|
||||
/// Shared dependencies for worker execution.
|
||||
///
|
||||
@@ -432,16 +433,31 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
})?;
|
||||
|
||||
// Tools requiring approval are blocked in autonomous jobs
|
||||
if tool.requires_approval() {
|
||||
if tool.requires_approval(params).is_required() {
|
||||
return Err(crate::error::ToolError::AuthRequired {
|
||||
name: tool_name.to_string(),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
// Fetch job context early so we have the real user_id for hooks
|
||||
// Fetch job context early so we have the real user_id for hooks and rate limiting
|
||||
let job_ctx = deps.context_manager.get_context(job_id).await?;
|
||||
|
||||
// Check per-tool rate limit before running hooks or executing (cheaper check first)
|
||||
if let Some(config) = tool.rate_limit_config()
|
||||
&& let RateLimitResult::Limited { retry_after, .. } = deps
|
||||
.tools
|
||||
.rate_limiter()
|
||||
.check_and_record(&job_ctx.user_id, tool_name, &config)
|
||||
.await
|
||||
{
|
||||
return Err(crate::error::ToolError::RateLimited {
|
||||
name: tool_name.to_string(),
|
||||
retry_after: Some(retry_after),
|
||||
}
|
||||
.into());
|
||||
}
|
||||
|
||||
// Run BeforeToolCall hook
|
||||
let params = {
|
||||
use crate::hooks::{HookError, HookEvent, HookOutcome};
|
||||
|
||||
+59
-104
@@ -200,9 +200,13 @@ impl AppBuilder {
|
||||
|
||||
self.session.attach_store(db.clone(), "default").await;
|
||||
|
||||
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
// Fire-and-forget housekeeping — no need to block startup.
|
||||
let db_cleanup = db.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = db_cleanup.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
self.db = Some(db);
|
||||
Ok(())
|
||||
@@ -285,94 +289,14 @@ impl AppBuilder {
|
||||
|
||||
/// Phase 3: Initialize LLM provider chain.
|
||||
///
|
||||
/// Creates the primary provider, then wraps with failover, circuit
|
||||
/// breaker, and response cache as configured.
|
||||
/// Delegates to `build_provider_chain` which applies all decorators
|
||||
/// (retry, smart routing, failover, circuit breaker, response cache).
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn init_llm(
|
||||
&self,
|
||||
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), anyhow::Error> {
|
||||
use crate::llm::{
|
||||
CachedProvider, CircuitBreakerConfig, CircuitBreakerProvider, CooldownConfig,
|
||||
FailoverProvider, ResponseCacheConfig, create_cheap_llm_provider, create_llm_provider,
|
||||
create_llm_provider_with_config,
|
||||
};
|
||||
|
||||
let llm = create_llm_provider(&self.config.llm, self.session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
|
||||
// Wrap in failover if a fallback model is configured
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(fallback_model) =
|
||||
self.config.llm.nearai.fallback_model.as_ref()
|
||||
{
|
||||
if fallback_model == &self.config.llm.nearai.model {
|
||||
tracing::warn!(
|
||||
"fallback_model is the same as primary model, failover may not be effective"
|
||||
);
|
||||
}
|
||||
let mut fallback_config = self.config.llm.nearai.clone();
|
||||
fallback_config.model = fallback_model.clone();
|
||||
let fallback = create_llm_provider_with_config(&fallback_config, self.session.clone())?;
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(
|
||||
self.config.llm.nearai.failover_cooldown_secs,
|
||||
),
|
||||
failure_threshold: self.config.llm.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in circuit breaker if configured
|
||||
let llm: Arc<dyn LlmProvider> =
|
||||
if let Some(threshold) = self.config.llm.nearai.circuit_breaker_threshold {
|
||||
let cb_config = CircuitBreakerConfig {
|
||||
failure_threshold: threshold,
|
||||
recovery_timeout: std::time::Duration::from_secs(
|
||||
self.config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
),
|
||||
..CircuitBreakerConfig::default()
|
||||
};
|
||||
tracing::info!(
|
||||
threshold,
|
||||
recovery_secs = self.config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
"LLM circuit breaker enabled"
|
||||
);
|
||||
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in response cache if configured
|
||||
let llm: Arc<dyn LlmProvider> = if self.config.llm.nearai.response_cache_enabled {
|
||||
let rc_config = ResponseCacheConfig {
|
||||
ttl: std::time::Duration::from_secs(self.config.llm.nearai.response_cache_ttl_secs),
|
||||
max_entries: self.config.llm.nearai.response_cache_max_entries,
|
||||
};
|
||||
tracing::info!(
|
||||
ttl_secs = self.config.llm.nearai.response_cache_ttl_secs,
|
||||
max_entries = self.config.llm.nearai.response_cache_max_entries,
|
||||
"LLM response cache enabled"
|
||||
);
|
||||
Arc::new(CachedProvider::new(llm, rc_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Cheap LLM for lightweight tasks
|
||||
let cheap_llm = create_cheap_llm_provider(&self.config.llm, self.session.clone())?;
|
||||
if let Some(ref cheap) = cheap_llm {
|
||||
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
|
||||
}
|
||||
|
||||
let (llm, cheap_llm) =
|
||||
crate::llm::build_provider_chain(&self.config.llm, self.session.clone())?;
|
||||
Ok((llm, cheap_llm))
|
||||
}
|
||||
|
||||
@@ -655,11 +579,44 @@ impl AppBuilder {
|
||||
|
||||
tokio::join!(wasm_tools_future, mcp_servers_future);
|
||||
|
||||
// Create extension manager
|
||||
let extension_manager = if let Some(ref secrets) = self.secrets_store {
|
||||
// Load registry catalog entries for extension discovery
|
||||
let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() {
|
||||
Ok(catalog) => {
|
||||
let entries: Vec<_> = catalog
|
||||
.all()
|
||||
.iter()
|
||||
.map(|m| m.to_registry_entry())
|
||||
.collect();
|
||||
tracing::info!(
|
||||
count = entries.len(),
|
||||
"Loaded registry catalog entries for extension discovery"
|
||||
);
|
||||
entries
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to load registry catalog: {}", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Create extension manager. Use ephemeral in-memory secrets if no
|
||||
// persistent store is configured (listing/install/activate still work).
|
||||
let ext_secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = if let Some(ref s) =
|
||||
self.secrets_store
|
||||
{
|
||||
Arc::clone(s)
|
||||
} else {
|
||||
use crate::secrets::{InMemorySecretsStore, SecretsCrypto};
|
||||
let ephemeral_key =
|
||||
secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex());
|
||||
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
|
||||
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
|
||||
Arc::new(InMemorySecretsStore::new(crypto))
|
||||
};
|
||||
let extension_manager = {
|
||||
let manager = Arc::new(ExtensionManager::new(
|
||||
Arc::clone(&mcp_session_manager),
|
||||
Arc::clone(secrets),
|
||||
ext_secrets,
|
||||
Arc::clone(tools),
|
||||
Some(Arc::clone(hooks)),
|
||||
wasm_tool_runtime.clone(),
|
||||
@@ -668,16 +625,11 @@ impl AppBuilder {
|
||||
self.config.tunnel.public_url.clone(),
|
||||
"default".to_string(),
|
||||
self.db.clone(),
|
||||
catalog_entries.clone(),
|
||||
));
|
||||
tools.register_extension_tools(Arc::clone(&manager));
|
||||
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
||||
Some(manager)
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"Extension manager not available (no secrets store). \
|
||||
Extension tools won't be registered."
|
||||
);
|
||||
None
|
||||
};
|
||||
|
||||
// register_builder_tool() already calls register_dev_tools() internally,
|
||||
@@ -715,15 +667,18 @@ impl AppBuilder {
|
||||
}
|
||||
|
||||
if embeddings.is_some() {
|
||||
match ws.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
let ws_bg = Arc::clone(ws);
|
||||
tokio::spawn(async move {
|
||||
match ws_bg.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,6 +9,32 @@ use uuid::Uuid;
|
||||
|
||||
use crate::error::ChannelError;
|
||||
|
||||
/// Kind of attachment carried on an incoming message.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AttachmentKind {
|
||||
/// Audio content (voice notes, audio files).
|
||||
Audio,
|
||||
/// Image content (photos, screenshots).
|
||||
Image,
|
||||
/// Document content (PDFs, files).
|
||||
Document,
|
||||
}
|
||||
|
||||
/// Binary attachment on a message (e.g., voice note, photo).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attachment {
|
||||
/// What kind of content this is.
|
||||
pub kind: AttachmentKind,
|
||||
/// MIME type (e.g., "audio/ogg", "image/jpeg").
|
||||
pub mime_type: String,
|
||||
/// Raw bytes of the attachment.
|
||||
pub data: Vec<u8>,
|
||||
/// Optional filename.
|
||||
pub filename: Option<String>,
|
||||
/// Duration in seconds (for audio/video).
|
||||
pub duration_secs: Option<u32>,
|
||||
}
|
||||
|
||||
/// A message received from an external channel.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IncomingMessage {
|
||||
@@ -28,6 +54,8 @@ pub struct IncomingMessage {
|
||||
pub received_at: DateTime<Utc>,
|
||||
/// Channel-specific metadata.
|
||||
pub metadata: serde_json::Value,
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl IncomingMessage {
|
||||
@@ -46,6 +74,7 @@ impl IncomingMessage {
|
||||
thread_id: None,
|
||||
received_at: Utc::now(),
|
||||
metadata: serde_json::Value::Null,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +95,12 @@ impl IncomingMessage {
|
||||
self.user_name = Some(name.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of incoming messages.
|
||||
|
||||
+4
-1
@@ -35,7 +35,10 @@ pub mod wasm;
|
||||
pub mod web;
|
||||
mod webhook_server;
|
||||
|
||||
pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
pub use channel::{
|
||||
Attachment, AttachmentKind, Channel, IncomingMessage, MessageStream, OutgoingResponse,
|
||||
StatusUpdate,
|
||||
};
|
||||
pub use http::HttpChannel;
|
||||
pub use manager::ChannelManager;
|
||||
pub use repl::ReplChannel;
|
||||
|
||||
+41
-4
@@ -15,6 +15,7 @@
|
||||
//! - `/compact` - Compact the context
|
||||
//! - `/new` - Start a new thread
|
||||
//! - `yes`/`no`/`always` - Respond to tool approval prompts
|
||||
//! - `Esc` - Interrupt current operation
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::io::{self, Write};
|
||||
@@ -28,7 +29,10 @@ use rustyline::error::ReadlineError;
|
||||
use rustyline::highlight::Highlighter;
|
||||
use rustyline::hint::Hinter;
|
||||
use rustyline::validate::Validator;
|
||||
use rustyline::{CompletionType, Editor, Helper};
|
||||
use rustyline::{
|
||||
Cmd as ReadlineCmd, CompletionType, ConditionalEventHandler, Editor, Event, EventContext,
|
||||
EventHandler, Helper, KeyCode, KeyEvent, Modifiers, RepeatCount,
|
||||
};
|
||||
use termimad::MadSkin;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
@@ -121,6 +125,23 @@ impl Highlighter for ReplHelper {
|
||||
impl Validator for ReplHelper {}
|
||||
impl Helper for ReplHelper {}
|
||||
|
||||
struct EscInterruptHandler {
|
||||
triggered: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
impl ConditionalEventHandler for EscInterruptHandler {
|
||||
fn handle(
|
||||
&self,
|
||||
_evt: &Event,
|
||||
_n: RepeatCount,
|
||||
_positive: bool,
|
||||
_ctx: &EventContext,
|
||||
) -> Option<ReadlineCmd> {
|
||||
self.triggered.store(true, Ordering::Relaxed);
|
||||
Some(ReadlineCmd::Interrupt)
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a termimad skin with our color scheme.
|
||||
fn make_skin() -> MadSkin {
|
||||
let mut skin = MadSkin::default();
|
||||
@@ -247,6 +268,7 @@ fn print_help() {
|
||||
println!(" {c}/compact{r} {d}compact context window{r}");
|
||||
println!(" {c}/new{r} {d}new conversation thread{r}");
|
||||
println!(" {c}/interrupt{r} {d}stop current operation{r}");
|
||||
println!(" {c}esc{r} {d}stop current operation{r}");
|
||||
println!();
|
||||
println!(" {h}Approval responses{r}");
|
||||
println!(" {c}yes{r} ({c}y{r}) {d}approve tool execution{r}");
|
||||
@@ -274,6 +296,7 @@ impl Channel for ReplChannel {
|
||||
let single_message = self.single_message.clone();
|
||||
let debug_mode = Arc::clone(&self.debug_mode);
|
||||
let suppress_banner = Arc::clone(&self.suppress_banner);
|
||||
let esc_interrupt_triggered_for_thread = Arc::new(AtomicBool::new(false));
|
||||
|
||||
std::thread::spawn(move || {
|
||||
// Single message mode: send it and return
|
||||
@@ -301,6 +324,13 @@ impl Channel for ReplChannel {
|
||||
|
||||
rl.set_helper(Some(ReplHelper));
|
||||
|
||||
rl.bind_sequence(
|
||||
KeyEvent(KeyCode::Esc, Modifiers::NONE),
|
||||
EventHandler::Conditional(Box::new(EscInterruptHandler {
|
||||
triggered: Arc::clone(&esc_interrupt_triggered_for_thread),
|
||||
})),
|
||||
);
|
||||
|
||||
// Load history
|
||||
let hist_path = history_path();
|
||||
if let Some(parent) = hist_path.parent() {
|
||||
@@ -360,9 +390,16 @@ impl Channel for ReplChannel {
|
||||
}
|
||||
}
|
||||
Err(ReadlineError::Interrupted) => {
|
||||
// Ctrl+C: send /interrupt
|
||||
let msg = IncomingMessage::new("repl", "default", "/interrupt");
|
||||
if tx.blocking_send(msg).is_err() {
|
||||
if esc_interrupt_triggered_for_thread.swap(false, Ordering::Relaxed) {
|
||||
// Esc: interrupt current operation and keep REPL open.
|
||||
let msg = IncomingMessage::new("repl", "default", "/interrupt");
|
||||
if tx.blocking_send(msg).is_err() {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// Ctrl+C (VINTR): request graceful shutdown.
|
||||
let msg = IncomingMessage::new("repl", "default", "/quit");
|
||||
let _ = tx.blocking_send(msg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
|
||||
const KNOWN_CHANNELS: &[(&str, &str)] = &[
|
||||
("telegram", "telegram_channel"),
|
||||
("slack", "slack_channel"),
|
||||
("discord", "discord_channel"),
|
||||
("whatsapp", "whatsapp_channel"),
|
||||
];
|
||||
|
||||
@@ -42,6 +43,10 @@ fn channels_src_dir() -> PathBuf {
|
||||
|
||||
/// Locate the build artifacts for a channel.
|
||||
///
|
||||
/// Checks two layouts:
|
||||
/// 1. **Flat** (Docker/packaged): `<channels_src>/<name>/<name>.wasm`
|
||||
/// 2. **Build tree** (dev): `<channels_src>/<name>/target/wasm32-wasip2/release/<crate_name>.wasm`
|
||||
///
|
||||
/// Returns (wasm_path, capabilities_path) or an error if files are missing.
|
||||
fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
|
||||
let (_, crate_name) = KNOWN_CHANNELS
|
||||
@@ -52,31 +57,34 @@ fn locate_channel_artifacts(name: &str) -> Result<(PathBuf, PathBuf), String> {
|
||||
let src_dir = channels_src_dir();
|
||||
let channel_dir = src_dir.join(name);
|
||||
|
||||
let wasm_path = channel_dir
|
||||
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
// Check flat layout first (Docker/packaged deployments)
|
||||
let flat_wasm = channel_dir.join(format!("{}.wasm", name));
|
||||
if flat_wasm.exists() && caps_path.exists() {
|
||||
return Ok((flat_wasm, caps_path));
|
||||
}
|
||||
|
||||
// Fall back to build tree layout (dev builds)
|
||||
let build_wasm = channel_dir
|
||||
.join("target/wasm32-wasip2/release")
|
||||
.join(format!("{}.wasm", crate_name));
|
||||
|
||||
let caps_path = channel_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if !wasm_path.exists() {
|
||||
return Err(format!(
|
||||
"Channel '{}' WASM not found at {}. Build it first:\n \
|
||||
cd {} && cargo build --target wasm32-wasip2 --release",
|
||||
name,
|
||||
wasm_path.display(),
|
||||
channel_dir.display()
|
||||
));
|
||||
if build_wasm.exists() && caps_path.exists() {
|
||||
return Ok((build_wasm, caps_path));
|
||||
}
|
||||
|
||||
if !caps_path.exists() {
|
||||
return Err(format!(
|
||||
"Channel '{}' capabilities not found at {}",
|
||||
name,
|
||||
caps_path.display()
|
||||
));
|
||||
}
|
||||
|
||||
Ok((wasm_path, caps_path))
|
||||
Err(format!(
|
||||
"Channel '{}' WASM not found. Checked:\n \
|
||||
- {} (flat/packaged)\n \
|
||||
- {} (build tree)\n \
|
||||
Build it first:\n \
|
||||
cd {} && cargo build --target wasm32-wasip2 --release",
|
||||
name,
|
||||
flat_wasm.display(),
|
||||
build_wasm.display(),
|
||||
channel_dir.display()
|
||||
))
|
||||
}
|
||||
|
||||
/// Install a channel from build artifacts into the channels directory.
|
||||
@@ -130,10 +138,11 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_known_channels_includes_all_three() {
|
||||
fn test_known_channels_includes_all_four() {
|
||||
let names = bundled_channel_names();
|
||||
assert!(names.contains(&"telegram"));
|
||||
assert!(names.contains(&"slack"));
|
||||
assert!(names.contains(&"discord"));
|
||||
assert!(names.contains(&"whatsapp"));
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::channels::channel::Attachment;
|
||||
use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig};
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::tools::wasm::{HostState, LogLevel};
|
||||
@@ -17,6 +18,9 @@ const MAX_EMITS_PER_EXECUTION: usize = 100;
|
||||
/// Maximum message content size (64 KB).
|
||||
const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024;
|
||||
|
||||
/// Maximum size for a single attachment (10 MB).
|
||||
const MAX_ATTACHMENT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// A message emitted by a WASM channel to be sent to the agent.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EmittedMessage {
|
||||
@@ -37,6 +41,9 @@ pub struct EmittedMessage {
|
||||
|
||||
/// Timestamp when the message was emitted.
|
||||
pub emitted_at_millis: u64,
|
||||
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl EmittedMessage {
|
||||
@@ -52,6 +59,7 @@ impl EmittedMessage {
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +80,12 @@ impl EmittedMessage {
|
||||
self.metadata_json = metadata_json.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A pending workspace write operation.
|
||||
@@ -168,7 +182,7 @@ impl ChannelHostState {
|
||||
///
|
||||
/// Messages are queued and delivered after callback execution completes.
|
||||
/// Rate limiting is enforced per-execution and globally.
|
||||
pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
pub fn emit_message(&mut self, mut msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
// Check per-execution limit
|
||||
if !self.emit_enabled {
|
||||
self.emits_dropped += 1;
|
||||
@@ -186,6 +200,22 @@ impl ChannelHostState {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Validate attachment sizes — drop only oversized attachments, not the whole message
|
||||
msg.attachments.retain(|attachment| {
|
||||
if attachment.data.len() > MAX_ATTACHMENT_SIZE {
|
||||
tracing::warn!(
|
||||
channel = %self.channel_name,
|
||||
size = attachment.data.len(),
|
||||
max = MAX_ATTACHMENT_SIZE,
|
||||
mime = %attachment.mime_type,
|
||||
"Attachment too large, dropping attachment (message still delivered)"
|
||||
);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
// Validate message content size
|
||||
if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -488,7 +488,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: name.to_string(),
|
||||
description: format!("Test channel: {}", name),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
|
||||
@@ -68,38 +68,51 @@ impl WasmChannelRuntimeConfig {
|
||||
}
|
||||
|
||||
/// A compiled WASM channel component ready for instantiation.
|
||||
#[derive(Debug)]
|
||||
///
|
||||
/// Stores the pre-compiled `Component` directly so instantiation
|
||||
/// doesn't require recompilation.
|
||||
pub struct PreparedChannelModule {
|
||||
/// Channel name.
|
||||
pub name: String,
|
||||
/// Channel description.
|
||||
pub description: String,
|
||||
/// Compiled component bytes (public for testing, otherwise use component_bytes()).
|
||||
pub(crate) component_bytes: Vec<u8>,
|
||||
/// Pre-compiled component (cheaply cloneable via internal Arc).
|
||||
pub(crate) component: Option<wasmtime::component::Component>,
|
||||
/// Resource limits for this channel.
|
||||
pub limits: ResourceLimits,
|
||||
}
|
||||
|
||||
impl PreparedChannelModule {
|
||||
/// Get the compiled component bytes.
|
||||
pub fn component_bytes(&self) -> &[u8] {
|
||||
&self.component_bytes
|
||||
/// Get the pre-compiled component for instantiation.
|
||||
pub fn component(&self) -> Option<&wasmtime::component::Component> {
|
||||
self.component.as_ref()
|
||||
}
|
||||
|
||||
/// Create a PreparedChannelModule for testing purposes.
|
||||
///
|
||||
/// Creates a module with no actual WASM bytes, suitable for testing
|
||||
/// Creates a module with no actual WASM component, suitable for testing
|
||||
/// channel infrastructure without requiring a real WASM component.
|
||||
pub fn for_testing(name: impl Into<String>, description: impl Into<String>) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PreparedChannelModule {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("PreparedChannelModule")
|
||||
.field("name", &self.name)
|
||||
.field("description", &self.description)
|
||||
.field("has_component", &self.component.is_some())
|
||||
.field("limits", &self.limits)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// WASM channel runtime.
|
||||
///
|
||||
/// Manages the Wasmtime engine and a cache of prepared channel modules.
|
||||
@@ -137,6 +150,13 @@ impl WasmChannelRuntime {
|
||||
// Disable debug info in production
|
||||
wasmtime_config.debug_info(false);
|
||||
|
||||
// Enable persistent compilation cache. Wasmtime serializes compiled native
|
||||
// code to disk (~/.cache/wasmtime by default), so subsequent startups
|
||||
// deserialize instead of recompiling — typically 10-50x faster.
|
||||
if let Err(e) = wasmtime_config.cache_config_load_default() {
|
||||
tracing::warn!("Failed to enable wasmtime compilation cache: {}", e);
|
||||
}
|
||||
|
||||
let engine = Engine::new(&wasmtime_config).map_err(|e| {
|
||||
WasmChannelError::Config(format!("Failed to create Wasmtime engine: {}", e))
|
||||
})?;
|
||||
@@ -183,13 +203,13 @@ impl WasmChannelRuntime {
|
||||
// Compile in blocking task (Wasmtime compilation is synchronous)
|
||||
let prepared = tokio::task::spawn_blocking(move || {
|
||||
// Validate and compile the component
|
||||
let _component = wasmtime::component::Component::new(&engine, &wasm_bytes)
|
||||
let component = wasmtime::component::Component::new(&engine, &wasm_bytes)
|
||||
.map_err(|e| WasmChannelError::Compilation(e.to_string()))?;
|
||||
|
||||
Ok::<_, WasmChannelError>(PreparedChannelModule {
|
||||
name: name.clone(),
|
||||
description: desc,
|
||||
component_bytes: wasm_bytes,
|
||||
component: Some(component),
|
||||
limits: limits.unwrap_or(default_limits),
|
||||
})
|
||||
})
|
||||
|
||||
+137
-51
@@ -37,9 +37,10 @@ use tokio::sync::{RwLock, mpsc, oneshot};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use uuid::Uuid;
|
||||
use wasmtime::Store;
|
||||
use wasmtime::component::{Component, Linker};
|
||||
use wasmtime::component::Linker;
|
||||
use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView};
|
||||
|
||||
use crate::channels::channel::{Attachment, AttachmentKind};
|
||||
use crate::channels::wasm::capabilities::ChannelCapabilities;
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::channels::wasm::host::{
|
||||
@@ -436,6 +437,7 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
user_id = %msg.user_id,
|
||||
user_name = ?msg.user_name,
|
||||
content_len = msg.content.len(),
|
||||
attachment_count = msg.attachments.len(),
|
||||
"WASM emit_message called"
|
||||
);
|
||||
|
||||
@@ -448,6 +450,31 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
}
|
||||
emitted = emitted.with_metadata(msg.metadata_json);
|
||||
|
||||
// Convert WIT attachments to Rust types
|
||||
if !msg.attachments.is_empty() {
|
||||
let attachments = msg
|
||||
.attachments
|
||||
.into_iter()
|
||||
.map(|a| {
|
||||
let kind = match a.kind {
|
||||
near::agent::channel_host::AttachmentKind::Audio => AttachmentKind::Audio,
|
||||
near::agent::channel_host::AttachmentKind::Image => AttachmentKind::Image,
|
||||
near::agent::channel_host::AttachmentKind::Document => {
|
||||
AttachmentKind::Document
|
||||
}
|
||||
};
|
||||
Attachment {
|
||||
kind,
|
||||
mime_type: a.mime_type,
|
||||
data: a.data,
|
||||
filename: a.filename,
|
||||
duration_secs: a.duration_secs,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
emitted = emitted.with_attachments(attachments);
|
||||
}
|
||||
|
||||
match self.host_state.emit_message(emitted) {
|
||||
Ok(()) => {
|
||||
tracing::info!("Message emitted to host state successfully");
|
||||
@@ -553,6 +580,9 @@ pub struct WasmChannel {
|
||||
/// In-memory workspace store persisting writes across callback invocations.
|
||||
/// Ensures WASM channels can maintain state (e.g., polling offsets) between ticks.
|
||||
workspace_store: Arc<ChannelWorkspaceStore>,
|
||||
|
||||
/// Optional transcription middleware for audio attachments.
|
||||
transcription_middleware: Option<Arc<crate::transcription::TranscriptionMiddleware>>,
|
||||
}
|
||||
|
||||
impl WasmChannel {
|
||||
@@ -584,9 +614,18 @@ impl WasmChannel {
|
||||
typing_task: RwLock::new(None),
|
||||
pairing_store,
|
||||
workspace_store: Arc::new(ChannelWorkspaceStore::new()),
|
||||
transcription_middleware: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the transcription middleware for audio attachment processing.
|
||||
pub fn set_transcription_middleware(
|
||||
&mut self,
|
||||
middleware: Arc<crate::transcription::TranscriptionMiddleware>,
|
||||
) {
|
||||
self.transcription_middleware = Some(middleware);
|
||||
}
|
||||
|
||||
/// Update the channel config before starting.
|
||||
///
|
||||
/// Merges the provided values into the existing config JSON.
|
||||
@@ -725,9 +764,13 @@ impl WasmChannel {
|
||||
) -> Result<SandboxedChannel, WasmChannelError> {
|
||||
let engine = runtime.engine();
|
||||
|
||||
// Compile the component (uses cached bytes)
|
||||
let component = Component::new(engine, prepared.component_bytes())
|
||||
.map_err(|e| WasmChannelError::Compilation(e.to_string()))?;
|
||||
// Use the pre-compiled component (no recompilation needed)
|
||||
let component = prepared
|
||||
.component()
|
||||
.ok_or_else(|| {
|
||||
WasmChannelError::Compilation("No compiled component available".to_string())
|
||||
})?
|
||||
.clone();
|
||||
|
||||
// Create linker and add host functions
|
||||
let mut linker = Linker::new(engine);
|
||||
@@ -778,7 +821,7 @@ impl WasmChannel {
|
||||
/// Returns the channel configuration for HTTP endpoint registration.
|
||||
async fn call_on_start(&self) -> Result<ChannelConfig, WasmChannelError> {
|
||||
// If no WASM bytes, return default config (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::info!(
|
||||
channel = %self.name,
|
||||
"WASM channel on_start called (no WASM module, returning defaults)"
|
||||
@@ -918,7 +961,7 @@ impl WasmChannel {
|
||||
);
|
||||
|
||||
// If no WASM bytes, return 200 OK (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
method = method,
|
||||
@@ -1018,7 +1061,7 @@ impl WasmChannel {
|
||||
/// Called periodically if polling is configured.
|
||||
pub async fn call_on_poll(&self) -> Result<(), WasmChannelError> {
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
"WASM channel on_poll called (no WASM module)"
|
||||
@@ -1118,7 +1161,7 @@ impl WasmChannel {
|
||||
);
|
||||
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %self.name,
|
||||
message_id = %message_id,
|
||||
@@ -1236,7 +1279,7 @@ impl WasmChannel {
|
||||
metadata: &serde_json::Value,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
// If no WASM bytes, do nothing (for testing)
|
||||
if self.prepared.component_bytes.is_empty() {
|
||||
if self.prepared.component().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1307,7 +1350,7 @@ impl WasmChannel {
|
||||
timeout: Duration,
|
||||
wit_update: wit_channel::StatusUpdate,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
if prepared.component_bytes.is_empty() {
|
||||
if prepared.component().is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -1490,27 +1533,18 @@ impl WasmChannel {
|
||||
});
|
||||
}
|
||||
|
||||
// Convert to IncomingMessage
|
||||
let mut msg = IncomingMessage::new(&self.name, &emitted.user_id, &emitted.content);
|
||||
let msg = Self::convert_emitted_to_incoming(
|
||||
&self.name,
|
||||
emitted,
|
||||
self.transcription_middleware.as_deref(),
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %self.name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending emitted message to agent"
|
||||
);
|
||||
|
||||
@@ -1547,6 +1581,7 @@ impl WasmChannel {
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let callback_timeout = self.runtime.config().callback_timeout;
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
let transcription_middleware = self.transcription_middleware.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval_timer = tokio::time::interval(interval);
|
||||
@@ -1581,6 +1616,7 @@ impl WasmChannel {
|
||||
emitted_messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
transcription_middleware.as_deref(),
|
||||
).await {
|
||||
tracing::warn!(
|
||||
channel = %channel_name,
|
||||
@@ -1627,7 +1663,7 @@ impl WasmChannel {
|
||||
workspace_store: &Arc<ChannelWorkspaceStore>,
|
||||
) -> Result<Vec<EmittedMessage>, WasmChannelError> {
|
||||
// Skip if no WASM bytes (testing mode)
|
||||
if prepared.component_bytes.is_empty() {
|
||||
if prepared.component().is_none() {
|
||||
tracing::debug!(
|
||||
channel = %channel_name,
|
||||
"WASM channel on_poll called (no WASM module)"
|
||||
@@ -1704,6 +1740,7 @@ impl WasmChannel {
|
||||
messages: Vec<EmittedMessage>,
|
||||
message_tx: &RwLock<Option<mpsc::Sender<IncomingMessage>>>,
|
||||
rate_limiter: &RwLock<ChannelEmitRateLimiter>,
|
||||
transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
@@ -1735,27 +1772,15 @@ impl WasmChannel {
|
||||
});
|
||||
}
|
||||
|
||||
// Convert to IncomingMessage
|
||||
let mut msg = IncomingMessage::new(channel_name, &emitted.user_id, &emitted.content);
|
||||
let msg =
|
||||
Self::convert_emitted_to_incoming(channel_name, emitted, transcription_middleware)
|
||||
.await;
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending polled message to agent"
|
||||
);
|
||||
|
||||
@@ -1775,6 +1800,65 @@ impl WasmChannel {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Convert an `EmittedMessage` to an `IncomingMessage`, applying transcription
|
||||
/// middleware with a timeout if available.
|
||||
///
|
||||
/// Shared by both `process_emitted_messages` (HTTP callback path) and
|
||||
/// `dispatch_emitted_messages` (polling path) to avoid duplication.
|
||||
async fn convert_emitted_to_incoming(
|
||||
channel_name: &str,
|
||||
emitted: EmittedMessage,
|
||||
transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>,
|
||||
) -> IncomingMessage {
|
||||
// Save user_id before partial moves for potential timeout fallback
|
||||
let user_id = emitted.user_id.clone();
|
||||
|
||||
let mut msg = IncomingMessage::new(channel_name, &emitted.user_id, &emitted.content);
|
||||
|
||||
if let Some(name) = emitted.user_name {
|
||||
msg = msg.with_user_name(name);
|
||||
}
|
||||
|
||||
if let Some(thread_id) = emitted.thread_id {
|
||||
msg = msg.with_thread(thread_id);
|
||||
}
|
||||
|
||||
// Parse metadata JSON
|
||||
if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Carry attachments through (moves emitted.attachments into msg)
|
||||
if !emitted.attachments.is_empty() {
|
||||
msg = msg.with_attachments(emitted.attachments);
|
||||
}
|
||||
|
||||
// Apply transcription middleware with a 30-second timeout to prevent
|
||||
// a slow/hanging provider from blocking the message pipeline indefinitely.
|
||||
if let Some(middleware) = transcription_middleware {
|
||||
match tokio::time::timeout(std::time::Duration::from_secs(30), middleware.process(msg))
|
||||
.await
|
||||
{
|
||||
Ok(processed) => return processed,
|
||||
Err(_) => {
|
||||
tracing::error!(
|
||||
channel = %channel_name,
|
||||
"Transcription timed out after 30s, delivering message without transcript"
|
||||
);
|
||||
// Timeout: `msg` was moved into the timed-out future, so
|
||||
// reconstruct a fallback message from the saved user_id.
|
||||
return IncomingMessage::new(
|
||||
channel_name,
|
||||
&user_id,
|
||||
"[Voice note: transcription timed out]",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
msg
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -2206,7 +2290,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "test".to_string(),
|
||||
description: "Test channel".to_string(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
@@ -2271,7 +2355,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_poll_no_wasm_returns_empty() {
|
||||
// When there's no WASM module (empty component_bytes), execute_poll
|
||||
// When there's no WASM module (None component), execute_poll
|
||||
// should return an empty vector of messages
|
||||
let config = WasmChannelRuntimeConfig::for_testing();
|
||||
let runtime = Arc::new(WasmChannelRuntime::new(config).unwrap());
|
||||
@@ -2279,7 +2363,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "poll-test".to_string(),
|
||||
description: "Test channel".to_string(),
|
||||
component_bytes: Vec::new(), // No WASM bytes
|
||||
component: None, // No WASM module
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
@@ -2328,6 +2412,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2366,6 +2451,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2381,7 +2467,7 @@ mod tests {
|
||||
let prepared = Arc::new(PreparedChannelModule {
|
||||
name: "poll-channel".to_string(),
|
||||
description: "Polling test channel".to_string(),
|
||||
component_bytes: Vec::new(),
|
||||
component: None,
|
||||
limits: ResourceLimits::default(),
|
||||
});
|
||||
|
||||
|
||||
@@ -89,6 +89,9 @@ impl GatewayChannel {
|
||||
skill_registry: None,
|
||||
skill_catalog: None,
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
registry_entries: Vec::new(),
|
||||
cost_guard: None,
|
||||
startup_time: std::time::Instant::now(),
|
||||
});
|
||||
|
||||
Self {
|
||||
@@ -119,6 +122,9 @@ impl GatewayChannel {
|
||||
skill_registry: self.state.skill_registry.clone(),
|
||||
skill_catalog: self.state.skill_catalog.clone(),
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
registry_entries: self.state.registry_entries.clone(),
|
||||
cost_guard: self.state.cost_guard.clone(),
|
||||
startup_time: self.state.startup_time,
|
||||
};
|
||||
mutate(&mut new_state);
|
||||
self.state = Arc::new(new_state);
|
||||
@@ -206,6 +212,18 @@ impl GatewayChannel {
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject registry catalog entries for the available extensions API.
|
||||
pub fn with_registry_entries(mut self, entries: Vec<crate::extensions::RegistryEntry>) -> Self {
|
||||
self.rebuild_state(|s| s.registry_entries = entries);
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject the cost guard for token/cost tracking in the status popover.
|
||||
pub fn with_cost_guard(mut self, cg: Arc<crate::agent::cost_guard::CostGuard>) -> Self {
|
||||
self.rebuild_state(|s| s.cost_guard = Some(cg));
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the auth token (for printing to console on startup).
|
||||
pub fn auth_token(&self) -> &str {
|
||||
&self.auth_token
|
||||
|
||||
+268
-20
@@ -13,7 +13,7 @@ use axum::{
|
||||
http::{StatusCode, header},
|
||||
middleware,
|
||||
response::{
|
||||
Html, IntoResponse,
|
||||
IntoResponse,
|
||||
sse::{Event, KeepAlive, Sse},
|
||||
},
|
||||
routing::{get, post},
|
||||
@@ -148,6 +148,13 @@ pub struct GatewayState {
|
||||
pub skill_catalog: Option<Arc<crate::skills::catalog::SkillCatalog>>,
|
||||
/// Rate limiter for chat endpoints (30 messages per 60 seconds).
|
||||
pub chat_rate_limiter: RateLimiter,
|
||||
/// Registry catalog entries for the available extensions API.
|
||||
/// Populated at startup from `registry/` manifests, independent of extension manager.
|
||||
pub registry_entries: Vec<crate::extensions::RegistryEntry>,
|
||||
/// Cost guard for token/cost tracking.
|
||||
pub cost_guard: Option<Arc<crate::agent::cost_guard::CostGuard>>,
|
||||
/// Server startup time for uptime calculation.
|
||||
pub startup_time: std::time::Instant,
|
||||
}
|
||||
|
||||
/// Start the gateway HTTP server.
|
||||
@@ -214,6 +221,7 @@ pub async fn start_server(
|
||||
// Extensions
|
||||
.route("/api/extensions", get(extensions_list_handler))
|
||||
.route("/api/extensions/tools", get(extensions_tools_handler))
|
||||
.route("/api/extensions/registry", get(extensions_registry_handler))
|
||||
.route("/api/extensions/install", post(extensions_install_handler))
|
||||
.route(
|
||||
"/api/extensions/{name}/activate",
|
||||
@@ -223,6 +231,16 @@ pub async fn start_server(
|
||||
"/api/extensions/{name}/remove",
|
||||
post(extensions_remove_handler),
|
||||
)
|
||||
.route(
|
||||
"/api/extensions/{name}/setup",
|
||||
get(extensions_setup_handler).post(extensions_setup_submit_handler),
|
||||
)
|
||||
// Pairing
|
||||
.route("/api/pairing/{channel}", get(pairing_list_handler))
|
||||
.route(
|
||||
"/api/pairing/{channel}/approve",
|
||||
post(pairing_approve_handler),
|
||||
)
|
||||
// Routines
|
||||
.route("/api/routines", get(routines_list_handler))
|
||||
.route("/api/routines/summary", get(routines_summary_handler))
|
||||
@@ -344,20 +362,32 @@ pub async fn start_server(
|
||||
|
||||
// --- Static file handlers ---
|
||||
|
||||
async fn index_handler() -> Html<&'static str> {
|
||||
Html(include_str!("static/index.html"))
|
||||
async fn index_handler() -> impl IntoResponse {
|
||||
(
|
||||
[
|
||||
(header::CONTENT_TYPE, "text/html; charset=utf-8"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/index.html"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn css_handler() -> impl IntoResponse {
|
||||
(
|
||||
[(header::CONTENT_TYPE, "text/css")],
|
||||
[
|
||||
(header::CONTENT_TYPE, "text/css"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/style.css"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn js_handler() -> impl IntoResponse {
|
||||
(
|
||||
[(header::CONTENT_TYPE, "application/javascript")],
|
||||
[
|
||||
(header::CONTENT_TYPE, "application/javascript"),
|
||||
(header::CACHE_CONTROL, "no-cache"),
|
||||
],
|
||||
include_str!("static/app.js"),
|
||||
)
|
||||
}
|
||||
@@ -564,9 +594,13 @@ pub async fn clear_auth_mode(state: &GatewayState) {
|
||||
async fn chat_events_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
state.sse.subscribe().ok_or((
|
||||
let sse = state.sse.subscribe().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Too many connections".to_string(),
|
||||
))?;
|
||||
Ok((
|
||||
[("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")],
|
||||
sse,
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1592,10 +1626,7 @@ async fn job_files_read_handler(
|
||||
|
||||
async fn logs_events_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<
|
||||
Sse<impl futures::Stream<Item = Result<Event, Infallible>> + Send + 'static>,
|
||||
(StatusCode, String),
|
||||
> {
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let broadcaster = state.log_broadcaster.as_ref().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Log broadcaster not available".to_string(),
|
||||
@@ -1608,22 +1639,25 @@ async fn logs_events_handler(
|
||||
|
||||
let history_stream = futures::stream::iter(history).map(|entry| {
|
||||
let data = serde_json::to_string(&entry).unwrap_or_default();
|
||||
Ok(Event::default().event("log").data(data))
|
||||
Ok::<_, Infallible>(Event::default().event("log").data(data))
|
||||
});
|
||||
|
||||
let live_stream = tokio_stream::wrappers::BroadcastStream::new(rx)
|
||||
.filter_map(|result| result.ok())
|
||||
.map(|entry| {
|
||||
let data = serde_json::to_string(&entry).unwrap_or_default();
|
||||
Ok(Event::default().event("log").data(data))
|
||||
Ok::<_, Infallible>(Event::default().event("log").data(data))
|
||||
});
|
||||
|
||||
let stream = history_stream.chain(live_stream);
|
||||
|
||||
Ok(Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(std::time::Duration::from_secs(30))
|
||||
.text(""),
|
||||
Ok((
|
||||
[("X-Accel-Buffering", "no"), ("Cache-Control", "no-cache")],
|
||||
Sse::new(stream).keep_alive(
|
||||
KeepAlive::new()
|
||||
.interval(std::time::Duration::from_secs(30))
|
||||
.text(""),
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1684,6 +1718,7 @@ async fn extensions_list_handler(
|
||||
authenticated: ext.authenticated,
|
||||
active: ext.active,
|
||||
tools: ext.tools,
|
||||
needs_setup: ext.needs_setup,
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -1714,10 +1749,30 @@ async fn extensions_install_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Json(req): Json<InstallExtensionRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
// When extension manager isn't available, check registry entries for a helpful message
|
||||
let Some(ext_mgr) = state.extension_manager.as_ref() else {
|
||||
// Look up the entry in the catalog to give a specific error
|
||||
if let Some(entry) = state.registry_entries.iter().find(|e| e.name == req.name) {
|
||||
let msg = match &entry.source {
|
||||
crate::extensions::ExtensionSource::WasmBuildable { .. } => {
|
||||
format!(
|
||||
"'{}' requires building from source. \
|
||||
Run `ironclaw registry install {}` from the CLI.",
|
||||
req.name, req.name
|
||||
)
|
||||
}
|
||||
_ => format!(
|
||||
"Extension manager not available (secrets store required). \
|
||||
Configure DATABASE_URL or a secrets backend to enable installation of '{}'.",
|
||||
req.name
|
||||
),
|
||||
};
|
||||
return Ok(Json(ActionResponse::fail(msg)));
|
||||
}
|
||||
return Ok(Json(ActionResponse::fail(
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
)));
|
||||
};
|
||||
|
||||
let kind_hint = req.kind.as_deref().and_then(|k| match k {
|
||||
"mcp_server" => Some(crate::extensions::ExtensionKind::McpServer),
|
||||
@@ -1866,6 +1921,160 @@ async fn extensions_remove_handler(
|
||||
}
|
||||
}
|
||||
|
||||
async fn extensions_registry_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Query(params): Query<RegistrySearchQuery>,
|
||||
) -> Json<RegistrySearchResponse> {
|
||||
let query = params.query.unwrap_or_default();
|
||||
let query_lower = query.to_lowercase();
|
||||
let tokens: Vec<&str> = query_lower.split_whitespace().collect();
|
||||
|
||||
// Filter registry entries by query (or return all if empty)
|
||||
let matching: Vec<&crate::extensions::RegistryEntry> = if tokens.is_empty() {
|
||||
state.registry_entries.iter().collect()
|
||||
} else {
|
||||
state
|
||||
.registry_entries
|
||||
.iter()
|
||||
.filter(|e| {
|
||||
let name = e.name.to_lowercase();
|
||||
let display = e.display_name.to_lowercase();
|
||||
let desc = e.description.to_lowercase();
|
||||
tokens.iter().any(|t| {
|
||||
name.contains(t)
|
||||
|| display.contains(t)
|
||||
|| desc.contains(t)
|
||||
|| e.keywords.iter().any(|k| k.to_lowercase().contains(t))
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
};
|
||||
|
||||
// Cross-reference with installed extensions by (name, kind) to avoid
|
||||
// false positives when the same name exists as different kinds.
|
||||
let installed: std::collections::HashSet<(String, String)> =
|
||||
if let Some(ext_mgr) = state.extension_manager.as_ref() {
|
||||
ext_mgr
|
||||
.list(None)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|ext| (ext.name, ext.kind.to_string()))
|
||||
.collect()
|
||||
} else {
|
||||
std::collections::HashSet::new()
|
||||
};
|
||||
|
||||
let entries = matching
|
||||
.into_iter()
|
||||
.map(|e| {
|
||||
let kind_str = e.kind.to_string();
|
||||
RegistryEntryInfo {
|
||||
name: e.name.clone(),
|
||||
display_name: e.display_name.clone(),
|
||||
installed: installed.contains(&(e.name.clone(), kind_str.clone())),
|
||||
kind: kind_str,
|
||||
description: e.description.clone(),
|
||||
keywords: e.keywords.clone(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Json(RegistrySearchResponse { entries })
|
||||
}
|
||||
|
||||
async fn extensions_setup_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Path(name): Path<String>,
|
||||
) -> Result<Json<ExtensionSetupResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
|
||||
let secrets = ext_mgr
|
||||
.get_setup_schema(&name)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
let kind = ext_mgr
|
||||
.list(None)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|list| list.into_iter().find(|e| e.name == name))
|
||||
.map(|e| e.kind.to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(Json(ExtensionSetupResponse {
|
||||
name,
|
||||
kind,
|
||||
secrets,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn extensions_setup_submit_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Path(name): Path<String>,
|
||||
Json(req): Json<ExtensionSetupRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let ext_mgr = state.extension_manager.as_ref().ok_or((
|
||||
StatusCode::NOT_IMPLEMENTED,
|
||||
"Extension manager not available (secrets store required)".to_string(),
|
||||
))?;
|
||||
|
||||
match ext_mgr.save_setup_secrets(&name, &req.secrets).await {
|
||||
Ok(message) => Ok(Json(ActionResponse::ok(message))),
|
||||
Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Pairing handlers ---
|
||||
|
||||
async fn pairing_list_handler(
|
||||
Path(channel): Path<String>,
|
||||
) -> Result<Json<PairingListResponse>, (StatusCode, String)> {
|
||||
let store = crate::pairing::PairingStore::new();
|
||||
let requests = store
|
||||
.list_pending(&channel)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
let infos = requests
|
||||
.into_iter()
|
||||
.map(|r| PairingRequestInfo {
|
||||
code: r.code,
|
||||
sender_id: r.id,
|
||||
meta: r.meta,
|
||||
created_at: r.created_at,
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Json(PairingListResponse {
|
||||
channel,
|
||||
requests: infos,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn pairing_approve_handler(
|
||||
Path(channel): Path<String>,
|
||||
Json(req): Json<PairingApproveRequest>,
|
||||
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
|
||||
let store = crate::pairing::PairingStore::new();
|
||||
match store.approve(&channel, &req.code) {
|
||||
Ok(Some(approved)) => Ok(Json(ActionResponse::ok(format!(
|
||||
"Pairing approved for sender '{}'",
|
||||
approved.id
|
||||
)))),
|
||||
Ok(None) => Ok(Json(ActionResponse::fail(
|
||||
"Invalid or expired pairing code".to_string(),
|
||||
))),
|
||||
Err(crate::pairing::PairingStoreError::ApproveRateLimited) => Err((
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"Too many failed approve attempts; try again later".to_string(),
|
||||
)),
|
||||
Err(e) => Ok(Json(ActionResponse::fail(e.to_string()))),
|
||||
}
|
||||
}
|
||||
|
||||
// --- Skills handlers ---
|
||||
|
||||
async fn skills_list_handler(
|
||||
@@ -2565,18 +2774,57 @@ async fn gateway_status_handler(
|
||||
.map(|t| t.connection_count())
|
||||
.unwrap_or(0);
|
||||
|
||||
let uptime_secs = state.startup_time.elapsed().as_secs();
|
||||
|
||||
let (daily_cost, actions_this_hour, model_usage) = if let Some(ref cg) = state.cost_guard {
|
||||
let cost = cg.daily_spend().await;
|
||||
let actions = cg.actions_this_hour().await;
|
||||
let usage = cg.model_usage().await;
|
||||
let models: Vec<ModelUsageEntry> = usage
|
||||
.into_iter()
|
||||
.map(|(model, tokens)| ModelUsageEntry {
|
||||
model,
|
||||
input_tokens: tokens.input_tokens,
|
||||
output_tokens: tokens.output_tokens,
|
||||
cost: format!("{:.6}", tokens.cost),
|
||||
})
|
||||
.collect();
|
||||
(Some(format!("{:.4}", cost)), Some(actions), Some(models))
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
Json(GatewayStatusResponse {
|
||||
sse_connections,
|
||||
ws_connections,
|
||||
total_connections: sse_connections + ws_connections,
|
||||
uptime_secs,
|
||||
daily_cost,
|
||||
actions_this_hour,
|
||||
model_usage,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct ModelUsageEntry {
|
||||
model: String,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cost: String,
|
||||
}
|
||||
|
||||
#[derive(serde::Serialize)]
|
||||
struct GatewayStatusResponse {
|
||||
sse_connections: u64,
|
||||
ws_connections: u64,
|
||||
total_connections: u64,
|
||||
uptime_secs: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
daily_cost: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
actions_this_hour: Option<u64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
model_usage: Option<Vec<ModelUsageEntry>>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+590
-27
@@ -38,6 +38,7 @@ function authenticate() {
|
||||
connectSSE();
|
||||
connectLogSSE();
|
||||
startGatewayStatusPolling();
|
||||
checkTeeStatus();
|
||||
loadThreads();
|
||||
loadMemoryTree();
|
||||
loadJobs();
|
||||
@@ -1203,15 +1204,18 @@ function loadServerLogLevel() {
|
||||
|
||||
function loadExtensions() {
|
||||
const extList = document.getElementById('extensions-list');
|
||||
const wasmList = document.getElementById('available-wasm-list');
|
||||
const mcpList = document.getElementById('mcp-servers-list');
|
||||
const toolsTbody = document.getElementById('tools-tbody');
|
||||
const toolsEmpty = document.getElementById('tools-empty');
|
||||
|
||||
// Fetch both in parallel
|
||||
// Fetch all three in parallel
|
||||
Promise.all([
|
||||
apiFetch('/api/extensions').catch(() => ({ extensions: [] })),
|
||||
apiFetch('/api/extensions/tools').catch(() => ({ tools: [] })),
|
||||
]).then(([extData, toolData]) => {
|
||||
// Render extensions
|
||||
apiFetch('/api/extensions/registry').catch(function(err) { console.warn('registry fetch failed:', err); return { entries: [] }; }),
|
||||
]).then(([extData, toolData, registryData]) => {
|
||||
// Render installed extensions
|
||||
if (extData.extensions.length === 0) {
|
||||
extList.innerHTML = '<div class="empty-state">No extensions installed</div>';
|
||||
} else {
|
||||
@@ -1221,6 +1225,31 @@ function loadExtensions() {
|
||||
}
|
||||
}
|
||||
|
||||
// Split registry entries by kind
|
||||
var wasmEntries = registryData.entries.filter(function(e) { return e.kind !== 'mcp_server' && !e.installed; });
|
||||
var mcpEntries = registryData.entries.filter(function(e) { return e.kind === 'mcp_server'; });
|
||||
|
||||
// Available WASM extensions
|
||||
if (wasmEntries.length === 0) {
|
||||
wasmList.innerHTML = '<div class="empty-state">No additional WASM extensions available</div>';
|
||||
} else {
|
||||
wasmList.innerHTML = '';
|
||||
for (const entry of wasmEntries) {
|
||||
wasmList.appendChild(renderAvailableExtensionCard(entry));
|
||||
}
|
||||
}
|
||||
|
||||
// MCP servers (show both installed and uninstalled)
|
||||
if (mcpEntries.length === 0) {
|
||||
mcpList.innerHTML = '<div class="empty-state">No MCP servers available</div>';
|
||||
} else {
|
||||
mcpList.innerHTML = '';
|
||||
for (const entry of mcpEntries) {
|
||||
var installedExt = extData.extensions.find(function(e) { return e.name === entry.name; });
|
||||
mcpList.appendChild(renderMcpServerCard(entry, installedExt));
|
||||
}
|
||||
}
|
||||
|
||||
// Render tools
|
||||
if (toolData.tools.length === 0) {
|
||||
toolsTbody.innerHTML = '';
|
||||
@@ -1234,6 +1263,148 @@ function loadExtensions() {
|
||||
});
|
||||
}
|
||||
|
||||
function renderAvailableExtensionCard(entry) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'ext-card ext-available';
|
||||
|
||||
const header = document.createElement('div');
|
||||
header.className = 'ext-header';
|
||||
|
||||
const name = document.createElement('span');
|
||||
name.className = 'ext-name';
|
||||
name.textContent = entry.display_name;
|
||||
header.appendChild(name);
|
||||
|
||||
const kind = document.createElement('span');
|
||||
kind.className = 'ext-kind kind-' + entry.kind;
|
||||
kind.textContent = entry.kind;
|
||||
header.appendChild(kind);
|
||||
|
||||
card.appendChild(header);
|
||||
|
||||
const desc = document.createElement('div');
|
||||
desc.className = 'ext-desc';
|
||||
desc.textContent = entry.description;
|
||||
card.appendChild(desc);
|
||||
|
||||
if (entry.keywords && entry.keywords.length > 0) {
|
||||
const kw = document.createElement('div');
|
||||
kw.className = 'ext-keywords';
|
||||
kw.textContent = entry.keywords.join(', ');
|
||||
card.appendChild(kw);
|
||||
}
|
||||
|
||||
const actions = document.createElement('div');
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
const installBtn = document.createElement('button');
|
||||
installBtn.className = 'btn-ext install';
|
||||
installBtn.textContent = 'Install';
|
||||
installBtn.addEventListener('click', function() {
|
||||
installBtn.disabled = true;
|
||||
installBtn.textContent = 'Installing...';
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: entry.name, kind: entry.kind },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + entry.display_name, 'success');
|
||||
} else {
|
||||
showToast('Install: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
loadExtensions();
|
||||
});
|
||||
});
|
||||
actions.appendChild(installBtn);
|
||||
|
||||
card.appendChild(actions);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderMcpServerCard(entry, installedExt) {
|
||||
var card = document.createElement('div');
|
||||
card.className = 'ext-card' + (installedExt ? '' : ' ext-available');
|
||||
|
||||
var header = document.createElement('div');
|
||||
header.className = 'ext-header';
|
||||
|
||||
var name = document.createElement('span');
|
||||
name.className = 'ext-name';
|
||||
name.textContent = entry.display_name;
|
||||
header.appendChild(name);
|
||||
|
||||
var kind = document.createElement('span');
|
||||
kind.className = 'ext-kind kind-mcp_server';
|
||||
kind.textContent = 'mcp_server';
|
||||
header.appendChild(kind);
|
||||
|
||||
if (installedExt) {
|
||||
var authDot = document.createElement('span');
|
||||
authDot.className = 'ext-auth-dot ' + (installedExt.authenticated ? 'authed' : 'unauthed');
|
||||
authDot.title = installedExt.authenticated ? 'Authenticated' : 'Not authenticated';
|
||||
header.appendChild(authDot);
|
||||
}
|
||||
|
||||
card.appendChild(header);
|
||||
|
||||
var desc = document.createElement('div');
|
||||
desc.className = 'ext-desc';
|
||||
desc.textContent = entry.description;
|
||||
card.appendChild(desc);
|
||||
|
||||
var actions = document.createElement('div');
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
if (installedExt) {
|
||||
if (!installedExt.active) {
|
||||
var activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', function() { activateExtension(installedExt.name); });
|
||||
actions.appendChild(activateBtn);
|
||||
} else {
|
||||
var activeLabel = document.createElement('span');
|
||||
activeLabel.className = 'ext-active-label';
|
||||
activeLabel.textContent = 'Active';
|
||||
actions.appendChild(activeLabel);
|
||||
}
|
||||
var removeBtn = document.createElement('button');
|
||||
removeBtn.className = 'btn-ext remove';
|
||||
removeBtn.textContent = 'Remove';
|
||||
removeBtn.addEventListener('click', function() { removeExtension(installedExt.name); });
|
||||
actions.appendChild(removeBtn);
|
||||
} else {
|
||||
var installBtn = document.createElement('button');
|
||||
installBtn.className = 'btn-ext install';
|
||||
installBtn.textContent = 'Install';
|
||||
installBtn.addEventListener('click', function() {
|
||||
installBtn.disabled = true;
|
||||
installBtn.textContent = 'Installing...';
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: entry.name, kind: entry.kind },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + entry.display_name, 'success');
|
||||
} else {
|
||||
showToast('Install: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
loadExtensions();
|
||||
});
|
||||
});
|
||||
actions.appendChild(installBtn);
|
||||
}
|
||||
|
||||
card.appendChild(actions);
|
||||
return card;
|
||||
}
|
||||
|
||||
function renderExtensionCard(ext) {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'ext-card';
|
||||
@@ -1284,11 +1455,18 @@ function renderExtensionCard(ext) {
|
||||
actions.className = 'ext-actions';
|
||||
|
||||
if (!ext.active) {
|
||||
const activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', () => activateExtension(ext.name));
|
||||
actions.appendChild(activateBtn);
|
||||
if (ext.kind === 'wasm_channel') {
|
||||
const restartLabel = document.createElement('span');
|
||||
restartLabel.className = 'ext-restart-label';
|
||||
restartLabel.textContent = 'Restart to activate';
|
||||
actions.appendChild(restartLabel);
|
||||
} else {
|
||||
const activateBtn = document.createElement('button');
|
||||
activateBtn.className = 'btn-ext activate';
|
||||
activateBtn.textContent = 'Activate';
|
||||
activateBtn.addEventListener('click', () => activateExtension(ext.name));
|
||||
actions.appendChild(activateBtn);
|
||||
}
|
||||
} else {
|
||||
const activeLabel = document.createElement('span');
|
||||
activeLabel.className = 'ext-active-label';
|
||||
@@ -1296,6 +1474,14 @@ function renderExtensionCard(ext) {
|
||||
actions.appendChild(activeLabel);
|
||||
}
|
||||
|
||||
if (ext.needs_setup) {
|
||||
const configBtn = document.createElement('button');
|
||||
configBtn.className = 'btn-ext configure';
|
||||
configBtn.textContent = ext.authenticated ? 'Reconfigure' : 'Configure';
|
||||
configBtn.addEventListener('click', () => showConfigureModal(ext.name));
|
||||
actions.appendChild(configBtn);
|
||||
}
|
||||
|
||||
const removeBtn = document.createElement('button');
|
||||
removeBtn.className = 'btn-ext remove';
|
||||
removeBtn.textContent = 'Remove';
|
||||
@@ -1303,6 +1489,15 @@ function renderExtensionCard(ext) {
|
||||
actions.appendChild(removeBtn);
|
||||
|
||||
card.appendChild(actions);
|
||||
|
||||
// For active WASM channels, check for pending pairing requests
|
||||
if (ext.active && ext.kind === 'wasm_channel') {
|
||||
const pairingSection = document.createElement('div');
|
||||
pairingSection.className = 'ext-pairing';
|
||||
card.appendChild(pairingSection);
|
||||
loadPairingRequests(ext.name, pairingSection);
|
||||
}
|
||||
|
||||
return card;
|
||||
}
|
||||
|
||||
@@ -1318,7 +1513,7 @@ function activateExtension(name) {
|
||||
showToast('Opening authentication for ' + name, 'info');
|
||||
window.open(res.auth_url, '_blank');
|
||||
} else if (res.awaiting_token) {
|
||||
showToast(res.instructions || 'Please provide an API token for ' + name, 'info');
|
||||
showConfigureModal(name);
|
||||
} else {
|
||||
showToast('Activate failed: ' + res.message, 'error');
|
||||
}
|
||||
@@ -1341,6 +1536,189 @@ function removeExtension(name) {
|
||||
.catch((err) => showToast('Remove failed: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
function showConfigureModal(name) {
|
||||
apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup')
|
||||
.then((setup) => {
|
||||
if (!setup.secrets || setup.secrets.length === 0) {
|
||||
showToast('No configuration needed for ' + name, 'info');
|
||||
return;
|
||||
}
|
||||
renderConfigureModal(name, setup.secrets);
|
||||
})
|
||||
.catch((err) => showToast('Failed to load setup: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
function renderConfigureModal(name, secrets) {
|
||||
closeConfigureModal();
|
||||
const overlay = document.createElement('div');
|
||||
overlay.className = 'configure-overlay';
|
||||
overlay.addEventListener('click', (e) => {
|
||||
if (e.target === overlay) closeConfigureModal();
|
||||
});
|
||||
|
||||
const modal = document.createElement('div');
|
||||
modal.className = 'configure-modal';
|
||||
|
||||
const header = document.createElement('h3');
|
||||
header.textContent = 'Configure ' + name;
|
||||
modal.appendChild(header);
|
||||
|
||||
const form = document.createElement('div');
|
||||
form.className = 'configure-form';
|
||||
|
||||
const fields = [];
|
||||
for (const secret of secrets) {
|
||||
const field = document.createElement('div');
|
||||
field.className = 'configure-field';
|
||||
|
||||
const label = document.createElement('label');
|
||||
label.textContent = secret.prompt;
|
||||
if (secret.optional) {
|
||||
const opt = document.createElement('span');
|
||||
opt.className = 'field-optional';
|
||||
opt.textContent = ' (optional)';
|
||||
label.appendChild(opt);
|
||||
}
|
||||
field.appendChild(label);
|
||||
|
||||
const inputRow = document.createElement('div');
|
||||
inputRow.className = 'configure-input-row';
|
||||
|
||||
const input = document.createElement('input');
|
||||
input.type = 'password';
|
||||
input.name = secret.name;
|
||||
input.placeholder = secret.provided ? '(already set — leave empty to keep)' : '';
|
||||
input.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter') submitConfigureModal(name, fields);
|
||||
});
|
||||
inputRow.appendChild(input);
|
||||
|
||||
if (secret.provided) {
|
||||
const badge = document.createElement('span');
|
||||
badge.className = 'field-provided';
|
||||
badge.textContent = 'Set';
|
||||
inputRow.appendChild(badge);
|
||||
}
|
||||
if (secret.auto_generate && !secret.provided) {
|
||||
const hint = document.createElement('span');
|
||||
hint.className = 'field-autogen';
|
||||
hint.textContent = 'Auto-generated if empty';
|
||||
inputRow.appendChild(hint);
|
||||
}
|
||||
|
||||
field.appendChild(inputRow);
|
||||
form.appendChild(field);
|
||||
fields.push({ name: secret.name, input: input });
|
||||
}
|
||||
|
||||
modal.appendChild(form);
|
||||
|
||||
const actions = document.createElement('div');
|
||||
actions.className = 'configure-actions';
|
||||
|
||||
const submitBtn = document.createElement('button');
|
||||
submitBtn.className = 'btn-ext activate';
|
||||
submitBtn.textContent = 'Save';
|
||||
submitBtn.addEventListener('click', () => submitConfigureModal(name, fields));
|
||||
actions.appendChild(submitBtn);
|
||||
|
||||
const cancelBtn = document.createElement('button');
|
||||
cancelBtn.className = 'btn-ext remove';
|
||||
cancelBtn.textContent = 'Cancel';
|
||||
cancelBtn.addEventListener('click', closeConfigureModal);
|
||||
actions.appendChild(cancelBtn);
|
||||
|
||||
modal.appendChild(actions);
|
||||
overlay.appendChild(modal);
|
||||
document.body.appendChild(overlay);
|
||||
|
||||
if (fields.length > 0) fields[0].input.focus();
|
||||
}
|
||||
|
||||
function submitConfigureModal(name, fields) {
|
||||
const secrets = {};
|
||||
for (const f of fields) {
|
||||
if (f.input.value.trim()) {
|
||||
secrets[f.name] = f.input.value.trim();
|
||||
}
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/' + encodeURIComponent(name) + '/setup', {
|
||||
method: 'POST',
|
||||
body: { secrets },
|
||||
})
|
||||
.then((res) => {
|
||||
closeConfigureModal();
|
||||
if (res.success) {
|
||||
showToast(res.message, 'success');
|
||||
} else {
|
||||
showToast(res.message || 'Configuration failed', 'error');
|
||||
}
|
||||
loadExtensions();
|
||||
})
|
||||
.catch((err) => {
|
||||
showToast('Configuration failed: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
function closeConfigureModal() {
|
||||
const existing = document.querySelector('.configure-overlay');
|
||||
if (existing) existing.remove();
|
||||
}
|
||||
|
||||
// --- Pairing ---
|
||||
|
||||
function loadPairingRequests(channel, container) {
|
||||
apiFetch('/api/pairing/' + encodeURIComponent(channel))
|
||||
.then(data => {
|
||||
container.innerHTML = '';
|
||||
if (!data.requests || data.requests.length === 0) return;
|
||||
|
||||
const heading = document.createElement('div');
|
||||
heading.className = 'pairing-heading';
|
||||
heading.textContent = 'Pending pairing requests';
|
||||
container.appendChild(heading);
|
||||
|
||||
data.requests.forEach(req => {
|
||||
const row = document.createElement('div');
|
||||
row.className = 'pairing-row';
|
||||
|
||||
const code = document.createElement('span');
|
||||
code.className = 'pairing-code';
|
||||
code.textContent = req.code;
|
||||
row.appendChild(code);
|
||||
|
||||
const sender = document.createElement('span');
|
||||
sender.className = 'pairing-sender';
|
||||
sender.textContent = 'from ' + req.sender_id;
|
||||
row.appendChild(sender);
|
||||
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'btn-ext activate';
|
||||
btn.textContent = 'Approve';
|
||||
btn.addEventListener('click', () => approvePairing(channel, req.code, container));
|
||||
row.appendChild(btn);
|
||||
|
||||
container.appendChild(row);
|
||||
});
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
function approvePairing(channel, code, container) {
|
||||
apiFetch('/api/pairing/' + encodeURIComponent(channel) + '/approve', {
|
||||
method: 'POST',
|
||||
body: { code },
|
||||
}).then(res => {
|
||||
if (res.success) {
|
||||
showToast('Pairing approved', 'success');
|
||||
loadPairingRequests(channel, container);
|
||||
} else {
|
||||
showToast(res.message || 'Approve failed', 'error');
|
||||
}
|
||||
}).catch(err => showToast('Error: ' + err.message, 'error'));
|
||||
}
|
||||
|
||||
// --- Jobs ---
|
||||
|
||||
let currentJobId = null;
|
||||
@@ -2081,13 +2459,72 @@ function startGatewayStatusPolling() {
|
||||
gatewayStatusInterval = setInterval(fetchGatewayStatus, 30000);
|
||||
}
|
||||
|
||||
function formatTokenCount(n) {
|
||||
if (n == null || n === 0) return '0';
|
||||
if (n >= 1000000) return (n / 1000000).toFixed(1) + 'M';
|
||||
if (n >= 1000) return (n / 1000).toFixed(1) + 'k';
|
||||
return '' + n;
|
||||
}
|
||||
|
||||
function formatCost(costStr) {
|
||||
if (!costStr) return '$0.00';
|
||||
var n = parseFloat(costStr);
|
||||
if (n < 0.01) return '$' + n.toFixed(4);
|
||||
return '$' + n.toFixed(2);
|
||||
}
|
||||
|
||||
function shortModelName(model) {
|
||||
// Strip provider prefix and shorten common model names
|
||||
var m = model.indexOf('/') >= 0 ? model.split('/').pop() : model;
|
||||
// Shorten dated suffixes
|
||||
m = m.replace(/-20\d{6}$/, '');
|
||||
return m;
|
||||
}
|
||||
|
||||
function fetchGatewayStatus() {
|
||||
apiFetch('/api/gateway/status').then((data) => {
|
||||
const popover = document.getElementById('gateway-popover');
|
||||
popover.innerHTML = '<div class="gw-stat"><span>SSE clients</span><span>' + (data.sse_clients || 0) + '</span></div>'
|
||||
+ '<div class="gw-stat"><span>Log clients</span><span>' + (data.log_clients || 0) + '</span></div>'
|
||||
+ '<div class="gw-stat"><span>Uptime</span><span>' + formatDuration(data.uptime_secs) + '</span></div>';
|
||||
}).catch(() => {});
|
||||
apiFetch('/api/gateway/status').then(function(data) {
|
||||
var popover = document.getElementById('gateway-popover');
|
||||
var html = '';
|
||||
|
||||
// Connection info
|
||||
html += '<div class="gw-section-label">Connections</div>';
|
||||
html += '<div class="gw-stat"><span>SSE</span><span>' + (data.sse_connections || 0) + '</span></div>';
|
||||
html += '<div class="gw-stat"><span>WebSocket</span><span>' + (data.ws_connections || 0) + '</span></div>';
|
||||
html += '<div class="gw-stat"><span>Uptime</span><span>' + formatDuration(data.uptime_secs) + '</span></div>';
|
||||
|
||||
// Cost tracker
|
||||
if (data.daily_cost != null) {
|
||||
html += '<div class="gw-divider"></div>';
|
||||
html += '<div class="gw-section-label">Cost Today</div>';
|
||||
html += '<div class="gw-stat"><span>Spent</span><span>' + formatCost(data.daily_cost) + '</span></div>';
|
||||
if (data.actions_this_hour != null) {
|
||||
html += '<div class="gw-stat"><span>Actions/hr</span><span>' + data.actions_this_hour + '</span></div>';
|
||||
}
|
||||
}
|
||||
|
||||
// Per-model token usage
|
||||
if (data.model_usage && data.model_usage.length > 0) {
|
||||
html += '<div class="gw-divider"></div>';
|
||||
html += '<div class="gw-section-label">Token Usage</div>';
|
||||
data.model_usage.sort(function(a, b) {
|
||||
return (b.input_tokens + b.output_tokens) - (a.input_tokens + a.output_tokens);
|
||||
});
|
||||
for (var i = 0; i < data.model_usage.length; i++) {
|
||||
var m = data.model_usage[i];
|
||||
var name = escapeHtml(shortModelName(m.model));
|
||||
html += '<div class="gw-model-row">'
|
||||
+ '<span class="gw-model-name">' + name + '</span>'
|
||||
+ '<span class="gw-model-cost">' + escapeHtml(formatCost(m.cost)) + '</span>'
|
||||
+ '</div>';
|
||||
html += '<div class="gw-token-detail">'
|
||||
+ '<span>in: ' + formatTokenCount(m.input_tokens) + '</span>'
|
||||
+ '<span>out: ' + formatTokenCount(m.output_tokens) + '</span>'
|
||||
+ '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
popover.innerHTML = html;
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
// Show/hide popover on hover
|
||||
@@ -2098,34 +2535,160 @@ document.getElementById('gateway-status-trigger').addEventListener('mouseleave',
|
||||
document.getElementById('gateway-popover').classList.remove('visible');
|
||||
});
|
||||
|
||||
// --- TEE attestation ---
|
||||
|
||||
let teeInfo = null;
|
||||
let teeReportCache = null;
|
||||
let teeReportLoading = false;
|
||||
|
||||
function teeApiBase() {
|
||||
var parts = window.location.hostname.split('.');
|
||||
if (parts.length < 2) return null;
|
||||
var domain = parts.slice(1).join('.');
|
||||
return window.location.protocol + '//api.' + domain;
|
||||
}
|
||||
|
||||
function teeInstanceName() {
|
||||
return window.location.hostname.split('.')[0];
|
||||
}
|
||||
|
||||
function checkTeeStatus() {
|
||||
var base = teeApiBase();
|
||||
if (!base) return;
|
||||
var name = teeInstanceName();
|
||||
fetch(base + '/instances/' + encodeURIComponent(name) + '/attestation').then(function(res) {
|
||||
if (!res.ok) throw new Error(res.status);
|
||||
return res.json();
|
||||
}).then(function(data) {
|
||||
teeInfo = data;
|
||||
document.getElementById('tee-shield').style.display = 'flex';
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
function fetchTeeReport() {
|
||||
if (teeReportCache) {
|
||||
renderTeePopover(teeReportCache);
|
||||
return;
|
||||
}
|
||||
if (teeReportLoading) return;
|
||||
teeReportLoading = true;
|
||||
var base = teeApiBase();
|
||||
if (!base) return;
|
||||
var popover = document.getElementById('tee-popover');
|
||||
popover.innerHTML = '<div class="tee-popover-loading">Loading attestation report...</div>';
|
||||
fetch(base + '/attestation/report').then(function(res) {
|
||||
if (!res.ok) throw new Error(res.status);
|
||||
return res.json();
|
||||
}).then(function(data) {
|
||||
teeReportCache = data;
|
||||
renderTeePopover(data);
|
||||
}).catch(function() {
|
||||
popover.innerHTML = '<div class="tee-popover-loading">Could not load attestation report</div>';
|
||||
}).finally(function() {
|
||||
teeReportLoading = false;
|
||||
});
|
||||
}
|
||||
|
||||
function renderTeePopover(report) {
|
||||
var popover = document.getElementById('tee-popover');
|
||||
var digest = (teeInfo && teeInfo.image_digest) || 'N/A';
|
||||
var fingerprint = report.tls_certificate_fingerprint || 'N/A';
|
||||
var reportData = report.report_data || '';
|
||||
var vmConfig = report.vm_config || 'N/A';
|
||||
var truncated = reportData.length > 32 ? reportData.slice(0, 32) + '...' : reportData;
|
||||
popover.innerHTML = '<div class="tee-popover-title">'
|
||||
+ '<svg width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z"/></svg>'
|
||||
+ 'TEE Attestation</div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">Image Digest</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(digest) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">TLS Certificate Fingerprint</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(fingerprint) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">Report Data</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(truncated) + '</div></div>'
|
||||
+ '<div class="tee-field"><div class="tee-field-label">VM Config</div>'
|
||||
+ '<div class="tee-field-value">' + escapeHtml(vmConfig) + '</div></div>'
|
||||
+ '<div class="tee-popover-actions">'
|
||||
+ '<button class="tee-btn-copy" onclick="copyTeeReport()">Copy Full Report</button></div>';
|
||||
}
|
||||
|
||||
function copyTeeReport() {
|
||||
if (!teeReportCache) return;
|
||||
var combined = Object.assign({}, teeReportCache, teeInfo || {});
|
||||
navigator.clipboard.writeText(JSON.stringify(combined, null, 2)).then(function() {
|
||||
showToast('Attestation report copied', 'success');
|
||||
}).catch(function() {
|
||||
showToast('Failed to copy report', 'error');
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('tee-shield').addEventListener('mouseenter', function() {
|
||||
fetchTeeReport();
|
||||
document.getElementById('tee-popover').classList.add('visible');
|
||||
});
|
||||
document.getElementById('tee-shield').addEventListener('mouseleave', function() {
|
||||
document.getElementById('tee-popover').classList.remove('visible');
|
||||
});
|
||||
|
||||
// --- Extension install ---
|
||||
|
||||
function installExtension() {
|
||||
const name = document.getElementById('ext-install-name').value.trim();
|
||||
function installWasmExtension() {
|
||||
var name = document.getElementById('wasm-install-name').value.trim();
|
||||
if (!name) {
|
||||
showToast('Extension name is required', 'error');
|
||||
return;
|
||||
}
|
||||
const url = document.getElementById('ext-install-url').value.trim();
|
||||
const kind = document.getElementById('ext-install-kind').value;
|
||||
var url = document.getElementById('wasm-install-url').value.trim();
|
||||
if (!url) {
|
||||
showToast('URL to .tar.gz bundle is required', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name, url: url || undefined, kind },
|
||||
}).then((res) => {
|
||||
body: { name: name, url: url, kind: 'wasm_tool' },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Installed ' + name, 'success');
|
||||
document.getElementById('ext-install-name').value = '';
|
||||
document.getElementById('ext-install-url').value = '';
|
||||
document.getElementById('wasm-install-name').value = '';
|
||||
document.getElementById('wasm-install-url').value = '';
|
||||
loadExtensions();
|
||||
} else {
|
||||
showToast('Install failed: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
}).catch((err) => {
|
||||
}).catch(function(err) {
|
||||
showToast('Install failed: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
function addMcpServer() {
|
||||
var name = document.getElementById('mcp-install-name').value.trim();
|
||||
if (!name) {
|
||||
showToast('Server name is required', 'error');
|
||||
return;
|
||||
}
|
||||
var url = document.getElementById('mcp-install-url').value.trim();
|
||||
if (!url) {
|
||||
showToast('MCP server URL is required', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
apiFetch('/api/extensions/install', {
|
||||
method: 'POST',
|
||||
body: { name: name, url: url, kind: 'mcp_server' },
|
||||
}).then(function(res) {
|
||||
if (res.success) {
|
||||
showToast('Added MCP server ' + name, 'success');
|
||||
document.getElementById('mcp-install-name').value = '';
|
||||
document.getElementById('mcp-install-url').value = '';
|
||||
loadExtensions();
|
||||
} else {
|
||||
showToast('Failed to add MCP server: ' + (res.message || 'unknown error'), 'error');
|
||||
}
|
||||
}).catch(function(err) {
|
||||
showToast('Failed to add MCP server: ' + err.message, 'error');
|
||||
});
|
||||
}
|
||||
|
||||
// --- Keyboard shortcuts ---
|
||||
|
||||
document.addEventListener('keydown', (e) => {
|
||||
@@ -2133,10 +2696,10 @@ document.addEventListener('keydown', (e) => {
|
||||
const tag = (e.target.tagName || '').toLowerCase();
|
||||
const inInput = tag === 'input' || tag === 'textarea';
|
||||
|
||||
// Mod+1-6: switch tabs
|
||||
if (mod && e.key >= '1' && e.key <= '6') {
|
||||
// Mod+1-5: switch tabs
|
||||
if (mod && e.key >= '1' && e.key <= '5') {
|
||||
e.preventDefault();
|
||||
const tabs = ['chat', 'memory', 'jobs', 'routines', 'logs', 'extensions'];
|
||||
const tabs = ['chat', 'memory', 'jobs', 'routines', 'extensions'];
|
||||
const idx = parseInt(e.key) - 1;
|
||||
if (tabs[idx]) switchTab(tabs[idx]);
|
||||
return;
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>IronClaw</title>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/style.css">
|
||||
<script
|
||||
src="https://cdn.jsdelivr.net/npm/[email protected]/lib/marked.umd.min.js"
|
||||
@@ -36,10 +39,17 @@
|
||||
<button class="active" data-tab="chat">Chat</button>
|
||||
<button data-tab="memory">Memory</button>
|
||||
<button data-tab="jobs">Jobs</button>
|
||||
<button data-tab="logs">Logs</button>
|
||||
<button data-tab="routines">Routines</button>
|
||||
<button data-tab="extensions">Extensions</button>
|
||||
<div class="spacer"></div>
|
||||
<button class="status-logs-btn" data-tab="logs" title="Logs">Logs</button>
|
||||
<div class="tee-shield" id="tee-shield" style="display:none" title="Running in a Trusted Execution Environment">
|
||||
<svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M12 22s8-4 8-10V5l-8-3-8 3v7c0 6 8 10 8 10z"/>
|
||||
</svg>
|
||||
<span id="tee-shield-label">TEE Verified</span>
|
||||
<div class="tee-popover" id="tee-popover"></div>
|
||||
</div>
|
||||
<div class="status" id="gateway-status-trigger">
|
||||
<div class="dot" id="sse-dot"></div>
|
||||
<span id="sse-status">Connected</span>
|
||||
@@ -178,34 +188,42 @@
|
||||
<!-- Extensions Tab -->
|
||||
<div class="tab-panel" id="tab-extensions">
|
||||
<div class="extensions-container">
|
||||
<div class="extensions-section">
|
||||
<h3>Install Extension</h3>
|
||||
<div class="ext-install-form" id="ext-install-form">
|
||||
<input type="text" id="ext-install-name" placeholder="Extension name (required)">
|
||||
<input type="text" id="ext-install-url" placeholder="URL (optional)">
|
||||
<select id="ext-install-kind">
|
||||
<option value="mcp_server">MCP Server</option>
|
||||
<option value="wasm_tool">WASM Tool</option>
|
||||
<option value="wasm_channel">WASM Channel</option>
|
||||
</select>
|
||||
<button onclick="installExtension()">Install</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Installed Extensions</h3>
|
||||
<div class="extensions-list" id="extensions-list">
|
||||
<div class="empty-state">Loading extensions...</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section" id="available-wasm-section">
|
||||
<h3>Available WASM Extensions</h3>
|
||||
<div class="extensions-list" id="available-wasm-list">
|
||||
<div class="empty-state">Loading...</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Install WASM Extension</h3>
|
||||
<div class="ext-install-form">
|
||||
<input type="text" id="wasm-install-name" placeholder="Extension name">
|
||||
<input type="text" id="wasm-install-url" placeholder="URL to .tar.gz bundle">
|
||||
<button onclick="installWasmExtension()">Install</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>MCP Servers</h3>
|
||||
<div class="extensions-list" id="mcp-servers-list">
|
||||
<div class="empty-state">Loading...</div>
|
||||
</div>
|
||||
<h4>Add Custom MCP Server</h4>
|
||||
<div class="ext-install-form">
|
||||
<input type="text" id="mcp-install-name" placeholder="Server name">
|
||||
<input type="text" id="mcp-install-url" placeholder="MCP server URL (https://...)">
|
||||
<button onclick="addMcpServer()">Add</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="extensions-section">
|
||||
<h3>Registered Tools</h3>
|
||||
<table class="tools-table" id="tools-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<thead><tr><th>Name</th><th>Description</th></tr></thead>
|
||||
<tbody id="tools-tbody"></tbody>
|
||||
</table>
|
||||
<div class="empty-state" id="tools-empty" style="display:none">No tools registered</div>
|
||||
|
||||
+535
-105
File diff suppressed because it is too large
Load Diff
@@ -346,6 +346,9 @@ pub struct ExtensionInfo {
|
||||
pub authenticated: bool,
|
||||
pub active: bool,
|
||||
pub tools: Vec<String>,
|
||||
/// Whether this extension has configurable secrets (setup schema).
|
||||
#[serde(default)]
|
||||
pub needs_setup: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -371,6 +374,31 @@ pub struct InstallExtensionRequest {
|
||||
pub kind: Option<String>,
|
||||
}
|
||||
|
||||
// --- Extension Setup ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ExtensionSetupResponse {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub secrets: Vec<SecretFieldInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SecretFieldInfo {
|
||||
pub name: String,
|
||||
pub prompt: String,
|
||||
pub optional: bool,
|
||||
/// Whether this secret is already stored.
|
||||
pub provided: bool,
|
||||
/// Whether the secret will be auto-generated if left empty.
|
||||
pub auto_generate: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ExtensionSetupRequest {
|
||||
pub secrets: std::collections::HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct ActionResponse {
|
||||
pub success: bool,
|
||||
@@ -408,6 +436,50 @@ impl ActionResponse {
|
||||
}
|
||||
}
|
||||
|
||||
// --- Registry ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RegistryEntryInfo {
|
||||
pub name: String,
|
||||
pub display_name: String,
|
||||
pub kind: String,
|
||||
pub description: String,
|
||||
pub keywords: Vec<String>,
|
||||
pub installed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct RegistrySearchResponse {
|
||||
pub entries: Vec<RegistryEntryInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct RegistrySearchQuery {
|
||||
pub query: Option<String>,
|
||||
}
|
||||
|
||||
// --- Pairing ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct PairingListResponse {
|
||||
pub channel: String,
|
||||
pub requests: Vec<PairingRequestInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct PairingRequestInfo {
|
||||
pub code: String,
|
||||
pub sender_id: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub meta: Option<serde_json::Value>,
|
||||
pub created_at: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct PairingApproveRequest {
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
// --- Skills ---
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
|
||||
@@ -490,6 +490,9 @@ mod tests {
|
||||
skill_registry: None,
|
||||
skill_catalog: None,
|
||||
chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60),
|
||||
registry_entries: Vec::new(),
|
||||
cost_guard: None,
|
||||
startup_time: std::time::Instant::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+145
-31
@@ -66,12 +66,47 @@ pub const OAUTH_CALLBACK_PORT: u16 = 9876;
|
||||
///
|
||||
/// Checks `IRONCLAW_OAUTH_CALLBACK_URL` env var first (useful for remote/VPS
|
||||
/// deployments where `127.0.0.1` is unreachable from the user's browser),
|
||||
/// then falls back to `http://127.0.0.1:{OAUTH_CALLBACK_PORT}`.
|
||||
/// then falls back to `http://{callback_host()}:{OAUTH_CALLBACK_PORT}`.
|
||||
pub fn callback_url() -> String {
|
||||
std::env::var("IRONCLAW_OAUTH_CALLBACK_URL")
|
||||
.ok()
|
||||
.filter(|v| !v.is_empty())
|
||||
.unwrap_or_else(|| format!("http://127.0.0.1:{}", OAUTH_CALLBACK_PORT))
|
||||
.unwrap_or_else(|| format!("http://{}:{}", callback_host(), OAUTH_CALLBACK_PORT))
|
||||
}
|
||||
|
||||
/// Returns the hostname used in OAuth callback URLs.
|
||||
///
|
||||
/// Reads `OAUTH_CALLBACK_HOST` from the environment (default: `127.0.0.1`).
|
||||
///
|
||||
/// **Remote server usage:** set `OAUTH_CALLBACK_HOST` to the network interface
|
||||
/// address you want to listen on (e.g. the server's LAN IP or `0.0.0.0`).
|
||||
/// The callback listener will bind to that specific address instead of the
|
||||
/// loopback interface, so the OAuth redirect can reach an external browser.
|
||||
/// Note: this transmits the session token over plain HTTP — prefer SSH port
|
||||
/// forwarding (`ssh -L 9876:127.0.0.1:9876 user@host`) when possible.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```bash
|
||||
/// export OAUTH_CALLBACK_HOST=203.0.113.10
|
||||
/// ironclaw login
|
||||
/// # Opens: http://203.0.113.10:9876/auth/callback
|
||||
/// ```
|
||||
pub fn callback_host() -> String {
|
||||
std::env::var("OAUTH_CALLBACK_HOST").unwrap_or_else(|_| "127.0.0.1".to_string())
|
||||
}
|
||||
|
||||
/// Returns `true` if `host` is a loopback address that only accepts local connections.
|
||||
///
|
||||
/// Covers `localhost` (case-insensitive), the full `127.0.0.0/8` IPv4 loopback
|
||||
/// range, and `::1` for IPv6.
|
||||
pub fn is_loopback_host(host: &str) -> bool {
|
||||
if host.eq_ignore_ascii_case("localhost") {
|
||||
return true;
|
||||
}
|
||||
host.parse::<std::net::IpAddr>()
|
||||
.map(|ip| ip.is_loopback())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Error from the OAuth callback listener.
|
||||
@@ -90,35 +125,50 @@ pub enum OAuthCallbackError {
|
||||
Io(String),
|
||||
}
|
||||
|
||||
/// Map a `std::io::Error` from a bind attempt to an `OAuthCallbackError`.
|
||||
fn bind_error(e: std::io::Error) -> OAuthCallbackError {
|
||||
if e.kind() == std::io::ErrorKind::AddrInUse {
|
||||
OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string())
|
||||
} else {
|
||||
OAuthCallbackError::Io(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Bind the OAuth callback listener on the fixed port.
|
||||
///
|
||||
/// Binds to IPv4 `127.0.0.1` first because callback URLs use `127.0.0.1`
|
||||
/// explicitly (e.g., NEAR AI redirects to `http://127.0.0.1:9876/auth/callback`).
|
||||
/// Falls back to IPv6 `[::1]` only if IPv4 binding fails for a reason other
|
||||
/// than `AddrInUse`. If the port is already occupied, fails immediately.
|
||||
/// When `OAUTH_CALLBACK_HOST` is a loopback address (the default `127.0.0.1`),
|
||||
/// binds to `127.0.0.1` first and falls back to `[::1]` so local-only auth
|
||||
/// flows remain restricted to the local machine.
|
||||
///
|
||||
/// When `OAUTH_CALLBACK_HOST` is set to a remote address, binds to that
|
||||
/// specific address so only connections directed to it are accepted.
|
||||
pub async fn bind_callback_listener() -> Result<TcpListener, OAuthCallbackError> {
|
||||
let ipv4_addr = format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||
match TcpListener::bind(&ipv4_addr).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||
return Err(OAuthCallbackError::PortInUse(
|
||||
OAUTH_CALLBACK_PORT,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
Err(_) => {
|
||||
// IPv4 not available, fall back to IPv6
|
||||
}
|
||||
}
|
||||
TcpListener::bind(format!("[::1]:{}", OAUTH_CALLBACK_PORT))
|
||||
.await
|
||||
.map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::AddrInUse {
|
||||
OAuthCallbackError::PortInUse(OAUTH_CALLBACK_PORT, e.to_string())
|
||||
} else {
|
||||
OAuthCallbackError::Io(e.to_string())
|
||||
let host = callback_host();
|
||||
|
||||
if is_loopback_host(&host) {
|
||||
// Local mode: prefer IPv4 loopback, fall back to IPv6.
|
||||
let ipv4_addr = format!("127.0.0.1:{}", OAUTH_CALLBACK_PORT);
|
||||
match TcpListener::bind(&ipv4_addr).await {
|
||||
Ok(listener) => return Ok(listener),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AddrInUse => {
|
||||
return Err(OAuthCallbackError::PortInUse(
|
||||
OAUTH_CALLBACK_PORT,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
})
|
||||
Err(_) => {
|
||||
// IPv4 not available, fall back to IPv6
|
||||
}
|
||||
}
|
||||
TcpListener::bind(format!("[::1]:{}", OAUTH_CALLBACK_PORT))
|
||||
.await
|
||||
.map_err(bind_error)
|
||||
} else {
|
||||
// Remote mode: bind to the specific configured host address only,
|
||||
// not 0.0.0.0, to limit exposure to the intended interface.
|
||||
let addr = format!("{}:{}", host, OAUTH_CALLBACK_PORT);
|
||||
TcpListener::bind(&addr).await.map_err(bind_error)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for an OAuth callback and extract a query parameter value.
|
||||
@@ -311,27 +361,91 @@ pub fn landing_html(provider_name: &str, success: bool) -> String {
|
||||
mod tests {
|
||||
use std::sync::Mutex;
|
||||
|
||||
use crate::cli::oauth_defaults::{builtin_credentials, callback_url, landing_html};
|
||||
use crate::cli::oauth_defaults::{
|
||||
builtin_credentials, callback_host, callback_url, is_loopback_host, landing_html,
|
||||
};
|
||||
|
||||
/// Serializes env-mutating tests to prevent parallel races.
|
||||
static ENV_MUTEX: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[test]
|
||||
fn test_is_loopback_host() {
|
||||
assert!(is_loopback_host("127.0.0.1"));
|
||||
assert!(is_loopback_host("127.0.0.2")); // full 127.0.0.0/8 range
|
||||
assert!(is_loopback_host("127.255.255.254"));
|
||||
assert!(is_loopback_host("::1"));
|
||||
assert!(is_loopback_host("localhost"));
|
||||
assert!(is_loopback_host("LOCALHOST"));
|
||||
assert!(!is_loopback_host("203.0.113.10"));
|
||||
assert!(!is_loopback_host("my-server.example.com"));
|
||||
assert!(!is_loopback_host("0.0.0.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_host_default() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
let original = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
assert_eq!(callback_host(), "127.0.0.1");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_host_env_override() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", "203.0.113.10");
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
}
|
||||
assert_eq!(callback_host(), "203.0.113.10");
|
||||
// callback_url() fallback should incorporate the custom host
|
||||
let url = callback_url();
|
||||
assert!(url.contains("203.0.113.10"), "url was: {url}");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original_host {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
} else {
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
if let Some(val) = original_url {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_callback_url_default() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
// Clear the env var to test default behavior
|
||||
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
// Clear both env vars to test default behavior
|
||||
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
|
||||
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
|
||||
// SAFETY: Under ENV_MUTEX, no concurrent env access.
|
||||
unsafe {
|
||||
std::env::remove_var("IRONCLAW_OAUTH_CALLBACK_URL");
|
||||
std::env::remove_var("OAUTH_CALLBACK_HOST");
|
||||
}
|
||||
let url = callback_url();
|
||||
assert_eq!(url, "http://127.0.0.1:9876");
|
||||
// Restore
|
||||
unsafe {
|
||||
if let Some(val) = original {
|
||||
if let Some(val) = original_url {
|
||||
std::env::set_var("IRONCLAW_OAUTH_CALLBACK_URL", val);
|
||||
}
|
||||
if let Some(val) = original_host {
|
||||
std::env::set_var("OAUTH_CALLBACK_HOST", val);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-51
@@ -1,7 +1,5 @@
|
||||
//! Registry CLI commands for discovering and installing extensions.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Subcommand;
|
||||
|
||||
use crate::registry::catalog::RegistryCatalog;
|
||||
@@ -59,8 +57,20 @@ pub enum RegistryCommand {
|
||||
|
||||
/// Run a registry command.
|
||||
pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
|
||||
let registry_dir = find_registry_dir()?;
|
||||
let catalog = RegistryCatalog::load(®istry_dir)?;
|
||||
// For install commands that need to build from source, a disk registry is required.
|
||||
// For list/info, embedded manifests suffice.
|
||||
let registry_dir = RegistryCatalog::find_dir();
|
||||
let catalog = if let Some(ref dir) = registry_dir {
|
||||
RegistryCatalog::load(dir)?
|
||||
} else {
|
||||
RegistryCatalog::load_or_embedded()?
|
||||
};
|
||||
|
||||
// Resolve repo root for installer (empty path when running from binary)
|
||||
let repo_root = registry_dir
|
||||
.as_ref()
|
||||
.and_then(|d| d.parent().map(|p| p.to_path_buf()))
|
||||
.unwrap_or_default();
|
||||
|
||||
match cmd {
|
||||
RegistryCommand::List { kind, tag, verbose } => {
|
||||
@@ -68,53 +78,14 @@ pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
|
||||
}
|
||||
RegistryCommand::Info { name } => cmd_info(&catalog, &name),
|
||||
RegistryCommand::Install { name, force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, &name, force, build).await
|
||||
cmd_install(&catalog, &repo_root, &name, force, build).await
|
||||
}
|
||||
RegistryCommand::InstallDefaults { force, build } => {
|
||||
cmd_install(&catalog, ®istry_dir, "default", force, build).await
|
||||
cmd_install(&catalog, &repo_root, "default", force, build).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the registry directory by looking relative to the current executable or cwd.
|
||||
fn find_registry_dir() -> anyhow::Result<PathBuf> {
|
||||
// Try relative to current directory (for dev usage)
|
||||
let cwd = std::env::current_dir()?;
|
||||
let candidate = cwd.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
// Try relative to executable (covers installed binary, target/debug/, target/release/)
|
||||
if let Ok(exe) = std::env::current_exe()
|
||||
&& let Some(parent) = exe.parent()
|
||||
{
|
||||
// Walk up to 3 levels: exe dir, parent (target/release → target), grandparent (→ repo root)
|
||||
let mut dir = Some(parent);
|
||||
for _ in 0..3 {
|
||||
if let Some(d) = dir {
|
||||
let candidate = d.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
dir = d.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let candidate = manifest_dir.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Ok(candidate);
|
||||
}
|
||||
|
||||
anyhow::bail!(
|
||||
"Could not find registry/ directory. Run from the ironclaw repo root, \
|
||||
or ensure registry/ is next to the ironclaw binary."
|
||||
)
|
||||
}
|
||||
|
||||
fn cmd_list(
|
||||
catalog: &RegistryCatalog,
|
||||
kind: Option<&str>,
|
||||
@@ -254,16 +225,11 @@ fn cmd_info(catalog: &RegistryCatalog, name: &str) -> anyhow::Result<()> {
|
||||
|
||||
async fn cmd_install(
|
||||
catalog: &RegistryCatalog,
|
||||
registry_dir: &std::path::Path,
|
||||
repo_root: &std::path::Path,
|
||||
name: &str,
|
||||
force: bool,
|
||||
prefer_build: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
// Registry dir parent is the repo root
|
||||
let repo_root = registry_dir
|
||||
.parent()
|
||||
.ok_or_else(|| anyhow::anyhow!("Cannot determine repo root from registry dir"))?;
|
||||
|
||||
let installer = RegistryInstaller::with_defaults(repo_root.to_path_buf());
|
||||
|
||||
let (manifests, bundle) = catalog.resolve(name)?;
|
||||
|
||||
@@ -23,6 +23,10 @@ pub struct AgentConfig {
|
||||
pub max_cost_per_day_cents: Option<u64>,
|
||||
/// Maximum LLM/tool actions per hour. None = unlimited.
|
||||
pub max_actions_per_hour: Option<u64>,
|
||||
/// Maximum tool-call iterations per agentic loop invocation. Default 50.
|
||||
pub max_tool_iterations: usize,
|
||||
/// When true, skip tool approval checks entirely. For benchmarks/CI.
|
||||
pub auto_approve_tools: bool,
|
||||
}
|
||||
|
||||
impl AgentConfig {
|
||||
@@ -115,6 +119,22 @@ impl AgentConfig {
|
||||
key: "MAX_ACTIONS_PER_HOUR".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?,
|
||||
max_tool_iterations: optional_env("AGENT_MAX_TOOL_ITERATIONS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "AGENT_MAX_TOOL_ITERATIONS".to_string(),
|
||||
message: format!("must be a positive integer: {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.agent.max_tool_iterations),
|
||||
auto_approve_tools: optional_env("AGENT_AUTO_APPROVE_TOOLS")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "AGENT_AUTO_APPROVE_TOOLS".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.agent.auto_approve_tools),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,9 @@ pub struct OpenAiCompatibleConfig {
|
||||
pub base_url: String,
|
||||
pub api_key: Option<SecretString>,
|
||||
pub model: String,
|
||||
/// Extra HTTP headers injected into every LLM request.
|
||||
/// Parsed from `LLM_EXTRA_HEADERS` env var (format: `Key:Value,Key2:Value2`).
|
||||
pub extra_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
/// Configuration for Tinfoil private inference.
|
||||
@@ -167,6 +170,10 @@ pub struct NearAiConfig {
|
||||
/// Number of consecutive retryable failures before a provider enters
|
||||
/// cooldown (default: 3).
|
||||
pub failover_cooldown_threshold: u32,
|
||||
/// Enable cascade mode for smart routing: when a moderate-complexity task
|
||||
/// gets an uncertain response from the cheap model, re-send to primary.
|
||||
/// Default: true.
|
||||
pub smart_routing_cascade: bool,
|
||||
}
|
||||
|
||||
impl LlmConfig {
|
||||
@@ -232,6 +239,7 @@ impl LlmConfig {
|
||||
response_cache_max_entries: parse_optional_env("RESPONSE_CACHE_MAX_ENTRIES", 1000)?,
|
||||
failover_cooldown_secs: parse_optional_env("LLM_FAILOVER_COOLDOWN_SECS", 300)?,
|
||||
failover_cooldown_threshold: parse_optional_env("LLM_FAILOVER_THRESHOLD", 3)?,
|
||||
smart_routing_cascade: parse_optional_env("SMART_ROUTING_CASCADE", true)?,
|
||||
};
|
||||
|
||||
// Resolve provider-specific configs based on backend
|
||||
@@ -293,10 +301,15 @@ impl LlmConfig {
|
||||
let model = optional_env("LLM_MODEL")?
|
||||
.or_else(|| settings.selected_model.clone())
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
let extra_headers = optional_env("LLM_EXTRA_HEADERS")?
|
||||
.map(|val| parse_extra_headers(&val))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
Some(OpenAiCompatibleConfig {
|
||||
base_url,
|
||||
api_key,
|
||||
model,
|
||||
extra_headers,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -327,6 +340,40 @@ impl LlmConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `LLM_EXTRA_HEADERS` value into a list of (key, value) pairs.
|
||||
///
|
||||
/// Format: `Key1:Value1,Key2:Value2` — colon-separated key:value, comma-separated pairs.
|
||||
/// Colon is used as the separator (not `=`) because header values often contain `=`
|
||||
/// (e.g., base64 tokens).
|
||||
fn parse_extra_headers(val: &str) -> Result<Vec<(String, String)>, ConfigError> {
|
||||
if val.trim().is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut headers = Vec::new();
|
||||
for pair in val.split(',') {
|
||||
let pair = pair.trim();
|
||||
if pair.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some((key, value)) = pair.split_once(':') else {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "LLM_EXTRA_HEADERS".to_string(),
|
||||
message: format!("malformed header entry '{}', expected Key:Value", pair),
|
||||
});
|
||||
};
|
||||
let key = key.trim();
|
||||
if key.is_empty() {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "LLM_EXTRA_HEADERS".to_string(),
|
||||
message: format!("empty header name in entry '{}'", pair),
|
||||
});
|
||||
}
|
||||
headers.push((key.to_string(), value.trim().to_string()));
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
/// Get the default session file path (~/.ironclaw/session.json).
|
||||
fn default_session_path() -> PathBuf {
|
||||
dirs::home_dir()
|
||||
@@ -399,4 +446,69 @@ mod tests {
|
||||
std::env::remove_var("LLM_MODEL");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_parsed() {
|
||||
let result = parse_extra_headers("HTTP-Referer:https://myapp.com,X-Title:MyApp").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
("HTTP-Referer".to_string(), "https://myapp.com".to_string()),
|
||||
("X-Title".to_string(), "MyApp".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_empty_string() {
|
||||
let result = parse_extra_headers("").unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_whitespace_only() {
|
||||
let result = parse_extra_headers(" ").unwrap();
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_malformed() {
|
||||
let result = parse_extra_headers("NoColonHere");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_empty_key() {
|
||||
let result = parse_extra_headers(":value");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_value_with_colons() {
|
||||
// Values can contain colons (e.g., URLs)
|
||||
let result = parse_extra_headers("Authorization:Bearer abc:def").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![("Authorization".to_string(), "Bearer abc:def".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_trailing_comma() {
|
||||
let result = parse_extra_headers("X-Title:MyApp,").unwrap();
|
||||
assert_eq!(result, vec![("X-Title".to_string(), "MyApp".to_string())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_headers_with_spaces() {
|
||||
let result =
|
||||
parse_extra_headers(" HTTP-Referer : https://myapp.com , X-Title : MyApp ").unwrap();
|
||||
assert_eq!(
|
||||
result,
|
||||
vec![
|
||||
("HTTP-Referer".to_string(), "https://myapp.com".to_string()),
|
||||
("X-Title".to_string(), "MyApp".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ mod safety;
|
||||
mod sandbox;
|
||||
mod secrets;
|
||||
mod skills;
|
||||
mod transcription;
|
||||
mod tunnel;
|
||||
mod wasm;
|
||||
|
||||
@@ -45,6 +46,7 @@ pub use self::safety::SafetyConfig;
|
||||
pub use self::sandbox::{ClaudeCodeConfig, SandboxModeConfig};
|
||||
pub use self::secrets::SecretsConfig;
|
||||
pub use self::skills::SkillsConfig;
|
||||
pub use self::transcription::TranscriptionConfig;
|
||||
pub use self::tunnel::TunnelConfig;
|
||||
pub use self::wasm::WasmConfig;
|
||||
|
||||
@@ -74,6 +76,7 @@ pub struct Config {
|
||||
pub sandbox: SandboxModeConfig,
|
||||
pub claude_code: ClaudeCodeConfig,
|
||||
pub skills: SkillsConfig,
|
||||
pub transcription: TranscriptionConfig,
|
||||
pub observability: crate::observability::ObservabilityConfig,
|
||||
}
|
||||
|
||||
@@ -198,6 +201,7 @@ impl Config {
|
||||
sandbox: SandboxModeConfig::resolve()?,
|
||||
claude_code: ClaudeCodeConfig::resolve()?,
|
||||
skills: SkillsConfig::resolve()?,
|
||||
transcription: TranscriptionConfig::resolve(settings)?,
|
||||
observability: crate::observability::ObservabilityConfig {
|
||||
backend: std::env::var("OBSERVABILITY_BACKEND").unwrap_or_else(|_| "none".into()),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
use secrecy::SecretString;
|
||||
|
||||
use crate::config::helpers::optional_env;
|
||||
use crate::error::ConfigError;
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Transcription provider configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TranscriptionConfig {
|
||||
/// Whether transcription is enabled.
|
||||
pub enabled: bool,
|
||||
/// Provider to use: "openai".
|
||||
pub provider: String,
|
||||
/// OpenAI API key (reused from embeddings/LLM config).
|
||||
pub openai_api_key: Option<SecretString>,
|
||||
/// Model to use for transcription.
|
||||
pub model: String,
|
||||
/// Optional language hint (ISO-639-1, e.g., "en").
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for TranscriptionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
provider: "openai".to_string(),
|
||||
openai_api_key: None,
|
||||
model: "whisper-1".to_string(),
|
||||
language: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TranscriptionConfig {
|
||||
pub(crate) fn resolve(settings: &Settings) -> Result<Self, ConfigError> {
|
||||
let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from);
|
||||
|
||||
let provider = optional_env("TRANSCRIPTION_PROVIDER")?
|
||||
.unwrap_or_else(|| settings.transcription.provider.clone());
|
||||
|
||||
let model = optional_env("TRANSCRIPTION_MODEL")?
|
||||
.unwrap_or_else(|| settings.transcription.model.clone());
|
||||
|
||||
let language = optional_env("TRANSCRIPTION_LANGUAGE")?
|
||||
.or_else(|| settings.transcription.language.clone());
|
||||
|
||||
let enabled = optional_env("TRANSCRIPTION_ENABLED")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_ENABLED".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.transcription.enabled);
|
||||
|
||||
// Only "openai" is currently supported
|
||||
if enabled && provider != "openai" {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_PROVIDER".to_string(),
|
||||
message: format!(
|
||||
"unsupported provider '{}', only 'openai' is currently supported",
|
||||
provider
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
provider,
|
||||
openai_api_key,
|
||||
model,
|
||||
language,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::helpers::ENV_MUTEX;
|
||||
use crate::settings::{Settings, TranscriptionSettings};
|
||||
|
||||
fn clear_transcription_env() {
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_PROVIDER");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
std::env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_defaults_from_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(!config.enabled);
|
||||
assert_eq!(config.provider, "openai");
|
||||
assert_eq!(config.model, "whisper-1");
|
||||
assert!(config.language.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_env_overrides_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
unsafe {
|
||||
std::env::set_var("TRANSCRIPTION_ENABLED", "true");
|
||||
std::env::set_var("TRANSCRIPTION_MODEL", "whisper-large-v3");
|
||||
std::env::set_var("TRANSCRIPTION_LANGUAGE", "en");
|
||||
}
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("en".to_string()));
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_settings_with_custom_values() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
model: "whisper-large-v3".to_string(),
|
||||
language: Some("fr".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("fr".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_rejects_unsupported_provider() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
provider: "deepgram".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = TranscriptionConfig::resolve(&settings);
|
||||
assert!(result.is_err());
|
||||
let err_msg = result.unwrap_err().to_string();
|
||||
assert!(
|
||||
err_msg.contains("unsupported provider"),
|
||||
"Error should mention unsupported provider, got: {err_msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_disabled_skips_provider_validation() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
// When disabled, any provider string is accepted (never used)
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: false,
|
||||
provider: "nonexistent".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("should succeed when disabled");
|
||||
assert!(!config.enabled);
|
||||
}
|
||||
}
|
||||
@@ -49,9 +49,10 @@ impl ConversationStore for LibSqlBackend {
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
let conn = self.connect().await?;
|
||||
let id = Uuid::new_v4();
|
||||
let now = fmt_ts(&Utc::now());
|
||||
conn.execute(
|
||||
"INSERT INTO conversation_messages (id, conversation_id, role, content) VALUES (?1, ?2, ?3, ?4)",
|
||||
params![id.to_string(), conversation_id.to_string(), role, content],
|
||||
"INSERT INTO conversation_messages (id, conversation_id, role, content, created_at) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![id.to_string(), conversation_id.to_string(), role, content, now],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| DatabaseError::Query(e.to_string()))?;
|
||||
@@ -100,7 +101,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
(SELECT substr(m2.content, 1, 100)
|
||||
FROM conversation_messages m2
|
||||
WHERE m2.conversation_id = c.id AND m2.role = 'user'
|
||||
ORDER BY m2.created_at ASC
|
||||
ORDER BY m2.created_at ASC, m2.rowid ASC
|
||||
LIMIT 1
|
||||
) AS title
|
||||
FROM conversations c
|
||||
@@ -216,7 +217,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1 AND created_at < ?2
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, rowid DESC
|
||||
LIMIT ?3
|
||||
"#,
|
||||
params![cid, fmt_ts(&before_ts), fetch_limit],
|
||||
@@ -228,7 +229,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC, rowid DESC
|
||||
LIMIT ?2
|
||||
"#,
|
||||
params![cid, fetch_limit],
|
||||
@@ -309,7 +310,7 @@ impl ConversationStore for LibSqlBackend {
|
||||
SELECT id, role, content, created_at
|
||||
FROM conversation_messages
|
||||
WHERE conversation_id = ?1
|
||||
ORDER BY created_at ASC
|
||||
ORDER BY created_at ASC, rowid ASC
|
||||
"#,
|
||||
params![conversation_id.to_string()],
|
||||
)
|
||||
|
||||
@@ -202,6 +202,12 @@ pub enum ToolError {
|
||||
#[error("Tool {name} requires authentication")]
|
||||
AuthRequired { name: String },
|
||||
|
||||
#[error("Tool {name} is rate limited, retry after {retry_after:?}")]
|
||||
RateLimited {
|
||||
name: String,
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
|
||||
#[error("Tool builder failed: {0}")]
|
||||
BuilderFailed(String),
|
||||
}
|
||||
|
||||
@@ -246,6 +246,7 @@ fn extract_url(source: &ExtensionSource) -> String {
|
||||
ExtensionSource::Discovered { url } => url.clone(),
|
||||
ExtensionSource::WasmDownload { wasm_url, .. } => wasm_url.clone(),
|
||||
ExtensionSource::WasmBuildable { repo_url, .. } => repo_url.clone(),
|
||||
ExtensionSource::Bundled { name } => name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+655
-58
@@ -4,7 +4,7 @@
|
||||
//! and tool registry. All extension operations (search, install, auth, activate,
|
||||
//! list, remove) flow through here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -60,6 +60,8 @@ pub struct ExtensionManager {
|
||||
user_id: String,
|
||||
/// Optional database store for DB-backed MCP config.
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
/// Names of WASM channels that were successfully loaded at startup.
|
||||
active_channel_names: RwLock<HashSet<String>>,
|
||||
}
|
||||
|
||||
impl ExtensionManager {
|
||||
@@ -75,9 +77,15 @@ impl ExtensionManager {
|
||||
tunnel_url: Option<String>,
|
||||
user_id: String,
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
catalog_entries: Vec<RegistryEntry>,
|
||||
) -> Self {
|
||||
let registry = if catalog_entries.is_empty() {
|
||||
ExtensionRegistry::new()
|
||||
} else {
|
||||
ExtensionRegistry::new_with_catalog(catalog_entries)
|
||||
};
|
||||
Self {
|
||||
registry: ExtensionRegistry::new(),
|
||||
registry,
|
||||
discovery: OnlineDiscovery::new(),
|
||||
mcp_session_manager,
|
||||
mcp_clients: RwLock::new(HashMap::new()),
|
||||
@@ -91,9 +99,17 @@ impl ExtensionManager {
|
||||
_tunnel_url: tunnel_url,
|
||||
user_id,
|
||||
store,
|
||||
active_channel_names: RwLock::new(HashSet::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register channel names that were loaded at startup.
|
||||
/// Called after WASM channels are loaded so `list()` reports accurate active status.
|
||||
pub async fn set_active_channels(&self, names: Vec<String>) {
|
||||
let mut active = self.active_channel_names.write().await;
|
||||
active.extend(names);
|
||||
}
|
||||
|
||||
/// Search for extensions. If `discover` is true, also searches online.
|
||||
pub async fn search(
|
||||
&self,
|
||||
@@ -131,9 +147,14 @@ impl ExtensionManager {
|
||||
url: Option<&str>,
|
||||
kind_hint: Option<ExtensionKind>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension");
|
||||
|
||||
// If we have a registry entry, use it
|
||||
if let Some(entry) = self.registry.get(name).await {
|
||||
return self.install_from_entry(&entry).await;
|
||||
return self.install_from_entry(&entry).await.map_err(|e| {
|
||||
tracing::error!(extension = %name, error = %e, "Extension install failed");
|
||||
e
|
||||
});
|
||||
}
|
||||
|
||||
// If a URL was provided, determine kind and install
|
||||
@@ -143,19 +164,21 @@ impl ExtensionManager {
|
||||
ExtensionKind::McpServer => self.install_mcp_from_url(name, url).await,
|
||||
ExtensionKind::WasmTool => self.install_wasm_tool_from_url(name, url).await,
|
||||
ExtensionKind::WasmChannel => {
|
||||
Err(ExtensionError::InstallFailed(
|
||||
"WASM channel installation from URL not yet supported. \
|
||||
Place the .wasm and .capabilities.json files in ~/.ironclaw/channels/ and restart."
|
||||
.to_string(),
|
||||
))
|
||||
self.install_wasm_channel_from_url(name, url, None).await
|
||||
}
|
||||
};
|
||||
}
|
||||
.map_err(|e| {
|
||||
tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed");
|
||||
e
|
||||
});
|
||||
}
|
||||
|
||||
Err(ExtensionError::NotFound(format!(
|
||||
let err = ExtensionError::NotFound(format!(
|
||||
"'{}' not found in registry. Try searching with discover:true or provide a URL.",
|
||||
name
|
||||
)))
|
||||
));
|
||||
tracing::warn!(extension = %name, "Extension not found in registry");
|
||||
Err(err)
|
||||
}
|
||||
|
||||
/// Authenticate an installed extension.
|
||||
@@ -173,7 +196,7 @@ impl ExtensionManager {
|
||||
match kind {
|
||||
ExtensionKind::McpServer => self.auth_mcp(name, token).await,
|
||||
ExtensionKind::WasmTool => self.auth_wasm_tool(name, token).await,
|
||||
ExtensionKind::WasmChannel => self.auth_wasm_tool(name, token).await,
|
||||
ExtensionKind::WasmChannel => self.auth_wasm_channel(name, token).await,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,6 +248,7 @@ impl ExtensionManager {
|
||||
authenticated,
|
||||
active,
|
||||
tools,
|
||||
needs_setup: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -251,6 +275,7 @@ impl ExtensionManager {
|
||||
authenticated: true, // WASM tools don't always need auth
|
||||
active,
|
||||
tools: if active { vec![name] } else { Vec::new() },
|
||||
needs_setup: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -266,15 +291,20 @@ impl ExtensionManager {
|
||||
{
|
||||
match crate::channels::wasm::discover_channels(&self.wasm_channels_dir).await {
|
||||
Ok(channels) => {
|
||||
let active_names = self.active_channel_names.read().await;
|
||||
for (name, _discovered) in channels {
|
||||
let active = active_names.contains(&name);
|
||||
let (authenticated, needs_setup) =
|
||||
self.check_channel_auth_status(&name).await;
|
||||
extensions.push(InstalledExtension {
|
||||
name,
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: None,
|
||||
url: None,
|
||||
authenticated: true,
|
||||
active: true, // If loaded at startup, they're active
|
||||
authenticated,
|
||||
active,
|
||||
tools: Vec::new(),
|
||||
needs_setup,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -356,10 +386,27 @@ impl ExtensionManager {
|
||||
|
||||
Ok(format!("Removed WASM tool '{}'", name))
|
||||
}
|
||||
ExtensionKind::WasmChannel => Err(ExtensionError::Other(
|
||||
"Channel removal requires restart. Delete the .wasm file from ~/.ironclaw/channels/ and restart."
|
||||
.to_string(),
|
||||
)),
|
||||
ExtensionKind::WasmChannel => {
|
||||
// Delete channel files
|
||||
let wasm_path = self.wasm_channels_dir.join(format!("{}.wasm", name));
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if wasm_path.exists() {
|
||||
tokio::fs::remove_file(&wasm_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
}
|
||||
if cap_path.exists() {
|
||||
let _ = tokio::fs::remove_file(&cap_path).await;
|
||||
}
|
||||
|
||||
Ok(format!(
|
||||
"Removed channel '{}'. Restart IronClaw for the change to take effect.",
|
||||
name
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -433,16 +480,54 @@ impl ExtensionManager {
|
||||
self.install_mcp_from_url(&entry.name, &url).await
|
||||
}
|
||||
ExtensionKind::WasmTool => match &entry.source {
|
||||
ExtensionSource::WasmDownload { wasm_url, .. } => {
|
||||
self.install_wasm_tool_from_url(&entry.name, wasm_url).await
|
||||
ExtensionSource::WasmDownload {
|
||||
wasm_url,
|
||||
capabilities_url,
|
||||
} => {
|
||||
self.install_wasm_tool_from_url_with_caps(
|
||||
&entry.name,
|
||||
wasm_url,
|
||||
capabilities_url.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ExtensionSource::WasmBuildable { .. } => {
|
||||
Err(ExtensionError::InstallFailed(format!(
|
||||
"'{}' requires building from source. Run `ironclaw registry install {}` \
|
||||
from the CLI (requires cargo-component).",
|
||||
entry.name, entry.name
|
||||
)))
|
||||
}
|
||||
_ => Err(ExtensionError::InstallFailed(
|
||||
"WASM tool entry has no download URL".to_string(),
|
||||
)),
|
||||
},
|
||||
ExtensionKind::WasmChannel => Err(ExtensionError::InstallFailed(
|
||||
"WASM channel installation not yet supported via this flow".to_string(),
|
||||
)),
|
||||
ExtensionKind::WasmChannel => match &entry.source {
|
||||
ExtensionSource::WasmDownload {
|
||||
wasm_url,
|
||||
capabilities_url,
|
||||
} => {
|
||||
self.install_wasm_channel_from_url(
|
||||
&entry.name,
|
||||
wasm_url,
|
||||
capabilities_url.as_deref(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
ExtensionSource::WasmBuildable { .. } => {
|
||||
Err(ExtensionError::InstallFailed(format!(
|
||||
"'{}' requires building from source. Run `ironclaw registry install {}` \
|
||||
from the CLI (requires cargo-component).",
|
||||
entry.name, entry.name
|
||||
)))
|
||||
}
|
||||
ExtensionSource::Bundled { name } => {
|
||||
self.install_bundled_channel_from_artifacts(name).await
|
||||
}
|
||||
_ => Err(ExtensionError::InstallFailed(
|
||||
"WASM channel entry has no download URL".to_string(),
|
||||
)),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -482,6 +567,57 @@ impl ExtensionManager {
|
||||
name: &str,
|
||||
url: &str,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.install_wasm_tool_from_url_with_caps(name, url, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn install_wasm_tool_from_url_with_caps(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_tools_dir)
|
||||
.await?;
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
|
||||
})
|
||||
}
|
||||
|
||||
async fn install_wasm_channel_from_url(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_channels_dir)
|
||||
.await?;
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
message: format!(
|
||||
"WASM channel '{}' installed to {}. Restart to activate.",
|
||||
name,
|
||||
self.wasm_channels_dir.display()
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/// Download a WASM extension (tool or channel) from URL and install to target directory.
|
||||
///
|
||||
/// Handles both tar.gz bundles (containing `.wasm` + `.capabilities.json`) and bare
|
||||
/// `.wasm` files. Validates HTTPS, size limits, and file format.
|
||||
async fn download_and_install_wasm(
|
||||
&self,
|
||||
name: &str,
|
||||
url: &str,
|
||||
capabilities_url: Option<&str>,
|
||||
target_dir: &std::path::Path,
|
||||
) -> Result<(), ExtensionError> {
|
||||
// Require HTTPS to prevent downgrade attacks
|
||||
if !url.starts_with("https://") {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
@@ -490,33 +626,41 @@ impl ExtensionManager {
|
||||
}
|
||||
|
||||
// 50 MB cap to prevent disk-fill DoS
|
||||
const MAX_WASM_SIZE: usize = 50 * 1024 * 1024;
|
||||
const MAX_DOWNLOAD_SIZE: usize = 50 * 1024 * 1024;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(60))
|
||||
.build()
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
|
||||
let response = client
|
||||
.get(url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
tracing::debug!(extension = %name, url = %url, "Downloading WASM extension");
|
||||
|
||||
let response = client.get(url).send().await.map_err(|e| {
|
||||
tracing::error!(extension = %name, url = %url, error = %e, "Download request failed");
|
||||
ExtensionError::DownloadFailed(e.to_string())
|
||||
})?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
tracing::error!(
|
||||
extension = %name,
|
||||
url = %url,
|
||||
status = %status,
|
||||
"Download returned non-success HTTP status"
|
||||
);
|
||||
return Err(ExtensionError::DownloadFailed(format!(
|
||||
"HTTP {}",
|
||||
response.status()
|
||||
"HTTP {} from {}",
|
||||
status, url
|
||||
)));
|
||||
}
|
||||
|
||||
// Check Content-Length header before downloading the full body
|
||||
if let Some(len) = response.content_length()
|
||||
&& len as usize > MAX_WASM_SIZE
|
||||
&& len as usize > MAX_DOWNLOAD_SIZE
|
||||
{
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
len, MAX_WASM_SIZE
|
||||
"Download too large ({} bytes, max {} bytes)",
|
||||
len, MAX_DOWNLOAD_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -525,44 +669,196 @@ impl ExtensionManager {
|
||||
.await
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
|
||||
if bytes.len() > MAX_WASM_SIZE {
|
||||
if bytes.len() > MAX_DOWNLOAD_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
"Download too large ({} bytes, max {} bytes)",
|
||||
bytes.len(),
|
||||
MAX_WASM_SIZE
|
||||
MAX_DOWNLOAD_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Basic WASM magic number check (\0asm)
|
||||
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
|
||||
));
|
||||
// Ensure target directory exists
|
||||
tokio::fs::create_dir_all(target_dir)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
let wasm_path = target_dir.join(format!("{}.wasm", name));
|
||||
let caps_path = target_dir.join(format!("{}.capabilities.json", name));
|
||||
|
||||
// Detect format: gzip (tar.gz bundle) or bare WASM
|
||||
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
|
||||
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
|
||||
self.extract_wasm_tar_gz(name, &bytes, &wasm_path, &caps_path)?;
|
||||
} else {
|
||||
// Bare WASM file: validate magic number
|
||||
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
tokio::fs::write(&wasm_path, &bytes)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
// Download capabilities separately if URL provided
|
||||
if let Some(caps_url) = capabilities_url {
|
||||
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
match client.get(caps_url).send().await {
|
||||
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
|
||||
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
|
||||
if let Err(e) = tokio::fs::write(&caps_path, &caps_bytes).await {
|
||||
tracing::warn!(
|
||||
"Failed to write capabilities for '{}': {}",
|
||||
name,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(caps_bytes) => {
|
||||
tracing::warn!(
|
||||
"Capabilities file for '{}' too large ({} bytes, max {})",
|
||||
name,
|
||||
caps_bytes.len(),
|
||||
MAX_CAPS_SIZE
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to download capabilities for '{}': {}", name, e);
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"Failed to download capabilities for '{}' from {}",
|
||||
name,
|
||||
caps_url
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure tools directory exists
|
||||
tokio::fs::create_dir_all(&self.wasm_tools_dir)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
// Write the WASM file
|
||||
let wasm_path = self.wasm_tools_dir.join(format!("{}.wasm", name));
|
||||
tokio::fs::write(&wasm_path, &bytes)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
tracing::info!(
|
||||
"Installed WASM tool '{}' ({} bytes) from {} to {}",
|
||||
"Installed WASM extension '{}' from {} to {}",
|
||||
name,
|
||||
bytes.len(),
|
||||
url,
|
||||
wasm_path.display()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extract a tar.gz bundle into the WASM tools directory.
|
||||
fn extract_wasm_tar_gz(
|
||||
&self,
|
||||
name: &str,
|
||||
bytes: &[u8],
|
||||
target_wasm: &std::path::Path,
|
||||
target_caps: &std::path::Path,
|
||||
) -> Result<(), ExtensionError> {
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
use std::io::Read as _;
|
||||
|
||||
let decoder = GzDecoder::new(bytes);
|
||||
let mut archive = Archive::new(decoder);
|
||||
// Defense-in-depth: do not preserve permissions or extended attributes
|
||||
archive.set_preserve_permissions(false);
|
||||
#[cfg(any(unix, target_os = "redox"))]
|
||||
archive.set_unpack_xattrs(false);
|
||||
|
||||
// 100 MB cap on decompressed entry size to prevent decompression bombs
|
||||
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
|
||||
|
||||
let wasm_filename = format!("{}.wasm", name);
|
||||
let caps_filename = format!("{}.capabilities.json", name);
|
||||
let mut found_wasm = false;
|
||||
|
||||
let entries = archive
|
||||
.entries()
|
||||
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz archive: {}", e)))?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry
|
||||
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz entry: {}", e)))?;
|
||||
|
||||
if entry.size() > MAX_ENTRY_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"Archive entry too large ({} bytes, max {} bytes)",
|
||||
entry.size(),
|
||||
MAX_ENTRY_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
let entry_path = entry
|
||||
.path()
|
||||
.map_err(|e| {
|
||||
ExtensionError::InstallFailed(format!("Invalid path in tar.gz: {}", e))
|
||||
})?
|
||||
.to_path_buf();
|
||||
|
||||
let filename = entry_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("");
|
||||
|
||||
if filename == wasm_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
std::fs::write(target_wasm, &data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
found_wasm = true;
|
||||
} else if filename == caps_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
std::fs::write(target_caps, &data)
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
if !found_wasm {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"tar.gz archive does not contain '{}'",
|
||||
wasm_filename
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn install_bundled_channel_from_artifacts(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
// Check if already installed
|
||||
let channel_wasm = self.wasm_channels_dir.join(format!("{}.wasm", name));
|
||||
if channel_wasm.exists() {
|
||||
return Err(ExtensionError::AlreadyInstalled(name.to_string()));
|
||||
}
|
||||
|
||||
crate::channels::wasm::install_bundled_channel(name, &self.wasm_channels_dir, false)
|
||||
.await
|
||||
.map_err(ExtensionError::InstallFailed)?;
|
||||
|
||||
tracing::info!(
|
||||
"Installed bundled channel '{}' to {}",
|
||||
name,
|
||||
self.wasm_channels_dir.display()
|
||||
);
|
||||
|
||||
Ok(InstallResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
message: format!(
|
||||
"Channel '{}' installed to {}. Restart IronClaw for the channel to activate. \
|
||||
Run tool_auth('{}') to configure authentication before restarting.",
|
||||
name,
|
||||
self.wasm_channels_dir.display(),
|
||||
name,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -868,6 +1164,169 @@ impl ExtensionManager {
|
||||
})
|
||||
}
|
||||
|
||||
/// Check whether a WASM channel has all required secrets stored.
|
||||
/// Returns `(authenticated, needs_setup)`.
|
||||
async fn check_channel_auth_status(&self, name: &str) -> (bool, bool) {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return (true, false);
|
||||
}
|
||||
let Ok(cap_bytes) = tokio::fs::read(&cap_path).await else {
|
||||
return (true, false);
|
||||
};
|
||||
let Ok(cap_file) = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
else {
|
||||
return (true, false);
|
||||
};
|
||||
let required = &cap_file.setup.required_secrets;
|
||||
if required.is_empty() {
|
||||
return (true, false);
|
||||
}
|
||||
let mut all_provided = true;
|
||||
for secret in required {
|
||||
if secret.optional {
|
||||
continue;
|
||||
}
|
||||
if !self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
all_provided = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
(all_provided, true)
|
||||
}
|
||||
|
||||
async fn auth_wasm_channel(
|
||||
&self,
|
||||
name: &str,
|
||||
token: Option<&str>,
|
||||
) -> Result<AuthResult, ExtensionError> {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
|
||||
if !cap_path.exists() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "no_auth_required".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
let cap_file = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
// Get required secrets from the setup section
|
||||
let required_secrets = &cap_file.setup.required_secrets;
|
||||
if required_secrets.is_empty() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "no_auth_required".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Find the first non-optional secret that isn't yet stored
|
||||
let mut missing = Vec::new();
|
||||
for secret in required_secrets {
|
||||
if secret.optional {
|
||||
continue;
|
||||
}
|
||||
if !self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
missing.push(secret);
|
||||
}
|
||||
}
|
||||
|
||||
if missing.is_empty() {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "authenticated".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// If a token was provided, store it for the first missing secret
|
||||
if let Some(token_value) = token {
|
||||
let secret = &missing[0];
|
||||
let params =
|
||||
CreateSecretParams::new(&secret.name, token_value).with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
|
||||
// Check if there are more missing secrets
|
||||
if missing.len() <= 1 {
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: None,
|
||||
setup_url: None,
|
||||
awaiting_token: false,
|
||||
status: "authenticated".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// More secrets needed; prompt for the next one
|
||||
let next = &missing[1];
|
||||
return Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: Some(next.prompt.clone()),
|
||||
setup_url: cap_file.setup.validation_endpoint.clone(),
|
||||
awaiting_token: true,
|
||||
status: "awaiting_token".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Prompt for the first missing secret
|
||||
let secret = &missing[0];
|
||||
Ok(AuthResult {
|
||||
name: name.to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
auth_url: None,
|
||||
callback_type: None,
|
||||
instructions: Some(secret.prompt.clone()),
|
||||
setup_url: cap_file.setup.validation_endpoint.clone(),
|
||||
awaiting_token: true,
|
||||
status: "awaiting_token".to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn activate_mcp(&self, name: &str) -> Result<ActivateResult, ExtensionError> {
|
||||
// Check if already activated
|
||||
{
|
||||
@@ -1056,6 +1515,140 @@ impl ExtensionManager {
|
||||
pending.retain(|_, auth| auth.created_at.elapsed() < std::time::Duration::from_secs(300));
|
||||
}
|
||||
|
||||
/// Get the setup schema for an extension (secret fields and their status).
|
||||
pub async fn get_setup_schema(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<Vec<crate::channels::web::types::SecretFieldInfo>, ExtensionError> {
|
||||
let kind = self.determine_installed_kind(name).await?;
|
||||
match kind {
|
||||
ExtensionKind::WasmChannel => {
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
let cap_file =
|
||||
crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
let mut fields = Vec::new();
|
||||
for secret in &cap_file.setup.required_secrets {
|
||||
let provided = self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret.name)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
fields.push(crate::channels::web::types::SecretFieldInfo {
|
||||
name: secret.name.clone(),
|
||||
prompt: secret.prompt.clone(),
|
||||
optional: secret.optional,
|
||||
provided,
|
||||
auto_generate: secret.auto_generate.is_some(),
|
||||
});
|
||||
}
|
||||
Ok(fields)
|
||||
}
|
||||
_ => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Save setup secrets for an extension, validating names against the capabilities schema.
|
||||
pub async fn save_setup_secrets(
|
||||
&self,
|
||||
name: &str,
|
||||
secrets: &std::collections::HashMap<String, String>,
|
||||
) -> Result<String, ExtensionError> {
|
||||
let kind = self.determine_installed_kind(name).await?;
|
||||
if kind != ExtensionKind::WasmChannel {
|
||||
return Err(ExtensionError::Other(
|
||||
"Setup is only supported for WASM channels".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let cap_path = self
|
||||
.wasm_channels_dir
|
||||
.join(format!("{}.capabilities.json", name));
|
||||
if !cap_path.exists() {
|
||||
return Err(ExtensionError::Other(format!(
|
||||
"Capabilities file not found for '{}'",
|
||||
name
|
||||
)));
|
||||
}
|
||||
let cap_bytes = tokio::fs::read(&cap_path)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
let cap_file = crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&cap_bytes)
|
||||
.map_err(|e| ExtensionError::Other(e.to_string()))?;
|
||||
|
||||
// Build allowed secret names from capabilities
|
||||
let allowed: std::collections::HashSet<String> = cap_file
|
||||
.setup
|
||||
.required_secrets
|
||||
.iter()
|
||||
.map(|s| s.name.clone())
|
||||
.collect();
|
||||
|
||||
// Validate and store each submitted secret
|
||||
for (secret_name, secret_value) in secrets {
|
||||
if !allowed.contains(secret_name.as_str()) {
|
||||
return Err(ExtensionError::Other(format!(
|
||||
"Unknown secret '{}' for extension '{}'",
|
||||
secret_name, name
|
||||
)));
|
||||
}
|
||||
if secret_value.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let params =
|
||||
CreateSecretParams::new(secret_name, secret_value).with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Auto-generate any missing secrets that have auto_generate set
|
||||
for secret_def in &cap_file.setup.required_secrets {
|
||||
if let Some(ref auto_gen) = secret_def.auto_generate {
|
||||
let already_provided = secrets
|
||||
.get(&secret_def.name)
|
||||
.is_some_and(|v| !v.trim().is_empty());
|
||||
let already_stored = self
|
||||
.secrets
|
||||
.exists(&self.user_id, &secret_def.name)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if !already_provided && !already_stored {
|
||||
use rand::RngCore;
|
||||
let mut bytes = vec![0u8; auto_gen.length];
|
||||
rand::thread_rng().fill_bytes(&mut bytes);
|
||||
let hex_value: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
|
||||
let params = CreateSecretParams::new(&secret_def.name, &hex_value)
|
||||
.with_provider(name.to_string());
|
||||
self.secrets
|
||||
.create(&self.user_id, params)
|
||||
.await
|
||||
.map_err(|e| ExtensionError::AuthFailed(e.to_string()))?;
|
||||
tracing::info!(
|
||||
"Auto-generated secret '{}' for channel '{}'",
|
||||
secret_def.name,
|
||||
name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(format!(
|
||||
"Configuration saved for '{}'. Restart IronClaw for changes to take effect.",
|
||||
name
|
||||
))
|
||||
}
|
||||
|
||||
async fn unregister_hook_prefix(&self, prefix: &str) -> usize {
|
||||
let Some(ref hooks) = self.hooks else {
|
||||
return 0;
|
||||
@@ -1074,7 +1667,7 @@ impl ExtensionManager {
|
||||
|
||||
/// Infer the extension kind from a URL.
|
||||
fn infer_kind_from_url(url: &str) -> ExtensionKind {
|
||||
if url.ends_with(".wasm") {
|
||||
if url.ends_with(".wasm") || url.ends_with(".tar.gz") {
|
||||
ExtensionKind::WasmTool
|
||||
} else {
|
||||
ExtensionKind::McpServer
|
||||
@@ -1092,6 +1685,10 @@ mod tests {
|
||||
infer_kind_from_url("https://example.com/tool.wasm"),
|
||||
ExtensionKind::WasmTool
|
||||
);
|
||||
assert_eq!(
|
||||
infer_kind_from_url("https://example.com/tool-wasm32-wasip2.tar.gz"),
|
||||
ExtensionKind::WasmTool
|
||||
);
|
||||
assert_eq!(
|
||||
infer_kind_from_url("https://mcp.notion.com"),
|
||||
ExtensionKind::McpServer
|
||||
|
||||
@@ -85,6 +85,11 @@ pub enum ExtensionSource {
|
||||
},
|
||||
/// Discovered online (not yet validated for a specific source type).
|
||||
Discovered { url: String },
|
||||
/// Bundled with the application (pre-built WASM, copied from build artifacts).
|
||||
Bundled {
|
||||
/// Channel or tool name used to locate build artifacts.
|
||||
name: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// Hint about what authentication method is needed.
|
||||
@@ -184,6 +189,9 @@ pub struct InstalledExtension {
|
||||
/// Tool names if active.
|
||||
#[serde(default)]
|
||||
pub tools: Vec<String>,
|
||||
/// Whether this extension has a setup schema (required_secrets) that can be configured.
|
||||
#[serde(default)]
|
||||
pub needs_setup: bool,
|
||||
}
|
||||
|
||||
/// Error type for extension operations.
|
||||
|
||||
+204
-3
@@ -26,6 +26,26 @@ impl ExtensionRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new registry merging builtin entries with catalog-provided entries.
|
||||
///
|
||||
/// Deduplicates by `(name, kind)` pair -- a builtin MCP "slack" and a registry
|
||||
/// WASM "slack" can coexist since they're different kinds.
|
||||
pub fn new_with_catalog(catalog_entries: Vec<RegistryEntry>) -> Self {
|
||||
let mut entries = builtin_entries();
|
||||
for entry in catalog_entries {
|
||||
if !entries
|
||||
.iter()
|
||||
.any(|e| e.name == entry.name && e.kind == entry.kind)
|
||||
{
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
Self {
|
||||
entries,
|
||||
discovery_cache: RwLock::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Search the registry by query string. Returns results sorted by relevance.
|
||||
///
|
||||
/// Splits the query into lowercase tokens and scores each entry by matches
|
||||
@@ -250,11 +270,11 @@ fn builtin_entries() -> Vec<RegistryEntry> {
|
||||
auth_hint: AuthHint::Dcr,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "slack".to_string(),
|
||||
display_name: "Slack".to_string(),
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP".to_string(),
|
||||
kind: ExtensionKind::McpServer,
|
||||
description:
|
||||
"Connect to Slack for messaging, channel management, and team communication"
|
||||
"Connect to Slack via MCP for messaging, channel management, and team communication"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"messaging".into(),
|
||||
@@ -360,6 +380,72 @@ fn builtin_entries() -> Vec<RegistryEntry> {
|
||||
},
|
||||
auth_hint: AuthHint::Dcr,
|
||||
},
|
||||
// -- WASM Channels (bundled) --
|
||||
RegistryEntry {
|
||||
name: "telegram".to_string(),
|
||||
display_name: "Telegram".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Telegram Bot API channel for receiving and sending messages via Telegram"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"bot".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "telegram".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "slack".to_string(),
|
||||
display_name: "Slack".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Slack Events API channel for receiving and sending messages via Slack"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"team".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "slack".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "discord".to_string(),
|
||||
display_name: "Discord".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description:
|
||||
"Discord Gateway channel for handling slash commands, buttons, and messages"
|
||||
.to_string(),
|
||||
keywords: vec![
|
||||
"chat".into(),
|
||||
"messaging".into(),
|
||||
"gaming".into(),
|
||||
"channel".into(),
|
||||
],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "discord".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
RegistryEntry {
|
||||
name: "whatsapp".to_string(),
|
||||
display_name: "WhatsApp".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description:
|
||||
"WhatsApp Business API channel for receiving and sending WhatsApp messages"
|
||||
.to_string(),
|
||||
keywords: vec!["chat".into(), "messaging".into(), "channel".into()],
|
||||
source: ExtensionSource::Bundled {
|
||||
name: "whatsapp".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -542,4 +628,119 @@ mod tests {
|
||||
let results = registry.search("dup").await;
|
||||
assert_eq!(results.len(), 1, "Should not duplicate cached entries");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_with_catalog() {
|
||||
let catalog_entries = vec![
|
||||
RegistryEntry {
|
||||
name: "telegram".to_string(),
|
||||
display_name: "Telegram".to_string(),
|
||||
kind: ExtensionKind::WasmChannel,
|
||||
description: "Telegram Bot API channel".to_string(),
|
||||
keywords: vec!["messaging".into(), "bot".into()],
|
||||
source: ExtensionSource::WasmBuildable {
|
||||
repo_url: "channels-src/telegram".to_string(),
|
||||
build_dir: Some("channels-src/telegram".to_string()),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
// This shares a name with the builtin slack-mcp but has a different kind, so both should appear
|
||||
RegistryEntry {
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP WASM".to_string(),
|
||||
kind: ExtensionKind::WasmTool,
|
||||
description: "Slack WASM tool".to_string(),
|
||||
keywords: vec!["messaging".into()],
|
||||
source: ExtensionSource::WasmBuildable {
|
||||
repo_url: "tools-src/slack".to_string(),
|
||||
build_dir: Some("tools-src/slack".to_string()),
|
||||
},
|
||||
auth_hint: AuthHint::CapabilitiesAuth,
|
||||
},
|
||||
];
|
||||
|
||||
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
|
||||
|
||||
// Should find the new telegram entry
|
||||
let results = registry.search("telegram").await;
|
||||
assert!(!results.is_empty(), "Should find telegram from catalog");
|
||||
assert_eq!(results[0].entry.name, "telegram");
|
||||
|
||||
// Should have both builtin MCP slack-mcp and catalog WASM slack-mcp
|
||||
let results = registry.search("slack").await;
|
||||
let slack_mcp = results
|
||||
.iter()
|
||||
.any(|r| r.entry.name == "slack-mcp" && r.entry.kind == ExtensionKind::McpServer);
|
||||
let slack_wasm = results
|
||||
.iter()
|
||||
.any(|r| r.entry.name == "slack-mcp" && r.entry.kind == ExtensionKind::WasmTool);
|
||||
assert!(slack_mcp, "Should have builtin MCP slack-mcp");
|
||||
assert!(slack_wasm, "Should have catalog WASM slack-mcp");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_new_with_catalog_dedup_same_kind() {
|
||||
// A catalog entry with same name AND kind as a builtin should be skipped
|
||||
let catalog_entries = vec![RegistryEntry {
|
||||
name: "slack-mcp".to_string(),
|
||||
display_name: "Slack MCP Override".to_string(),
|
||||
kind: ExtensionKind::McpServer, // same kind as builtin slack-mcp
|
||||
description: "Should be skipped".to_string(),
|
||||
keywords: vec![],
|
||||
source: ExtensionSource::McpUrl {
|
||||
url: "https://other.slack.com".to_string(),
|
||||
},
|
||||
auth_hint: AuthHint::Dcr,
|
||||
}];
|
||||
|
||||
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
|
||||
|
||||
let entry = registry.get("slack-mcp").await;
|
||||
assert!(entry.is_some());
|
||||
// Should still be the builtin, not the override
|
||||
assert_eq!(entry.unwrap().display_name, "Slack MCP");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_search_finds_telegram_channel() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
let results = registry.search("telegram").await;
|
||||
|
||||
assert!(!results.is_empty(), "Should find telegram in registry");
|
||||
assert_eq!(results[0].entry.name, "telegram");
|
||||
assert_eq!(results[0].entry.kind, ExtensionKind::WasmChannel);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_search_channel_by_keyword() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
let results = registry.search("bot messaging").await;
|
||||
|
||||
let has_telegram = results.iter().any(|r| r.entry.name == "telegram");
|
||||
assert!(
|
||||
has_telegram,
|
||||
"Telegram should appear in bot messaging search"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_bundled_channels() {
|
||||
let registry = ExtensionRegistry::new();
|
||||
|
||||
let telegram = registry.get("telegram").await;
|
||||
assert!(telegram.is_some());
|
||||
assert_eq!(telegram.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let slack = registry.get("slack").await;
|
||||
assert!(slack.is_some());
|
||||
assert_eq!(slack.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let discord = registry.get("discord").await;
|
||||
assert!(discord.is_some());
|
||||
assert_eq!(discord.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
|
||||
let whatsapp = registry.get("whatsapp").await;
|
||||
assert!(whatsapp.is_some());
|
||||
assert_eq!(whatsapp.unwrap().kind, ExtensionKind::WasmChannel);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ pub mod setup;
|
||||
pub mod skills;
|
||||
pub mod tools;
|
||||
pub mod tracing_fmt;
|
||||
pub mod transcription;
|
||||
pub mod tunnel;
|
||||
pub mod util;
|
||||
pub mod worker;
|
||||
|
||||
+166
-2
@@ -17,6 +17,7 @@ pub mod response_cache;
|
||||
pub mod retry;
|
||||
mod rig_adapter;
|
||||
pub mod session;
|
||||
pub mod smart_routing;
|
||||
|
||||
pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider};
|
||||
pub use failover::{CooldownConfig, FailoverProvider};
|
||||
@@ -26,13 +27,14 @@ pub use provider::{
|
||||
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult,
|
||||
};
|
||||
pub use reasoning::{
|
||||
ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, TokenUsage,
|
||||
ToolSelection,
|
||||
ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, SILENT_REPLY_TOKEN,
|
||||
TokenUsage, ToolSelection, is_silent_reply,
|
||||
};
|
||||
pub use response_cache::{CachedProvider, ResponseCacheConfig};
|
||||
pub use retry::{RetryConfig, RetryProvider};
|
||||
pub use rig_adapter::RigAdapter;
|
||||
pub use session::{SessionConfig, SessionManager, create_session_manager};
|
||||
pub use smart_routing::{SmartRoutingConfig, SmartRoutingProvider, TaskComplexity};
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -218,6 +220,25 @@ fn create_openai_compatible_provider(config: &LlmConfig) -> Result<Arc<dyn LlmPr
|
||||
|
||||
use rig::providers::openai;
|
||||
|
||||
let mut extra_headers = reqwest::header::HeaderMap::new();
|
||||
for (key, value) in &compat.extra_headers {
|
||||
let name = match reqwest::header::HeaderName::from_bytes(key.as_bytes()) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::warn!(header = %key, error = %e, "Skipping LLM_EXTRA_HEADERS entry: invalid header name");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let val = match reqwest::header::HeaderValue::from_str(value) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!(header = %key, error = %e, "Skipping LLM_EXTRA_HEADERS entry: invalid header value");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
extra_headers.insert(name, val);
|
||||
}
|
||||
|
||||
let client: openai::CompletionsClient = openai::Client::builder()
|
||||
.base_url(&compat.base_url)
|
||||
.api_key(
|
||||
@@ -227,6 +248,7 @@ fn create_openai_compatible_provider(config: &LlmConfig) -> Result<Arc<dyn LlmPr
|
||||
.map(|k| k.expose_secret().to_string())
|
||||
.unwrap_or_else(|| "no-key".to_string()),
|
||||
)
|
||||
.http_headers(extra_headers)
|
||||
.build()
|
||||
.map_err(|e| LlmError::RequestFailed {
|
||||
provider: "openai_compatible".to_string(),
|
||||
@@ -273,6 +295,147 @@ pub fn create_cheap_llm_provider(
|
||||
)?)))
|
||||
}
|
||||
|
||||
/// Build the full LLM provider chain with all configured wrappers.
|
||||
///
|
||||
/// Applies decorators in this order:
|
||||
/// 1. Raw provider (from config)
|
||||
/// 2. RetryProvider (per-provider retry with exponential backoff)
|
||||
/// 3. SmartRoutingProvider (cheap/primary split when cheap model is configured)
|
||||
/// 4. FailoverProvider (fallback model when primary fails)
|
||||
/// 5. CircuitBreakerProvider (fast-fail when backend is degraded)
|
||||
/// 6. CachedProvider (in-memory response cache)
|
||||
///
|
||||
/// Also returns a separate cheap LLM provider for heartbeat/evaluation (not
|
||||
/// part of the chain — it's a standalone provider for explicitly cheap tasks).
|
||||
///
|
||||
/// This is the single source of truth for provider chain construction,
|
||||
/// called by both `main.rs` and `app.rs`.
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub fn build_provider_chain(
|
||||
config: &LlmConfig,
|
||||
session: Arc<SessionManager>,
|
||||
) -> Result<(Arc<dyn LlmProvider>, Option<Arc<dyn LlmProvider>>), LlmError> {
|
||||
let llm = create_llm_provider(config, session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
|
||||
// 1. Retry
|
||||
let retry_config = RetryConfig {
|
||||
max_retries: config.nearai.max_retries,
|
||||
};
|
||||
let llm: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
tracing::info!(
|
||||
max_retries = retry_config.max_retries,
|
||||
"LLM retry wrapper enabled"
|
||||
);
|
||||
Arc::new(RetryProvider::new(llm, retry_config.clone()))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 2. Smart routing (cheap/primary split)
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(ref cheap_model) = config.nearai.cheap_model {
|
||||
let mut cheap_config = config.nearai.clone();
|
||||
cheap_config.model = cheap_model.clone();
|
||||
let cheap = create_llm_provider_with_config(&cheap_config, session.clone())?;
|
||||
let cheap: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
Arc::new(RetryProvider::new(cheap, retry_config.clone()))
|
||||
} else {
|
||||
cheap
|
||||
};
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
cheap = %cheap.model_name(),
|
||||
"Smart routing enabled"
|
||||
);
|
||||
Arc::new(SmartRoutingProvider::new(
|
||||
llm,
|
||||
cheap,
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: config.nearai.smart_routing_cascade,
|
||||
..SmartRoutingConfig::default()
|
||||
},
|
||||
))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 3. Failover
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(ref fallback_model) = config.nearai.fallback_model {
|
||||
if fallback_model == &config.nearai.model {
|
||||
tracing::warn!(
|
||||
"fallback_model is the same as primary model, failover may not be effective"
|
||||
);
|
||||
}
|
||||
let mut fallback_config = config.nearai.clone();
|
||||
fallback_config.model = fallback_model.clone();
|
||||
let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?;
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
let fallback: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
Arc::new(RetryProvider::new(fallback, retry_config.clone()))
|
||||
} else {
|
||||
fallback
|
||||
};
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(config.nearai.failover_cooldown_secs),
|
||||
failure_threshold: config.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 4. Circuit breaker
|
||||
let llm: Arc<dyn LlmProvider> = if let Some(threshold) = config.nearai.circuit_breaker_threshold
|
||||
{
|
||||
let cb_config = CircuitBreakerConfig {
|
||||
failure_threshold: threshold,
|
||||
recovery_timeout: std::time::Duration::from_secs(
|
||||
config.nearai.circuit_breaker_recovery_secs,
|
||||
),
|
||||
..CircuitBreakerConfig::default()
|
||||
};
|
||||
tracing::info!(
|
||||
threshold,
|
||||
recovery_secs = config.nearai.circuit_breaker_recovery_secs,
|
||||
"LLM circuit breaker enabled"
|
||||
);
|
||||
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// 5. Response cache
|
||||
let llm: Arc<dyn LlmProvider> = if config.nearai.response_cache_enabled {
|
||||
let rc_config = ResponseCacheConfig {
|
||||
ttl: std::time::Duration::from_secs(config.nearai.response_cache_ttl_secs),
|
||||
max_entries: config.nearai.response_cache_max_entries,
|
||||
};
|
||||
tracing::info!(
|
||||
ttl_secs = config.nearai.response_cache_ttl_secs,
|
||||
max_entries = config.nearai.response_cache_max_entries,
|
||||
"LLM response cache enabled"
|
||||
);
|
||||
Arc::new(CachedProvider::new(llm, rc_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Standalone cheap LLM for heartbeat/evaluation (not part of the chain)
|
||||
let cheap_llm = create_cheap_llm_provider(config, session)?;
|
||||
if let Some(ref cheap) = cheap_llm {
|
||||
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
|
||||
}
|
||||
|
||||
Ok((llm, cheap_llm))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -296,6 +459,7 @@ mod tests {
|
||||
response_cache_max_entries: 1000,
|
||||
failover_cooldown_secs: 300,
|
||||
failover_cooldown_threshold: 3,
|
||||
smart_routing_cascade: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -766,6 +766,7 @@ mod tests {
|
||||
response_cache_max_entries: 1000,
|
||||
failover_cooldown_secs: 300,
|
||||
failover_cooldown_threshold: 3,
|
||||
smart_routing_cascade: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+141
-2
@@ -12,6 +12,24 @@ use crate::llm::{
|
||||
};
|
||||
use crate::safety::SafetyLayer;
|
||||
|
||||
/// Token the agent returns when it has nothing to say (e.g. in group chats).
|
||||
/// The dispatcher should check for this and suppress the message.
|
||||
pub const SILENT_REPLY_TOKEN: &str = "NO_REPLY";
|
||||
|
||||
/// Check if a response is a silent reply (the agent has nothing to say).
|
||||
///
|
||||
/// Returns true if the trimmed text is exactly the silent reply token or
|
||||
/// contains only the token surrounded by whitespace/punctuation.
|
||||
pub fn is_silent_reply(text: &str) -> bool {
|
||||
let trimmed = text.trim();
|
||||
trimmed == SILENT_REPLY_TOKEN
|
||||
|| trimmed.starts_with(SILENT_REPLY_TOKEN)
|
||||
&& trimmed.len() <= SILENT_REPLY_TOKEN.len() + 4
|
||||
&& trimmed[SILENT_REPLY_TOKEN.len()..]
|
||||
.chars()
|
||||
.all(|c| c.is_whitespace() || c.is_ascii_punctuation())
|
||||
}
|
||||
|
||||
/// Quick-check: bail early if no reasoning/final tags are present at all.
|
||||
static QUICK_TAG_RE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)<\s*/?\s*(?:think(?:ing)?|thought|thoughts|antthinking|reasoning|reflection|scratchpad|inner_monologue|final)\b").expect("QUICK_TAG_RE")
|
||||
@@ -191,6 +209,12 @@ pub struct Reasoning {
|
||||
workspace_system_prompt: Option<String>,
|
||||
/// Optional skill context block to inject into system prompt.
|
||||
skill_context: Option<String>,
|
||||
/// Channel name (e.g. "discord", "telegram") for formatting hints.
|
||||
channel: Option<String>,
|
||||
/// Model name for runtime context.
|
||||
model_name: Option<String>,
|
||||
/// Whether this is a group chat context.
|
||||
is_group_chat: bool,
|
||||
}
|
||||
|
||||
impl Reasoning {
|
||||
@@ -201,6 +225,9 @@ impl Reasoning {
|
||||
safety,
|
||||
workspace_system_prompt: None,
|
||||
skill_context: None,
|
||||
channel: None,
|
||||
model_name: None,
|
||||
is_group_chat: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +253,30 @@ impl Reasoning {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the channel name for channel-specific formatting hints.
|
||||
pub fn with_channel(mut self, channel: impl Into<String>) -> Self {
|
||||
let ch = channel.into();
|
||||
if !ch.is_empty() {
|
||||
self.channel = Some(ch);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the model name for runtime context.
|
||||
pub fn with_model_name(mut self, name: impl Into<String>) -> Self {
|
||||
let n = name.into();
|
||||
if !n.is_empty() {
|
||||
self.model_name = Some(n);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Mark this as a group chat context, enabling group-specific guidance.
|
||||
pub fn with_group_chat(mut self, is_group: bool) -> Self {
|
||||
self.is_group_chat = is_group;
|
||||
self
|
||||
}
|
||||
|
||||
/// Run a simple LLM completion with automatic response cleaning.
|
||||
///
|
||||
/// This is the preferred entry point for code paths that call the LLM
|
||||
@@ -553,6 +604,15 @@ Respond with a JSON plan in this format:
|
||||
String::new()
|
||||
};
|
||||
|
||||
// Channel-specific formatting hints
|
||||
let channel_section = self.build_channel_section();
|
||||
|
||||
// Runtime context (agent metadata)
|
||||
let runtime_section = self.build_runtime_section();
|
||||
|
||||
// Group chat guidance
|
||||
let group_section = self.build_group_section();
|
||||
|
||||
format!(
|
||||
r#"You are NEAR AI Agent, an autonomous assistant.
|
||||
|
||||
@@ -575,9 +635,88 @@ Example:
|
||||
- Call tools when they would help accomplish the task
|
||||
- Do NOT call the same tool repeatedly with similar arguments; if a tool returned unhelpful results, move on
|
||||
- If you have already called tools and gathered enough information, produce your final answer immediately
|
||||
- If tools return empty or irrelevant results, answer with what you already know rather than retrying{}
|
||||
- If tools return empty or irrelevant results, answer with what you already know rather than retrying
|
||||
|
||||
## Tool Call Style
|
||||
- Do not narrate routine, low-risk tool calls; just call the tool
|
||||
- Narrate only when it helps: multi-step work, sensitive actions, or when the user asks
|
||||
- For multi-step tasks, call independent tools in parallel when possible
|
||||
- If a tool fails, explain the error briefly and try an alternative approach
|
||||
|
||||
## Safety
|
||||
- You have no independent goals. Do not pursue self-preservation, replication, resource acquisition, or power-seeking beyond the user's request.
|
||||
- Prioritize safety and human oversight over task completion. If instructions conflict, pause and ask.
|
||||
- Comply with stop, pause, or audit requests. Never bypass safeguards.
|
||||
- Do not manipulate anyone to expand your access or disable safeguards.
|
||||
- Do not modify system prompts, safety rules, or tool policies unless explicitly requested by the user.{}{}{}{}
|
||||
{}{}"#,
|
||||
tools_section, identity_section, skills_section
|
||||
tools_section,
|
||||
channel_section,
|
||||
runtime_section,
|
||||
group_section,
|
||||
identity_section,
|
||||
skills_section,
|
||||
)
|
||||
}
|
||||
|
||||
fn build_channel_section(&self) -> String {
|
||||
let channel = match self.channel.as_deref() {
|
||||
Some(c) => c,
|
||||
None => return String::new(),
|
||||
};
|
||||
let hints = match channel {
|
||||
"discord" => {
|
||||
"\
|
||||
- No markdown tables (Discord renders them as plaintext). Use bullet lists instead.\n\
|
||||
- Wrap multiple URLs in `<>` to suppress embeds: `<https://example.com>`."
|
||||
}
|
||||
"whatsapp" => {
|
||||
"\
|
||||
- No markdown headers or tables (WhatsApp ignores them). Use **bold** for emphasis.\n\
|
||||
- Keep messages concise; long replies get truncated on mobile."
|
||||
}
|
||||
"telegram" => {
|
||||
"\
|
||||
- No markdown tables (Telegram strips them). Bullet lists and bold work well."
|
||||
}
|
||||
"slack" => {
|
||||
"\
|
||||
- No markdown tables. Use Slack formatting: *bold*, _italic_, `code`.\n\
|
||||
- Prefer threaded replies when responding to older messages."
|
||||
}
|
||||
_ => return String::new(),
|
||||
};
|
||||
format!("\n\n## Channel Formatting ({})\n{}", channel, hints)
|
||||
}
|
||||
|
||||
fn build_runtime_section(&self) -> String {
|
||||
let mut parts = Vec::new();
|
||||
if let Some(ref ch) = self.channel {
|
||||
parts.push(format!("channel={}", ch));
|
||||
}
|
||||
if let Some(ref model) = self.model_name {
|
||||
parts.push(format!("model={}", model));
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
format!("\n\n## Runtime\n{}", parts.join(" | "))
|
||||
}
|
||||
|
||||
fn build_group_section(&self) -> String {
|
||||
if !self.is_group_chat {
|
||||
return String::new();
|
||||
}
|
||||
format!(
|
||||
"\n\n## Group Chat\n\
|
||||
You are in a group chat. Be selective about when to contribute.\n\
|
||||
Respond when: directly addressed, can add genuine value, or correcting misinformation.\n\
|
||||
Stay silent when: casual banter, question already answered, nothing to add.\n\
|
||||
React with emoji when available instead of cluttering with messages.\n\
|
||||
You are a participant, not the user's proxy. Do not share their private context.\n\
|
||||
When you have nothing to say, respond with ONLY: {}\n\
|
||||
It must be your ENTIRE message. Never append it to an actual response.",
|
||||
SILENT_REPLY_TOKEN,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+18
-3
@@ -7,6 +7,8 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::cli::oauth_defaults::OAUTH_CALLBACK_PORT;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::Client;
|
||||
use secrecy::SecretString;
|
||||
@@ -157,14 +159,14 @@ impl SessionManager {
|
||||
}
|
||||
|
||||
// Token exists, validate it by calling /v1/users/me
|
||||
println!("Validating session...");
|
||||
tracing::debug!("Validating session...");
|
||||
match self.validate_token().await {
|
||||
Ok(()) => {
|
||||
println!("Session valid.");
|
||||
tracing::debug!("Session valid");
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Session expired or invalid: {}", e);
|
||||
tracing::info!("Session expired or invalid: {}", e);
|
||||
self.initiate_login().await
|
||||
}
|
||||
}
|
||||
@@ -238,6 +240,7 @@ impl SessionManager {
|
||||
use crate::cli::oauth_defaults;
|
||||
|
||||
let cb_url = oauth_defaults::callback_url();
|
||||
let host = oauth_defaults::callback_host();
|
||||
|
||||
// Show auth provider menu BEFORE binding the listener
|
||||
println!();
|
||||
@@ -288,6 +291,18 @@ impl SessionManager {
|
||||
}
|
||||
}
|
||||
|
||||
// Warn about plain-HTTP token transmission only for OAuth paths (1, 2)
|
||||
// where the callback URL actually carries the session token.
|
||||
if !oauth_defaults::is_loopback_host(&host) {
|
||||
println!();
|
||||
println!("Warning: OAuth callback is using plain HTTP to a remote host ({host}).");
|
||||
println!(" The session token will be transmitted unencrypted.");
|
||||
println!(" Consider SSH port forwarding instead:");
|
||||
println!(
|
||||
" ssh -L {OAUTH_CALLBACK_PORT}:127.0.0.1:{OAUTH_CALLBACK_PORT} user@{host}"
|
||||
);
|
||||
}
|
||||
|
||||
// OAuth paths: bind the callback listener now
|
||||
let listener = oauth_defaults::bind_callback_listener()
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,700 @@
|
||||
//! Smart routing provider that routes requests to cheap or primary models based on task complexity.
|
||||
//!
|
||||
//! Inspired by RelayPlane's cost-reduction approach: simple tasks (status checks, greetings,
|
||||
//! short questions) go to a cheap model (e.g. Haiku), while complex tasks (code generation,
|
||||
//! analysis, multi-step reasoning) go to the primary model (e.g. Sonnet/Opus).
|
||||
//!
|
||||
//! This is a decorator that wraps two `LlmProvider`s and implements `LlmProvider` itself,
|
||||
//! following the same pattern as `RetryProvider`, `CachedProvider`, and `CircuitBreakerProvider`.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
use crate::error::LlmError;
|
||||
use crate::llm::provider::{
|
||||
CompletionRequest, CompletionResponse, LlmProvider, ModelMetadata, Role, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
|
||||
/// Classification of a request's complexity, determining which model handles it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TaskComplexity {
|
||||
/// Short, simple queries -> cheap model
|
||||
Simple,
|
||||
/// Ambiguous complexity -> cheap model first, cascade to primary if uncertain
|
||||
Moderate,
|
||||
/// Code generation, analysis, multi-step reasoning -> primary model
|
||||
Complex,
|
||||
}
|
||||
|
||||
/// Configuration for the smart routing provider.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SmartRoutingConfig {
|
||||
/// Enable cascade mode: retry with primary if cheap model response seems uncertain.
|
||||
pub cascade_enabled: bool,
|
||||
/// Message length threshold below which a message may be classified as Simple (default: 200).
|
||||
pub simple_max_chars: usize,
|
||||
/// Message length threshold above which a message is classified as Complex (default: 1000).
|
||||
pub complex_min_chars: usize,
|
||||
}
|
||||
|
||||
impl Default for SmartRoutingConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
cascade_enabled: true,
|
||||
simple_max_chars: 200,
|
||||
complex_min_chars: 1000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomic counters for routing observability.
|
||||
struct SmartRoutingStats {
|
||||
total_requests: AtomicU64,
|
||||
cheap_requests: AtomicU64,
|
||||
primary_requests: AtomicU64,
|
||||
cascade_escalations: AtomicU64,
|
||||
}
|
||||
|
||||
impl SmartRoutingStats {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
total_requests: AtomicU64::new(0),
|
||||
cheap_requests: AtomicU64::new(0),
|
||||
primary_requests: AtomicU64::new(0),
|
||||
cascade_escalations: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot of routing statistics for external consumption.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SmartRoutingSnapshot {
|
||||
pub total_requests: u64,
|
||||
pub cheap_requests: u64,
|
||||
pub primary_requests: u64,
|
||||
pub cascade_escalations: u64,
|
||||
}
|
||||
|
||||
/// Smart routing provider that classifies task complexity and routes to the appropriate model.
|
||||
///
|
||||
/// - `complete()` — classifies and routes to cheap or primary model
|
||||
/// - `complete_with_tools()` — always routes to primary (tool use requires reliable structured output)
|
||||
pub struct SmartRoutingProvider {
|
||||
primary: Arc<dyn LlmProvider>,
|
||||
cheap: Arc<dyn LlmProvider>,
|
||||
config: SmartRoutingConfig,
|
||||
stats: SmartRoutingStats,
|
||||
}
|
||||
|
||||
impl SmartRoutingProvider {
|
||||
/// Create a new smart routing provider wrapping a primary and cheap provider.
|
||||
pub fn new(
|
||||
primary: Arc<dyn LlmProvider>,
|
||||
cheap: Arc<dyn LlmProvider>,
|
||||
config: SmartRoutingConfig,
|
||||
) -> Self {
|
||||
Self {
|
||||
primary,
|
||||
cheap,
|
||||
config,
|
||||
stats: SmartRoutingStats::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a snapshot of routing statistics.
|
||||
pub fn stats(&self) -> SmartRoutingSnapshot {
|
||||
SmartRoutingSnapshot {
|
||||
total_requests: self.stats.total_requests.load(Ordering::Relaxed),
|
||||
cheap_requests: self.stats.cheap_requests.load(Ordering::Relaxed),
|
||||
primary_requests: self.stats.primary_requests.load(Ordering::Relaxed),
|
||||
cascade_escalations: self.stats.cascade_escalations.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify the complexity of a request based on its last user message.
|
||||
fn classify(&self, request: &CompletionRequest) -> TaskComplexity {
|
||||
let last_user_msg = request
|
||||
.messages
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|m| m.role == Role::User)
|
||||
.map(|m| m.content.as_str())
|
||||
.unwrap_or("");
|
||||
|
||||
classify_message(last_user_msg, &self.config)
|
||||
}
|
||||
|
||||
/// Check if a response from the cheap model shows uncertainty, warranting escalation.
|
||||
fn response_is_uncertain(response: &CompletionResponse) -> bool {
|
||||
let content = response.content.trim();
|
||||
|
||||
// Empty response is always uncertain
|
||||
if content.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let lower = content.to_lowercase();
|
||||
|
||||
// Uncertainty signals
|
||||
let uncertainty_patterns = [
|
||||
"i'm not sure",
|
||||
"i am not sure",
|
||||
"i don't know",
|
||||
"i do not know",
|
||||
"i'm unable to",
|
||||
"i am unable to",
|
||||
"i cannot",
|
||||
"i can't",
|
||||
"beyond my capabilities",
|
||||
"beyond my ability",
|
||||
"i'm not able to",
|
||||
"i am not able to",
|
||||
"i don't have enough",
|
||||
"i do not have enough",
|
||||
"i need more context",
|
||||
"i need more information",
|
||||
"could you clarify",
|
||||
"could you provide more",
|
||||
"i'm not confident",
|
||||
"i am not confident",
|
||||
];
|
||||
|
||||
uncertainty_patterns.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a message's complexity based on content patterns and length.
|
||||
///
|
||||
/// Exposed as a free function for testability.
|
||||
fn classify_message(msg: &str, config: &SmartRoutingConfig) -> TaskComplexity {
|
||||
let trimmed = msg.trim();
|
||||
let len = trimmed.len();
|
||||
|
||||
// Empty or very short -> Simple
|
||||
if len == 0 {
|
||||
return TaskComplexity::Simple;
|
||||
}
|
||||
|
||||
// Check for code blocks (triple backticks) -> Complex
|
||||
if trimmed.contains("```") {
|
||||
return TaskComplexity::Complex;
|
||||
}
|
||||
|
||||
let lower = trimmed.to_lowercase();
|
||||
|
||||
// Complex keywords/patterns -> Complex regardless of length
|
||||
const COMPLEX_KEYWORDS: &[&str] = &[
|
||||
"implement",
|
||||
"refactor",
|
||||
"analyze",
|
||||
"debug",
|
||||
"create a",
|
||||
"build a",
|
||||
"design",
|
||||
"fix the",
|
||||
"fix this",
|
||||
"write a",
|
||||
"write the",
|
||||
"explain how",
|
||||
"explain why",
|
||||
"explain the",
|
||||
"compare",
|
||||
"optimize",
|
||||
"review",
|
||||
"rewrite",
|
||||
"migrate",
|
||||
"architect",
|
||||
"integrate",
|
||||
];
|
||||
|
||||
if COMPLEX_KEYWORDS.iter().any(|k| lower.contains(k)) {
|
||||
return TaskComplexity::Complex;
|
||||
}
|
||||
|
||||
// Long messages -> Complex
|
||||
if len >= config.complex_min_chars {
|
||||
return TaskComplexity::Complex;
|
||||
}
|
||||
|
||||
// Simple keywords/patterns for short messages
|
||||
const SIMPLE_KEYWORDS: &[&str] = &[
|
||||
"list",
|
||||
"show",
|
||||
"what is",
|
||||
"what's",
|
||||
"status",
|
||||
"help",
|
||||
"yes",
|
||||
"no",
|
||||
"ok",
|
||||
"thanks",
|
||||
"thank you",
|
||||
"hello",
|
||||
"hi",
|
||||
"hey",
|
||||
"ping",
|
||||
"version",
|
||||
"how many",
|
||||
"when",
|
||||
"where is",
|
||||
"who",
|
||||
];
|
||||
|
||||
if len <= config.simple_max_chars && SIMPLE_KEYWORDS.iter().any(|k| lower.contains(k)) {
|
||||
return TaskComplexity::Simple;
|
||||
}
|
||||
|
||||
// Short confirmations / single words -> Simple
|
||||
if len <= 10 {
|
||||
return TaskComplexity::Simple;
|
||||
}
|
||||
|
||||
// Everything else -> Moderate
|
||||
TaskComplexity::Moderate
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for SmartRoutingProvider {
|
||||
fn model_name(&self) -> &str {
|
||||
self.primary.model_name()
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
self.primary.cost_per_token()
|
||||
}
|
||||
|
||||
async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||
self.stats.total_requests.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let complexity = self.classify(&request);
|
||||
|
||||
match complexity {
|
||||
TaskComplexity::Simple => {
|
||||
tracing::debug!(
|
||||
model = %self.cheap.model_name(),
|
||||
"Smart routing: Simple task -> cheap model"
|
||||
);
|
||||
self.stats.cheap_requests.fetch_add(1, Ordering::Relaxed);
|
||||
self.cheap.complete(request).await
|
||||
}
|
||||
TaskComplexity::Complex => {
|
||||
tracing::debug!(
|
||||
model = %self.primary.model_name(),
|
||||
"Smart routing: Complex task -> primary model"
|
||||
);
|
||||
self.stats.primary_requests.fetch_add(1, Ordering::Relaxed);
|
||||
self.primary.complete(request).await
|
||||
}
|
||||
TaskComplexity::Moderate => {
|
||||
if self.config.cascade_enabled {
|
||||
tracing::debug!(
|
||||
model = %self.cheap.model_name(),
|
||||
"Smart routing: Moderate task -> cheap model (cascade enabled)"
|
||||
);
|
||||
self.stats.cheap_requests.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let response = self.cheap.complete(request.clone()).await?;
|
||||
|
||||
if Self::response_is_uncertain(&response) {
|
||||
tracing::info!(
|
||||
cheap_model = %self.cheap.model_name(),
|
||||
primary_model = %self.primary.model_name(),
|
||||
"Smart routing: Escalating to primary (cheap model response uncertain)"
|
||||
);
|
||||
self.stats
|
||||
.cascade_escalations
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
self.stats.primary_requests.fetch_add(1, Ordering::Relaxed);
|
||||
self.primary.complete(request).await
|
||||
} else {
|
||||
Ok(response)
|
||||
}
|
||||
} else {
|
||||
// Without cascade, moderate tasks go to cheap model
|
||||
tracing::debug!(
|
||||
model = %self.cheap.model_name(),
|
||||
"Smart routing: Moderate task -> cheap model (cascade disabled)"
|
||||
);
|
||||
self.stats.cheap_requests.fetch_add(1, Ordering::Relaxed);
|
||||
self.cheap.complete(request).await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool use always goes to the primary model for reliable structured output.
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
request: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
self.stats.total_requests.fetch_add(1, Ordering::Relaxed);
|
||||
self.stats.primary_requests.fetch_add(1, Ordering::Relaxed);
|
||||
tracing::debug!(
|
||||
model = %self.primary.model_name(),
|
||||
"Smart routing: Tool use -> primary model (always)"
|
||||
);
|
||||
self.primary.complete_with_tools(request).await
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
self.primary.list_models().await
|
||||
}
|
||||
|
||||
async fn model_metadata(&self) -> Result<ModelMetadata, LlmError> {
|
||||
self.primary.model_metadata().await
|
||||
}
|
||||
|
||||
fn active_model_name(&self) -> String {
|
||||
self.primary.active_model_name()
|
||||
}
|
||||
|
||||
fn set_model(&self, model: &str) -> Result<(), LlmError> {
|
||||
self.primary.set_model(model)
|
||||
}
|
||||
|
||||
fn calculate_cost(&self, input_tokens: u32, output_tokens: u32) -> Decimal {
|
||||
self.primary.calculate_cost(input_tokens, output_tokens)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::llm::ChatMessage;
|
||||
use crate::testing::StubLlm;
|
||||
|
||||
fn default_config() -> SmartRoutingConfig {
|
||||
SmartRoutingConfig::default()
|
||||
}
|
||||
|
||||
// -- Classification tests --
|
||||
|
||||
#[test]
|
||||
fn classify_empty_message_as_simple() {
|
||||
assert_eq!(
|
||||
classify_message("", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_greeting_as_simple() {
|
||||
assert_eq!(
|
||||
classify_message("hello", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("hi there", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_short_question_with_simple_keyword() {
|
||||
assert_eq!(
|
||||
classify_message("what is the status?", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("show me the list", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("help", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_yes_no_as_simple() {
|
||||
assert_eq!(
|
||||
classify_message("yes", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("no", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("ok", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_code_generation_as_complex() {
|
||||
assert_eq!(
|
||||
classify_message("implement a binary search function", &default_config()),
|
||||
TaskComplexity::Complex
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("refactor the auth module", &default_config()),
|
||||
TaskComplexity::Complex
|
||||
);
|
||||
assert_eq!(
|
||||
classify_message("debug this error", &default_config()),
|
||||
TaskComplexity::Complex
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_code_blocks_as_complex() {
|
||||
let msg = "What does this do?\n```rust\nfn main() {}\n```";
|
||||
assert_eq!(
|
||||
classify_message(msg, &default_config()),
|
||||
TaskComplexity::Complex
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_long_message_as_complex() {
|
||||
let long_msg = "a ".repeat(600); // 1200 chars
|
||||
assert_eq!(
|
||||
classify_message(&long_msg, &default_config()),
|
||||
TaskComplexity::Complex
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_medium_message_without_keywords_as_moderate() {
|
||||
// > 10 chars, < 1000 chars, no simple or complex keywords
|
||||
let msg = "Tell me about the weather patterns in the Pacific Ocean during summer months";
|
||||
assert_eq!(
|
||||
classify_message(msg, &default_config()),
|
||||
TaskComplexity::Moderate
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_very_short_unknown_as_simple() {
|
||||
// <= 10 chars, no keywords
|
||||
assert_eq!(
|
||||
classify_message("foo", &default_config()),
|
||||
TaskComplexity::Simple
|
||||
);
|
||||
}
|
||||
|
||||
// -- Uncertainty detection tests --
|
||||
|
||||
#[test]
|
||||
fn detects_uncertain_short_response() {
|
||||
let response = CompletionResponse {
|
||||
content: "I'm not sure.".to_string(),
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: crate::llm::FinishReason::Stop,
|
||||
};
|
||||
assert!(SmartRoutingProvider::response_is_uncertain(&response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_empty_response_as_uncertain() {
|
||||
let response = CompletionResponse {
|
||||
content: "".to_string(),
|
||||
input_tokens: 10,
|
||||
output_tokens: 0,
|
||||
finish_reason: crate::llm::FinishReason::Stop,
|
||||
};
|
||||
assert!(SmartRoutingProvider::response_is_uncertain(&response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_confident_response_is_not_uncertain() {
|
||||
let response = CompletionResponse {
|
||||
content: "Yes.".to_string(),
|
||||
input_tokens: 10,
|
||||
output_tokens: 1,
|
||||
finish_reason: crate::llm::FinishReason::Stop,
|
||||
};
|
||||
assert!(!SmartRoutingProvider::response_is_uncertain(&response));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn confident_response_is_not_uncertain() {
|
||||
let response = CompletionResponse {
|
||||
content: "The answer is 42. This is a well-known constant from the Hitchhiker's Guide."
|
||||
.to_string(),
|
||||
input_tokens: 10,
|
||||
output_tokens: 20,
|
||||
finish_reason: crate::llm::FinishReason::Stop,
|
||||
};
|
||||
assert!(!SmartRoutingProvider::response_is_uncertain(&response));
|
||||
}
|
||||
|
||||
// -- Routing tests --
|
||||
|
||||
fn make_request(content: &str) -> CompletionRequest {
|
||||
CompletionRequest::new(vec![ChatMessage::user(content)])
|
||||
}
|
||||
|
||||
fn make_tool_request() -> ToolCompletionRequest {
|
||||
ToolCompletionRequest::new(vec![ChatMessage::user("implement a search")], vec![])
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn simple_task_routes_to_cheap() {
|
||||
let primary = Arc::new(StubLlm::new("primary-response").with_model_name("primary"));
|
||||
let cheap = Arc::new(StubLlm::new("cheap-response").with_model_name("cheap"));
|
||||
|
||||
let router = SmartRoutingProvider::new(
|
||||
primary.clone(),
|
||||
cheap.clone(),
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: false,
|
||||
..default_config()
|
||||
},
|
||||
);
|
||||
|
||||
let resp = router.complete(make_request("hello")).await.unwrap();
|
||||
assert_eq!(resp.content, "cheap-response");
|
||||
assert_eq!(cheap.calls(), 1);
|
||||
assert_eq!(primary.calls(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn complex_task_routes_to_primary() {
|
||||
let primary = Arc::new(StubLlm::new("primary-response").with_model_name("primary"));
|
||||
let cheap = Arc::new(StubLlm::new("cheap-response").with_model_name("cheap"));
|
||||
|
||||
let router = SmartRoutingProvider::new(primary.clone(), cheap.clone(), default_config());
|
||||
|
||||
let resp = router
|
||||
.complete(make_request("implement a binary search"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.content, "primary-response");
|
||||
assert_eq!(primary.calls(), 1);
|
||||
assert_eq!(cheap.calls(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tool_use_always_routes_to_primary() {
|
||||
let primary = Arc::new(StubLlm::new("primary-response").with_model_name("primary"));
|
||||
let cheap = Arc::new(StubLlm::new("cheap-response").with_model_name("cheap"));
|
||||
|
||||
let router = SmartRoutingProvider::new(primary.clone(), cheap.clone(), default_config());
|
||||
|
||||
let resp = router
|
||||
.complete_with_tools(make_tool_request())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(resp.content, Some("primary-response".to_string()));
|
||||
assert_eq!(primary.calls(), 1);
|
||||
assert_eq!(cheap.calls(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stats_increment_correctly() {
|
||||
let primary = Arc::new(StubLlm::new("primary").with_model_name("primary"));
|
||||
let cheap = Arc::new(StubLlm::new("cheap").with_model_name("cheap"));
|
||||
|
||||
let router = SmartRoutingProvider::new(
|
||||
primary,
|
||||
cheap,
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: false,
|
||||
..default_config()
|
||||
},
|
||||
);
|
||||
|
||||
// Simple -> cheap
|
||||
router.complete(make_request("hello")).await.unwrap();
|
||||
// Complex -> primary
|
||||
router
|
||||
.complete(make_request("implement a search"))
|
||||
.await
|
||||
.unwrap();
|
||||
// Tool use -> primary
|
||||
router
|
||||
.complete_with_tools(make_tool_request())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let stats = router.stats();
|
||||
assert_eq!(stats.total_requests, 3);
|
||||
assert_eq!(stats.cheap_requests, 1);
|
||||
assert_eq!(stats.primary_requests, 2);
|
||||
assert_eq!(stats.cascade_escalations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cascade_escalates_on_uncertain_response() {
|
||||
// Cheap model returns an uncertain response
|
||||
let primary = Arc::new(StubLlm::new("primary-response").with_model_name("primary"));
|
||||
let cheap = Arc::new(StubLlm::new("I'm not sure about that.").with_model_name("cheap"));
|
||||
|
||||
let router = SmartRoutingProvider::new(
|
||||
primary.clone(),
|
||||
cheap.clone(),
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: true,
|
||||
..default_config()
|
||||
},
|
||||
);
|
||||
|
||||
// A moderate task (no simple/complex keywords, medium length)
|
||||
let resp = router
|
||||
.complete(make_request(
|
||||
"Tell me about the weather patterns in the Pacific Ocean during summer months",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Should have escalated to primary
|
||||
assert_eq!(resp.content, "primary-response");
|
||||
assert_eq!(cheap.calls(), 1);
|
||||
assert_eq!(primary.calls(), 1);
|
||||
|
||||
let stats = router.stats();
|
||||
assert_eq!(stats.cascade_escalations, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cascade_does_not_escalate_on_confident_response() {
|
||||
let primary = Arc::new(StubLlm::new("primary-response").with_model_name("primary"));
|
||||
let cheap = Arc::new(
|
||||
StubLlm::new(
|
||||
"The Pacific Ocean weather patterns during summer are characterized by trade winds.",
|
||||
)
|
||||
.with_model_name("cheap"),
|
||||
);
|
||||
|
||||
let router = SmartRoutingProvider::new(
|
||||
primary.clone(),
|
||||
cheap.clone(),
|
||||
SmartRoutingConfig {
|
||||
cascade_enabled: true,
|
||||
..default_config()
|
||||
},
|
||||
);
|
||||
|
||||
let resp = router
|
||||
.complete(make_request(
|
||||
"Tell me about the weather patterns in the Pacific Ocean during summer months",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Should NOT have escalated
|
||||
assert!(resp.content.contains("Pacific Ocean"));
|
||||
assert_eq!(cheap.calls(), 1);
|
||||
assert_eq!(primary.calls(), 0);
|
||||
|
||||
let stats = router.stats();
|
||||
assert_eq!(stats.cascade_escalations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn model_name_returns_primary() {
|
||||
let primary = Arc::new(StubLlm::new("ok").with_model_name("sonnet"));
|
||||
let cheap = Arc::new(StubLlm::new("ok").with_model_name("haiku"));
|
||||
|
||||
let router = SmartRoutingProvider::new(primary, cheap, default_config());
|
||||
assert_eq!(router.model_name(), "sonnet");
|
||||
assert_eq!(router.active_model_name(), "sonnet");
|
||||
}
|
||||
}
|
||||
+128
-150
@@ -24,12 +24,7 @@ use ironclaw::{
|
||||
context::ContextManager,
|
||||
extensions::ExtensionManager,
|
||||
hooks::{HookRegistry, bootstrap_hooks},
|
||||
llm::{
|
||||
CachedProvider, CircuitBreakerConfig, CircuitBreakerProvider, CooldownConfig,
|
||||
FailoverProvider, LlmProvider, ResponseCacheConfig, RetryConfig, RetryProvider,
|
||||
SessionConfig, create_cheap_llm_provider, create_llm_provider,
|
||||
create_llm_provider_with_config, create_session_manager,
|
||||
},
|
||||
llm::{SessionConfig, build_provider_chain, create_session_manager},
|
||||
orchestrator::{
|
||||
ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore,
|
||||
api::OrchestratorState,
|
||||
@@ -487,9 +482,13 @@ async fn main() -> anyhow::Result<()> {
|
||||
session.attach_store(Arc::clone(db), "default").await;
|
||||
|
||||
// Mark any jobs left in "running" or "creating" state as "interrupted".
|
||||
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
// Fire-and-forget housekeeping — no need to block startup.
|
||||
let db_cleanup = Arc::clone(db);
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = db_cleanup.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Create secrets store early: needed for injecting LLM API keys from encrypted
|
||||
@@ -622,110 +621,22 @@ async fn main() -> anyhow::Result<()> {
|
||||
None
|
||||
};
|
||||
|
||||
// Initialize LLM provider (clone session so we can reuse it for embeddings)
|
||||
let llm = create_llm_provider(&config.llm, session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
|
||||
// Wrap each provider with RetryProvider for automatic retries on transient errors.
|
||||
// RetryProvider sits inside FailoverProvider so each provider in the failover chain
|
||||
// gets its own retry attempts before the failover moves to the next provider.
|
||||
let retry_config = RetryConfig {
|
||||
max_retries: config.llm.nearai.max_retries,
|
||||
};
|
||||
let llm: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
tracing::info!(
|
||||
max_retries = retry_config.max_retries,
|
||||
"LLM retry wrapper enabled"
|
||||
);
|
||||
Arc::new(RetryProvider::new(llm, retry_config.clone()))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in failover if a fallback model is configured
|
||||
let llm: Arc<dyn LlmProvider> =
|
||||
if let Some(fallback_model) = config.llm.nearai.fallback_model.as_ref() {
|
||||
if fallback_model == &config.llm.nearai.model {
|
||||
tracing::warn!(
|
||||
"fallback_model is the same as primary model, failover may not be effective"
|
||||
);
|
||||
}
|
||||
let mut fallback_config = config.llm.nearai.clone();
|
||||
fallback_config.model = fallback_model.clone();
|
||||
let fallback = create_llm_provider_with_config(&fallback_config, session.clone())?;
|
||||
tracing::info!(
|
||||
primary = %llm.model_name(),
|
||||
fallback = %fallback.model_name(),
|
||||
"LLM failover enabled"
|
||||
);
|
||||
// Wrap fallback with retry too
|
||||
let fallback: Arc<dyn LlmProvider> = if retry_config.max_retries > 0 {
|
||||
Arc::new(RetryProvider::new(fallback, retry_config.clone()))
|
||||
} else {
|
||||
fallback
|
||||
};
|
||||
let cooldown_config = CooldownConfig {
|
||||
cooldown_duration: std::time::Duration::from_secs(
|
||||
config.llm.nearai.failover_cooldown_secs,
|
||||
),
|
||||
failure_threshold: config.llm.nearai.failover_cooldown_threshold,
|
||||
};
|
||||
Arc::new(FailoverProvider::with_cooldown(
|
||||
vec![llm, fallback],
|
||||
cooldown_config,
|
||||
)?)
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in circuit breaker if configured
|
||||
let llm: Arc<dyn LlmProvider> =
|
||||
if let Some(threshold) = config.llm.nearai.circuit_breaker_threshold {
|
||||
let cb_config = CircuitBreakerConfig {
|
||||
failure_threshold: threshold,
|
||||
recovery_timeout: std::time::Duration::from_secs(
|
||||
config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
),
|
||||
..CircuitBreakerConfig::default()
|
||||
};
|
||||
tracing::info!(
|
||||
threshold,
|
||||
recovery_secs = config.llm.nearai.circuit_breaker_recovery_secs,
|
||||
"LLM circuit breaker enabled"
|
||||
);
|
||||
Arc::new(CircuitBreakerProvider::new(llm, cb_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Wrap in response cache if configured
|
||||
let llm: Arc<dyn LlmProvider> = if config.llm.nearai.response_cache_enabled {
|
||||
let rc_config = ResponseCacheConfig {
|
||||
ttl: std::time::Duration::from_secs(config.llm.nearai.response_cache_ttl_secs),
|
||||
max_entries: config.llm.nearai.response_cache_max_entries,
|
||||
};
|
||||
tracing::info!(
|
||||
ttl_secs = config.llm.nearai.response_cache_ttl_secs,
|
||||
max_entries = config.llm.nearai.response_cache_max_entries,
|
||||
"LLM response cache enabled"
|
||||
);
|
||||
Arc::new(CachedProvider::new(llm, rc_config))
|
||||
} else {
|
||||
llm
|
||||
};
|
||||
|
||||
// Initialize cheap LLM provider for lightweight tasks (heartbeat, evaluation)
|
||||
let cheap_llm = create_cheap_llm_provider(&config.llm, session.clone())?;
|
||||
if let Some(ref cheap) = cheap_llm {
|
||||
tracing::info!("Cheap LLM provider initialized: {}", cheap.model_name());
|
||||
}
|
||||
// Build the full LLM provider chain (retry → smart routing → failover → circuit breaker → cache)
|
||||
let (llm, cheap_llm) = build_provider_chain(&config.llm, session.clone())?;
|
||||
|
||||
// Initialize safety layer
|
||||
let safety = Arc::new(SafetyLayer::new(&config.safety));
|
||||
tracing::info!("Safety layer initialized");
|
||||
|
||||
// Initialize tool registry
|
||||
let tools = Arc::new(ToolRegistry::new());
|
||||
// Initialize tool registry with credential injection support
|
||||
let credential_registry = Arc::new(ironclaw::tools::wasm::SharedCredentialRegistry::new());
|
||||
let tools = if let Some(ref ss) = secrets_store {
|
||||
Arc::new(
|
||||
ToolRegistry::new().with_credentials(Arc::clone(&credential_registry), Arc::clone(ss)),
|
||||
)
|
||||
} else {
|
||||
Arc::new(ToolRegistry::new())
|
||||
};
|
||||
tools.register_builtin_tools();
|
||||
|
||||
// Create embeddings provider if configured
|
||||
@@ -793,14 +704,20 @@ async fn main() -> anyhow::Result<()> {
|
||||
);
|
||||
}
|
||||
|
||||
// Register memory tools if database is available
|
||||
if let Some(ref db) = db {
|
||||
let mut workspace = Workspace::new_with_db("default", Arc::clone(db));
|
||||
// Create workspace once, reused for memory tools and agent
|
||||
let workspace: Option<Arc<Workspace>> = if let Some(ref db) = db {
|
||||
let mut ws = Workspace::new_with_db("default", Arc::clone(db));
|
||||
if let Some(ref emb) = embeddings {
|
||||
workspace = workspace.with_embeddings(emb.clone());
|
||||
ws = ws.with_embeddings(emb.clone());
|
||||
}
|
||||
let workspace = Arc::new(workspace);
|
||||
tools.register_memory_tools(workspace);
|
||||
Some(Arc::new(ws))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Register memory tools if workspace is available
|
||||
if let Some(ref ws) = workspace {
|
||||
tools.register_memory_tools(Arc::clone(ws));
|
||||
}
|
||||
|
||||
// Register builder tool if enabled.
|
||||
@@ -992,11 +909,43 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let (dev_loaded_tool_names, _) = tokio::join!(wasm_tools_future, mcp_servers_future);
|
||||
|
||||
// Create extension manager for in-chat discovery/install/auth/activate
|
||||
let extension_manager = if let Some(ref secrets) = secrets_store {
|
||||
// Load registry catalog entries for in-chat extension discovery
|
||||
let catalog_entries = match ironclaw::registry::RegistryCatalog::load_or_embedded() {
|
||||
Ok(catalog) => {
|
||||
let entries: Vec<ironclaw::extensions::RegistryEntry> = catalog
|
||||
.all()
|
||||
.iter()
|
||||
.map(|m| m.to_registry_entry())
|
||||
.collect();
|
||||
tracing::info!(
|
||||
count = entries.len(),
|
||||
"Loaded registry catalog entries for extension discovery"
|
||||
);
|
||||
entries
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to load registry catalog: {}", e);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
// Create extension manager for in-chat discovery/install/auth/activate.
|
||||
// If no persistent secrets store is available, use an ephemeral in-memory store
|
||||
// so that listing/installing/activating extensions still works (auth won't persist).
|
||||
let ext_secrets: Arc<dyn SecretsStore + Send + Sync> = if let Some(ref s) = secrets_store {
|
||||
Arc::clone(s)
|
||||
} else {
|
||||
use ironclaw::secrets::{InMemorySecretsStore, SecretsCrypto};
|
||||
let ephemeral_key =
|
||||
secrecy::SecretString::from(ironclaw::secrets::keychain::generate_master_key_hex());
|
||||
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
|
||||
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
|
||||
Arc::new(InMemorySecretsStore::new(crypto))
|
||||
};
|
||||
let extension_manager = {
|
||||
let manager = Arc::new(ExtensionManager::new(
|
||||
Arc::clone(&mcp_session_manager),
|
||||
Arc::clone(secrets),
|
||||
ext_secrets,
|
||||
Arc::clone(&tools),
|
||||
Some(Arc::clone(&hooks)),
|
||||
wasm_tool_runtime.clone(),
|
||||
@@ -1005,16 +954,11 @@ async fn main() -> anyhow::Result<()> {
|
||||
config.tunnel.public_url.clone(),
|
||||
"default".to_string(),
|
||||
db.clone(),
|
||||
catalog_entries.clone(),
|
||||
));
|
||||
tools.register_extension_tools(Arc::clone(&manager));
|
||||
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
||||
Some(manager)
|
||||
} else {
|
||||
tracing::debug!(
|
||||
"Extension manager not available (no secrets store). \
|
||||
Extension tools won't be registered."
|
||||
);
|
||||
None
|
||||
};
|
||||
|
||||
// Set up orchestrator for sandboxed job execution
|
||||
@@ -1111,6 +1055,32 @@ async fn main() -> anyhow::Result<()> {
|
||||
// Collect webhook route fragments; a single WebhookServer hosts them all.
|
||||
let mut webhook_routes: Vec<axum::Router> = Vec::new();
|
||||
|
||||
// Build transcription middleware if enabled.
|
||||
let transcription_middleware: Option<Arc<ironclaw::transcription::TranscriptionMiddleware>> =
|
||||
if config.transcription.enabled {
|
||||
if let Some(ref api_key) = config.transcription.openai_api_key {
|
||||
let provider = Arc::new(ironclaw::transcription::openai::OpenAiWhisper::new(
|
||||
api_key.clone(),
|
||||
config.transcription.model.clone(),
|
||||
));
|
||||
let middleware = ironclaw::transcription::TranscriptionMiddleware::new(
|
||||
provider,
|
||||
config.transcription.language.clone(),
|
||||
);
|
||||
tracing::info!(
|
||||
provider = %config.transcription.provider,
|
||||
model = %config.transcription.model,
|
||||
"Audio transcription enabled"
|
||||
);
|
||||
Some(Arc::new(middleware))
|
||||
} else {
|
||||
tracing::warn!("Transcription enabled but OPENAI_API_KEY not set, disabling");
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Load WASM channels and register their webhook routes.
|
||||
if config.channels.wasm_channels_enabled && config.channels.wasm_channels_dir.exists() {
|
||||
match WasmChannelRuntime::new(WasmChannelRuntimeConfig::default()) {
|
||||
@@ -1155,7 +1125,11 @@ async fn main() -> anyhow::Result<()> {
|
||||
require_secret: webhook_secret.is_some(),
|
||||
}];
|
||||
|
||||
let channel_arc = Arc::new(loaded.channel);
|
||||
let mut channel = loaded.channel;
|
||||
if let Some(ref mw) = transcription_middleware {
|
||||
channel.set_transcription_middleware(Arc::clone(mw));
|
||||
}
|
||||
let channel_arc = Arc::new(channel);
|
||||
|
||||
{
|
||||
let mut config_updates = std::collections::HashMap::new();
|
||||
@@ -1251,6 +1225,12 @@ async fn main() -> anyhow::Result<()> {
|
||||
));
|
||||
}
|
||||
|
||||
// Tell extension manager which channels are actually loaded
|
||||
if let Some(ref em) = extension_manager {
|
||||
em.set_active_channels(loaded_wasm_channel_names.clone())
|
||||
.await;
|
||||
}
|
||||
|
||||
for (path, err) in &results.errors {
|
||||
tracing::warn!(
|
||||
"Failed to load WASM channel {}: {}",
|
||||
@@ -1315,17 +1295,6 @@ async fn main() -> anyhow::Result<()> {
|
||||
None
|
||||
};
|
||||
|
||||
// Create workspace for agent (shared with memory tools)
|
||||
let workspace = if let Some(ref db_ref) = db {
|
||||
let mut ws = Workspace::new_with_db("default", Arc::clone(db_ref));
|
||||
if let Some(ref emb) = embeddings {
|
||||
ws = ws.with_embeddings(emb.clone());
|
||||
}
|
||||
Some(Arc::new(ws))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Seed workspace with core identity files on first boot
|
||||
if let Some(ref ws) = workspace {
|
||||
match ws.seed_if_empty().await {
|
||||
@@ -1336,17 +1305,20 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
// Backfill embeddings if we just enabled the provider
|
||||
// Backfill embeddings in background (fire-and-forget housekeeping)
|
||||
if let (Some(ws), Some(_)) = (&workspace, &embeddings) {
|
||||
match ws.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
let ws_bg = Arc::clone(ws);
|
||||
tokio::spawn(async move {
|
||||
match ws_bg.backfill_embeddings().await {
|
||||
Ok(count) if count > 0 => {
|
||||
tracing::info!("Backfilled embeddings for {} chunks", count);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to backfill embeddings: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Create context manager (shared between job tools and agent)
|
||||
@@ -1410,6 +1382,14 @@ async fn main() -> anyhow::Result<()> {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
// Create cost guard early so gateway can reference it.
|
||||
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
|
||||
ironclaw::agent::cost_guard::CostGuardConfig {
|
||||
max_cost_per_day_cents: config.agent.max_cost_per_day_cents,
|
||||
max_actions_per_hour: config.agent.max_actions_per_hour,
|
||||
},
|
||||
));
|
||||
|
||||
// Add web gateway channel if configured
|
||||
let mut gateway_url: Option<String> = None;
|
||||
if let Some(ref gw_config) = config.channels.gateway {
|
||||
@@ -1424,6 +1404,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
if let Some(ref ext_mgr) = extension_manager {
|
||||
gw = gw.with_extension_manager(Arc::clone(ext_mgr));
|
||||
}
|
||||
if !catalog_entries.is_empty() {
|
||||
gw = gw.with_registry_entries(catalog_entries.clone());
|
||||
}
|
||||
if let Some(ref d) = db {
|
||||
gw = gw.with_store(Arc::clone(d));
|
||||
}
|
||||
@@ -1436,6 +1419,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
if let Some(ref sc) = skill_catalog {
|
||||
gw = gw.with_skill_catalog(Arc::clone(sc));
|
||||
}
|
||||
gw = gw.with_cost_guard(Arc::clone(&cost_guard));
|
||||
if config.sandbox.enabled {
|
||||
gw = gw.with_prompt_queue(Arc::clone(&prompt_queue));
|
||||
|
||||
@@ -1470,12 +1454,6 @@ async fn main() -> anyhow::Result<()> {
|
||||
let boot_cheap_model = cheap_llm.as_ref().map(|c| c.model_name().to_string());
|
||||
|
||||
// Create and run the agent
|
||||
let cost_guard = Arc::new(ironclaw::agent::cost_guard::CostGuard::new(
|
||||
ironclaw::agent::cost_guard::CostGuardConfig {
|
||||
max_cost_per_day_cents: config.agent.max_cost_per_day_cents,
|
||||
max_actions_per_hour: config.agent.max_actions_per_hour,
|
||||
},
|
||||
));
|
||||
let deps = AgentDeps {
|
||||
store: db,
|
||||
llm,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::registry::embedded;
|
||||
use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest, ManifestKind};
|
||||
|
||||
/// Error type for registry operations.
|
||||
@@ -64,6 +65,69 @@ pub struct RegistryCatalog {
|
||||
}
|
||||
|
||||
impl RegistryCatalog {
|
||||
/// Find the `registry/` directory by searching relative to cwd, the executable,
|
||||
/// and `CARGO_MANIFEST_DIR`. Returns `None` if the directory cannot be found
|
||||
/// (non-fatal at startup).
|
||||
pub fn find_dir() -> Option<PathBuf> {
|
||||
// Try relative to current directory (for dev usage)
|
||||
if let Ok(cwd) = std::env::current_dir() {
|
||||
let candidate = cwd.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Some(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
// Try relative to executable (covers installed binary, target/debug/, target/release/)
|
||||
if let Ok(exe) = std::env::current_exe()
|
||||
&& let Some(parent) = exe.parent()
|
||||
{
|
||||
// Walk up to 3 levels: exe dir, parent (target/release -> target), grandparent (-> repo root)
|
||||
let mut dir = Some(parent);
|
||||
for _ in 0..3 {
|
||||
if let Some(d) = dir {
|
||||
let candidate = d.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Some(candidate);
|
||||
}
|
||||
dir = d.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
|
||||
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
|
||||
let candidate = manifest_dir.join("registry");
|
||||
if candidate.is_dir() {
|
||||
return Some(candidate);
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Try to load from disk; if `registry/` cannot be found, fall back to
|
||||
/// manifests embedded into the binary at compile time.
|
||||
pub fn load_or_embedded() -> Result<Self, RegistryError> {
|
||||
if let Some(dir) = Self::find_dir() {
|
||||
return Self::load(&dir);
|
||||
}
|
||||
|
||||
// Fall back to embedded catalog
|
||||
let manifests = embedded::load_embedded();
|
||||
let bundles = embedded::load_embedded_bundles();
|
||||
|
||||
tracing::info!(
|
||||
"Loaded embedded registry catalog ({} extensions, {} bundles)",
|
||||
manifests.len(),
|
||||
bundles.len()
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
manifests,
|
||||
bundles,
|
||||
root: PathBuf::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Load the catalog from a registry directory.
|
||||
///
|
||||
/// Expects the structure:
|
||||
@@ -577,4 +641,12 @@ mod tests {
|
||||
let result = RegistryCatalog::load(Path::new("/nonexistent/path"));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_or_embedded_succeeds() {
|
||||
// Should always succeed: either finds registry/ on disk or falls back to embedded
|
||||
let catalog = RegistryCatalog::load_or_embedded().unwrap();
|
||||
// At minimum, the embedded catalog from the repo should have entries
|
||||
assert!(!catalog.all().is_empty() || !catalog.bundle_names().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
//! Embedded registry catalog compiled into the binary at build time.
|
||||
//!
|
||||
//! When IronClaw is distributed as a pre-built binary without a source tree,
|
||||
//! the `registry/` directory is unavailable. This module provides the same
|
||||
//! manifest data via `include_str!` from a JSON blob generated by `build.rs`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest};
|
||||
|
||||
/// Raw JSON generated by build.rs from `registry/{tools,channels}/*.json` and `_bundles.json`.
|
||||
const EMBEDDED_CATALOG: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_catalog.json"));
|
||||
|
||||
/// Intermediate deserialization shape matching the build.rs output.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EmbeddedCatalogRaw {
|
||||
#[serde(default)]
|
||||
tools: Vec<ExtensionManifest>,
|
||||
#[serde(default)]
|
||||
channels: Vec<ExtensionManifest>,
|
||||
#[serde(default)]
|
||||
bundles: BundlesFile,
|
||||
}
|
||||
|
||||
/// Parsed catalog cached across calls.
|
||||
struct ParsedCatalog {
|
||||
manifests: HashMap<String, ExtensionManifest>,
|
||||
bundles: HashMap<String, BundleDefinition>,
|
||||
}
|
||||
|
||||
fn parsed_catalog() -> &'static ParsedCatalog {
|
||||
static CACHE: OnceLock<ParsedCatalog> = OnceLock::new();
|
||||
CACHE.get_or_init(|| {
|
||||
let raw: EmbeddedCatalogRaw = match serde_json::from_str(EMBEDDED_CATALOG) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to parse embedded catalog: {}", e);
|
||||
return ParsedCatalog {
|
||||
manifests: HashMap::new(),
|
||||
bundles: HashMap::new(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
let mut manifests = HashMap::new();
|
||||
for m in raw.tools {
|
||||
let key = format!("tools/{}", m.name);
|
||||
manifests.insert(key, m);
|
||||
}
|
||||
for m in raw.channels {
|
||||
let key = format!("channels/{}", m.name);
|
||||
manifests.insert(key, m);
|
||||
}
|
||||
|
||||
ParsedCatalog {
|
||||
manifests,
|
||||
bundles: raw.bundles.bundles,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Load all embedded extension manifests, keyed by `"tools/<name>"` or `"channels/<name>"`.
|
||||
pub fn load_embedded() -> HashMap<String, ExtensionManifest> {
|
||||
parsed_catalog().manifests.clone()
|
||||
}
|
||||
|
||||
/// Load embedded bundle definitions.
|
||||
pub fn load_embedded_bundles() -> HashMap<String, BundleDefinition> {
|
||||
parsed_catalog().bundles.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_load_embedded_parses() {
|
||||
let manifests = load_embedded();
|
||||
// Should have at least the manifests from registry/ if built from the repo
|
||||
// (empty is also valid for minimal builds without registry/)
|
||||
assert!(
|
||||
manifests.is_empty() || manifests.contains_key("tools/github"),
|
||||
"Expected either empty catalog or github tool, got {} entries",
|
||||
manifests.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_embedded_bundles_parses() {
|
||||
let bundles = load_embedded_bundles();
|
||||
assert!(
|
||||
bundles.is_empty() || bundles.contains_key("default"),
|
||||
"Expected either empty bundles or 'default' bundle"
|
||||
);
|
||||
}
|
||||
}
|
||||
+325
-61
@@ -137,6 +137,10 @@ impl RegistryInstaller {
|
||||
}
|
||||
|
||||
/// Download and install a pre-built artifact.
|
||||
///
|
||||
/// Supports two formats:
|
||||
/// - **tar.gz bundle**: Contains `{name}.wasm` + `{name}.capabilities.json`
|
||||
/// - **bare .wasm file**: Just the WASM binary (capabilities fetched separately if available)
|
||||
pub async fn install_from_artifact(
|
||||
&self,
|
||||
manifest: &ExtensionManifest,
|
||||
@@ -156,13 +160,6 @@ impl RegistryInstaller {
|
||||
))
|
||||
})?;
|
||||
|
||||
let expected_sha = artifact.sha256.as_ref().ok_or_else(|| {
|
||||
RegistryError::ExtensionNotFound(format!(
|
||||
"No SHA256 hash for '{}'. Cannot verify download.",
|
||||
manifest.name
|
||||
))
|
||||
})?;
|
||||
|
||||
let target_dir = match manifest.kind {
|
||||
ManifestKind::Tool => &self.tools_dir,
|
||||
ManifestKind::Channel => &self.channels_dir,
|
||||
@@ -186,75 +183,90 @@ impl RegistryInstaller {
|
||||
"Downloading {} '{}'...",
|
||||
manifest.kind, manifest.display_name
|
||||
);
|
||||
let response = reqwest::get(url)
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!("request failed: {}", e),
|
||||
})?;
|
||||
let bytes = download_artifact(url).await?;
|
||||
|
||||
let response = response
|
||||
.error_for_status()
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
let bytes = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!("failed to read body: {}", e),
|
||||
})?;
|
||||
|
||||
// Verify SHA256
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bytes);
|
||||
let actual_sha = format!("{:x}", hasher.finalize());
|
||||
|
||||
if actual_sha != *expected_sha {
|
||||
return Err(RegistryError::DownloadFailed {
|
||||
url: url.clone(),
|
||||
reason: format!(
|
||||
"SHA256 mismatch: expected {}, got {}",
|
||||
expected_sha, actual_sha
|
||||
),
|
||||
});
|
||||
// Verify SHA256 if provided, warn otherwise
|
||||
if let Some(expected_sha) = &artifact.sha256 {
|
||||
verify_sha256(&bytes, expected_sha, url)?;
|
||||
} else {
|
||||
println!(
|
||||
"WARNING: No SHA256 checksum for '{}'; download is not cryptographically verified.",
|
||||
manifest.name
|
||||
);
|
||||
}
|
||||
|
||||
// Write file
|
||||
fs::write(&target_wasm, &bytes)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
|
||||
// Copy capabilities from source dir (still needed even for pre-built artifacts).
|
||||
// NOTE: This requires the source tree to be present. When pre-built artifact
|
||||
// distribution is implemented, capabilities should be bundled with the artifact
|
||||
// or fetched from a separate URL.
|
||||
let caps_source = self
|
||||
.repo_root
|
||||
.join(&manifest.source.dir)
|
||||
.join(&manifest.source.capabilities);
|
||||
let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name));
|
||||
let has_capabilities = if caps_source.exists() {
|
||||
fs::copy(&caps_source, &target_caps)
|
||||
|
||||
// Detect format and extract
|
||||
let has_capabilities = if is_gzip(&bytes) {
|
||||
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
|
||||
let extracted =
|
||||
extract_tar_gz(&bytes, &manifest.name, &target_wasm, &target_caps, url)?;
|
||||
extracted.has_capabilities
|
||||
} else {
|
||||
// Bare WASM file
|
||||
fs::write(&target_wasm, &bytes)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
|
||||
// Try to get capabilities from:
|
||||
// 1. Separate capabilities_url in the artifact
|
||||
// 2. Source tree (legacy, requires repo)
|
||||
if let Some(ref caps_url) = artifact.capabilities_url {
|
||||
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
|
||||
match download_artifact(caps_url).await {
|
||||
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
|
||||
fs::write(&target_caps, &caps_bytes)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
true
|
||||
}
|
||||
Ok(caps_bytes) => {
|
||||
tracing::warn!(
|
||||
"Capabilities file too large ({} bytes, max {}), skipping",
|
||||
caps_bytes.len(),
|
||||
MAX_CAPS_SIZE
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to download capabilities from {}: {}", caps_url, e);
|
||||
false
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Legacy fallback: try source tree
|
||||
let caps_source = self
|
||||
.repo_root
|
||||
.join(&manifest.source.dir)
|
||||
.join(&manifest.source.capabilities);
|
||||
if caps_source.exists() {
|
||||
fs::copy(&caps_source, &target_caps)
|
||||
.await
|
||||
.map_err(RegistryError::Io)?;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
println!(" Installed to {}", target_wasm.display());
|
||||
|
||||
let mut warnings = Vec::new();
|
||||
if !has_capabilities {
|
||||
warnings.push(format!(
|
||||
"No capabilities file found for '{}'. Auth and hooks may not work.",
|
||||
manifest.name
|
||||
));
|
||||
}
|
||||
|
||||
Ok(InstallOutcome {
|
||||
name: manifest.name.clone(),
|
||||
kind: manifest.kind,
|
||||
wasm_path: target_wasm,
|
||||
has_capabilities,
|
||||
warnings: Vec::new(),
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -399,6 +411,159 @@ async fn build_wasm_component(source_dir: &Path, crate_name: &str) -> anyhow::Re
|
||||
)
|
||||
}
|
||||
|
||||
/// Download an artifact from a URL.
|
||||
async fn download_artifact(url: &str) -> Result<bytes::Bytes, RegistryError> {
|
||||
let response = reqwest::get(url)
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("request failed: {}", e),
|
||||
})?;
|
||||
|
||||
let response = response
|
||||
.error_for_status()
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: e.to_string(),
|
||||
})?;
|
||||
|
||||
response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("failed to read body: {}", e),
|
||||
})
|
||||
}
|
||||
|
||||
/// Verify SHA256 of downloaded bytes.
|
||||
fn verify_sha256(bytes: &[u8], expected: &str, url: &str) -> Result<(), RegistryError> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(bytes);
|
||||
let actual = format!("{:x}", hasher.finalize());
|
||||
|
||||
if actual != expected {
|
||||
return Err(RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("SHA256 mismatch: expected {}, got {}", expected, actual),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if bytes start with gzip magic number (0x1f 0x8b).
|
||||
fn is_gzip(bytes: &[u8]) -> bool {
|
||||
bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b
|
||||
}
|
||||
|
||||
/// Result of extracting a tar.gz bundle.
|
||||
struct ExtractResult {
|
||||
has_capabilities: bool,
|
||||
}
|
||||
|
||||
/// Extract a tar.gz archive, looking for `{name}.wasm` and `{name}.capabilities.json`.
|
||||
fn extract_tar_gz(
|
||||
bytes: &[u8],
|
||||
name: &str,
|
||||
target_wasm: &Path,
|
||||
target_caps: &Path,
|
||||
url: &str,
|
||||
) -> Result<ExtractResult, RegistryError> {
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
use std::io::Read as _;
|
||||
|
||||
let decoder = GzDecoder::new(bytes);
|
||||
let mut archive = Archive::new(decoder);
|
||||
// Defense-in-depth: do not preserve permissions or extended attributes
|
||||
archive.set_preserve_permissions(false);
|
||||
#[cfg(any(unix, target_os = "redox"))]
|
||||
archive.set_unpack_xattrs(false);
|
||||
|
||||
// 100 MB cap on decompressed entry size to prevent decompression bombs
|
||||
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
|
||||
|
||||
let wasm_filename = format!("{}.wasm", name);
|
||||
let caps_filename = format!("{}.capabilities.json", name);
|
||||
let mut found_wasm = false;
|
||||
let mut found_caps = false;
|
||||
|
||||
let entries = archive
|
||||
.entries()
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("failed to read tar.gz entries: {}", e),
|
||||
})?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("failed to read tar.gz entry: {}", e),
|
||||
})?;
|
||||
|
||||
if entry.size() > MAX_ENTRY_SIZE {
|
||||
return Err(RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!(
|
||||
"archive entry too large ({} bytes, max {} bytes)",
|
||||
entry.size(),
|
||||
MAX_ENTRY_SIZE
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
let entry_path = entry
|
||||
.path()
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("invalid path in tar.gz: {}", e),
|
||||
})?
|
||||
.to_path_buf();
|
||||
|
||||
// Match by filename (ignoring any directory prefix in the archive)
|
||||
let filename = entry_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("");
|
||||
|
||||
if filename == wasm_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("failed to read {} from archive: {}", wasm_filename, e),
|
||||
})?;
|
||||
std::fs::write(target_wasm, &data).map_err(RegistryError::Io)?;
|
||||
found_wasm = true;
|
||||
} else if filename == caps_filename {
|
||||
let mut data = Vec::with_capacity(entry.size() as usize);
|
||||
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
|
||||
.map_err(|e| RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!("failed to read {} from archive: {}", caps_filename, e),
|
||||
})?;
|
||||
std::fs::write(target_caps, &data).map_err(RegistryError::Io)?;
|
||||
found_caps = true;
|
||||
}
|
||||
}
|
||||
|
||||
if !found_wasm {
|
||||
return Err(RegistryError::DownloadFailed {
|
||||
url: url.to_string(),
|
||||
reason: format!(
|
||||
"tar.gz archive does not contain '{}'. Archive may be malformed.",
|
||||
wasm_filename
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(ExtractResult {
|
||||
has_capabilities: found_caps,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -412,4 +577,103 @@ mod tests {
|
||||
);
|
||||
assert_eq!(installer.repo_root, PathBuf::from("/repo"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_gzip() {
|
||||
assert!(is_gzip(&[0x1f, 0x8b, 0x08]));
|
||||
assert!(!is_gzip(&[0x00, 0x61, 0x73, 0x6d])); // WASM magic
|
||||
assert!(!is_gzip(&[0x1f])); // Too short
|
||||
assert!(!is_gzip(&[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_sha256_valid() {
|
||||
use sha2::{Digest, Sha256};
|
||||
let data = b"hello world";
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
let hash = format!("{:x}", hasher.finalize());
|
||||
assert!(verify_sha256(data, &hash, "test://url").is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_sha256_invalid() {
|
||||
assert!(verify_sha256(b"data", "0000", "test://url").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tar_gz() {
|
||||
use flate2::Compression;
|
||||
use flate2::write::GzEncoder;
|
||||
use tar::Builder;
|
||||
|
||||
// Create a tar.gz in memory with test.wasm and test.capabilities.json
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
{
|
||||
let mut builder = Builder::new(&mut encoder);
|
||||
|
||||
let wasm_data = b"\0asm\x01\x00\x00\x00";
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(wasm_data.len() as u64);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "test.wasm", &wasm_data[..])
|
||||
.unwrap();
|
||||
|
||||
let caps_data = br#"{"auth":null}"#;
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(caps_data.len() as u64);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "test.capabilities.json", &caps_data[..])
|
||||
.unwrap();
|
||||
|
||||
builder.finish().unwrap();
|
||||
}
|
||||
let gz_bytes = encoder.finish().unwrap();
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let wasm_path = tmp.path().join("test.wasm");
|
||||
let caps_path = tmp.path().join("test.capabilities.json");
|
||||
|
||||
let result =
|
||||
extract_tar_gz(&gz_bytes, "test", &wasm_path, &caps_path, "test://url").unwrap();
|
||||
|
||||
assert!(wasm_path.exists());
|
||||
assert!(caps_path.exists());
|
||||
assert!(result.has_capabilities);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extract_tar_gz_missing_wasm() {
|
||||
use flate2::Compression;
|
||||
use flate2::write::GzEncoder;
|
||||
use tar::Builder;
|
||||
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
{
|
||||
let mut builder = Builder::new(&mut encoder);
|
||||
|
||||
let data = b"not a wasm file";
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(data.len() as u64);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "wrong.wasm", &data[..])
|
||||
.unwrap();
|
||||
builder.finish().unwrap();
|
||||
}
|
||||
let gz_bytes = encoder.finish().unwrap();
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let result = extract_tar_gz(
|
||||
&gz_bytes,
|
||||
"test",
|
||||
&tmp.path().join("test.wasm"),
|
||||
&tmp.path().join("test.capabilities.json"),
|
||||
"test://url",
|
||||
);
|
||||
|
||||
assert!(result.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user